mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Seperate dbs
This commit is contained in:
mrfry
parent
728931d56e
commit
4e34267d44
9 changed files with 149 additions and 82 deletions
|
|
@ -57,11 +57,13 @@ const idvStatFile = 'stats/idvstats'
|
|||
const todosFile = 'data/todos.json'
|
||||
|
||||
// other constants
|
||||
const line = '====================================================' // lol
|
||||
const maxVeteranPwGetCount = 10
|
||||
const addPWPerDay = 3 // every x day a user can give a pw
|
||||
const maxPWCount = 6 // maximum pw give opportunities a user can have at once
|
||||
const addPWCount = 1 // how many pw gen opportunities to add each time
|
||||
const daysAfterUserGetsPWs = 5 // days after user gets pw-s
|
||||
const minimumAlowwedSessions = 1 // how many sessions are allowed for a user
|
||||
|
||||
// stuff gotten from server.js
|
||||
let userDB
|
||||
|
|
@ -79,19 +81,27 @@ function GetApp(): ModuleType {
|
|||
// files in public dirs
|
||||
const recivedFiles = publicDir + 'recivedfiles'
|
||||
const uloadFiles = publicDir + 'f'
|
||||
// FIXME: this to seperate file?
|
||||
const dataFiles: Array<DataFile> = [
|
||||
{
|
||||
path: `${publicDir}oldData.json`,
|
||||
name: 'oldData',
|
||||
shouldSave: (recData: RecievedData): boolean => {
|
||||
return recData.version.includes('2.0.')
|
||||
return recData.version.startsWith('2.0.')
|
||||
},
|
||||
},
|
||||
{
|
||||
path: `${publicDir}data.json`,
|
||||
name: 'newData',
|
||||
shouldSave: (recData: RecievedData): boolean => {
|
||||
return recData.version.includes('2.1.')
|
||||
return recData.version.startsWith('2.1.')
|
||||
},
|
||||
},
|
||||
{
|
||||
path: `${publicDir}fromwebsiteData.json`,
|
||||
name: 'fromwebsiteData',
|
||||
shouldSave: (recData: RecievedData): boolean => {
|
||||
return recData.version === 'WEBSITE'
|
||||
},
|
||||
},
|
||||
]
|
||||
|
|
@ -99,6 +109,11 @@ function GetApp(): ModuleType {
|
|||
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
|
||||
const versionFile = publicDir + 'version'
|
||||
|
||||
let domain = url.split('.') // [ "https://api", "frylabs", "net" ]
|
||||
domain.shift() // [ "frylabs", "net" ]
|
||||
domain = domain.join('.') // "frylabs.net"
|
||||
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
|
||||
|
||||
app.use(
|
||||
bodyParser.urlencoded({
|
||||
limit: '10mb',
|
||||
|
|
@ -246,7 +261,21 @@ function GetApp(): ModuleType {
|
|||
|
||||
// -------------------------------------------------------------
|
||||
|
||||
app.get('/getDbs', (req: any, res: any) => {
|
||||
logger.LogReq(req)
|
||||
|
||||
res.json(
|
||||
dataFiles.map((df) => {
|
||||
return {
|
||||
path: df.path.split('/').pop(),
|
||||
name: df.name,
|
||||
}
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
app.get('/updateTodo', (req: any, res: any) => {
|
||||
logger.LogReq(req)
|
||||
const userId = req.session.user.id
|
||||
const id = req.query.id
|
||||
const todos = utils.ReadJSON(todosFile)
|
||||
|
|
@ -632,13 +661,12 @@ function GetApp(): ModuleType {
|
|||
if (user) {
|
||||
const sessionID = uuidv4()
|
||||
|
||||
// FIXME: Users now can only log in in one session, this might be too strict.
|
||||
const existingSessions = dbtools.Select(userDB, 'sessions', {
|
||||
userID: user.id,
|
||||
isScript: isScript ? 1 : 0,
|
||||
})
|
||||
|
||||
if (existingSessions.length > 0) {
|
||||
if (existingSessions.length >= minimumAlowwedSessions) {
|
||||
logger.Log(
|
||||
`Multiple ${isScript ? 'script' : 'website'} sessions ( ${
|
||||
existingSessions.length
|
||||
|
|
@ -675,9 +703,9 @@ function GetApp(): ModuleType {
|
|||
})
|
||||
|
||||
// https://www.npmjs.com/package/cookie
|
||||
// TODO: cookie age
|
||||
// FIXME: cookies are not configured coorectly
|
||||
res.cookie('sessionID', sessionID, {
|
||||
domain: '.frylabs.net', // TODO: use url. url: "https://api.frylabs.net"
|
||||
domain: domain,
|
||||
expires: new Date(
|
||||
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
|
||||
),
|
||||
|
|
@ -724,7 +752,6 @@ function GetApp(): ModuleType {
|
|||
dbtools.Delete(userDB, 'sessions', {
|
||||
id: sessionID,
|
||||
})
|
||||
// TODO: remove old sessions every once in a while
|
||||
res.clearCookie('sessionID').json({
|
||||
result: 'success',
|
||||
})
|
||||
|
|
@ -829,9 +856,14 @@ function GetApp(): ModuleType {
|
|||
app.get('/allqr.txt', function(req: any, res: any) {
|
||||
res.set('Content-Type', 'text/plain')
|
||||
const stringifiedData = questionDbs.map((qdb) => {
|
||||
return dataToString(qdb.data)
|
||||
let result = ''
|
||||
result += '\n' + line
|
||||
result += ` Questions in ${qdb.name}: `
|
||||
result += line + '\n'
|
||||
result += dataToString(qdb.data)
|
||||
result += '\n' + line + line + '\n'
|
||||
return result
|
||||
})
|
||||
// TODO: test this
|
||||
res.send(stringifiedData.join('\n\n'))
|
||||
res.end()
|
||||
logger.LogReq(req)
|
||||
|
|
@ -949,27 +981,39 @@ function GetApp(): ModuleType {
|
|||
|
||||
app.post('/isAdding', function(req: any, res: any) {
|
||||
logger.LogReq(req)
|
||||
|
||||
const user: User = req.session.user
|
||||
|
||||
const dryRun = testUsers.includes(user.id)
|
||||
|
||||
try {
|
||||
processIncomingRequest(req.body, questionDbs, dryRun, user)
|
||||
.then((resultArray) => {
|
||||
logResult(req.body, resultArray)
|
||||
logResult(req.body, resultArray, user.id, dryRun)
|
||||
res.json({
|
||||
success: resultArray.length > 0, // TODO check for -1s
|
||||
success: resultArray.length > 0,
|
||||
newQuestions: resultArray,
|
||||
})
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.Log(
|
||||
'Couldnt process incoming request!',
|
||||
'Error during processing incoming request',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
res.json({
|
||||
success: false,
|
||||
})
|
||||
})
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error during getting incoming request processor promises ',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
res.json({
|
||||
success: false,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
app.get('/ask', function(req: any, res) {
|
||||
if (Object.keys(req.query).length === 0) {
|
||||
|
|
@ -996,8 +1040,6 @@ function GetApp(): ModuleType {
|
|||
|
||||
searchDatas(questionDbs, question, subj, recData)
|
||||
.then((result) => {
|
||||
// TODO: test
|
||||
console.log(result)
|
||||
res.json({
|
||||
result: result,
|
||||
success: true,
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ function GetApp(): ModuleType {
|
|||
// app, '/*.mp4', 'video/mp4', 'stuff/video'
|
||||
function appGetFileType(app, wildcard, contentType, pageToRender) {
|
||||
app.get(wildcard, function(req, res) {
|
||||
const path = decodeURI(req.url)
|
||||
const path = decodeURIComponent(req.url)
|
||||
let fp: any = path
|
||||
if (path.includes('?')) {
|
||||
fp = path.split('?')
|
||||
|
|
@ -151,7 +151,7 @@ function GetApp(): ModuleType {
|
|||
})
|
||||
|
||||
app.get('/*', function(req, res) {
|
||||
const parsedUrl = decodeURI(req.url)
|
||||
const parsedUrl = decodeURIComponent(req.url)
|
||||
let curr =
|
||||
listedFiles +
|
||||
'/' +
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ export interface ModuleType {
|
|||
dailyAction?: Function
|
||||
}
|
||||
|
||||
// TODO: add more props based on db
|
||||
export interface User {
|
||||
id: number
|
||||
pw: string
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import logger from '../utils/logger'
|
|||
import { searchData, createQuestion } from '../utils/classes'
|
||||
import idStats from '../utils/ids'
|
||||
import utils from '../utils/utils'
|
||||
import { addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
|
||||
// types
|
||||
import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
|
||||
|
|
@ -51,14 +51,27 @@ interface Result {
|
|||
|
||||
export function logResult(
|
||||
recievedData: RecievedData,
|
||||
result: Array<Result>
|
||||
result: Array<Result>,
|
||||
userId: number,
|
||||
dryRun?: boolean
|
||||
): void {
|
||||
let subjRow = '\t' + recievedData.subj
|
||||
if (recievedData.id) {
|
||||
subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
|
||||
}
|
||||
logger.Log(subjRow)
|
||||
logger.Log('\t' + recievedData.subj)
|
||||
|
||||
if (dryRun) {
|
||||
logger.Log('\tDry run')
|
||||
}
|
||||
|
||||
let idRow = '\t'
|
||||
if (recievedData.version) {
|
||||
idRow += 'Version: ' + logger.logHashed(recievedData.version)
|
||||
}
|
||||
if (recievedData.id) {
|
||||
idRow += ', CID: ' + logger.logHashed(recievedData.id)
|
||||
}
|
||||
idRow += `, User #${logger.logHashed(userId.toString())}`
|
||||
logger.Log(idRow)
|
||||
|
||||
if (result.length > 0) {
|
||||
result.forEach((res: Result) => {
|
||||
const allQLength = recievedData.quiz.length
|
||||
let msg = `${res.qdbName}: `
|
||||
|
|
@ -69,11 +82,11 @@ export function logResult(
|
|||
} else {
|
||||
msg += `No new data ( ${allQLength} )`
|
||||
}
|
||||
if (recievedData.version !== undefined) {
|
||||
msg += '. Version: ' + recievedData.version
|
||||
}
|
||||
logger.Log('\t' + msg, color)
|
||||
})
|
||||
} else {
|
||||
logger.Log('\tNo db-s passed shouldSave!', logger.GetColor('red'))
|
||||
}
|
||||
}
|
||||
|
||||
export function processIncomingRequest(
|
||||
|
|
@ -90,7 +103,7 @@ export function processIncomingRequest(
|
|||
}
|
||||
|
||||
try {
|
||||
let towrite = logger.GetDateString() + '\n'
|
||||
let towrite = utils.GetDateString() + '\n'
|
||||
towrite +=
|
||||
'------------------------------------------------------------------------------\n'
|
||||
if (typeof recievedData === 'object') {
|
||||
|
|
@ -153,15 +166,15 @@ function processIncomingRequestUsingDb(
|
|||
logger.DebugLog(currentQuestion, 'actions', 3)
|
||||
recievedQuestions.push(currentQuestion)
|
||||
questionSearchPromises.push(
|
||||
searchData(qdb.data, currentQuestion, recievedData.subj)
|
||||
searchData(qdb, currentQuestion, recievedData.subj)
|
||||
)
|
||||
})
|
||||
|
||||
Promise.all(questionSearchPromises)
|
||||
.then((results) => {
|
||||
.then((results: Array<SearchResult>) => {
|
||||
const allQuestions = [] // all new questions here that do not have result
|
||||
results.forEach((result, i) => {
|
||||
const add = result.every((res) => {
|
||||
const add = result.result.every((res) => {
|
||||
return res.match < minMatchToAmmountToAdd
|
||||
})
|
||||
if (add) {
|
||||
|
|
@ -192,8 +205,6 @@ function processIncomingRequestUsingDb(
|
|||
currWrites = 0
|
||||
logger.DebugLog('Writing data.json', 'actions', 1)
|
||||
utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
|
||||
} else if (dryRun) {
|
||||
logger.Log('\tDry run')
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,11 +7,15 @@ import {
|
|||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
|
||||
// TODO
|
||||
interface SearchResultQuestion extends Question {
|
||||
match: number
|
||||
}
|
||||
|
||||
export interface SearchResult {
|
||||
result: Array<SearchResultQuestion>
|
||||
dbName: string
|
||||
}
|
||||
|
||||
const searchDataWorkerFile = './src/utils/classes.ts'
|
||||
|
||||
const assert = (val) => {
|
||||
|
|
@ -217,7 +221,10 @@ function createQuestion(
|
|||
}
|
||||
|
||||
function compareImage(data: QuestionData, data2: QuestionData) {
|
||||
return compareString(data.images.join(' '), data2.images.join(' '))
|
||||
// TODO: img comparing (hashed images vs images)
|
||||
const imgs1 = data.hashedImages ? data.hashedImages : data.images
|
||||
const imgs2 = data2.hashedImages ? data2.hashedImages : data2.images
|
||||
return compareString(imgs1.join(' '), imgs2.join(' '))
|
||||
}
|
||||
|
||||
function compareData(q1: Question, q2: Question) {
|
||||
|
|
@ -412,23 +419,22 @@ function searchDatas(
|
|||
question: any,
|
||||
subjName: string,
|
||||
questionData?: QuestionData
|
||||
): Promise<Array<Array<SearchResultQuestion>>> {
|
||||
): Promise<Array<SearchResult>> {
|
||||
return Promise.all(
|
||||
data.map((db) => {
|
||||
return searchData(db.data, question, subjName, questionData)
|
||||
data.map((db: QuestionDb) => {
|
||||
return searchData(db, question, subjName, questionData)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// TODO: remove questionData, make question only Question type
|
||||
// FIXME: remove questionData, make question only Question type
|
||||
function searchData(
|
||||
data: Array<Subject>,
|
||||
qdb: QuestionDb,
|
||||
question: any,
|
||||
subjName: string,
|
||||
questionData?: QuestionData
|
||||
): Promise<Array<SearchResultQuestion>> {
|
||||
): Promise<SearchResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
assert(data)
|
||||
assert(question)
|
||||
logger.DebugLog('Searching for question', 'qdb search', 1)
|
||||
logger.DebugLog('Question:', 'qdb search', 2)
|
||||
|
|
@ -447,7 +453,7 @@ function searchData(
|
|||
question = simplifyQuestion(question)
|
||||
|
||||
const worker = workerTs(searchDataWorkerFile, {
|
||||
workerData: { data, subjName, question, questionData },
|
||||
workerData: { data: qdb.data, subjName, question, questionData },
|
||||
})
|
||||
|
||||
worker.on('error', (err) => {
|
||||
|
|
@ -478,7 +484,10 @@ function searchData(
|
|||
'qdb search',
|
||||
1
|
||||
)
|
||||
resolve(result)
|
||||
resolve({
|
||||
result: result,
|
||||
dbName: qdb.name,
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -494,10 +503,10 @@ function dataToString(data: Array<Subject>): string {
|
|||
// ------------------------------------------------------------------------
|
||||
|
||||
function searchWorker(
|
||||
data: any,
|
||||
subjName: any,
|
||||
question: any,
|
||||
questionData: any
|
||||
data: Array<Subject>,
|
||||
subjName: string,
|
||||
question: Question,
|
||||
questionData?: QuestionData
|
||||
): any {
|
||||
let result = []
|
||||
|
||||
|
|
|
|||
|
|
@ -126,7 +126,6 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
|
|||
const cols = Object.keys(columns)
|
||||
.reduce((acc, key) => {
|
||||
const item = columns[key]
|
||||
// FIXME: array, and push stuff, then join()
|
||||
const flags = []
|
||||
const toCheck = {
|
||||
primary: 'PRIMARY KEY',
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ const hr =
|
|||
'---------------------------------------------------------------------------------'
|
||||
|
||||
export default {
|
||||
GetDateString: GetDateString,
|
||||
getColoredDateString: getColoredDateString,
|
||||
Log: Log,
|
||||
DebugLog: DebugLog,
|
||||
GetColor: GetColor,
|
||||
|
|
@ -70,7 +70,7 @@ function setNewLogfileName(): void {
|
|||
logFileName = utils.GetDateString(true)
|
||||
}
|
||||
|
||||
function GetDateString(): string {
|
||||
function getColoredDateString(): string {
|
||||
const date = new Date()
|
||||
const dateString = utils.GetDateString()
|
||||
return GetRandomColor(date.getHours().toString()) + dateString + C()
|
||||
|
|
@ -99,7 +99,7 @@ function Log(msg: string | object, color?: string): void {
|
|||
let log = msg
|
||||
if (typeof msg !== 'object') {
|
||||
const delimiter = DELIM + C(color)
|
||||
log = C(color) + GetDateString() + delimiter + msg + C()
|
||||
log = getColoredDateString() + delimiter + C(color) + msg + C()
|
||||
}
|
||||
|
||||
console.log(log)
|
||||
|
|
@ -109,7 +109,6 @@ function Log(msg: string | object, color?: string): void {
|
|||
)
|
||||
}
|
||||
|
||||
// FIXME: express.Request type, but with .cookies and .session
|
||||
function LogReq(
|
||||
req: any /*express.Request*/,
|
||||
toFile?: boolean,
|
||||
|
|
@ -162,7 +161,7 @@ function LogReq(
|
|||
if (!toFile) {
|
||||
Log(logEntry)
|
||||
} else {
|
||||
const defLogs = GetDateString() + dl + logEntry
|
||||
const defLogs = getColoredDateString() + dl + logEntry
|
||||
|
||||
utils.AppendToFile(defLogs, vlogDir + logFileName)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,14 +17,22 @@ import fs from 'fs'
|
|||
import logger from '../utils/logger'
|
||||
|
||||
interface URLFormatOptions {
|
||||
query: any
|
||||
pathname?: string
|
||||
query?: any
|
||||
}
|
||||
|
||||
// TODO
|
||||
function formatUrl(options: URLFormatOptions): string {
|
||||
console.log(options.pathname, options.query)
|
||||
return options.pathname + JSON.stringify(options.query)
|
||||
const queryString = options.query
|
||||
? '?' +
|
||||
Object.keys(options.query)
|
||||
.map((key) => {
|
||||
return `${key}=${encodeURIComponent(options.query[key])}`
|
||||
})
|
||||
.join('&')
|
||||
: ''
|
||||
const path = options.pathname || ''
|
||||
return path + queryString
|
||||
}
|
||||
|
||||
function GetDateString(noTime?: boolean): string {
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3
|
||||
Subproject commit 309436b2270ebf2398945b2cd707d1f133fe8ec1
|
||||
Loading…
Add table
Add a link
Reference in a new issue