From 4e34267d44475285934e3c0dbaf26cbabb193704 Mon Sep 17 00:00:00 2001 From: mrfry Date: Thu, 26 Nov 2020 09:16:12 +0100 Subject: [PATCH] Seperate dbs --- src/modules/api/api.ts | 94 +++++++++++++++++++++++++++----------- src/modules/stuff/stuff.ts | 4 +- src/types/basicTypes.ts | 1 - src/utils/actions.ts | 67 +++++++++++++++------------ src/utils/classes.ts | 39 ++++++++++------ src/utils/dbtools.ts | 1 - src/utils/logger.ts | 9 ++-- src/utils/utils.ts | 14 ++++-- submodules/qmining-page | 2 +- 9 files changed, 149 insertions(+), 82 deletions(-) diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index 77a21e8..242cac8 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -57,11 +57,13 @@ const idvStatFile = 'stats/idvstats' const todosFile = 'data/todos.json' // other constants +const line = '====================================================' // lol const maxVeteranPwGetCount = 10 const addPWPerDay = 3 // every x day a user can give a pw const maxPWCount = 6 // maximum pw give opportunities a user can have at once const addPWCount = 1 // how many pw gen opportunities to add each time const daysAfterUserGetsPWs = 5 // days after user gets pw-s +const minimumAlowwedSessions = 1 // how many sessions are allowed for a user // stuff gotten from server.js let userDB @@ -79,19 +81,27 @@ function GetApp(): ModuleType { // files in public dirs const recivedFiles = publicDir + 'recivedfiles' const uloadFiles = publicDir + 'f' + // FIXME: this to seperate file? const dataFiles: Array = [ { path: `${publicDir}oldData.json`, name: 'oldData', shouldSave: (recData: RecievedData): boolean => { - return recData.version.includes('2.0.') + return recData.version.startsWith('2.0.') }, }, { path: `${publicDir}data.json`, name: 'newData', shouldSave: (recData: RecievedData): boolean => { - return recData.version.includes('2.1.') + return recData.version.startsWith('2.1.') + }, + }, + { + path: `${publicDir}fromwebsiteData.json`, + name: 'fromwebsiteData', + shouldSave: (recData: RecievedData): boolean => { + return recData.version === 'WEBSITE' }, }, ] @@ -99,6 +109,11 @@ function GetApp(): ModuleType { const userSpecificMotdFile = publicDir + 'userSpecificMotd.json' const versionFile = publicDir + 'version' + let domain = url.split('.') // [ "https://api", "frylabs", "net" ] + domain.shift() // [ "frylabs", "net" ] + domain = domain.join('.') // "frylabs.net" + logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) + app.use( bodyParser.urlencoded({ limit: '10mb', @@ -246,7 +261,21 @@ function GetApp(): ModuleType { // ------------------------------------------------------------- + app.get('/getDbs', (req: any, res: any) => { + logger.LogReq(req) + + res.json( + dataFiles.map((df) => { + return { + path: df.path.split('/').pop(), + name: df.name, + } + }) + ) + }) + app.get('/updateTodo', (req: any, res: any) => { + logger.LogReq(req) const userId = req.session.user.id const id = req.query.id const todos = utils.ReadJSON(todosFile) @@ -632,13 +661,12 @@ function GetApp(): ModuleType { if (user) { const sessionID = uuidv4() - // FIXME: Users now can only log in in one session, this might be too strict. const existingSessions = dbtools.Select(userDB, 'sessions', { userID: user.id, isScript: isScript ? 1 : 0, }) - if (existingSessions.length > 0) { + if (existingSessions.length >= minimumAlowwedSessions) { logger.Log( `Multiple ${isScript ? 'script' : 'website'} sessions ( ${ existingSessions.length @@ -675,9 +703,9 @@ function GetApp(): ModuleType { }) // https://www.npmjs.com/package/cookie - // TODO: cookie age + // FIXME: cookies are not configured coorectly res.cookie('sessionID', sessionID, { - domain: '.frylabs.net', // TODO: use url. url: "https://api.frylabs.net" + domain: domain, expires: new Date( new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000 ), @@ -724,7 +752,6 @@ function GetApp(): ModuleType { dbtools.Delete(userDB, 'sessions', { id: sessionID, }) - // TODO: remove old sessions every once in a while res.clearCookie('sessionID').json({ result: 'success', }) @@ -829,9 +856,14 @@ function GetApp(): ModuleType { app.get('/allqr.txt', function(req: any, res: any) { res.set('Content-Type', 'text/plain') const stringifiedData = questionDbs.map((qdb) => { - return dataToString(qdb.data) + let result = '' + result += '\n' + line + result += ` Questions in ${qdb.name}: ` + result += line + '\n' + result += dataToString(qdb.data) + result += '\n' + line + line + '\n' + return result }) - // TODO: test this res.send(stringifiedData.join('\n\n')) res.end() logger.LogReq(req) @@ -949,26 +981,38 @@ function GetApp(): ModuleType { app.post('/isAdding', function(req: any, res: any) { logger.LogReq(req) - const user: User = req.session.user - const dryRun = testUsers.includes(user.id) - processIncomingRequest(req.body, questionDbs, dryRun, user) - .then((resultArray) => { - logResult(req.body, resultArray) - res.json({ - success: resultArray.length > 0, // TODO check for -1s - newQuestions: resultArray, + try { + processIncomingRequest(req.body, questionDbs, dryRun, user) + .then((resultArray) => { + logResult(req.body, resultArray, user.id, dryRun) + res.json({ + success: resultArray.length > 0, + newQuestions: resultArray, + }) }) + .catch((err) => { + logger.Log( + 'Error during processing incoming request', + logger.GetColor('redbg') + ) + console.error(err) + res.json({ + success: false, + }) + }) + } catch (err) { + logger.Log( + 'Error during getting incoming request processor promises ', + logger.GetColor('redbg') + ) + console.error(err) + res.json({ + success: false, }) - .catch((err) => { - logger.Log( - 'Couldnt process incoming request!', - logger.GetColor('redbg') - ) - console.error(err) - }) + } }) app.get('/ask', function(req: any, res) { @@ -996,8 +1040,6 @@ function GetApp(): ModuleType { searchDatas(questionDbs, question, subj, recData) .then((result) => { - // TODO: test - console.log(result) res.json({ result: result, success: true, diff --git a/src/modules/stuff/stuff.ts b/src/modules/stuff/stuff.ts index 697b18a..8f9ba4e 100644 --- a/src/modules/stuff/stuff.ts +++ b/src/modules/stuff/stuff.ts @@ -74,7 +74,7 @@ function GetApp(): ModuleType { // app, '/*.mp4', 'video/mp4', 'stuff/video' function appGetFileType(app, wildcard, contentType, pageToRender) { app.get(wildcard, function(req, res) { - const path = decodeURI(req.url) + const path = decodeURIComponent(req.url) let fp: any = path if (path.includes('?')) { fp = path.split('?') @@ -151,7 +151,7 @@ function GetApp(): ModuleType { }) app.get('/*', function(req, res) { - const parsedUrl = decodeURI(req.url) + const parsedUrl = decodeURIComponent(req.url) let curr = listedFiles + '/' + diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index 044d9dd..acffeed 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -38,7 +38,6 @@ export interface ModuleType { dailyAction?: Function } -// TODO: add more props based on db export interface User { id: number pw: string diff --git a/src/utils/actions.ts b/src/utils/actions.ts index df25997..a50fdda 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -24,7 +24,7 @@ import logger from '../utils/logger' import { searchData, createQuestion } from '../utils/classes' import idStats from '../utils/ids' import utils from '../utils/utils' -import { addQuestion, getSubjNameWithoutYear } from './classes' +import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes' // types import { QuestionDb, Question, User, DataFile } from '../types/basicTypes' @@ -51,29 +51,42 @@ interface Result { export function logResult( recievedData: RecievedData, - result: Array + result: Array, + userId: number, + dryRun?: boolean ): void { - let subjRow = '\t' + recievedData.subj - if (recievedData.id) { - subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')' - } - logger.Log(subjRow) + logger.Log('\t' + recievedData.subj) - result.forEach((res: Result) => { - const allQLength = recievedData.quiz.length - let msg = `${res.qdbName}: ` - let color = logger.GetColor('green') - if (res.newQuestions > 0) { - color = logger.GetColor('blue') - msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` - } else { - msg += `No new data ( ${allQLength} )` - } - if (recievedData.version !== undefined) { - msg += '. Version: ' + recievedData.version - } - logger.Log('\t' + msg, color) - }) + if (dryRun) { + logger.Log('\tDry run') + } + + let idRow = '\t' + if (recievedData.version) { + idRow += 'Version: ' + logger.logHashed(recievedData.version) + } + if (recievedData.id) { + idRow += ', CID: ' + logger.logHashed(recievedData.id) + } + idRow += `, User #${logger.logHashed(userId.toString())}` + logger.Log(idRow) + + if (result.length > 0) { + result.forEach((res: Result) => { + const allQLength = recievedData.quiz.length + let msg = `${res.qdbName}: ` + let color = logger.GetColor('green') + if (res.newQuestions > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` + } else { + msg += `No new data ( ${allQLength} )` + } + logger.Log('\t' + msg, color) + }) + } else { + logger.Log('\tNo db-s passed shouldSave!', logger.GetColor('red')) + } } export function processIncomingRequest( @@ -90,7 +103,7 @@ export function processIncomingRequest( } try { - let towrite = logger.GetDateString() + '\n' + let towrite = utils.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' if (typeof recievedData === 'object') { @@ -153,15 +166,15 @@ function processIncomingRequestUsingDb( logger.DebugLog(currentQuestion, 'actions', 3) recievedQuestions.push(currentQuestion) questionSearchPromises.push( - searchData(qdb.data, currentQuestion, recievedData.subj) + searchData(qdb, currentQuestion, recievedData.subj) ) }) Promise.all(questionSearchPromises) - .then((results) => { + .then((results: Array) => { const allQuestions = [] // all new questions here that do not have result results.forEach((result, i) => { - const add = result.every((res) => { + const add = result.result.every((res) => { return res.match < minMatchToAmmountToAdd }) if (add) { @@ -192,8 +205,6 @@ function processIncomingRequestUsingDb( currWrites = 0 logger.DebugLog('Writing data.json', 'actions', 1) utils.WriteFile(JSON.stringify(qdb.data), qdb.path) - } else if (dryRun) { - logger.Log('\tDry run') } } diff --git a/src/utils/classes.ts b/src/utils/classes.ts index ca3aa23..a0d051a 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -7,11 +7,15 @@ import { Subject, } from '../types/basicTypes' -// TODO interface SearchResultQuestion extends Question { match: number } +export interface SearchResult { + result: Array + dbName: string +} + const searchDataWorkerFile = './src/utils/classes.ts' const assert = (val) => { @@ -217,7 +221,10 @@ function createQuestion( } function compareImage(data: QuestionData, data2: QuestionData) { - return compareString(data.images.join(' '), data2.images.join(' ')) + // TODO: img comparing (hashed images vs images) + const imgs1 = data.hashedImages ? data.hashedImages : data.images + const imgs2 = data2.hashedImages ? data2.hashedImages : data2.images + return compareString(imgs1.join(' '), imgs2.join(' ')) } function compareData(q1: Question, q2: Question) { @@ -412,23 +419,22 @@ function searchDatas( question: any, subjName: string, questionData?: QuestionData -): Promise>> { +): Promise> { return Promise.all( - data.map((db) => { - return searchData(db.data, question, subjName, questionData) + data.map((db: QuestionDb) => { + return searchData(db, question, subjName, questionData) }) ) } -// TODO: remove questionData, make question only Question type +// FIXME: remove questionData, make question only Question type function searchData( - data: Array, + qdb: QuestionDb, question: any, subjName: string, questionData?: QuestionData -): Promise> { +): Promise { return new Promise((resolve, reject) => { - assert(data) assert(question) logger.DebugLog('Searching for question', 'qdb search', 1) logger.DebugLog('Question:', 'qdb search', 2) @@ -447,7 +453,7 @@ function searchData( question = simplifyQuestion(question) const worker = workerTs(searchDataWorkerFile, { - workerData: { data, subjName, question, questionData }, + workerData: { data: qdb.data, subjName, question, questionData }, }) worker.on('error', (err) => { @@ -478,7 +484,10 @@ function searchData( 'qdb search', 1 ) - resolve(result) + resolve({ + result: result, + dbName: qdb.name, + }) }) }) } @@ -494,10 +503,10 @@ function dataToString(data: Array): string { // ------------------------------------------------------------------------ function searchWorker( - data: any, - subjName: any, - question: any, - questionData: any + data: Array, + subjName: string, + question: Question, + questionData?: QuestionData ): any { let result = [] diff --git a/src/utils/dbtools.ts b/src/utils/dbtools.ts index 797f1ef..abe1b00 100644 --- a/src/utils/dbtools.ts +++ b/src/utils/dbtools.ts @@ -126,7 +126,6 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any { const cols = Object.keys(columns) .reduce((acc, key) => { const item = columns[key] - // FIXME: array, and push stuff, then join() const flags = [] const toCheck = { primary: 'PRIMARY KEY', diff --git a/src/utils/logger.ts b/src/utils/logger.ts index b5faa7f..c682e9f 100755 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -22,7 +22,7 @@ const hr = '---------------------------------------------------------------------------------' export default { - GetDateString: GetDateString, + getColoredDateString: getColoredDateString, Log: Log, DebugLog: DebugLog, GetColor: GetColor, @@ -70,7 +70,7 @@ function setNewLogfileName(): void { logFileName = utils.GetDateString(true) } -function GetDateString(): string { +function getColoredDateString(): string { const date = new Date() const dateString = utils.GetDateString() return GetRandomColor(date.getHours().toString()) + dateString + C() @@ -99,7 +99,7 @@ function Log(msg: string | object, color?: string): void { let log = msg if (typeof msg !== 'object') { const delimiter = DELIM + C(color) - log = C(color) + GetDateString() + delimiter + msg + C() + log = getColoredDateString() + delimiter + C(color) + msg + C() } console.log(log) @@ -109,7 +109,6 @@ function Log(msg: string | object, color?: string): void { ) } -// FIXME: express.Request type, but with .cookies and .session function LogReq( req: any /*express.Request*/, toFile?: boolean, @@ -162,7 +161,7 @@ function LogReq( if (!toFile) { Log(logEntry) } else { - const defLogs = GetDateString() + dl + logEntry + const defLogs = getColoredDateString() + dl + logEntry utils.AppendToFile(defLogs, vlogDir + logFileName) } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 22277c9..ee8ec15 100755 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -17,14 +17,22 @@ import fs from 'fs' import logger from '../utils/logger' interface URLFormatOptions { - query: any pathname?: string + query?: any } -// TODO function formatUrl(options: URLFormatOptions): string { console.log(options.pathname, options.query) - return options.pathname + JSON.stringify(options.query) + const queryString = options.query + ? '?' + + Object.keys(options.query) + .map((key) => { + return `${key}=${encodeURIComponent(options.query[key])}` + }) + .join('&') + : '' + const path = options.pathname || '' + return path + queryString } function GetDateString(noTime?: boolean): string { diff --git a/submodules/qmining-page b/submodules/qmining-page index 4a7ad2d..309436b 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3 +Subproject commit 309436b2270ebf2398945b2cd707d1f133fe8ec1