From f87e165084c506d381a584a52a5b8e8316a4f20b Mon Sep 17 00:00:00 2001 From: mrfry Date: Tue, 22 Dec 2020 16:11:01 +0100 Subject: [PATCH] Domain specific data files --- src/modules/api/api.ts | 99 ++++++++++++++++++--------- src/sharedViews/404.ejs | 2 +- src/standaloneUtils/dataSimplifyer.js | 29 ++++++++ src/types/basicTypes.ts | 12 +++- src/utils/actions.ts | 60 +++++++++++----- src/utils/classes.ts | 27 ++++---- src/utils/logger.ts | 4 +- src/utils/workerPool.ts | 10 +-- submodules/qmining-page | 2 +- 9 files changed, 174 insertions(+), 71 deletions(-) create mode 100644 src/standaloneUtils/dataSimplifyer.js diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index 04bf815..a8c2883 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -33,9 +33,8 @@ import { processIncomingRequest, logResult, backupData, + shouldSaveDataFile, loadJSON, - getQuestionDbsWithoutFunct, - RecievedData, Result, } from '../../utils/actions' import dbtools from '../../utils/dbtools' @@ -48,7 +47,13 @@ import { } from '../../utils/workerPool' import { SetupData } from '../../server' -import { ModuleType, User, DataFile, Request } from '../../types/basicTypes' +import { + ModuleType, + User, + DataFile, + Request, + QuestionDb, +} from '../../types/basicTypes' // files const msgFile = 'stats/msgs' @@ -65,6 +70,7 @@ const todosFile = 'data/todos.json' const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js' const rootRedirectToFile = 'data/apiRootRedirectTo' const recievedQuestionFile = 'stats/recievedQuestions' +const dbsFile = 'data/dbs.json' // other constants const line = '====================================================' // lol @@ -90,30 +96,7 @@ function GetApp(): ModuleType { // files in public dirs const recivedFiles = publicDir + 'recivedfiles' const uloadFiles = publicDir + 'f' - // FIXME: this to seperate file? - const dataFiles: Array = [ - { - path: `${publicDir}oldData.json`, - name: 'oldData', - shouldSave: (recData: RecievedData): boolean => { - return recData.version.startsWith('2.0.') - }, - }, - { - path: `${publicDir}data.json`, - name: 'newData', - shouldSave: (recData: RecievedData): boolean => { - return recData.version.startsWith('2.1.') - }, - }, - { - path: `${publicDir}fromwebsiteData.json`, - name: 'fromwebsiteData', - shouldSave: (recData: RecievedData): boolean => { - return recData.version === 'WEBSITE' - }, - }, - ] + const dataFiles: Array = utils.ReadJSON(dbsFile) const motdFile = publicDir + 'motd' const userSpecificMotdFile = publicDir + 'userSpecificMotd.json' @@ -161,7 +144,7 @@ function GetApp(): ModuleType { }) ) - const questionDbs = loadJSON(dataFiles) + const questionDbs = loadJSON(dataFiles, publicDir) let version = '' let rootRedirectURL = '' let motd = '' @@ -169,7 +152,7 @@ function GetApp(): ModuleType { // FIXME: check type from file let testUsers: any = [] - initWorkerPool(getQuestionDbsWithoutFunct(questionDbs)) + initWorkerPool(questionDbs) function mergeObjSum(a, b) { const res = { ...b } @@ -931,7 +914,61 @@ function GetApp(): ModuleType { const dryRun = testUsers.includes(user.id) try { - processIncomingRequest(req.body, questionDbs, dryRun, user) + let maxIndex = -1 + const suitedQuestionDbs = questionDbs.filter((qdb) => { + if (maxIndex < qdb.index) { + maxIndex = qdb.index + } + return shouldSaveDataFile(qdb, req.body) + }, []) + + if (suitedQuestionDbs.length === 0) { + const location = req.body.location.split('/')[2] + // TODO: should check if location is a not empty string + logger.Log( + `No suitable questiondbs found for ${location}, creating a new one...` + ) + const newDb: DataFile = { + path: `${location}.json`, + name: location, + shouldSave: { + location: { + val: location, + }, + }, + } + + utils.WriteFile( + JSON.stringify( + [ + ...utils.ReadJSON(dbsFile), + newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime + ], + null, + 2 + ), + dbsFile + ) + + // "loading" new db + const loadedNewDb: QuestionDb = { + ...newDb, + data: [], + path: publicDir + newDb.path, + index: maxIndex, + } + utils.WriteFile('[]', loadedNewDb.path) + + suitedQuestionDbs.push(loadedNewDb) + questionDbs.push(loadedNewDb) + // TODO: problem: new dbs wont get to workers before trying to search with them. + msgAllWorker({ + newdb: loadedNewDb, + type: 'newdb', + }) + } + + processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user) .then((resultArray: Array) => { logResult(req.body, resultArray, user.id, dryRun) @@ -947,7 +984,7 @@ function GetApp(): ModuleType { if (totalNewQuestions > 0) { msgAllWorker({ - qdbs: getQuestionDbsWithoutFunct(questionDbs), + qdbs: questionDbs, type: 'update', }) } diff --git a/src/sharedViews/404.ejs b/src/sharedViews/404.ejs index 4126305..64be699 100755 --- a/src/sharedViews/404.ejs +++ b/src/sharedViews/404.ejs @@ -18,7 +18,7 @@

404

- +
diff --git a/src/standaloneUtils/dataSimplifyer.js b/src/standaloneUtils/dataSimplifyer.js new file mode 100644 index 0000000..5520cdb --- /dev/null +++ b/src/standaloneUtils/dataSimplifyer.js @@ -0,0 +1,29 @@ +const fs = require('fs') +const { simplifyString } = require('./src/utils/classes.js') + +const file = './publicDirs/qminingPublic/data.json' + +const data = JSON.parse(fs.readFileSync(file, 'utf8')) +const res = [] + +data.forEach((subj) => { + const questions = [] + subj.Questions.forEach((question) => { + const res = {} + if (question.Q) { + res.Q = simplifyString(question.Q) + } + if (question.A) { + res.A = simplifyString(question.A) + } + res.data = question.data + + questions.push(res) + }) + res.push({ + Name: subj.Name, + Questions: questions, + }) +}) + +fs.writeFileSync(file + '.res', JSON.stringify(res)) diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index 19f0d9c..1759d87 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -20,12 +20,20 @@ export interface Subject { export interface DataFile { path: string name: string - shouldSave: (recData: any) => boolean + shouldSave: { + location?: { + val: string + } + version?: { + compare: string + val: string + } + } } export interface QuestionDb extends DataFile { data: Array - index: Number + index: number } export interface User { diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 4434ec0..092d4eb 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -43,6 +43,7 @@ export interface RecievedData { id: string version: string scriptVersion: string + location: string } export interface Result { @@ -86,7 +87,7 @@ export function logResult( logger.Log('\t' + msg, color) }) } else { - logger.Log('\tNo db-s passed shouldSave!', logger.GetColor('red')) + logger.Log('\tResults length is zero!', logger.GetColor('red')) } } @@ -130,9 +131,7 @@ export function processIncomingRequest( // FIXME: this many promises and stuff might be unnecesarry const promises: Array> = questionDbs.reduce((acc, qdb) => { - if (qdb.shouldSave(recievedData)) { - acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) - } + acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) return acc }, []) return Promise.all(promises) @@ -262,17 +261,53 @@ function processIncomingRequestUsingDb( }) } -export function loadJSON(dataFiles: Array): Array { +export function shouldSaveDataFile( + df: DataFile, + recievedData: RecievedData +): Boolean { + if (df.shouldSave.version) { + const { compare, val } = df.shouldSave.version + if (compare === 'equals') { + return recievedData.version === val + } else if (compare === 'startsWith') { + return recievedData.version.startsWith(val) + } + } + + if (df.shouldSave.version) { + const { compare, val } = df.shouldSave.version + if (compare === 'equals') { + return recievedData.version === val + } else if (compare === 'startsWith') { + return recievedData.version.startsWith(val) + } + } + + if (df.shouldSave.location) { + const { val } = df.shouldSave.location + return recievedData.location.includes(val) + } + + return false +} + +export function loadJSON( + dataFiles: Array, + dataDir: string +): Array { return dataFiles.reduce((acc, dataFile, index) => { - if (!utils.FileExists(dataFile.path)) { - utils.WriteFile(JSON.stringify([]), dataFile.path) + const dataPath = dataDir + dataFile.path + + if (!utils.FileExists(dataPath)) { + utils.WriteFile(JSON.stringify([]), dataPath) } try { acc.push({ ...dataFile, + path: dataPath, index: index, - data: JSON.parse(utils.ReadFile(dataFile.path)), + data: JSON.parse(utils.ReadFile(dataPath)), }) } catch (err) { console.error(err) @@ -305,12 +340,3 @@ export function backupData(questionDbs: Array): void { } }) } - -export function getQuestionDbsWithoutFunct( - questionDbs: Array // FIXME: type for dis -): Array { - return questionDbs.map((qdb) => { - const { shouldSave, ...res } = qdb // eslint-disable-line - return res - }) -} diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 399cb4d..158b7c9 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -509,11 +509,11 @@ if (!isMainThread) { const index = msg.index const searchIn = msg.data.searchIn - console.log( - `[THREAD #${workerIndex}]: staring work${ - !isNaN(index) ? ` on job index #${index}` : '' - }` - ) + // console.log( + // `[THREAD #${workerIndex}]: staring work${ + // !isNaN(index) ? ` on job index #${index}` : '' + // }` + // ) let searchResult = [] @@ -549,18 +549,21 @@ if (!isMainThread) { result: sortedResult, }) - console.log( - `[THREAD #${workerIndex}]: Work ${ - !isNaN(index) ? `#${index}` : '' - }done!` - ) + // console.log( + // `[THREAD #${workerIndex}]: Work ${ + // !isNaN(index) ? `#${index}` : '' + // }done!` + // ) } else if (msg.type === 'update') { qdbs = msg.qdbs - console.log(`[THREAD #${workerIndex}]: update`) + // console.log(`[THREAD #${workerIndex}]: update`) + } else if (msg.type === 'newdb') { + qdbs.push(msg.newdb) + // console.log(`[THREAD #${workerIndex}]: newdb`) } }) } else { - console.log('[THREAD]: Main thread!') + // console.log('[THREAD]: Main thread!') } // ------------------------------------------------------------------------ diff --git a/src/utils/logger.ts b/src/utils/logger.ts index f6107e2..70d3579 100755 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -32,7 +32,7 @@ export default { logHashed: logHashed, hr: hr, C: C, - setNewLogfileName, + setNewLogfileName: setNewLogfileName, } const DELIM = C('green') + '|' + C() @@ -166,7 +166,7 @@ function LogReq(req: Request, toFile?: boolean, statusCode?: string): void { if (!toFile) { Log(logEntry) } else { - const defLogs = getColoredDateString() + dl + logEntry + const defLogs = utils.GetDateString() + dl + logEntry utils.AppendToFile(defLogs, vlogDir + logFileName) } diff --git a/src/utils/workerPool.ts b/src/utils/workerPool.ts index 9fd0fc5..37eef90 100644 --- a/src/utils/workerPool.ts +++ b/src/utils/workerPool.ts @@ -27,7 +27,7 @@ export function doALongTask(obj: any): Promise { resolve(res) // TODO: check if result is really a result, and want to release port pool.release(client) - console.log('[RELEASE]: #' + client.index) + // console.log('[RELEASE]: #' + client.index) }) }) .catch(function(err) { @@ -56,9 +56,9 @@ export function initWorkerPool(initData: any): void { } }, destroy: function(client) { - console.log('[DESTROY]') + // console.log('[DESTROY]') client.worker.terminate() - console.log('[DESTROYED] #' + client.index) + // console.log('[DESTROYED] #' + client.index) }, } @@ -115,9 +115,9 @@ function getAWorker(i, initData) { // --------------------------------------------------------------------------- function doSomething(client, obj) { - const { index, worker } = client + const { /* index, */ worker } = client return new Promise((resolve) => { - console.log('[ACCUIRE]: #' + index) + // console.log('[ACCUIRE]: #' + index) worker.postMessage(obj) worker.once('message', (msg) => { resolve(msg) diff --git a/submodules/qmining-page b/submodules/qmining-page index bc776b3..d798a83 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit bc776b3307c975f826afb1338874330924d601ba +Subproject commit d798a8322d33cdaf0b38771f78a5fb7b7207eefc