diff --git a/src/middlewares/reqlogger.middleware.ts b/src/middlewares/reqlogger.middleware.ts index 3f32289..24a5c0f 100644 --- a/src/middlewares/reqlogger.middleware.ts +++ b/src/middlewares/reqlogger.middleware.ts @@ -3,6 +3,7 @@ import logger from '../utils/logger' interface Options { loggableKeywords: Array loggableModules: Array + exceptions: Array } export default function(options: Options): any { diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index cb30773..d982af3 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -34,6 +34,7 @@ import { logResult, backupData, shouldSaveDataFile, + shouldSearchDataFile, loadJSON, Result, } from '../../utils/actions' @@ -930,10 +931,62 @@ function GetApp(): ModuleType { } }) + function getNewQdb(location, maxIndex) { + // TODO: should check if location is a not empty string + logger.Log( + `No suitable questiondbs found for ${location}, creating a new one...` + ) + const newDb: DataFile = { + path: `${location}.json`, + name: location, + shouldSearch: { + location: { + val: location, + }, + }, + shouldSave: { + location: { + val: location, + }, + }, + } + + utils.WriteFile( + JSON.stringify( + [ + ...utils.ReadJSON(dbsFile), + newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime + ], + null, + 2 + ), + dbsFile + ) + + // "loading" new db + const loadedNewDb: QuestionDb = { + ...newDb, + data: [], + path: publicDir + newDb.path, + index: maxIndex, + } + utils.WriteFile('[]', loadedNewDb.path) + + questionDbs.push(loadedNewDb) + // TODO: problem: new dbs wont get to workers before trying to search with them. + msgAllWorker({ + newdb: loadedNewDb, + type: 'newdb', + }) + + return loadedNewDb + } + app.post('/isAdding', function(req: Request, res: any) { logger.LogReq(req) const user: User = req.session.user const dryRun = testUsers.includes(user.id) + const location = req.body.location.split('/')[2] try { let maxIndex = -1 @@ -945,49 +998,7 @@ function GetApp(): ModuleType { }, []) if (suitedQuestionDbs.length === 0) { - const location = req.body.location.split('/')[2] - // TODO: should check if location is a not empty string - logger.Log( - `No suitable questiondbs found for ${location}, creating a new one...` - ) - const newDb: DataFile = { - path: `${location}.json`, - name: location, - shouldSave: { - location: { - val: location, - }, - }, - } - - utils.WriteFile( - JSON.stringify( - [ - ...utils.ReadJSON(dbsFile), - newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime - ], - null, - 2 - ), - dbsFile - ) - - // "loading" new db - const loadedNewDb: QuestionDb = { - ...newDb, - data: [], - path: publicDir + newDb.path, - index: maxIndex, - } - utils.WriteFile('[]', loadedNewDb.path) - - suitedQuestionDbs.push(loadedNewDb) - questionDbs.push(loadedNewDb) - // TODO: problem: new dbs wont get to workers before trying to search with them. - msgAllWorker({ - newdb: loadedNewDb, - type: 'newdb', - }) + suitedQuestionDbs.push(getNewQdb(location, maxIndex)) } processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user) @@ -1048,16 +1059,16 @@ function GetApp(): ModuleType { } } - function saveQuestion(questions, subj, location, userid) { + function saveQuestion(questions, subj, testUrl, userid) { // TODO: clear folder every now and then, check if saved questions exist const questionsToSave = { questions: questions, subj: subj, userid: userid, - location: location, + testUrl: testUrl, date: new Date(), } - const fname = `${utils.GetDateString()}_${userid}_${location}.json` + const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json` const subject = getSubjNameWithoutYear(subj) const subjPath = `${savedQuestionsDir}/${subject}` const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}` @@ -1072,7 +1083,7 @@ function GetApp(): ModuleType { fname: fname, subj: subj, userid: userid, - location: location, + testUrl: testUrl, date: new Date(), }) utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath) @@ -1094,12 +1105,14 @@ function GetApp(): ModuleType { return } const subj: any = req.body.subj || '' - const location = req.body.location.split('/')[2] + const testUrl = req.body.testUrl + ? req.body.testUrl.split('/')[2] + : undefined writeAskData(req.body) const resultPromises = req.body.questions.map((question) => { - return getResult(question, subj, null, req.query) + return getResult(question, subj, null, req.body, testUrl) }) Promise.all(resultPromises).then((results) => { @@ -1120,7 +1133,7 @@ function GetApp(): ModuleType { return acc }, []) - saveQuestion(saveableQuestions, subj, location, user.id) + saveQuestion(saveableQuestions, subj, testUrl, user.id) }) }) @@ -1155,18 +1168,33 @@ function GetApp(): ModuleType { } }) - function getResult(question, subj, recData, recievedData) { + function getResult(question, subj, recData, recievedData, testUrl?) { return new Promise((resolve) => { + let searchIn = testUrl + ? questionDbs.reduce((acc, qdb, i) => { + if (shouldSearchDataFile(qdb, testUrl)) { + acc.push(i) + } + return acc + }, []) + : 'all' + + if (searchIn.length === 0) { + searchIn = 'all' + } + // searchIn could be [0], [1], ... to search every db in different thread. Put this into a // forEach(qdbs) to achieve this doALongTask({ type: 'work', data: { - searchIn: 'all', + searchIn: searchIn, + testUrl: testUrl, question: question, subjName: subj, questionData: recData, searchInAllIfNoResult: true, + searchTillMatchPercent: 30, }, }) .then((taskResult) => { diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index 1759d87..470fce3 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -20,6 +20,13 @@ export interface Subject { export interface DataFile { path: string name: string + shouldSearch: + | string + | { + location?: { + val: string + } + } shouldSave: { location?: { val: string diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 08370b8..f259311 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -263,6 +263,22 @@ function processIncomingRequestUsingDb( }) } +export function shouldSearchDataFile(df: DataFile, testUrl: string): Boolean { + if (typeof df.shouldSearch === 'string' && df.shouldSearch === 'always') { + return true + } + + if (typeof df.shouldSearch === 'object') { + if (df.shouldSearch.location) { + const { val } = df.shouldSearch.location + return testUrl.includes(val) + } + } + + // FIXME: always return true if it gets here? + return true +} + export function shouldSaveDataFile( df: DataFile, recievedData: RecievedData diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 60abe67..ca0af48 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -315,7 +315,7 @@ function questionToString(question: Question) { // --------------------------------------------------------------------------------------------------------- // Subject // --------------------------------------------------------------------------------------------------------- -function searchQuestion( +function searchSubject( subj: Subject, question: Question, subjName: string, @@ -465,7 +465,7 @@ function doSearch( .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) ) { logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2) - const subjRes = searchQuestion( + const subjRes = searchSubject( subj, questionToSearch, subjName, @@ -473,14 +473,13 @@ function doSearch( ) result = result.concat(subjRes) if (searchTillMatchPercent) { - // TODO: this should return true to continue searching? - return subjRes.every((sr) => { - return sr.match <= searchTillMatchPercent + return !subjRes.some((sr) => { + return sr.match >= searchTillMatchPercent }) } return true } - // TODO: data.every should always return something! true or false? + return true }) if (searchInAllIfNoResult) { @@ -495,18 +494,19 @@ function doSearch( 1 ) data.every((subj) => { - const subjRes = searchQuestion( + const subjRes = searchSubject( subj, questionToSearch, subjName, searchTillMatchPercent ) result = result.concat(subjRes) + if (searchTillMatchPercent) { - // TODO: this should return true to continue searching? - return subjRes.every((sr) => { - return sr.match < searchTillMatchPercent + const continueSearching = !subjRes.some((sr) => { + return sr.match >= searchTillMatchPercent }) + return continueSearching } return true }) @@ -546,11 +546,10 @@ if (!isMainThread) { questionData, searchTillMatchPercent, searchInAllIfNoResult, + searchIn, + index, } = msg.data - const index = msg.index - const searchIn = msg.data.searchIn - // console.log( // `[THREAD #${workerIndex}]: staring work${ // !isNaN(index) ? ` on job index #${index}` : ''