From 19b8fdc5abc996331aaecc80e6d7b8ed4eb82194 Mon Sep 17 00:00:00 2001 From: mrfry Date: Sat, 19 Dec 2020 12:07:31 +0100 Subject: [PATCH] Workers can now search in multiple specified db-s --- devel/tests/testScripts/ask.sh | 3 +- devel/tests/testScripts/postTestData.sh | 2 + src/modules/api/api.ts | 68 ++++++++++--------------- src/utils/actions.ts | 7 ++- src/utils/classes.ts | 29 ++++++++--- 5 files changed, 57 insertions(+), 52 deletions(-) diff --git a/devel/tests/testScripts/ask.sh b/devel/tests/testScripts/ask.sh index 5ff590d..7fe2ea2 100755 --- a/devel/tests/testScripts/ask.sh +++ b/devel/tests/testScripts/ask.sh @@ -1,5 +1,6 @@ #!/bin/bash url=$(head -n 1 ../serverAddress) +# url='https://api.frylabs.net' echo "Server url: $url" simpleType='\{"type":"simple"\}' @@ -45,7 +46,7 @@ fi echo "Result:" ../bin/hr.sh -res=$(curl -H "Content-Type: application/json" -L -s -X GET "${url}/ask?q=${q}&data=${data}&subj=${subj}") +res=$(curl --cookie "sessionID=e0ac328d-86cc-4dbf-a00b-213bec6011e7" -H "Content-Type: application/json" -L -s -X GET "${url}/ask?q=${q}&data=${data}&subj=${subj}") echo "$res" | jq if [ "$?" -ne 0 ]; then echo "jq error" diff --git a/devel/tests/testScripts/postTestData.sh b/devel/tests/testScripts/postTestData.sh index 7170e38..80aef73 100755 --- a/devel/tests/testScripts/postTestData.sh +++ b/devel/tests/testScripts/postTestData.sh @@ -1,5 +1,6 @@ #!/bin/bash url=$(head -n 1 ../serverAddress) +# url='https://api.frylabs.net' echo "Server url: $url" if [ "$#" -ne 1 ]; then @@ -13,6 +14,7 @@ else echo "$data" ../bin/hr.sh curl \ + --cookie "sessionID=e0ac328d-86cc-4dbf-a00b-213bec6011e7" \ -H "Content-Type: application/json" \ -X POST --data "$data" \ "$url/isAdding" diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index 3ba48a5..bcde7f1 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -34,8 +34,9 @@ import { logResult, backupData, loadJSON, - RecievedData, getQuestionDbsWithoutFunct, + RecievedData, + Result, } from '../../utils/actions' import dbtools from '../../utils/dbtools' import auth from '../../middlewares/auth.middleware' @@ -705,8 +706,6 @@ function GetApp(): ModuleType { return new Date(a).getTime() - new Date(b).getTime() }) - // TODO: check if sort works, and only the first few gets deleted. - const diff = existingSessions.length - minimumAlowwedSessions if (diff > 0) { logger.Log( @@ -1030,16 +1029,25 @@ function GetApp(): ModuleType { try { processIncomingRequest(req.body, questionDbs, dryRun, user) - .then((resultArray) => { + .then((resultArray: Array) => { logResult(req.body, resultArray, user.id, dryRun) + + const totalNewQuestions = resultArray.reduce((acc, sres) => { + return acc + sres.newQuestions + }, 0) + res.json({ success: resultArray.length > 0, newQuestions: resultArray, + totalNewQuestions: totalNewQuestions, }) - msgAllWorker({ - qdbs: getQuestionDbsWithoutFunct(questionDbs), - type: 'update', - }) + + if (totalNewQuestions > 0) { + msgAllWorker({ + qdbs: getQuestionDbsWithoutFunct(questionDbs), + type: 'update', + }) + } }) .catch((err) => { logger.Log( @@ -1078,48 +1086,28 @@ function GetApp(): ModuleType { const question = req.query.q const recData: any = req.query.data - const promises = [] - - questionDbs.map((qdb) => { - promises.push( - doALongTask({ - type: 'work', - data: { - searchIn: [qdb.index], // TODO: search in all - question: question, - subjName: subj, - questionData: recData, - }, - }) - ) + doALongTask({ + type: 'work', + data: { + searchIn: 'all', + question: question, + subjName: subj, + questionData: recData, + }, }) - - Promise.all(promises) .then((result) => { + console.log(result) try { - const mergedResult = result.reduce((acc, dbRes) => { - return [...acc, ...dbRes.result] - }, []) - const sortedResult = mergedResult.sort((q1, q2) => { - if (q1.match < q2.match) { - return 1 - } else if (q1.match > q2.match) { - return -1 - } else { - return 0 - } - }) - res.json({ - result: sortedResult, + result: result, success: true, }) logger.DebugLog( - `Question result length: ${mergedResult.length}`, + `Question result length: ${result.length}`, 'ask', 1 ) - logger.DebugLog(mergedResult, 'ask', 2) + logger.DebugLog(result, 'ask', 2) } catch (err) { console.error(err) logger.Log( diff --git a/src/utils/actions.ts b/src/utils/actions.ts index dc01021..4434ec0 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -22,7 +22,7 @@ const dataLockFile = './data/lockData' import logger from '../utils/logger' import { createQuestion } from '../utils/classes' -import { doALongTask } from './workerPool' +import { doALongTask } from './workerPool' import idStats from '../utils/ids' import utils from '../utils/utils' import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes' @@ -45,7 +45,7 @@ export interface RecievedData { scriptVersion: string } -interface Result { +export interface Result { qdbName: string newQuestions: number } @@ -128,6 +128,7 @@ export function processIncomingRequest( return new Promise((resolve) => resolve([])) } + // FIXME: this many promises and stuff might be unnecesarry const promises: Array> = questionDbs.reduce((acc, qdb) => { if (qdb.shouldSave(recievedData)) { acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) @@ -202,8 +203,6 @@ function processIncomingRequestUsingDb( ) logger.DebugLog(currentQuestion, 'actions', 3) addQuestion(qdb.data, sName, currentQuestion) - // TODO: check if it really adds it, not only just some clone (questionDbs in api.ts - // modifies too) }) currWrites++ diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 17e6ca7..c6667d1 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -495,33 +495,48 @@ if (!isMainThread) { parentPort.on('message', (msg) => { if (msg.type === 'work') { - const { searchIn, subjName, question, questionData } = msg.data + const { subjName, question, questionData } = msg.data const index = msg.index + const searchIn = msg.data.searchIn + console.log( `[THREAD #${workerIndex}]: staring work${ !isNaN(index) ? ` on job index #${index}` : '' }` ) - let searchResult = null - const currQdb = qdbs.find((qdb) => { - return searchIn[0] === qdb.index - }) + let searchResult = [] try { - searchResult = doSearch(currQdb.data, subjName, question, questionData) + qdbs.forEach((qdb) => { + if (searchIn === 'all' || searchIn.includes(qdb.index)) { + const res = doSearch(qdb.data, subjName, question, questionData) + searchResult = [...searchResult, ...res] + } + }) } catch (err) { logger.Log('Error in worker thread!', logger.GetColor('redbg')) console.error(err) } + // sorting + const sortedResult = searchResult.sort((q1, q2) => { + if (q1.match < q2.match) { + return 1 + } else if (q1.match > q2.match) { + return -1 + } else { + return 0 + } + }) + // ONDONE: parentPort.postMessage({ msg: `From thread #${workerIndex}: job ${ !isNaN(index) ? `#${index}` : '' }done`, workerIndex: workerIndex, - result: searchResult, + result: sortedResult, }) console.log(