Workers can now search in multiple specified db-s

devel/tests/testScripts/ask.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 url=$(head -n 1 ../serverAddress)
+# url='https://api.frylabs.net'
 echo "Server url: $url"
 
 simpleType='\{"type":"simple"\}'
@@ -45,7 +46,7 @@ fi
 
 echo "Result:"
 ../bin/hr.sh
-res=$(curl -H "Content-Type: application/json" -L -s -X GET "${url}/ask?q=${q}&data=${data}&subj=${subj}")
+res=$(curl --cookie "sessionID=e0ac328d-86cc-4dbf-a00b-213bec6011e7" -H "Content-Type: application/json" -L -s -X GET "${url}/ask?q=${q}&data=${data}&subj=${subj}")
 echo "$res" | jq
 if [ "$?" -ne 0 ]; then
   echo "jq error"

devel/tests/testScripts/postTestData.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 url=$(head -n 1 ../serverAddress)
+# url='https://api.frylabs.net'
 echo "Server url: $url"
 
 if [ "$#" -ne 1 ]; then
@@ -13,6 +14,7 @@ else
   echo "$data"
   ../bin/hr.sh
   curl \
+     --cookie "sessionID=e0ac328d-86cc-4dbf-a00b-213bec6011e7" \
     -H "Content-Type: application/json" \
     -X POST --data "$data" \
     "$url/isAdding"

src/modules/api/api.ts

@@ -34,8 +34,9 @@ import {
   logResult,
   backupData,
   loadJSON,
-  RecievedData,
   getQuestionDbsWithoutFunct,
+  RecievedData,
+  Result,
 } from '../../utils/actions'
 import dbtools from '../../utils/dbtools'
 import auth from '../../middlewares/auth.middleware'
@@ -705,8 +706,6 @@ function GetApp(): ModuleType {
           return new Date(a).getTime() - new Date(b).getTime()
         })
 
-      // TODO: check if sort works, and only the first few gets deleted.
-
       const diff = existingSessions.length - minimumAlowwedSessions
       if (diff > 0) {
         logger.Log(
@@ -1030,16 +1029,25 @@ function GetApp(): ModuleType {
 
     try {
       processIncomingRequest(req.body, questionDbs, dryRun, user)
-        .then((resultArray) => {
+        .then((resultArray: Array<Result>) => {
           logResult(req.body, resultArray, user.id, dryRun)
+
+          const totalNewQuestions = resultArray.reduce((acc, sres) => {
+            return acc + sres.newQuestions
+          }, 0)
+
           res.json({
             success: resultArray.length > 0,
             newQuestions: resultArray,
+            totalNewQuestions: totalNewQuestions,
           })
-          msgAllWorker({
+
-            qdbs: getQuestionDbsWithoutFunct(questionDbs),
+          if (totalNewQuestions > 0) {
-            type: 'update',
+            msgAllWorker({
-          })
+              qdbs: getQuestionDbsWithoutFunct(questionDbs),
+              type: 'update',
+            })
+          }
         })
         .catch((err) => {
           logger.Log(
@@ -1078,48 +1086,28 @@ function GetApp(): ModuleType {
         const question = req.query.q
         const recData: any = req.query.data
 
-        const promises = []
+        doALongTask({
-
+          type: 'work',
-        questionDbs.map((qdb) => {
+          data: {
-          promises.push(
+            searchIn: 'all',
-            doALongTask({
+            question: question,
-              type: 'work',
+            subjName: subj,
-              data: {
+            questionData: recData,
-                searchIn: [qdb.index], // TODO: search in all
+          },
-                question: question,
-                subjName: subj,
-                questionData: recData,
-              },
-            })
-          )
         })
-
-        Promise.all(promises)
           .then((result) => {
+            console.log(result)
             try {
-              const mergedResult = result.reduce((acc, dbRes) => {
-                return [...acc, ...dbRes.result]
-              }, [])
-              const sortedResult = mergedResult.sort((q1, q2) => {
-                if (q1.match < q2.match) {
-                  return 1
-                } else if (q1.match > q2.match) {
-                  return -1
-                } else {
-                  return 0
-                }
-              })
-
               res.json({
-                result: sortedResult,
+                result: result,
                 success: true,
               })
               logger.DebugLog(
-                `Question result length: ${mergedResult.length}`,
+                `Question result length: ${result.length}`,
                 'ask',
                 1
               )
-              logger.DebugLog(mergedResult, 'ask', 2)
+              logger.DebugLog(result, 'ask', 2)
             } catch (err) {
               console.error(err)
               logger.Log(

src/utils/actions.ts

@@ -22,7 +22,7 @@ const dataLockFile = './data/lockData'
 
 import logger from '../utils/logger'
 import { createQuestion } from '../utils/classes'
-import { doALongTask  } from './workerPool'
+import { doALongTask } from './workerPool'
 import idStats from '../utils/ids'
 import utils from '../utils/utils'
 import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
@@ -45,7 +45,7 @@ export interface RecievedData {
   scriptVersion: string
 }
 
-interface Result {
+export interface Result {
   qdbName: string
   newQuestions: number
 }
@@ -128,6 +128,7 @@ export function processIncomingRequest(
     return new Promise((resolve) => resolve([]))
   }
 
+  // FIXME: this many promises and stuff might be unnecesarry
   const promises: Array<Promise<Result>> = questionDbs.reduce((acc, qdb) => {
     if (qdb.shouldSave(recievedData)) {
       acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user))
@@ -202,8 +203,6 @@ function processIncomingRequestUsingDb(
                 )
                 logger.DebugLog(currentQuestion, 'actions', 3)
                 addQuestion(qdb.data, sName, currentQuestion)
-                // TODO: check if it really adds it, not only just some clone (questionDbs in api.ts
-                // modifies too)
               })
 
               currWrites++

src/utils/classes.ts

@@ -495,33 +495,48 @@ if (!isMainThread) {
 
   parentPort.on('message', (msg) => {
     if (msg.type === 'work') {
-      const { searchIn, subjName, question, questionData } = msg.data
+      const { subjName, question, questionData } = msg.data
       const index = msg.index
+      const searchIn = msg.data.searchIn
+
       console.log(
         `[THREAD #${workerIndex}]: staring work${
           !isNaN(index) ? ` on job index #${index}` : ''
         }`
       )
 
-      let searchResult = null
+      let searchResult = []
-      const currQdb = qdbs.find((qdb) => {
-        return searchIn[0] === qdb.index
-      })
 
       try {
-        searchResult = doSearch(currQdb.data, subjName, question, questionData)
+        qdbs.forEach((qdb) => {
+          if (searchIn === 'all' || searchIn.includes(qdb.index)) {
+            const res = doSearch(qdb.data, subjName, question, questionData)
+            searchResult = [...searchResult, ...res]
+          }
+        })
       } catch (err) {
         logger.Log('Error in worker thread!', logger.GetColor('redbg'))
         console.error(err)
       }
 
+      // sorting
+      const sortedResult = searchResult.sort((q1, q2) => {
+        if (q1.match < q2.match) {
+          return 1
+        } else if (q1.match > q2.match) {
+          return -1
+        } else {
+          return 0
+        }
+      })
+
       // ONDONE:
       parentPort.postMessage({
         msg: `From thread #${workerIndex}: job ${
           !isNaN(index) ? `#${index}` : ''
         }done`,
         workerIndex: workerIndex,
-        result: searchResult,
+        result: sortedResult,
       })
 
       console.log(