Adding questions using worker pools

devel/tests/testScripts/ask.sh

@@ -45,7 +45,7 @@ fi
 
 echo "Result:"
 ../bin/hr.sh
-res=$(curl -L -s -X GET "${url}/ask?q=${q}&data=${data}&subj=${subj}")
+res=$(curl -H "Content-Type: application/json" -L -s -X GET "${url}/ask?q=${q}&data=${data}&subj=${subj}")
 echo "$res" | jq
 if [ "$?" -ne 0 ]; then
   echo "jq error"

src/modules/api/api.ts

@@ -1067,16 +1067,31 @@ function GetApp(): ModuleType {
       if (req.query.q && req.query.data) {
         const subj: any = req.query.subj || ''
         const question = req.query.q
-        const recData: any = req.query.data
+        let recData: any = req.query.data
+        if (typeof recData === 'string') {
+          try {
+            recData = JSON.parse(recData)
+          } catch (err) {
+            logger.Log(
+              'Error parsing recData in /ask!',
+              logger.GetColor('redbg')
+            )
+            console.error(err)
+          }
+        }
 
         const promises = []
 
-        questionDbs.map((qdb, i) => {
+        questionDbs.map((qdb) => {
           promises.push(
-            doALongTask(i, {
+            doALongTask({
               type: 'work',
-              index: i,
-              data: { qdb: qdb.data, question, subjName: subj, recData },
+              data: {
+                qdb: qdb.data,
+                question,
+                subjName: subj,
+                questionData: recData,
+              },
             })
           )
         })
@@ -1102,11 +1117,11 @@ function GetApp(): ModuleType {
                 success: true,
               })
               logger.DebugLog(
-                `Question result length: ${result.length}`,
+                `Question result length: ${mergedResult.length}`,
                 'ask',
                 1
               )
-              logger.DebugLog(result, 'ask', 2)
+              logger.DebugLog(mergedResult, 'ask', 2)
             } catch (err) {
               console.error(err)
               logger.Log(

src/utils/actions.ts

@@ -21,7 +21,8 @@ const recDataFile = './stats/recdata'
 const dataLockFile = './data/lockData'
 
 import logger from '../utils/logger'
-import { searchData, createQuestion } from '../utils/classes'
+import { createQuestion } from '../utils/classes'
+import { doALongTask } from './workerPool'
 import idStats from '../utils/ids'
 import utils from '../utils/utils'
 import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
@@ -166,7 +167,14 @@ function processIncomingRequestUsingDb(
         logger.DebugLog(currentQuestion, 'actions', 3)
         recievedQuestions.push(currentQuestion)
         questionSearchPromises.push(
-          searchData(qdb, currentQuestion, recievedData.subj)
+          doALongTask({
+            type: 'work',
+            data: {
+              qdb: qdb.data,
+              question: currentQuestion,
+              subjName: recievedData.subj,
+            },
+          })
         )
       })
 

src/utils/classes.ts

@@ -585,7 +585,6 @@ const workerTs = (file: string, wkOpts: any) => {
 
 if (!isMainThread) {
   const workerIndex = workerData.workerIndex
-  // TODO: check if thread independent
 
   logger.Log(
     `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
@@ -595,7 +594,11 @@ if (!isMainThread) {
     if (msg.type === 'work') {
       const { qdb, subjName, question, questionData } = msg.data
       const index = msg.index
-      console.log(`[THREAD #${workerIndex}]: staring work on ${index}`)
+      console.log(
+        `[THREAD #${workerIndex}]: staring work${
+          !isNaN(index) ? ` on job index #${index}` : ''
+        }`
+      )
 
       let searchResult = null
       try {
@@ -607,12 +610,18 @@ if (!isMainThread) {
 
       // ONDONE:
       parentPort.postMessage({
-        msg: `From thread #${workerIndex}: job ${index} done`,
+        msg: `From thread #${workerIndex}: job ${
+          !isNaN(index) ? `#${index}` : ''
+        }done`,
         workerIndex: workerIndex,
         result: searchResult,
       })
 
-      console.log(`[THREAD #${workerIndex}]: Work ${index} done!`)
+      console.log(
+        `[THREAD #${workerIndex}]: Work ${
+          !isNaN(index) ? `#${index}` : ''
+        }done!`
+      )
     } else if (msg.type === 'update') {
       if (msg.data.workerIndex !== workerIndex) {
         // TODO

src/utils/workerPool.ts

@@ -18,7 +18,7 @@ let workers: any = null
 //   console.log('[MSGFROMCLIENT]', res)
 // })
 
-export function doALongTask(i: Number, obj: any): Promise<any> {
+export function doALongTask(obj: any): Promise<any> {
   return new Promise((resolve) => {
     pool
       .acquire()
@@ -47,11 +47,12 @@ export function initWorkerPool(): void {
   workers = []
   const factory = {
     create: function() {
-      const worker = getAWorker(workers.length)
+      const currInd = workers.length
+      const worker = getAWorker(currInd)
       workers.push(worker)
       return {
         worker: worker,
-        index: workers.length,
+        index: currInd,
       }
     },
     destroy: function(client) {