Threads store their own qdb. It gets updated when new questions are added to reduce message sizes between threads

src/modules/api/api.ts

@@ -35,11 +35,16 @@ import {
   backupData,
   loadJSON,
   RecievedData,
+  getQuestionDbsWithoutFunct,
 } from '../../utils/actions'
 import dbtools from '../../utils/dbtools'
 import auth from '../../middlewares/auth.middleware'
 import { dataToString } from '../../utils/classes'
-import { initWorkerPool, doALongTask } from '../../utils/workerPool'
+import {
+  initWorkerPool,
+  doALongTask,
+  msgAllWorker,
+} from '../../utils/workerPool'
 
 import { SetupData } from '../../server'
 import { ModuleType, User, DataFile, Request } from '../../types/basicTypes'
@@ -76,8 +81,6 @@ let publicdirs = []
 function GetApp(): ModuleType {
   const app = express()
 
-  initWorkerPool()
-
   const publicDir = publicdirs[0]
   if (!publicDir) {
     throw new Error(`No public dir! ( API )`)
@@ -166,6 +169,8 @@ function GetApp(): ModuleType {
   // FIXME: check type from file
   let testUsers: any = []
 
+  initWorkerPool(getQuestionDbsWithoutFunct(questionDbs))
+
   function mergeObjSum(a, b) {
     const res = { ...b }
     Object.keys(a).forEach((key) => {
@@ -1031,6 +1036,10 @@ function GetApp(): ModuleType {
             success: resultArray.length > 0,
             newQuestions: resultArray,
           })
+          msgAllWorker({
+            qdbs: getQuestionDbsWithoutFunct(questionDbs),
+            type: 'update',
+          })
         })
         .catch((err) => {
           logger.Log(
@@ -1067,18 +1076,7 @@ function GetApp(): ModuleType {
       if (req.query.q && req.query.data) {
         const subj: any = req.query.subj || ''
         const question = req.query.q
-        let recData: any = req.query.data
+        const recData: any = req.query.data
-        if (typeof recData === 'string') {
-          try {
-            recData = JSON.parse(recData)
-          } catch (err) {
-            logger.Log(
-              'Error parsing recData in /ask!',
-              logger.GetColor('redbg')
-            )
-            console.error(err)
-          }
-        }
 
         const promises = []
 
@@ -1087,8 +1085,8 @@ function GetApp(): ModuleType {
             doALongTask({
               type: 'work',
               data: {
-                qdb: qdb.data,
+                searchIn: [qdb.index], // TODO: search in all
-                question,
+                question: question,
                 subjName: subj,
                 questionData: recData,
               },

src/types/basicTypes.ts

@@ -25,6 +25,7 @@ export interface DataFile {
 
 export interface QuestionDb extends DataFile {
   data: Array<Subject>
+  index: Number
 }
 
 export interface User {

src/utils/actions.ts

@@ -22,7 +22,7 @@ const dataLockFile = './data/lockData'
 
 import logger from '../utils/logger'
 import { createQuestion } from '../utils/classes'
-import { doALongTask } from './workerPool'
+import { doALongTask  } from './workerPool'
 import idStats from '../utils/ids'
 import utils from '../utils/utils'
 import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
@@ -170,6 +170,7 @@ function processIncomingRequestUsingDb(
           doALongTask({
             type: 'work',
             data: {
+              searchIn: [qdb.index],
               qdb: qdb.data,
               question: currentQuestion,
               subjName: recievedData.subj,
@@ -201,6 +202,8 @@ function processIncomingRequestUsingDb(
                 )
                 logger.DebugLog(currentQuestion, 'actions', 3)
                 addQuestion(qdb.data, sName, currentQuestion)
+                // TODO: check if it really adds it, not only just some clone (questionDbs in api.ts
+                // modifies too)
               })
 
               currWrites++
@@ -261,7 +264,7 @@ function processIncomingRequestUsingDb(
 }
 
 export function loadJSON(dataFiles: Array<DataFile>): Array<QuestionDb> {
-  return dataFiles.reduce((acc, dataFile) => {
+  return dataFiles.reduce((acc, dataFile, index) => {
     if (!utils.FileExists(dataFile.path)) {
       utils.WriteFile(JSON.stringify([]), dataFile.path)
     }
@@ -269,6 +272,7 @@ export function loadJSON(dataFiles: Array<DataFile>): Array<QuestionDb> {
     try {
       acc.push({
         ...dataFile,
+        index: index,
         data: JSON.parse(utils.ReadFile(dataFile.path)),
       })
     } catch (err) {
@@ -302,3 +306,12 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
     }
   })
 }
+
+export function getQuestionDbsWithoutFunct(
+  questionDbs: Array<QuestionDb> // FIXME: type for dis
+): Array<any> {
+  return questionDbs.map((qdb) => {
+    const { shouldSave, ...res } = qdb // eslint-disable-line
+    return res
+  })
+}

src/utils/classes.ts

@@ -1,11 +1,6 @@
-import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'
+import { isMainThread, parentPort, workerData } from 'worker_threads'
 import logger from './logger'
-import {
+import { Question, QuestionData, Subject } from '../types/basicTypes'
-  Question,
-  QuestionDb,
-  QuestionData,
-  Subject,
-} from '../types/basicTypes'
 
 interface SearchResultQuestion extends Question {
   match: number
@@ -16,8 +11,6 @@ export interface SearchResult {
   dbName: string
 }
 
-const searchDataWorkerFile = './src/utils/classes.ts'
-
 const assert = (val) => {
   if (!val) {
     throw new Error('Assertion failed')
@@ -216,7 +209,7 @@ function createQuestion(
   return {
     Q: simplifyQuestion(question),
     A: answer ? simplifyAnswer(answer) : null,
-    data,
+    data: data,
   }
 }
 
@@ -414,83 +407,13 @@ function prepareQuestion(
     preparedQuestion = question
   } else {
     // FIXME data was checkedif its null, it should be never null. check if its really never null
-    const parsedData = typeof data === 'object' ? data : JSON.parse(data)
+    const parsedData = typeof data === 'object' ? data : JSON.parse(data) // TODO: put in try?
     preparedQuestion = createQuestion(question, null, parsedData)
   }
 
   return simplifyQuestion(preparedQuestion)
 }
 
-function searchData(
-  qdb: QuestionDb,
-  question: Question | string,
-  subjName: string,
-  questionData?: QuestionData | string
-): Promise<SearchResult> {
-  // FIXME subjName was checkedif its null, it should be never null. check if its really never null
-  return new Promise((resolve, reject) => {
-    assert(question)
-    logger.DebugLog('Searching for question', 'qdb search', 1)
-
-    const preparedQuestion = prepareQuestion(question, questionData)
-
-    logger.DebugLog('Question:', 'qdb search', 2)
-    logger.DebugLog(preparedQuestion, 'qdb search', 2)
-    logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2)
-
-    const worker = workerTs(searchDataWorkerFile, {
-      workerData: {
-        data: qdb.data,
-        subjName,
-        question: preparedQuestion,
-        questionData,
-      },
-    })
-
-    worker.on('error', (err) => {
-      logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
-      console.error(err)
-      reject(err)
-    })
-
-    worker.on('exit', (code) => {
-      logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
-      if (code !== 0) {
-        logger.Log(
-          'Search Data Worker error! Exit code is not 0',
-          logger.GetColor('redbg')
-        )
-        reject(new Error('Search Data Worker error! Exit code is not 0'))
-      }
-    })
-
-    worker.on('message', (result) => {
-      // TODO: remove (?)
-      if (typeof result === 'string') {
-        try {
-          console.log(JSON.parse(result))
-        } catch (err) {
-          console.log(result)
-        }
-      }
-      logger.DebugLog(`Worker message arrived`, 'worker', 2)
-      logger.DebugLog(result, 'worker', 3)
-      logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-      logger.DebugLog(result, 'ask', 2)
-
-      logger.DebugLog(
-        `QDB search result length: ${result.length}`,
-        'qdb search',
-        1
-      )
-      resolve({
-        result: result,
-        dbName: qdb.name,
-      })
-    })
-  })
-}
-
 function dataToString(data: Array<Subject>): string {
   const result = []
   data.forEach((subj) => {
@@ -509,10 +432,7 @@ function doSearch(
 ): any {
   let result = []
 
-  const questionToSearch =
+  const questionToSearch = prepareQuestion(question, questionData)
-    typeof question === 'string'
-      ? createQuestion(question, null, questionData || { type: 'simple' })
-      : question
 
   assert(questionToSearch.data)
 
@@ -565,26 +485,9 @@ function doSearch(
   return result
 }
 
-const workerTs = (file: string, wkOpts: any) => {
-  wkOpts.eval = true
-  if (!wkOpts.workerData) {
-    wkOpts.workerData = {}
-  }
-  wkOpts.workerData.__filename = file
-  return new Worker(
-    `
-            const wk = require('worker_threads');
-            require('ts-node').register();
-            let file = wk.workerData.__filename;
-            delete wk.workerData.__filename;
-            require(file);
-        `,
-    wkOpts
-  )
-}
-
 if (!isMainThread) {
-  const workerIndex = workerData.workerIndex
+  const { workerIndex } = workerData
+  let qdbs: Array<any> = workerData.initData
 
   logger.Log(
     `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
@@ -592,7 +495,7 @@ if (!isMainThread) {
 
   parentPort.on('message', (msg) => {
     if (msg.type === 'work') {
-      const { qdb, subjName, question, questionData } = msg.data
+      const { searchIn, subjName, question, questionData } = msg.data
       const index = msg.index
       console.log(
         `[THREAD #${workerIndex}]: staring work${
@@ -601,8 +504,12 @@ if (!isMainThread) {
       )
 
       let searchResult = null
+      const currQdb = qdbs.find((qdb) => {
+        return searchIn[0] === qdb.index
+      })
+
       try {
-        searchResult = doSearch(qdb, subjName, question, questionData)
+        searchResult = doSearch(currQdb.data, subjName, question, questionData)
       } catch (err) {
         logger.Log('Error in worker thread!', logger.GetColor('redbg'))
         console.error(err)
@@ -623,12 +530,8 @@ if (!isMainThread) {
         }done!`
       )
     } else if (msg.type === 'update') {
-      if (msg.data.workerIndex !== workerIndex) {
+      qdbs = msg.qdbs
-        // TODO
+      console.log(`[THREAD #${workerIndex}]: update`)
-        // qdbs = msg.qdb
-        console.log(`[THREAD #${workerIndex}]: update`, msg.data)
-        console.log(`[THREAD #${workerIndex}]: From ... to ${msg.data.result}`)
-      }
     }
   })
 } else {
@@ -642,6 +545,5 @@ export {
   getSubjNameWithoutYear,
   createQuestion,
   addQuestion,
-  searchData,
   dataToString,
 }

src/utils/workerPool.ts

@@ -39,7 +39,7 @@ export function doALongTask(obj: any): Promise<any> {
   })
 }
 
-export function initWorkerPool(): void {
+export function initWorkerPool(initData: any): void {
   if (workers && pool) {
     logger.Log('WORKER AND POOL ALREADY EXISTS', logger.GetColor('redbg'))
     return
@@ -48,7 +48,7 @@ export function initWorkerPool(): void {
   const factory = {
     create: function() {
       const currInd = workers.length
-      const worker = getAWorker(currInd)
+      const worker = getAWorker(currInd, initData)
       workers.push(worker)
       return {
         worker: worker,
@@ -71,21 +71,19 @@ export function initWorkerPool(): void {
   pool = genericPool.createPool(factory, opts)
 }
 
-export function msgAll(data: any): void {
+export function msgAllWorker(data: any): void {
   workers.forEach((worker) => {
-    worker.postMessage({
+    worker.postMessage(data)
-      type: 'update',
-      data,
-    })
   })
 }
 
 // ---------------------------------------------------------------------------
 
-function getAWorker(i) {
+function getAWorker(i, initData) {
   const worker = workerTs(workerFile, {
     workerData: {
       workerIndex: i,
+      initData: initData,
     },
   })
 
@@ -105,7 +103,10 @@ function getAWorker(i) {
   })
 
   worker.on('exit', (code) => {
-    logger.Log(`[MAIN]: worker #${i} exit code: ${code}`)
+    logger.Log(
+      `[MAIN]: worker #${i} exit code: ${code}`,
+      code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
+    )
   })
   return worker
 }