Problems converting process incoming request to use new searchData worker

src/modules/api/api.js

@@ -22,7 +22,6 @@
 const express = require('express')
 const bodyParser = require('body-parser')
 const busboy = require('connect-busboy')
-const { Worker } = require('worker_threads')
 const { v4: uuidv4 } = require('uuid')
 const fs = require('fs')
 const app = express()
@@ -33,10 +32,9 @@ const utils = require('../../utils/utils.js')
 const actions = require('../../utils/actions.js')
 const dbtools = require('../../utils/dbtools.js')
 const auth = require('../../middlewares/auth.middleware.js')
-const { dataToString } = require('../../utils/classes.js')
+const { dataToString, searchData } = require('../../utils/classes.js')
 
 // files
-const searchDataWorkerFile = './src/utils/searchData.js'
 const msgFile = 'stats/msgs'
 const passwordFile = 'data/dataEditorPasswords.json'
 const dataEditsLog = 'stats/dataEdits'
@@ -763,17 +761,21 @@ function GetApp() {
           )
         }
 
-        const worker = new Worker(searchDataWorkerFile, {
+        searchData(data, question, subj, recData)
-          workerData: {
+          .then((result) => {
-            data,
+            res.json({
-            question,
+              result: result,
-            subj,
+              success: true,
-            recData,
-          },
             })
-
+            logger.DebugLog(
-        worker.on('error', (err) => {
+              `Question result length: ${result.length}`,
-          logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
+              'ask',
+              1
+            )
+            logger.DebugLog(result, 'ask', 2)
+          })
+          .catch((err) => {
+            logger.Log('Search Data error!', logger.GetColor('redbg'))
             console.error(err)
             res.json({
               message: `There was an error processing the question: ${err.message}`,
@@ -782,34 +784,6 @@ function GetApp() {
               success: false,
             })
           })
-
-        worker.on('exit', (code) => {
-          logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
-          if (code !== 0) {
-            logger.Log(
-              'Search Data Worker error! Exit code is not 0',
-              logger.GetColor('redbg')
-            )
-          }
-        })
-
-        worker.on('message', (workerMsg) => {
-          const result = workerMsg
-          res.json({
-            result: result,
-            success: true,
-          })
-          logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-          logger.DebugLog(result, 'ask', 2)
-        })
-
-        // let result = data.Search(question, subj, recData)
-        // res.json({
-        //   result: result,
-        //   success: true,
-        // })
-        // logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-        // logger.DebugLog(result, 'ask', 2)
       } else {
         logger.DebugLog(`Invalid question`, 'ask', 1)
         res.json({

src/utils/actions.js

@@ -24,10 +24,9 @@ module.exports = {
 
 const dataFile = './publicDirs/qminingPublic/data.json'
 const recDataFile = './stats/recdata'
-const processDataWorkerFile = './src/utils/processData.js'
 
-const { Worker } = require('worker_threads')
 const logger = require('../utils/logger.js')
+const { searchData } = require('../utils/classes.js')
 const idStats = require('../utils/ids.js')
 const utils = require('../utils/utils.js')
 const { addQuestion, getSubjNameWithoutYear } = require('./classes.js')
@@ -74,35 +73,24 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
       logger.DebugLog(data, 'actions', 3)
       let allQLength = data.quiz.length
 
-      const worker = new Worker(processDataWorkerFile, {
+      logger.DebugLog('Question:', 'actions', 2)
-        workerData: {
+      logger.DebugLog(question, 'actions', 2)
-          data: data,
+      let currentQuestion = createQuestion(
-          qdb: qdb,
+        question.Q,
-        },
+        question.A,
-      })
+        question.data
-
-      worker.on('error', (err) => {
-        logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
-        console.error(err)
-        reject(err)
-      })
-
-      worker.on('exit', (code) => {
-        logger.DebugLog('Process Data exit, code: ' + code, 'actions', 1)
-        if (code !== 0) {
-          logger.Log(
-            'Process Data Worker error! Exit code is not 0',
-            logger.GetColor('redbg')
       )
-          reject(new Error('Process Data Worker exit code is not 0!'))
+      logger.DebugLog(
-        }
+        'Searching for question in subj ' + data.subj,
-      })
+        'actions',
+        3
+      )
+      logger.DebugLog(currentQuestion, 'actions', 3)
 
-      worker.on('message', (workerMsg) => {
+      searchData(qdb, currentQuestion, data.subj)
+        .then((allQuestions) => {
           logger.DebugLog('Message from processData', 'actions', 1)
-        logger.DebugLog(workerMsg, 'actions', 3)
+          logger.DebugLog(allQuestions, 'actions', 3)
-
-        const allQuestions = workerMsg
 
           try {
             let color = logger.GetColor('green')
@@ -164,9 +152,16 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
               'Error while processing processData worker result!',
               logger.GetColor('redbg')
             )
-          reject(new Error('Error while processing processData worker result!'))
+            reject(
+              new Error('Error while processing processData worker result!')
+            )
           }
         })
+        .catch((err) => {
+          logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
+          console.error(err)
+          reject(err)
+        })
     } catch (err) {
       console.log(err)
       logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))

src/utils/classes.js

@@ -1,14 +1,12 @@
-var debugLogger = null
+const {
+  Worker,
+  isMainThread,
+  parentPort,
+  workerData,
+} = require('worker_threads')
+const searchDataWorkerFile = './src/utils/searchData.js'
 
-function initLogger(logger) {
+const logger = require('../../utils/logger.js')
-  debugLogger = logger
-}
-
-function debugLog(msg, name, lvl) {
-  if (debugLogger) {
-    debugLogger(msg, name, lvl)
-  }
-}
 
 const commonUselessAnswerParts = [
   'A helyes válasz az ',
@@ -124,6 +122,8 @@ function answerPreProcessor(value) {
 function removeAnswerLetters(value) {
   assert(value)
 
+  console.log(value)
+
   let val = value.split('. ')
   if (val[0].length < 2 && val.length > 1) {
     val.shift()
@@ -138,36 +138,31 @@ function simplifyQA(value, mods) {
     return
   }
 
-  const reducer = (res, fn) => {
+  return mods.reduce((res, fn) => {
     return fn(res)
-  }
+  }, value)
-
-  return mods.reduce(reducer, value)
 }
 
 function simplifyAnswer(value) {
   return simplifyQA(value, [
-    removeSpecialChars.bind(this),
+    removeSpecialChars,
-    removeUnnecesarySpaces.bind(this),
+    removeUnnecesarySpaces,
-    answerPreProcessor.bind(this),
+    answerPreProcessor,
-    removeAnswerLetters.bind(this),
+    removeAnswerLetters,
   ])
 }
 
 function simplifyQuestion(question) {
   if (typeof q === 'string') {
-    return simplifyQA(question, [
+    return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces])
-      removeSpecialChars.bind(this),
-      removeUnnecesarySpaces.bind(this),
-    ])
   } else {
     question.Q = simplifyQA(question.Q, [
-      removeSpecialChars.bind(this),
+      removeSpecialChars,
-      removeUnnecesarySpaces.bind(this),
+      removeUnnecesarySpaces,
     ])
     question.A = simplifyQA(question.A, [
-      removeSpecialChars.bind(this),
+      removeSpecialChars,
-      removeUnnecesarySpaces.bind(this),
+      removeUnnecesarySpaces,
     ])
     return question
   }
@@ -198,16 +193,20 @@ function compareData(q1, q2) {
       } else if (dataType === 'image') {
         return compareImage(q1.data, q2.data)
       } else {
-        debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1)
+        logger.DebugLog(
-        debugLog(q1, 'Compare question data', 2)
+          `Unhandled data type ${dataType}`,
+          'Compare question data',
+          1
+        )
+        logger.DebugLog(q1, 'Compare question data', 2)
       }
     } else {
       return 0
     }
   } catch (error) {
-    debugLog('Error comparing data', 'Compare question data', 1)
+    logger.DebugLog('Error comparing data', 'Compare question data', 1)
-    debugLog(error.message, 'Compare question data', 1)
+    logger.DebugLog(error.message, 'Compare question data', 1)
-    debugLog(error, 'Compare question data', 2)
+    logger.DebugLog(error, 'Compare question data', 2)
   }
   return 0
 }
@@ -327,8 +326,8 @@ function subjectToString(subj) {
 // QuestionDB
 // ---------------------------------------------------------------------------------------------------------
 function addQuestion(data, subj, question) {
-  debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
+  logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
-  debugLog(question, 'qdb add', 3)
+  logger.DebugLog(question, 'qdb add', 3)
   assert(data)
   assert(subj)
   assert(question)
@@ -345,10 +344,10 @@ function addQuestion(data, subj, question) {
   }
 
   if (i < data.length) {
-    debugLog('Adding new question to existing subject', 'qdb add', 1)
+    logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
     data[i].Questions.push(question)
   } else {
-    debugLog('Creating new subject for question', 'qdb add', 1)
+    logger.DebugLog('Creating new subject for question', 'qdb add', 1)
     data.push({
       Name: subj,
       Questions: [question],
@@ -357,72 +356,58 @@ function addQuestion(data, subj, question) {
 }
 
 function searchData(data, question, subjName, questionData) {
+  return new Promise((resolve, reject) => {
     assert(data)
     assert(question)
-  debugLog('Searching for question', 'qdb search', 1)
+    logger.DebugLog('Searching for question', 'qdb search', 1)
-  debugLog('Question:', 'qdb search', 2)
+    logger.DebugLog('Question:', 'qdb search', 2)
-  debugLog(question, 'qdb search', 2)
+    logger.DebugLog(question, 'qdb search', 2)
-  debugLog(`Subject name: ${subjName}`, 'qdb search', 2)
+    logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2)
-  debugLog('Data:', 'qdb search', 2)
+    logger.DebugLog('Data:', 'qdb search', 2)
-  debugLog(questionData || question.data, 'qdb search', 2)
+    logger.DebugLog(questionData || question.data, 'qdb search', 2)
 
     if (!questionData) {
       questionData = question.data || { type: 'simple' }
     }
     if (!subjName) {
       subjName = ''
-    debugLog('No subject name as param!', 'qdb search', 1)
+      logger.DebugLog('No subject name as param!', 'qdb search', 1)
     }
     question = simplifyQuestion(question)
 
-  let result = []
+    const worker = new Worker(searchDataWorkerFile, {
-  data.forEach((subj) => {
+      workerData: { data, subjName, question, questionData },
-    if (
+    })
-      subjName
+
-        .toLowerCase()
+    worker.on('error', (err) => {
-        .includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
+      logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
-    ) {
+      console.error(err)
-      debugLog(`Searching in ${subj.Name} `, 2)
+      reject(err)
-      result = result.concat(
+    })
-        searchQuestion(subj, question, questionData, subjName)
+
+    worker.on('exit', (code) => {
+      logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
+      if (code !== 0) {
+        logger.Log(
+          'Search Data Worker error! Exit code is not 0',
+          logger.GetColor('redbg')
         )
+        reject(new Error('Search Data Worker error! Exit code is not 0'))
       }
     })
 
-  // FIXME: try to remove this? but this is also a good backup plan so idk
+    worker.on('message', (result) => {
-  if (result.length === 0) {
+      logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-    debugLog(
+      logger.DebugLog(result, 'ask', 2)
-      'Reqults length is zero when comparing names, trying all subjects',
+
+      logger.DebugLog(
+        `QDB search result length: ${result.length}`,
         'qdb search',
         1
       )
-    data.forEach((subj) => {
+      resolve(result)
-      result = result.concat(
-        searchQuestion(subj, question, questionData, subjName)
-      )
     })
-    if (result.length > 0) {
-      debugLog(
-        `FIXME: '${subjName}' gave no result but '' did!`,
-        'qdb search',
-        1
-      )
-      console.error(`FIXME: '${subjName}' gave no result but '' did!`)
-    }
-  }
-
-  result = result.sort((q1, q2) => {
-    if (q1.match < q2.match) {
-      return 1
-    } else if (q1.match > q2.match) {
-      return -1
-    } else {
-      return 0
-    }
   })
-
-  debugLog(`QDB search result length: ${result.length}`, 'qdb search', 1)
-  return result
 }
 
 function addSubject(data, subj) {
@@ -458,9 +443,60 @@ function dataToString(data) {
   return result.join('\n\n')
 }
 
+if (!isMainThread) {
+  const { data, subjName, question, questionData } = workerData
+  let result = []
+
+  data.forEach((subj) => {
+    if (
+      subjName
+        .toLowerCase()
+        .includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
+    ) {
+      logger.DebugLog(`Searching in ${subj.Name} `, 2)
+      result = result.concat(
+        searchQuestion(subj, question, questionData, subjName)
+      )
+    }
+  })
+
+  // FIXME: try to remove this? but this is also a good backup plan so idk
+  if (result.length === 0) {
+    logger.DebugLog(
+      'Reqults length is zero when comparing names, trying all subjects',
+      'qdb search',
+      1
+    )
+    data.forEach((subj) => {
+      result = result.concat(
+        searchQuestion(subj, question, questionData, subjName)
+      )
+    })
+    if (result.length > 0) {
+      logger.DebugLog(
+        `FIXME: '${subjName}' gave no result but '' did!`,
+        'qdb search',
+        1
+      )
+      console.error(`FIXME: '${subjName}' gave no result but '' did!`)
+    }
+  }
+
+  result = result.sort((q1, q2) => {
+    if (q1.match < q2.match) {
+      return 1
+    } else if (q1.match > q2.match) {
+      return -1
+    } else {
+      return 0
+    }
+  })
+
+  parentPort.postMessage(result)
+}
+
 module.exports = {
   minMatchAmmount,
-  initLogger,
   getSubjNameWithoutYear,
   createQuestion,
   addQuestion,

submodules/moodle-test-userscript

@@ -1 +1 @@
-Subproject commit 443203848c09e20a812f0e1f664ba9519cf28708
+Subproject commit f8d4bf2a414d2973582c08bfa8b8b8f19389b2e9