Problems converting process incoming request to use new searchData worker

src/modules/api/api.js

@@ -22,7 +22,6 @@
 const express = require('express')
 const bodyParser = require('body-parser')
 const busboy = require('connect-busboy')
-const { Worker } = require('worker_threads')
 const { v4: uuidv4 } = require('uuid')
 const fs = require('fs')
 const app = express()
@@ -33,10 +32,9 @@ const utils = require('../../utils/utils.js')
 const actions = require('../../utils/actions.js')
 const dbtools = require('../../utils/dbtools.js')
 const auth = require('../../middlewares/auth.middleware.js')
-const { dataToString } = require('../../utils/classes.js')
+const { dataToString, searchData } = require('../../utils/classes.js')
 
 // files
-const searchDataWorkerFile = './src/utils/searchData.js'
 const msgFile = 'stats/msgs'
 const passwordFile = 'data/dataEditorPasswords.json'
 const dataEditsLog = 'stats/dataEdits'
@@ -763,53 +761,29 @@ function GetApp() {
           )
         }
 
-        const worker = new Worker(searchDataWorkerFile, {
+        searchData(data, question, subj, recData)
-          workerData: {
+          .then((result) => {
-            data,
+            res.json({
-            question,
+              result: result,
-            subj,
+              success: true,
-            recData,
+            })
-          },
+            logger.DebugLog(
-        })
+              `Question result length: ${result.length}`,
-
+              'ask',
-        worker.on('error', (err) => {
+              1
-          logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
-          console.error(err)
-          res.json({
-            message: `There was an error processing the question: ${err.message}`,
-            result: [],
-            recievedData: JSON.stringify(req.query),
-            success: false,
-          })
-        })
-
-        worker.on('exit', (code) => {
-          logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
-          if (code !== 0) {
-            logger.Log(
-              'Search Data Worker error! Exit code is not 0',
-              logger.GetColor('redbg')
             )
-          }
+            logger.DebugLog(result, 'ask', 2)
-        })
+          })
-
+          .catch((err) => {
-        worker.on('message', (workerMsg) => {
+            logger.Log('Search Data error!', logger.GetColor('redbg'))
-          const result = workerMsg
+            console.error(err)
-          res.json({
+            res.json({
-            result: result,
+              message: `There was an error processing the question: ${err.message}`,
-            success: true,
+              result: [],
+              recievedData: JSON.stringify(req.query),
+              success: false,
+            })
           })
-          logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-          logger.DebugLog(result, 'ask', 2)
-        })
-
-        // let result = data.Search(question, subj, recData)
-        // res.json({
-        //   result: result,
-        //   success: true,
-        // })
-        // logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-        // logger.DebugLog(result, 'ask', 2)
       } else {
         logger.DebugLog(`Invalid question`, 'ask', 1)
         res.json({

src/utils/actions.js

@@ -24,10 +24,9 @@ module.exports = {
 
 const dataFile = './publicDirs/qminingPublic/data.json'
 const recDataFile = './stats/recdata'
-const processDataWorkerFile = './src/utils/processData.js'
 
-const { Worker } = require('worker_threads')
 const logger = require('../utils/logger.js')
+const { searchData } = require('../utils/classes.js')
 const idStats = require('../utils/ids.js')
 const utils = require('../utils/utils.js')
 const { addQuestion, getSubjNameWithoutYear } = require('./classes.js')
@@ -74,99 +73,95 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
       logger.DebugLog(data, 'actions', 3)
       let allQLength = data.quiz.length
 
-      const worker = new Worker(processDataWorkerFile, {
+      logger.DebugLog('Question:', 'actions', 2)
-        workerData: {
+      logger.DebugLog(question, 'actions', 2)
-          data: data,
+      let currentQuestion = createQuestion(
-          qdb: qdb,
+        question.Q,
-        },
+        question.A,
-      })
+        question.data
+      )
+      logger.DebugLog(
+        'Searching for question in subj ' + data.subj,
+        'actions',
+        3
+      )
+      logger.DebugLog(currentQuestion, 'actions', 3)
 
-      worker.on('error', (err) => {
+      searchData(qdb, currentQuestion, data.subj)
-        logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
+        .then((allQuestions) => {
-        console.error(err)
+          logger.DebugLog('Message from processData', 'actions', 1)
-        reject(err)
+          logger.DebugLog(allQuestions, 'actions', 3)
-      })
 
-      worker.on('exit', (code) => {
+          try {
-        logger.DebugLog('Process Data exit, code: ' + code, 'actions', 1)
+            let color = logger.GetColor('green')
-        if (code !== 0) {
+            let msg = ''
-          logger.Log(
+            if (allQuestions.length > 0) {
-            'Process Data Worker error! Exit code is not 0',
+              color = logger.GetColor('blue')
-            logger.GetColor('redbg')
+              msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
-          )
+              allQuestions.forEach((currentQuestion) => {
-          reject(new Error('Process Data Worker exit code is not 0!'))
+                const sName = getSubjNameWithoutYear(data.subj)
-        }
+                logger.DebugLog(
-      })
+                  'Adding question with subjName: ' + sName + ' :',
+                  'actions',
+                  3
+                )
+                logger.DebugLog(currentQuestion, 'actions', 3)
+                addQuestion(qdb, sName, currentQuestion)
+              })
 
-      worker.on('message', (workerMsg) => {
+              currWrites++
-        logger.DebugLog('Message from processData', 'actions', 1)
-        logger.DebugLog(workerMsg, 'actions', 3)
-
-        const allQuestions = workerMsg
-
-        try {
-          let color = logger.GetColor('green')
-          let msg = ''
-          if (allQuestions.length > 0) {
-            color = logger.GetColor('blue')
-            msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
-            allQuestions.forEach((currentQuestion) => {
-              const sName = getSubjNameWithoutYear(data.subj)
               logger.DebugLog(
-                'Adding question with subjName: ' + sName + ' :',
+                'currWrites for data.json: ' + currWrites,
                 'actions',
-                3
+                1
               )
-              logger.DebugLog(currentQuestion, 'actions', 3)
+              if (currWrites >= writeAfter && !dryRun) {
-              addQuestion(qdb, sName, currentQuestion)
+                currWrites = 0
-            })
+                logger.DebugLog('Writing data.json', 'actions', 1)
-
+                utils.WriteFile(JSON.stringify(qdb), dataFile)
-            currWrites++
+                logger.Log('\tData file written', color)
-            logger.DebugLog(
+              } else if (dryRun) {
-              'currWrites for data.json: ' + currWrites,
+                logger.Log('\tDry run')
-              'actions',
+              }
-              1
+            } else {
-            )
+              msg += `No new data ( ${allQLength} )`
-            if (currWrites >= writeAfter && !dryRun) {
-              currWrites = 0
-              logger.DebugLog('Writing data.json', 'actions', 1)
-              utils.WriteFile(JSON.stringify(qdb), dataFile)
-              logger.Log('\tData file written', color)
-            } else if (dryRun) {
-              logger.Log('\tDry run')
             }
-          } else {
-            msg += `No new data ( ${allQLength} )`
-          }
 
-          let subjRow = '\t' + data.subj
+            let subjRow = '\t' + data.subj
-          if (data.id) {
+            if (data.id) {
-            subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')'
+              subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')'
-            idStats.LogId(user.id, data.subj, allQuestions.length, allQLength)
+              idStats.LogId(user.id, data.subj, allQuestions.length, allQLength)
-          }
+            }
-          logger.Log(subjRow)
+            logger.Log(subjRow)
-          if (data.version !== undefined) {
+            if (data.version !== undefined) {
-            msg += '. Version: ' + data.version
+              msg += '. Version: ' + data.version
-          }
+            }
 
-          logger.Log('\t' + msg, color)
+            logger.Log('\t' + msg, color)
-          logger.DebugLog('New Questions:', 'actions', 2)
+            logger.DebugLog('New Questions:', 'actions', 2)
-          logger.DebugLog(allQuestions, 'actions', 2)
+            logger.DebugLog(allQuestions, 'actions', 2)
 
-          logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
+            logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
-          resolve({
+            resolve({
-            newQuestions: allQLength.length,
+              newQuestions: allQLength.length,
-            newDb: qdb,
+              newDb: qdb,
-          })
+            })
-        } catch (error) {
+          } catch (error) {
-          console.log(error)
+            console.log(error)
-          logger.Log(
+            logger.Log(
-            'Error while processing processData worker result!',
+              'Error while processing processData worker result!',
-            logger.GetColor('redbg')
+              logger.GetColor('redbg')
-          )
+            )
-          reject(new Error('Error while processing processData worker result!'))
+            reject(
-        }
+              new Error('Error while processing processData worker result!')
-      })
+            )
+          }
+        })
+        .catch((err) => {
+          logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
+          console.error(err)
+          reject(err)
+        })
     } catch (err) {
       console.log(err)
       logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))

src/utils/classes.js

@@ -1,14 +1,12 @@
-var debugLogger = null
+const {
+  Worker,
+  isMainThread,
+  parentPort,
+  workerData,
+} = require('worker_threads')
+const searchDataWorkerFile = './src/utils/searchData.js'
 
-function initLogger(logger) {
+const logger = require('../../utils/logger.js')
-  debugLogger = logger
-}
-
-function debugLog(msg, name, lvl) {
-  if (debugLogger) {
-    debugLogger(msg, name, lvl)
-  }
-}
 
 const commonUselessAnswerParts = [
   'A helyes válasz az ',
@@ -124,6 +122,8 @@ function answerPreProcessor(value) {
 function removeAnswerLetters(value) {
   assert(value)
 
+  console.log(value)
+
   let val = value.split('. ')
   if (val[0].length < 2 && val.length > 1) {
     val.shift()
@@ -138,36 +138,31 @@ function simplifyQA(value, mods) {
     return
   }
 
-  const reducer = (res, fn) => {
+  return mods.reduce((res, fn) => {
     return fn(res)
-  }
+  }, value)
-
-  return mods.reduce(reducer, value)
 }
 
 function simplifyAnswer(value) {
   return simplifyQA(value, [
-    removeSpecialChars.bind(this),
+    removeSpecialChars,
-    removeUnnecesarySpaces.bind(this),
+    removeUnnecesarySpaces,
-    answerPreProcessor.bind(this),
+    answerPreProcessor,
-    removeAnswerLetters.bind(this),
+    removeAnswerLetters,
   ])
 }
 
 function simplifyQuestion(question) {
   if (typeof q === 'string') {
-    return simplifyQA(question, [
+    return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces])
-      removeSpecialChars.bind(this),
-      removeUnnecesarySpaces.bind(this),
-    ])
   } else {
     question.Q = simplifyQA(question.Q, [
-      removeSpecialChars.bind(this),
+      removeSpecialChars,
-      removeUnnecesarySpaces.bind(this),
+      removeUnnecesarySpaces,
     ])
     question.A = simplifyQA(question.A, [
-      removeSpecialChars.bind(this),
+      removeSpecialChars,
-      removeUnnecesarySpaces.bind(this),
+      removeUnnecesarySpaces,
     ])
     return question
   }
@@ -198,16 +193,20 @@ function compareData(q1, q2) {
       } else if (dataType === 'image') {
         return compareImage(q1.data, q2.data)
       } else {
-        debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1)
+        logger.DebugLog(
-        debugLog(q1, 'Compare question data', 2)
+          `Unhandled data type ${dataType}`,
+          'Compare question data',
+          1
+        )
+        logger.DebugLog(q1, 'Compare question data', 2)
       }
     } else {
       return 0
     }
   } catch (error) {
-    debugLog('Error comparing data', 'Compare question data', 1)
+    logger.DebugLog('Error comparing data', 'Compare question data', 1)
-    debugLog(error.message, 'Compare question data', 1)
+    logger.DebugLog(error.message, 'Compare question data', 1)
-    debugLog(error, 'Compare question data', 2)
+    logger.DebugLog(error, 'Compare question data', 2)
   }
   return 0
 }
@@ -327,8 +326,8 @@ function subjectToString(subj) {
 // QuestionDB
 // ---------------------------------------------------------------------------------------------------------
 function addQuestion(data, subj, question) {
-  debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
+  logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
-  debugLog(question, 'qdb add', 3)
+  logger.DebugLog(question, 'qdb add', 3)
   assert(data)
   assert(subj)
   assert(question)
@@ -345,10 +344,10 @@ function addQuestion(data, subj, question) {
   }
 
   if (i < data.length) {
-    debugLog('Adding new question to existing subject', 'qdb add', 1)
+    logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
     data[i].Questions.push(question)
   } else {
-    debugLog('Creating new subject for question', 'qdb add', 1)
+    logger.DebugLog('Creating new subject for question', 'qdb add', 1)
     data.push({
       Name: subj,
       Questions: [question],
@@ -357,72 +356,58 @@ function addQuestion(data, subj, question) {
 }
 
 function searchData(data, question, subjName, questionData) {
-  assert(data)
+  return new Promise((resolve, reject) => {
-  assert(question)
+    assert(data)
-  debugLog('Searching for question', 'qdb search', 1)
+    assert(question)
-  debugLog('Question:', 'qdb search', 2)
+    logger.DebugLog('Searching for question', 'qdb search', 1)
-  debugLog(question, 'qdb search', 2)
+    logger.DebugLog('Question:', 'qdb search', 2)
-  debugLog(`Subject name: ${subjName}`, 'qdb search', 2)
+    logger.DebugLog(question, 'qdb search', 2)
-  debugLog('Data:', 'qdb search', 2)
+    logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2)
-  debugLog(questionData || question.data, 'qdb search', 2)
+    logger.DebugLog('Data:', 'qdb search', 2)
+    logger.DebugLog(questionData || question.data, 'qdb search', 2)
 
-  if (!questionData) {
+    if (!questionData) {
-    questionData = question.data || { type: 'simple' }
+      questionData = question.data || { type: 'simple' }
-  }
-  if (!subjName) {
-    subjName = ''
-    debugLog('No subject name as param!', 'qdb search', 1)
-  }
-  question = simplifyQuestion(question)
-
-  let result = []
-  data.forEach((subj) => {
-    if (
-      subjName
-        .toLowerCase()
-        .includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
-    ) {
-      debugLog(`Searching in ${subj.Name} `, 2)
-      result = result.concat(
-        searchQuestion(subj, question, questionData, subjName)
-      )
     }
-  })
+    if (!subjName) {
+      subjName = ''
+      logger.DebugLog('No subject name as param!', 'qdb search', 1)
+    }
+    question = simplifyQuestion(question)
 
-  // FIXME: try to remove this? but this is also a good backup plan so idk
+    const worker = new Worker(searchDataWorkerFile, {
-  if (result.length === 0) {
+      workerData: { data, subjName, question, questionData },
-    debugLog(
-      'Reqults length is zero when comparing names, trying all subjects',
-      'qdb search',
-      1
-    )
-    data.forEach((subj) => {
-      result = result.concat(
-        searchQuestion(subj, question, questionData, subjName)
-      )
     })
-    if (result.length > 0) {
+
-      debugLog(
+    worker.on('error', (err) => {
-        `FIXME: '${subjName}' gave no result but '' did!`,
+      logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
+      console.error(err)
+      reject(err)
+    })
+
+    worker.on('exit', (code) => {
+      logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
+      if (code !== 0) {
+        logger.Log(
+          'Search Data Worker error! Exit code is not 0',
+          logger.GetColor('redbg')
+        )
+        reject(new Error('Search Data Worker error! Exit code is not 0'))
+      }
+    })
+
+    worker.on('message', (result) => {
+      logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
+      logger.DebugLog(result, 'ask', 2)
+
+      logger.DebugLog(
+        `QDB search result length: ${result.length}`,
         'qdb search',
         1
       )
-      console.error(`FIXME: '${subjName}' gave no result but '' did!`)
+      resolve(result)
-    }
+    })
-  }
-
-  result = result.sort((q1, q2) => {
-    if (q1.match < q2.match) {
-      return 1
-    } else if (q1.match > q2.match) {
-      return -1
-    } else {
-      return 0
-    }
   })
-
-  debugLog(`QDB search result length: ${result.length}`, 'qdb search', 1)
-  return result
 }
 
 function addSubject(data, subj) {
@@ -458,9 +443,60 @@ function dataToString(data) {
   return result.join('\n\n')
 }
 
+if (!isMainThread) {
+  const { data, subjName, question, questionData } = workerData
+  let result = []
+
+  data.forEach((subj) => {
+    if (
+      subjName
+        .toLowerCase()
+        .includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
+    ) {
+      logger.DebugLog(`Searching in ${subj.Name} `, 2)
+      result = result.concat(
+        searchQuestion(subj, question, questionData, subjName)
+      )
+    }
+  })
+
+  // FIXME: try to remove this? but this is also a good backup plan so idk
+  if (result.length === 0) {
+    logger.DebugLog(
+      'Reqults length is zero when comparing names, trying all subjects',
+      'qdb search',
+      1
+    )
+    data.forEach((subj) => {
+      result = result.concat(
+        searchQuestion(subj, question, questionData, subjName)
+      )
+    })
+    if (result.length > 0) {
+      logger.DebugLog(
+        `FIXME: '${subjName}' gave no result but '' did!`,
+        'qdb search',
+        1
+      )
+      console.error(`FIXME: '${subjName}' gave no result but '' did!`)
+    }
+  }
+
+  result = result.sort((q1, q2) => {
+    if (q1.match < q2.match) {
+      return 1
+    } else if (q1.match > q2.match) {
+      return -1
+    } else {
+      return 0
+    }
+  })
+
+  parentPort.postMessage(result)
+}
+
 module.exports = {
   minMatchAmmount,
-  initLogger,
   getSubjNameWithoutYear,
   createQuestion,
   addQuestion,

submodules/moodle-test-userscript

@@ -1 +1 @@
-Subproject commit 443203848c09e20a812f0e1f664ba9519cf28708
+Subproject commit f8d4bf2a414d2973582c08bfa8b8b8f19389b2e9