searchData is now using workers, fixed random question letter in question

src/server.js

@@ -42,8 +42,6 @@ const reqlogger = require('./middlewares/reqlogger.middleware.js')
 const extraModulesFile = './src/extraModules.json'
 const modulesFile = './src/modules.json'
 const usersDBPath = 'data/dbs/users.db'
-const { initLogger } = require('./utils/classes.js')
-initLogger(logger.DebugLog)
 const idStats = require('./utils/ids.js')
 idStats.Load()
 

src/utils/actions.js

@@ -26,7 +26,7 @@ const dataFile = './publicDirs/qminingPublic/data.json'
 const recDataFile = './stats/recdata'
 
 const logger = require('../utils/logger.js')
-const { searchData } = require('../utils/classes.js')
+const { searchData, createQuestion } = require('../utils/classes.js')
 const idStats = require('../utils/ids.js')
 const utils = require('../utils/utils.js')
 const { addQuestion, getSubjNameWithoutYear } = require('./classes.js')
@@ -72,25 +72,32 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
       logger.DebugLog('recievedData JSON  parsed', 'actions', 1)
       logger.DebugLog(data, 'actions', 3)
       let allQLength = data.quiz.length
+      const questionSearchPromises = []
+      data.quiz.forEach((question) => {
+        logger.DebugLog('Question:', 'actions', 2)
+        logger.DebugLog(question, 'actions', 2)
+        let currentQuestion = createQuestion(
+          question.Q,
+          question.A,
+          question.data
+        )
+        logger.DebugLog(
+          'Searching for question in subj ' + data.subj,
+          'actions',
+          3
+        )
+        logger.DebugLog(currentQuestion, 'actions', 3)
+        questionSearchPromises.push(searchData(qdb, currentQuestion, data.subj))
+      })
 
-      logger.DebugLog('Question:', 'actions', 2)
-      logger.DebugLog(question, 'actions', 2)
-      let currentQuestion = createQuestion(
-        question.Q,
-        question.A,
-        question.data
-      )
-      logger.DebugLog(
-        'Searching for question in subj ' + data.subj,
-        'actions',
-        3
-      )
-      logger.DebugLog(currentQuestion, 'actions', 3)
-
-      searchData(qdb, currentQuestion, data.subj)
-        .then((allQuestions) => {
-          logger.DebugLog('Message from processData', 'actions', 1)
-          logger.DebugLog(allQuestions, 'actions', 3)
+      Promise.all(questionSearchPromises)
+        .then((results) => {
+          const allQuestions = [] // all new questions here that do not have result
+          results.forEach((result) => {
+            if (result.length === 0) {
+              allQuestions.push(result)
+            }
+          })
 
           try {
             let color = logger.GetColor('green')
@@ -158,9 +165,11 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
           }
         })
         .catch((err) => {
-          logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
+          logger.Log(
+            'Error while searching for questions in ProcessIncomingRequest!',
+            logger.GetColor('redbg')
+          )
           console.error(err)
-          reject(err)
         })
     } catch (err) {
       console.log(err)

src/utils/classes.js

@@ -4,9 +4,16 @@ const {
   parentPort,
   workerData,
 } = require('worker_threads')
-const searchDataWorkerFile = './src/utils/searchData.js'
 
-const logger = require('../../utils/logger.js')
+const logger = require('./logger.js')
+
+const searchDataWorkerFile = './src/utils/classes.js'
+
+const assert = (val) => {
+  if (!val) {
+    throw new Error('Assertion failed')
+  }
+}
 
 const commonUselessAnswerParts = [
   'A helyes válasz az ',
@@ -18,17 +25,12 @@ const commonUselessAnswerParts = [
   'The correct answer is:',
   "'",
 ]
+
 const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
 const specialChars = ['&', '\\+']
 const lengthDiffMultiplier = 10 /* Percent minus for length difference */
 const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */
 
-const assert = (val) => {
-  if (!val) {
-    throw new Error('Assertion failed')
-  }
-}
-
 // ---------------------------------------------------------------------------------------------------------
 // String Utils
 // ---------------------------------------------------------------------------------------------------------
@@ -153,16 +155,22 @@ function simplifyAnswer(value) {
 }
 
 function simplifyQuestion(question) {
-  if (typeof q === 'string') {
-    return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces])
+  if (typeof question === 'string') {
+    return simplifyQA(question, [
+      removeSpecialChars,
+      removeUnnecesarySpaces,
+      removeAnswerLetters,
+    ])
   } else {
     question.Q = simplifyQA(question.Q, [
       removeSpecialChars,
       removeUnnecesarySpaces,
+      removeAnswerLetters,
     ])
     question.A = simplifyQA(question.A, [
       removeSpecialChars,
       removeUnnecesarySpaces,
+      removeAnswerLetters,
     ])
     return question
   }
@@ -374,6 +382,7 @@ function searchData(data, question, subjName, questionData) {
       logger.DebugLog('No subject name as param!', 'qdb search', 1)
     }
     question = simplifyQuestion(question)
+    console.log('RESULT: ' + question)
 
     const worker = new Worker(searchDataWorkerFile, {
       workerData: { data, subjName, question, questionData },
@@ -443,6 +452,7 @@ function dataToString(data) {
   return result.join('\n\n')
 }
 
+// ------------------------------------------------------------------------
 if (!isMainThread) {
   const { data, subjName, question, questionData } = workerData
   let result = []
@@ -493,7 +503,9 @@ if (!isMainThread) {
   })
 
   parentPort.postMessage(result)
+  process.exit(0)
 }
+// ------------------------------------------------------------------------
 
 module.exports = {
   minMatchAmmount,

src/utils/processData.js

@@ -1,46 +0,0 @@
-const { isMainThread, parentPort, workerData } = require('worker_threads')
-const logger = require('../utils/logger.js')
-const { createQuestion, searchData } = require('./classes.js')
-
-const minMatchAmmountToAdd = 90 // FIXME: test this value
-
-if (!isMainThread) {
-  logger.DebugLog('Starting worker thread', 'processdata', 1)
-  logger.DebugLog(workerData, 'processdata', 3)
-
-  parentPort.postMessage(ProcessData(workerData.data, workerData.qdb))
-} else {
-  logger.Log(
-    'Porcess data should not run on main thread!',
-    logger.GetColor('redbg')
-  )
-}
-
-function ProcessData(data, qdb) {
-  let allQuestions = []
-
-  data.quiz.forEach((question) => {
-    logger.DebugLog('Question:', 'actions', 2)
-    logger.DebugLog(question, 'actions', 2)
-    let currentQuestion = createQuestion(question.Q, question.A, question.data)
-    logger.DebugLog('Searching for question in subj ' + data.subj, 'actions', 3)
-    logger.DebugLog(currentQuestion, 'actions', 3)
-
-    let sames = searchData(qdb, currentQuestion, data.subj)
-    logger.DebugLog('Same questions:', 'actions', 2)
-    logger.DebugLog('Length: ' + sames.length, 'actions', 2)
-    logger.DebugLog(sames, 'actions', 3)
-    // if it didnt find any question, or every found questions match is lower thatn 80
-    let isNew =
-      sames.length === 0 ||
-      sames.every((searchResItem) => {
-        return searchResItem.match < minMatchAmmountToAdd
-      })
-    logger.DebugLog('isNew: ' + isNew, 'actions', 2)
-    if (isNew) {
-      allQuestions.push(currentQuestion)
-    }
-  })
-
-  return allQuestions
-}

src/utils/searchData.js

@@ -1,16 +0,0 @@
-const { isMainThread, parentPort, workerData } = require('worker_threads')
-const logger = require('../utils/logger.js')
-const { searchData } = require('./classes.js')
-
-if (!isMainThread) {
-  logger.DebugLog('Starting worker thread', 'searchdata', 1)
-  logger.DebugLog(workerData, 'searchdata', 3)
-  const { data, question, subj, recData } = workerData
-
-  parentPort.postMessage(searchData(data, question, subj, recData))
-} else {
-  logger.Log(
-    'Porcess data should not run on main thread!',
-    logger.GetColor('redbg')
-  )
-}