Added worker to search for stuff in db (2) #10

.eslintrc.js

@@ -14,7 +14,7 @@ module.exports = {
   rules: {
     'no-undef': ['error'],
     eqeqeq: ['warn', 'smart'],
-    'no-unused-vars': 'off',
+    'no-unused-vars': 'warn',
     'no-prototype-builtins': 'off',
     'id-length': ['warn', { exceptions: ['i', 'j'] }],
   },

src/modules/api/api.js

@@ -22,6 +22,7 @@
 const express = require('express')
 const bodyParser = require('body-parser')
 const busboy = require('connect-busboy')
+const { Worker } = require('worker_threads')
 const { v4: uuidv4 } = require('uuid')
 const fs = require('fs')
 const app = express()
@@ -34,6 +35,7 @@ const dbtools = require('../../utils/dbtools.js')
 const auth = require('../../middlewares/auth.middleware.js')
 
 // files
+const searchDataWorkerFile = './src/utils/searchData.js'
 const msgFile = 'stats/msgs'
 const passwordFile = 'data/dataEditorPasswords.json'
 const dataEditsLog = 'stats/dataEdits'
@@ -512,7 +514,7 @@ function GetApp() {
   })
 
   app.post('/postfeedbackfile', function(req, res) {
-    UploadFile(req, res, uloadFiles, (fn) => {
+    UploadFile(req, res, uloadFiles, () => {
       res.json({ success: true })
     })
 
@@ -590,14 +592,14 @@ function GetApp() {
     }
   }
 
-  app.route('/fosuploader').post(function(req, res, next) {
+  app.route('/fosuploader').post(function(req, res) {
     UploadFile(req, res, uloadFiles, (fn) => {
       res.redirect('/f/' + fn)
     })
   })
 
-  app.route('/badtestsender').post(function(req, res, next) {
+  app.route('/badtestsender').post(function(req, res) {
-    UploadFile(req, res, recivedFiles, (fn) => {
+    UploadFile(req, res, recivedFiles, () => {
       res.redirect('back')
     })
     logger.LogReq(req)
@@ -759,14 +761,43 @@ function GetApp() {
             logger.GetColor('redbg')
           )
         }
-        let result = data.Search(question, subj, recData)
 
-        res.json({
+        const worker = new Worker(searchDataWorkerFile, {
-          result: result,
+          workerData: {
-          success: true,
+            data,
+            question,
+            subj,
+            recData,
+          },
+        })
+
+        worker.on('error', (err) => {
+          logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
+          console.error(err)
+          // TODO: handle error
+        })
+
+        worker.on('exit', (code) => {
+          logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
+          if (code !== 0) {
+            logger.Log(
+              'Search Data Worker error! Exit code is not 0',
+              logger.GetColor('redbg')
+            )
+            // TODO: handle error
+          }
+        })
+
+        // let result = data.Search(question, subj, recData)
+        worker.on('message', (workerMsg) => {
+          const result = workerMsg
+          res.json({
+            result: result,
+            success: true,
+          })
+          logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
+          logger.DebugLog(result, 'ask', 2)
         })
-        logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
-        logger.DebugLog(result, 'ask', 2)
       } else {
         logger.DebugLog(`Invalid question`, 'ask', 1)
         res.json({
@@ -930,6 +961,6 @@ exports.name = 'API'
 exports.getApp = GetApp
 exports.setup = (data) => {
   userDB = data.userDB
-  url = data.url
+  url = data.url // eslint-disable-line
   publicdirs = data.publicdirs
 }

src/utils/actions.js

@@ -91,7 +91,7 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun) {
       })
 
       worker.on('exit', (code) => {
-        logger.DebugLog('ProcessData exit, code: ' + code, 'actions', 1)
+        logger.DebugLog('Process Data exit, code: ' + code, 'actions', 1)
         if (code !== 0) {
           logger.Log(
             'Process Data Worker error! Exit code is not 0',

src/utils/processData.js

@@ -1,12 +1,4 @@
-const dataFile = './qminingPublic/data.json'
+const { isMainThread, parentPort, workerData } = require('worker_threads')
-const recDataFile = './stats/recdata'
-
-const {
-  Worker,
-  isMainThread,
-  parentPort,
-  workerData,
-} = require('worker_threads')
 const logger = require('../utils/logger.js')
 const actions = require('../utils/actions.js')
 const classes = require('./classes.js')
@@ -16,7 +8,7 @@ const minMatchAmmountToAdd = 90 // FIXME: test this value
 
 if (!isMainThread) {
   logger.DebugLog('Starting worker thread', 'processdata', 1)
-  console.log(workerData)
+  logger.DebugLog(workerData, 'processdata', 1)
 
   parentPort.postMessage(
     ProcessData(workerData.data, actions.LoadJSONFromObject(workerData.qdb))

src/utils/searchData.js

@@ -0,0 +1,22 @@
+const { isMainThread, parentPort, workerData } = require('worker_threads')
+const logger = require('../utils/logger.js')
+const actions = require('../utils/actions.js')
+
+if (!isMainThread) {
+  logger.DebugLog('Starting worker thread', 'searchdata', 1)
+  logger.DebugLog(workerData, 'searchdata', 1)
+  const { data, question, subj, recData } = workerData
+
+  parentPort.postMessage(
+    SearchData(actions.LoadJSONFromObject(data), question, subj, recData)
+  )
+} else {
+  logger.Log(
+    'Porcess data should not run on main thread!',
+    logger.GetColor('redbg')
+  )
+}
+
+function SearchData(data, question, subj, recData) {
+  return data.Search(question, subj, recData)
+}