diff --git a/src/server.js b/src/server.js index dc1005f..8455f41 100755 --- a/src/server.js +++ b/src/server.js @@ -42,8 +42,6 @@ const reqlogger = require('./middlewares/reqlogger.middleware.js') const extraModulesFile = './src/extraModules.json' const modulesFile = './src/modules.json' const usersDBPath = 'data/dbs/users.db' -const { initLogger } = require('./utils/classes.js') -initLogger(logger.DebugLog) const idStats = require('./utils/ids.js') idStats.Load() diff --git a/src/utils/actions.js b/src/utils/actions.js index ee0304c..7acd72e 100755 --- a/src/utils/actions.js +++ b/src/utils/actions.js @@ -26,7 +26,7 @@ const dataFile = './publicDirs/qminingPublic/data.json' const recDataFile = './stats/recdata' const logger = require('../utils/logger.js') -const { searchData } = require('../utils/classes.js') +const { searchData, createQuestion } = require('../utils/classes.js') const idStats = require('../utils/ids.js') const utils = require('../utils/utils.js') const { addQuestion, getSubjNameWithoutYear } = require('./classes.js') @@ -72,25 +72,32 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) { logger.DebugLog('recievedData JSON parsed', 'actions', 1) logger.DebugLog(data, 'actions', 3) let allQLength = data.quiz.length + const questionSearchPromises = [] + data.quiz.forEach((question) => { + logger.DebugLog('Question:', 'actions', 2) + logger.DebugLog(question, 'actions', 2) + let currentQuestion = createQuestion( + question.Q, + question.A, + question.data + ) + logger.DebugLog( + 'Searching for question in subj ' + data.subj, + 'actions', + 3 + ) + logger.DebugLog(currentQuestion, 'actions', 3) + questionSearchPromises.push(searchData(qdb, currentQuestion, data.subj)) + }) - logger.DebugLog('Question:', 'actions', 2) - logger.DebugLog(question, 'actions', 2) - let currentQuestion = createQuestion( - question.Q, - question.A, - question.data - ) - logger.DebugLog( - 'Searching for question in subj ' + data.subj, - 'actions', - 3 - ) - logger.DebugLog(currentQuestion, 'actions', 3) - - searchData(qdb, currentQuestion, data.subj) - .then((allQuestions) => { - logger.DebugLog('Message from processData', 'actions', 1) - logger.DebugLog(allQuestions, 'actions', 3) + Promise.all(questionSearchPromises) + .then((results) => { + const allQuestions = [] // all new questions here that do not have result + results.forEach((result) => { + if (result.length === 0) { + allQuestions.push(result) + } + }) try { let color = logger.GetColor('green') @@ -158,9 +165,11 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) { } }) .catch((err) => { - logger.Log('Process Data Worker error!', logger.GetColor('redbg')) + logger.Log( + 'Error while searching for questions in ProcessIncomingRequest!', + logger.GetColor('redbg') + ) console.error(err) - reject(err) }) } catch (err) { console.log(err) diff --git a/src/utils/classes.js b/src/utils/classes.js index ecac78c..5b43612 100755 --- a/src/utils/classes.js +++ b/src/utils/classes.js @@ -4,9 +4,16 @@ const { parentPort, workerData, } = require('worker_threads') -const searchDataWorkerFile = './src/utils/searchData.js' -const logger = require('../../utils/logger.js') +const logger = require('./logger.js') + +const searchDataWorkerFile = './src/utils/classes.js' + +const assert = (val) => { + if (!val) { + throw new Error('Assertion failed') + } +} const commonUselessAnswerParts = [ 'A helyes válasz az ', @@ -18,17 +25,12 @@ const commonUselessAnswerParts = [ 'The correct answer is:', "'", ] + const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.'] const specialChars = ['&', '\\+'] const lengthDiffMultiplier = 10 /* Percent minus for length difference */ const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */ -const assert = (val) => { - if (!val) { - throw new Error('Assertion failed') - } -} - // --------------------------------------------------------------------------------------------------------- // String Utils // --------------------------------------------------------------------------------------------------------- @@ -153,16 +155,22 @@ function simplifyAnswer(value) { } function simplifyQuestion(question) { - if (typeof q === 'string') { - return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces]) + if (typeof question === 'string') { + return simplifyQA(question, [ + removeSpecialChars, + removeUnnecesarySpaces, + removeAnswerLetters, + ]) } else { question.Q = simplifyQA(question.Q, [ removeSpecialChars, removeUnnecesarySpaces, + removeAnswerLetters, ]) question.A = simplifyQA(question.A, [ removeSpecialChars, removeUnnecesarySpaces, + removeAnswerLetters, ]) return question } @@ -374,6 +382,7 @@ function searchData(data, question, subjName, questionData) { logger.DebugLog('No subject name as param!', 'qdb search', 1) } question = simplifyQuestion(question) + console.log('RESULT: ' + question) const worker = new Worker(searchDataWorkerFile, { workerData: { data, subjName, question, questionData }, @@ -443,6 +452,7 @@ function dataToString(data) { return result.join('\n\n') } +// ------------------------------------------------------------------------ if (!isMainThread) { const { data, subjName, question, questionData } = workerData let result = [] @@ -493,7 +503,9 @@ if (!isMainThread) { }) parentPort.postMessage(result) + process.exit(0) } +// ------------------------------------------------------------------------ module.exports = { minMatchAmmount, diff --git a/src/utils/processData.js b/src/utils/processData.js deleted file mode 100644 index a6fd2f9..0000000 --- a/src/utils/processData.js +++ /dev/null @@ -1,46 +0,0 @@ -const { isMainThread, parentPort, workerData } = require('worker_threads') -const logger = require('../utils/logger.js') -const { createQuestion, searchData } = require('./classes.js') - -const minMatchAmmountToAdd = 90 // FIXME: test this value - -if (!isMainThread) { - logger.DebugLog('Starting worker thread', 'processdata', 1) - logger.DebugLog(workerData, 'processdata', 3) - - parentPort.postMessage(ProcessData(workerData.data, workerData.qdb)) -} else { - logger.Log( - 'Porcess data should not run on main thread!', - logger.GetColor('redbg') - ) -} - -function ProcessData(data, qdb) { - let allQuestions = [] - - data.quiz.forEach((question) => { - logger.DebugLog('Question:', 'actions', 2) - logger.DebugLog(question, 'actions', 2) - let currentQuestion = createQuestion(question.Q, question.A, question.data) - logger.DebugLog('Searching for question in subj ' + data.subj, 'actions', 3) - logger.DebugLog(currentQuestion, 'actions', 3) - - let sames = searchData(qdb, currentQuestion, data.subj) - logger.DebugLog('Same questions:', 'actions', 2) - logger.DebugLog('Length: ' + sames.length, 'actions', 2) - logger.DebugLog(sames, 'actions', 3) - // if it didnt find any question, or every found questions match is lower thatn 80 - let isNew = - sames.length === 0 || - sames.every((searchResItem) => { - return searchResItem.match < minMatchAmmountToAdd - }) - logger.DebugLog('isNew: ' + isNew, 'actions', 2) - if (isNew) { - allQuestions.push(currentQuestion) - } - }) - - return allQuestions -} diff --git a/src/utils/searchData.js b/src/utils/searchData.js deleted file mode 100644 index 41b5b12..0000000 --- a/src/utils/searchData.js +++ /dev/null @@ -1,16 +0,0 @@ -const { isMainThread, parentPort, workerData } = require('worker_threads') -const logger = require('../utils/logger.js') -const { searchData } = require('./classes.js') - -if (!isMainThread) { - logger.DebugLog('Starting worker thread', 'searchdata', 1) - logger.DebugLog(workerData, 'searchdata', 3) - const { data, question, subj, recData } = workerData - - parentPort.postMessage(searchData(data, question, subj, recData)) -} else { - logger.Log( - 'Porcess data should not run on main thread!', - logger.GetColor('redbg') - ) -}