From 63b08244dcb976200c2692e15efbe0cf0e72453d Mon Sep 17 00:00:00 2001 From: mrfry Date: Wed, 14 Oct 2020 10:22:21 +0200 Subject: [PATCH] Problems converting process incoming request to use new searchData worker --- src/modules/api/api.js | 70 +++------- src/utils/actions.js | 165 +++++++++++----------- src/utils/classes.js | 224 +++++++++++++++++------------- submodules/moodle-test-userscript | 2 +- 4 files changed, 233 insertions(+), 228 deletions(-) diff --git a/src/modules/api/api.js b/src/modules/api/api.js index 77da458..cc0dc00 100644 --- a/src/modules/api/api.js +++ b/src/modules/api/api.js @@ -22,7 +22,6 @@ const express = require('express') const bodyParser = require('body-parser') const busboy = require('connect-busboy') -const { Worker } = require('worker_threads') const { v4: uuidv4 } = require('uuid') const fs = require('fs') const app = express() @@ -33,10 +32,9 @@ const utils = require('../../utils/utils.js') const actions = require('../../utils/actions.js') const dbtools = require('../../utils/dbtools.js') const auth = require('../../middlewares/auth.middleware.js') -const { dataToString } = require('../../utils/classes.js') +const { dataToString, searchData } = require('../../utils/classes.js') // files -const searchDataWorkerFile = './src/utils/searchData.js' const msgFile = 'stats/msgs' const passwordFile = 'data/dataEditorPasswords.json' const dataEditsLog = 'stats/dataEdits' @@ -763,53 +761,29 @@ function GetApp() { ) } - const worker = new Worker(searchDataWorkerFile, { - workerData: { - data, - question, - subj, - recData, - }, - }) - - worker.on('error', (err) => { - logger.Log('Search Data Worker error!', logger.GetColor('redbg')) - console.error(err) - res.json({ - message: `There was an error processing the question: ${err.message}`, - result: [], - recievedData: JSON.stringify(req.query), - success: false, - }) - }) - - worker.on('exit', (code) => { - logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1) - if (code !== 0) { - logger.Log( - 'Search Data Worker error! Exit code is not 0', - logger.GetColor('redbg') + searchData(data, question, subj, recData) + .then((result) => { + res.json({ + result: result, + success: true, + }) + logger.DebugLog( + `Question result length: ${result.length}`, + 'ask', + 1 ) - } - }) - - worker.on('message', (workerMsg) => { - const result = workerMsg - res.json({ - result: result, - success: true, + logger.DebugLog(result, 'ask', 2) + }) + .catch((err) => { + logger.Log('Search Data error!', logger.GetColor('redbg')) + console.error(err) + res.json({ + message: `There was an error processing the question: ${err.message}`, + result: [], + recievedData: JSON.stringify(req.query), + success: false, + }) }) - logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) - logger.DebugLog(result, 'ask', 2) - }) - - // let result = data.Search(question, subj, recData) - // res.json({ - // result: result, - // success: true, - // }) - // logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) - // logger.DebugLog(result, 'ask', 2) } else { logger.DebugLog(`Invalid question`, 'ask', 1) res.json({ diff --git a/src/utils/actions.js b/src/utils/actions.js index 76deb9d..ee0304c 100755 --- a/src/utils/actions.js +++ b/src/utils/actions.js @@ -24,10 +24,9 @@ module.exports = { const dataFile = './publicDirs/qminingPublic/data.json' const recDataFile = './stats/recdata' -const processDataWorkerFile = './src/utils/processData.js' -const { Worker } = require('worker_threads') const logger = require('../utils/logger.js') +const { searchData } = require('../utils/classes.js') const idStats = require('../utils/ids.js') const utils = require('../utils/utils.js') const { addQuestion, getSubjNameWithoutYear } = require('./classes.js') @@ -74,99 +73,95 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) { logger.DebugLog(data, 'actions', 3) let allQLength = data.quiz.length - const worker = new Worker(processDataWorkerFile, { - workerData: { - data: data, - qdb: qdb, - }, - }) + logger.DebugLog('Question:', 'actions', 2) + logger.DebugLog(question, 'actions', 2) + let currentQuestion = createQuestion( + question.Q, + question.A, + question.data + ) + logger.DebugLog( + 'Searching for question in subj ' + data.subj, + 'actions', + 3 + ) + logger.DebugLog(currentQuestion, 'actions', 3) - worker.on('error', (err) => { - logger.Log('Process Data Worker error!', logger.GetColor('redbg')) - console.error(err) - reject(err) - }) + searchData(qdb, currentQuestion, data.subj) + .then((allQuestions) => { + logger.DebugLog('Message from processData', 'actions', 1) + logger.DebugLog(allQuestions, 'actions', 3) - worker.on('exit', (code) => { - logger.DebugLog('Process Data exit, code: ' + code, 'actions', 1) - if (code !== 0) { - logger.Log( - 'Process Data Worker error! Exit code is not 0', - logger.GetColor('redbg') - ) - reject(new Error('Process Data Worker exit code is not 0!')) - } - }) + try { + let color = logger.GetColor('green') + let msg = '' + if (allQuestions.length > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` + allQuestions.forEach((currentQuestion) => { + const sName = getSubjNameWithoutYear(data.subj) + logger.DebugLog( + 'Adding question with subjName: ' + sName + ' :', + 'actions', + 3 + ) + logger.DebugLog(currentQuestion, 'actions', 3) + addQuestion(qdb, sName, currentQuestion) + }) - worker.on('message', (workerMsg) => { - logger.DebugLog('Message from processData', 'actions', 1) - logger.DebugLog(workerMsg, 'actions', 3) - - const allQuestions = workerMsg - - try { - let color = logger.GetColor('green') - let msg = '' - if (allQuestions.length > 0) { - color = logger.GetColor('blue') - msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` - allQuestions.forEach((currentQuestion) => { - const sName = getSubjNameWithoutYear(data.subj) + currWrites++ logger.DebugLog( - 'Adding question with subjName: ' + sName + ' :', + 'currWrites for data.json: ' + currWrites, 'actions', - 3 + 1 ) - logger.DebugLog(currentQuestion, 'actions', 3) - addQuestion(qdb, sName, currentQuestion) - }) - - currWrites++ - logger.DebugLog( - 'currWrites for data.json: ' + currWrites, - 'actions', - 1 - ) - if (currWrites >= writeAfter && !dryRun) { - currWrites = 0 - logger.DebugLog('Writing data.json', 'actions', 1) - utils.WriteFile(JSON.stringify(qdb), dataFile) - logger.Log('\tData file written', color) - } else if (dryRun) { - logger.Log('\tDry run') + if (currWrites >= writeAfter && !dryRun) { + currWrites = 0 + logger.DebugLog('Writing data.json', 'actions', 1) + utils.WriteFile(JSON.stringify(qdb), dataFile) + logger.Log('\tData file written', color) + } else if (dryRun) { + logger.Log('\tDry run') + } + } else { + msg += `No new data ( ${allQLength} )` } - } else { - msg += `No new data ( ${allQLength} )` - } - let subjRow = '\t' + data.subj - if (data.id) { - subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')' - idStats.LogId(user.id, data.subj, allQuestions.length, allQLength) - } - logger.Log(subjRow) - if (data.version !== undefined) { - msg += '. Version: ' + data.version - } + let subjRow = '\t' + data.subj + if (data.id) { + subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')' + idStats.LogId(user.id, data.subj, allQuestions.length, allQLength) + } + logger.Log(subjRow) + if (data.version !== undefined) { + msg += '. Version: ' + data.version + } - logger.Log('\t' + msg, color) - logger.DebugLog('New Questions:', 'actions', 2) - logger.DebugLog(allQuestions, 'actions', 2) + logger.Log('\t' + msg, color) + logger.DebugLog('New Questions:', 'actions', 2) + logger.DebugLog(allQuestions, 'actions', 2) - logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) - resolve({ - newQuestions: allQLength.length, - newDb: qdb, - }) - } catch (error) { - console.log(error) - logger.Log( - 'Error while processing processData worker result!', - logger.GetColor('redbg') - ) - reject(new Error('Error while processing processData worker result!')) - } - }) + logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) + resolve({ + newQuestions: allQLength.length, + newDb: qdb, + }) + } catch (error) { + console.log(error) + logger.Log( + 'Error while processing processData worker result!', + logger.GetColor('redbg') + ) + reject( + new Error('Error while processing processData worker result!') + ) + } + }) + .catch((err) => { + logger.Log('Process Data Worker error!', logger.GetColor('redbg')) + console.error(err) + reject(err) + }) } catch (err) { console.log(err) logger.Log('Couldnt parse JSON data', logger.GetColor('redbg')) diff --git a/src/utils/classes.js b/src/utils/classes.js index 891b9c9..ecac78c 100755 --- a/src/utils/classes.js +++ b/src/utils/classes.js @@ -1,14 +1,12 @@ -var debugLogger = null +const { + Worker, + isMainThread, + parentPort, + workerData, +} = require('worker_threads') +const searchDataWorkerFile = './src/utils/searchData.js' -function initLogger(logger) { - debugLogger = logger -} - -function debugLog(msg, name, lvl) { - if (debugLogger) { - debugLogger(msg, name, lvl) - } -} +const logger = require('../../utils/logger.js') const commonUselessAnswerParts = [ 'A helyes válasz az ', @@ -124,6 +122,8 @@ function answerPreProcessor(value) { function removeAnswerLetters(value) { assert(value) + console.log(value) + let val = value.split('. ') if (val[0].length < 2 && val.length > 1) { val.shift() @@ -138,36 +138,31 @@ function simplifyQA(value, mods) { return } - const reducer = (res, fn) => { + return mods.reduce((res, fn) => { return fn(res) - } - - return mods.reduce(reducer, value) + }, value) } function simplifyAnswer(value) { return simplifyQA(value, [ - removeSpecialChars.bind(this), - removeUnnecesarySpaces.bind(this), - answerPreProcessor.bind(this), - removeAnswerLetters.bind(this), + removeSpecialChars, + removeUnnecesarySpaces, + answerPreProcessor, + removeAnswerLetters, ]) } function simplifyQuestion(question) { if (typeof q === 'string') { - return simplifyQA(question, [ - removeSpecialChars.bind(this), - removeUnnecesarySpaces.bind(this), - ]) + return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces]) } else { question.Q = simplifyQA(question.Q, [ - removeSpecialChars.bind(this), - removeUnnecesarySpaces.bind(this), + removeSpecialChars, + removeUnnecesarySpaces, ]) question.A = simplifyQA(question.A, [ - removeSpecialChars.bind(this), - removeUnnecesarySpaces.bind(this), + removeSpecialChars, + removeUnnecesarySpaces, ]) return question } @@ -198,16 +193,20 @@ function compareData(q1, q2) { } else if (dataType === 'image') { return compareImage(q1.data, q2.data) } else { - debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1) - debugLog(q1, 'Compare question data', 2) + logger.DebugLog( + `Unhandled data type ${dataType}`, + 'Compare question data', + 1 + ) + logger.DebugLog(q1, 'Compare question data', 2) } } else { return 0 } } catch (error) { - debugLog('Error comparing data', 'Compare question data', 1) - debugLog(error.message, 'Compare question data', 1) - debugLog(error, 'Compare question data', 2) + logger.DebugLog('Error comparing data', 'Compare question data', 1) + logger.DebugLog(error.message, 'Compare question data', 1) + logger.DebugLog(error, 'Compare question data', 2) } return 0 } @@ -327,8 +326,8 @@ function subjectToString(subj) { // QuestionDB // --------------------------------------------------------------------------------------------------------- function addQuestion(data, subj, question) { - debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1) - debugLog(question, 'qdb add', 3) + logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1) + logger.DebugLog(question, 'qdb add', 3) assert(data) assert(subj) assert(question) @@ -345,10 +344,10 @@ function addQuestion(data, subj, question) { } if (i < data.length) { - debugLog('Adding new question to existing subject', 'qdb add', 1) + logger.DebugLog('Adding new question to existing subject', 'qdb add', 1) data[i].Questions.push(question) } else { - debugLog('Creating new subject for question', 'qdb add', 1) + logger.DebugLog('Creating new subject for question', 'qdb add', 1) data.push({ Name: subj, Questions: [question], @@ -357,72 +356,58 @@ function addQuestion(data, subj, question) { } function searchData(data, question, subjName, questionData) { - assert(data) - assert(question) - debugLog('Searching for question', 'qdb search', 1) - debugLog('Question:', 'qdb search', 2) - debugLog(question, 'qdb search', 2) - debugLog(`Subject name: ${subjName}`, 'qdb search', 2) - debugLog('Data:', 'qdb search', 2) - debugLog(questionData || question.data, 'qdb search', 2) + return new Promise((resolve, reject) => { + assert(data) + assert(question) + logger.DebugLog('Searching for question', 'qdb search', 1) + logger.DebugLog('Question:', 'qdb search', 2) + logger.DebugLog(question, 'qdb search', 2) + logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2) + logger.DebugLog('Data:', 'qdb search', 2) + logger.DebugLog(questionData || question.data, 'qdb search', 2) - if (!questionData) { - questionData = question.data || { type: 'simple' } - } - if (!subjName) { - subjName = '' - debugLog('No subject name as param!', 'qdb search', 1) - } - question = simplifyQuestion(question) - - let result = [] - data.forEach((subj) => { - if ( - subjName - .toLowerCase() - .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) - ) { - debugLog(`Searching in ${subj.Name} `, 2) - result = result.concat( - searchQuestion(subj, question, questionData, subjName) - ) + if (!questionData) { + questionData = question.data || { type: 'simple' } } - }) + if (!subjName) { + subjName = '' + logger.DebugLog('No subject name as param!', 'qdb search', 1) + } + question = simplifyQuestion(question) - // FIXME: try to remove this? but this is also a good backup plan so idk - if (result.length === 0) { - debugLog( - 'Reqults length is zero when comparing names, trying all subjects', - 'qdb search', - 1 - ) - data.forEach((subj) => { - result = result.concat( - searchQuestion(subj, question, questionData, subjName) - ) + const worker = new Worker(searchDataWorkerFile, { + workerData: { data, subjName, question, questionData }, }) - if (result.length > 0) { - debugLog( - `FIXME: '${subjName}' gave no result but '' did!`, + + worker.on('error', (err) => { + logger.Log('Search Data Worker error!', logger.GetColor('redbg')) + console.error(err) + reject(err) + }) + + worker.on('exit', (code) => { + logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1) + if (code !== 0) { + logger.Log( + 'Search Data Worker error! Exit code is not 0', + logger.GetColor('redbg') + ) + reject(new Error('Search Data Worker error! Exit code is not 0')) + } + }) + + worker.on('message', (result) => { + logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) + logger.DebugLog(result, 'ask', 2) + + logger.DebugLog( + `QDB search result length: ${result.length}`, 'qdb search', 1 ) - console.error(`FIXME: '${subjName}' gave no result but '' did!`) - } - } - - result = result.sort((q1, q2) => { - if (q1.match < q2.match) { - return 1 - } else if (q1.match > q2.match) { - return -1 - } else { - return 0 - } + resolve(result) + }) }) - - debugLog(`QDB search result length: ${result.length}`, 'qdb search', 1) - return result } function addSubject(data, subj) { @@ -458,9 +443,60 @@ function dataToString(data) { return result.join('\n\n') } +if (!isMainThread) { + const { data, subjName, question, questionData } = workerData + let result = [] + + data.forEach((subj) => { + if ( + subjName + .toLowerCase() + .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) + ) { + logger.DebugLog(`Searching in ${subj.Name} `, 2) + result = result.concat( + searchQuestion(subj, question, questionData, subjName) + ) + } + }) + + // FIXME: try to remove this? but this is also a good backup plan so idk + if (result.length === 0) { + logger.DebugLog( + 'Reqults length is zero when comparing names, trying all subjects', + 'qdb search', + 1 + ) + data.forEach((subj) => { + result = result.concat( + searchQuestion(subj, question, questionData, subjName) + ) + }) + if (result.length > 0) { + logger.DebugLog( + `FIXME: '${subjName}' gave no result but '' did!`, + 'qdb search', + 1 + ) + console.error(`FIXME: '${subjName}' gave no result but '' did!`) + } + } + + result = result.sort((q1, q2) => { + if (q1.match < q2.match) { + return 1 + } else if (q1.match > q2.match) { + return -1 + } else { + return 0 + } + }) + + parentPort.postMessage(result) +} + module.exports = { minMatchAmmount, - initLogger, getSubjNameWithoutYear, createQuestion, addQuestion, diff --git a/submodules/moodle-test-userscript b/submodules/moodle-test-userscript index 4432038..f8d4bf2 160000 --- a/submodules/moodle-test-userscript +++ b/submodules/moodle-test-userscript @@ -1 +1 @@ -Subproject commit 443203848c09e20a812f0e1f664ba9519cf28708 +Subproject commit f8d4bf2a414d2973582c08bfa8b8b8f19389b2e9