diff --git a/src/modules/api/api.js b/src/modules/api/api.js index c3c16da..338d95d 100644 --- a/src/modules/api/api.js +++ b/src/modules/api/api.js @@ -33,6 +33,7 @@ const utils = require('../../utils/utils.js') const actions = require('../../utils/actions.js') const dbtools = require('../../utils/dbtools.js') const auth = require('../../middlewares/auth.middleware.js') +const { searchData } = require('../../utils/classes.js') // files const searchDataWorkerFile = './src/utils/searchData.js' @@ -762,42 +763,49 @@ function GetApp() { ) } - const worker = new Worker(searchDataWorkerFile, { - workerData: { - data, - question, - subj, - recData, - }, - }) + // const worker = new Worker(searchDataWorkerFile, { + // workerData: { + // data, + // question, + // subj, + // recData, + // }, + // }) - worker.on('error', (err) => { - logger.Log('Search Data Worker error!', logger.GetColor('redbg')) - console.error(err) - // TODO: handle error - }) + // worker.on('error', (err) => { + // logger.Log('Search Data Worker error!', logger.GetColor('redbg')) + // console.error(err) + // // TODO: handle error + // }) - worker.on('exit', (code) => { - logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1) - if (code !== 0) { - logger.Log( - 'Search Data Worker error! Exit code is not 0', - logger.GetColor('redbg') - ) - // TODO: handle error - } - }) + // worker.on('exit', (code) => { + // logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1) + // if (code !== 0) { + // logger.Log( + // 'Search Data Worker error! Exit code is not 0', + // logger.GetColor('redbg') + // ) + // // TODO: handle error + // } + // }) - // let result = data.Search(question, subj, recData) - worker.on('message', (workerMsg) => { - const result = workerMsg - res.json({ - result: result, - success: true, - }) - logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) - logger.DebugLog(result, 'ask', 2) + // // let result = data.Search(question, subj, recData) + // worker.on('message', (workerMsg) => { + // const result = workerMsg + // res.json({ + // result: result, + // success: true, + // }) + // logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) + // logger.DebugLog(result, 'ask', 2) + // }) + let result = searchData(data, question, subj, recData) + res.json({ + result: result, + success: true, }) + logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) + logger.DebugLog(result, 'ask', 2) } else { logger.DebugLog(`Invalid question`, 'ask', 1) res.json({ diff --git a/src/server.js b/src/server.js index 068b1ac..bf273a0 100755 --- a/src/server.js +++ b/src/server.js @@ -35,13 +35,15 @@ const http = require('http') const https = require('https') const cors = require('cors') const cookieParser = require('cookie-parser') -const { v4: uuidv4 } = require('uuid'); +const { v4: uuidv4 } = require('uuid') const dbtools = require('./utils/dbtools.js') const reqlogger = require('./middlewares/reqlogger.middleware.js') const extraModulesFile = './src/extraModules.json' const modulesFile = './src/modules.json' const usersDBPath = 'data/dbs/users.db' +const { initLogger } = require('./utils/classes.js') +initLogger(logger.DebugLog) if (!utils.FileExists(usersDBPath)) { throw new Error('No user DB exists yet! please run utils/dbSetup.js first!') diff --git a/src/utils/actions.js b/src/utils/actions.js index c87924b..928af67 100755 --- a/src/utils/actions.js +++ b/src/utils/actions.js @@ -20,7 +20,6 @@ Question Server module.exports = { ProcessIncomingRequest: ProcessIncomingRequest, LoadJSON: LoadJSON, - LoadJSONFromObject: LoadJSONFromObject, } const dataFile = './qminingPublic/data.json' @@ -32,8 +31,7 @@ const logger = require('../utils/logger.js') const idStats = require('../utils/ids.js') idStats.Load() // FIXME: dont always load when actions.js is used const utils = require('../utils/utils.js') -const classes = require('./classes.js') -classes.initLogger(logger.DebugLog) +const { addQuestion, getSubjNameWithoutYear } = require('./classes.js') // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db @@ -103,15 +101,9 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun) { worker.on('message', (workerMsg) => { logger.DebugLog('Message from processData', 'actions', 1) - logger.DebugLog(workerMsg, 'actions', 1) + logger.DebugLog(workerMsg, 'actions', 3) - const allQuestions = workerMsg.map((resultQuestion) => { - return new classes.Question( - resultQuestion.Q, - resultQuestion.A, - resultQuestion.data - ) - }) + const allQuestions = workerMsg try { let color = logger.GetColor('green') @@ -120,14 +112,14 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun) { color = logger.GetColor('blue') msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` allQuestions.forEach((currentQuestion) => { - const sName = classes.SUtils.GetSubjNameWithoutYear(data.subj) + const sName = getSubjNameWithoutYear(data.subj) logger.DebugLog( 'Adding question with subjName: ' + sName + ' :', 'actions', 3 ) logger.DebugLog(currentQuestion, 'actions', 3) - qdb.AddQuestion(sName, currentQuestion) + addQuestion(qdb, sName, currentQuestion) }) currWrites++ @@ -195,30 +187,11 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun) { // loading stuff function LoadJSON(dataFile) { var data = JSON.parse(utils.ReadFile(dataFile)) - return LoadJSONFromObject(data) -} - -function LoadJSONFromObject(data) { - try { - var result = new classes.QuestionDB() - var rt = [] - - for (var i = 0; i < data.Subjects.length; i++) { - let subject = new classes.Subject(data.Subjects[i].Name) - var j = 0 - for (j = 0; j < data.Subjects[i].Questions.length; j++) { - var currQ = data.Subjects[i].Questions[j] - subject.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data)) - } - rt.push({ - name: data.Subjects[i].Name, - count: j, - }) - result.AddSubject(subject) - } - return result - } catch (err) { - logger.Log('Error loading sutff', logger.GetColor('redbg'), true) - console.log(err) + if (!data.Subjects) { + logger.Log( + "data.Subjects is undefined! Couldn't load data!", + logger.GetColor('redbg') + ) } + return data.Subjects } diff --git a/src/utils/classes.js b/src/utils/classes.js index 040a5e8..1ebe09c 100755 --- a/src/utils/classes.js +++ b/src/utils/classes.js @@ -145,7 +145,6 @@ function simplifyQA(value, mods) { return mods.reduce(reducer, value) } -// TODO: simplify answer before setting function simplifyAnswer(value) { return simplifyQA(value, [ removeSpecialChars.bind(this), @@ -190,17 +189,17 @@ function compareImage(data, data2) { return compareString(data.images.join(' '), data2.images.join(' ')) } -function compareData(data, qObj) { +function compareData(q1, q2) { try { - if (qObj.data.type === data.type) { - let dataType = qObj.data.type + if (q1.data.type === q2.data.type) { + let dataType = q1.data.type if (dataType === 'simple') { return -1 } else if (dataType === 'image') { - return compareImage(qObj.data) + return compareImage(q1.data, q2.data) } else { debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1) - debugLog(qObj, 'Compare question data', 2) + debugLog(q1, 'Compare question data', 2) } } else { return 0 @@ -221,11 +220,11 @@ function compareAnswer(q1, q2) { return compareString(q1.A, q2.A) } -function compareQuestionObj(q1, q2, data) { +function compareQuestionObj(q1, q1subjName, q2, q2subjName, data) { assert(data) assert(q1) + assert(typeof q1 === 'object') assert(q2) - assert(typeof q2 === 'object') let qObj if (typeof q2 === 'string') { @@ -237,10 +236,10 @@ function compareQuestionObj(q1, q2, data) { qObj = q2 } - const qMatch = compareQuestion(q1, qObj.Q) - const aMatch = compareAnswer(q1.A, qObj.A) + const qMatch = compareQuestion(q1, qObj) + const aMatch = compareAnswer(q1, qObj) // -1 if botth questions are simple - const dMatch = compareData(q1.data, qObj.data) + const dMatch = compareData(q1, qObj) let avg = -1 if (qObj.A) { @@ -261,6 +260,7 @@ function compareQuestionObj(q1, q2, data) { qMatch: qMatch, aMatch: aMatch, dMatch: dMatch, + matchedSubjName: q2subjName, avg: avg, } } @@ -278,12 +278,18 @@ function questionToString(question) { // --------------------------------------------------------------------------------------------------------- // Subject // --------------------------------------------------------------------------------------------------------- -function searchQuestion(questions, question, questionData) { +function searchQuestion(subj, question, questionData, subjName) { assert(question) var result = [] - questions.forEach((currentQuestion) => { - let percent = compareQuestionObj(currentQuestion, question, questionData) + subj.Questions.forEach((currentQuestion) => { + let percent = compareQuestionObj( + currentQuestion, + subjName, + question, + subj.Name, + questionData + ) if (percent.avg > minMatchAmmount) { result.push({ question: currentQuestion, @@ -294,7 +300,7 @@ function searchQuestion(questions, question, questionData) { }) // TODO: check if sorting is correct! - result.sort((q1, q2) => { + result = result.sort((q1, q2) => { return q1.match < q2.match }) @@ -327,7 +333,9 @@ function addQuestion(data, subj, question) { var i = 0 while ( i < data.length && - !subj.toLowerCase().includes(getSubjNameWithoutYear(data[i]).toLowerCase()) + !subj + .toLowerCase() + .includes(getSubjNameWithoutYear(data[i].Name).toLowerCase()) ) { i++ } @@ -335,10 +343,7 @@ function addQuestion(data, subj, question) { if (i < data.length) { debugLog('Adding new question to existing subject', 'qdb add', 1) result = [...data] - result[i].Questions = { - ...data[i].Questions, - question, - } + result[i].Questions = [...data[i].Questions, question] } else { debugLog('Creating new subject for question', 'qdb add', 1) result = [ @@ -380,7 +385,9 @@ function searchData(data, question, subjName, questionData) { .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) ) { debugLog(`Searching in ${subj.Name} `, 2) - result = result.concat(subj.Search(question, questionData)) + result = result.concat( + searchQuestion(subj, question, questionData, subjName) + ) } }) @@ -393,7 +400,7 @@ function searchData(data, question, subjName, questionData) { ) data.forEach((subj) => { result = result.concat( - searchQuestion(subj.Questions, question, questionData) + searchQuestion(subj, question, questionData, subjName) ) }) if (result.length > 0) { @@ -407,7 +414,7 @@ function searchData(data, question, subjName, questionData) { } // TODO: check if sorting is correct! - result.sort((q1, q2) => { + result = result.sort((q1, q2) => { return q1.match < q2.match }) diff --git a/src/utils/processData.js b/src/utils/processData.js index 6a79c3e..a6fd2f9 100644 --- a/src/utils/processData.js +++ b/src/utils/processData.js @@ -1,18 +1,14 @@ const { isMainThread, parentPort, workerData } = require('worker_threads') const logger = require('../utils/logger.js') -const actions = require('../utils/actions.js') -const classes = require('./classes.js') -classes.initLogger(logger.DebugLog) +const { createQuestion, searchData } = require('./classes.js') const minMatchAmmountToAdd = 90 // FIXME: test this value if (!isMainThread) { logger.DebugLog('Starting worker thread', 'processdata', 1) - logger.DebugLog(workerData, 'processdata', 1) + logger.DebugLog(workerData, 'processdata', 3) - parentPort.postMessage( - ProcessData(workerData.data, actions.LoadJSONFromObject(workerData.qdb)) - ) + parentPort.postMessage(ProcessData(workerData.data, workerData.qdb)) } else { logger.Log( 'Porcess data should not run on main thread!', @@ -26,15 +22,11 @@ function ProcessData(data, qdb) { data.quiz.forEach((question) => { logger.DebugLog('Question:', 'actions', 2) logger.DebugLog(question, 'actions', 2) - let currentQuestion = new classes.Question( - question.Q, - question.A, - question.data - ) + let currentQuestion = createQuestion(question.Q, question.A, question.data) logger.DebugLog('Searching for question in subj ' + data.subj, 'actions', 3) logger.DebugLog(currentQuestion, 'actions', 3) - let sames = qdb.Search(currentQuestion, data.subj) + let sames = searchData(qdb, currentQuestion, data.subj) logger.DebugLog('Same questions:', 'actions', 2) logger.DebugLog('Length: ' + sames.length, 'actions', 2) logger.DebugLog(sames, 'actions', 3) diff --git a/src/utils/searchData.js b/src/utils/searchData.js index 5377975..83991e4 100644 --- a/src/utils/searchData.js +++ b/src/utils/searchData.js @@ -1,22 +1,16 @@ const { isMainThread, parentPort, workerData } = require('worker_threads') const logger = require('../utils/logger.js') -const actions = require('../utils/actions.js') +const { searchData } = require('../../utils/classes.js') if (!isMainThread) { logger.DebugLog('Starting worker thread', 'searchdata', 1) - logger.DebugLog(workerData, 'searchdata', 1) + logger.DebugLog(workerData, 'searchdata', 3) const { data, question, subj, recData } = workerData - parentPort.postMessage( - SearchData(actions.LoadJSONFromObject(data), question, subj, recData) - ) + parentPort.postMessage(searchData(data, question, subj, recData)) } else { logger.Log( 'Porcess data should not run on main thread!', logger.GetColor('redbg') ) } - -function SearchData(data, question, subj, recData) { - return data.Search(question, subj, recData) -}