diff --git a/.eslintrc.js b/.eslintrc.js index 4bfeb18..1c82364 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -12,10 +12,10 @@ module.exports = { SharedArrayBuffer: 'readonly', }, rules: { - 'no-undef': ['warn'], + 'no-undef': ['error'], eqeqeq: ['warn', 'smart'], 'no-unused-vars': 'off', 'no-prototype-builtins': 'off', - 'id-length': ['warn', { exceptions: [] }], + 'id-length': ['warn', { exceptions: ['i', 'j'] }], }, } diff --git a/src/modules/api/api.js b/src/modules/api/api.js index ead9a3d..5c8e2a4 100644 --- a/src/modules/api/api.js +++ b/src/modules/api/api.js @@ -22,7 +22,7 @@ const express = require('express') const bodyParser = require('body-parser') const busboy = require('connect-busboy') -const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported +const { v4: uuidv4 } = require('uuid') const fs = require('fs') const app = express() diff --git a/src/server.js b/src/server.js index 56f8587..068b1ac 100755 --- a/src/server.js +++ b/src/server.js @@ -35,7 +35,7 @@ const http = require('http') const https = require('https') const cors = require('cors') const cookieParser = require('cookie-parser') -const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported +const { v4: uuidv4 } = require('uuid'); const dbtools = require('./utils/dbtools.js') const reqlogger = require('./middlewares/reqlogger.middleware.js') diff --git a/src/utils/actions.js b/src/utils/actions.js index 82ec66d..f4bd96c 100755 --- a/src/utils/actions.js +++ b/src/utils/actions.js @@ -20,11 +20,14 @@ Question Server module.exports = { ProcessIncomingRequest: ProcessIncomingRequest, LoadJSON: LoadJSON, + LoadJSONFromObject: LoadJSONFromObject, } const dataFile = './qminingPublic/data.json' const recDataFile = './stats/recdata' +const processDataWorkerFile = './src/utils/processData.js' +const { Worker } = require('worker_threads') const logger = require('../utils/logger.js') const idStats = require('../utils/ids.js') idStats.Load() // FIXME: dont always load when actions.js is used @@ -33,7 +36,6 @@ const classes = require('./classes.js') classes.initLogger(logger.DebugLog) // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db -const minMatchAmmountToAdd = 90 // FIXME: test this value const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate var currWrites = 0 @@ -59,105 +61,131 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun) { '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) logger.DebugLog('recDataFile written', 'actions', 1) - } catch (e) { + } catch (err) { logger.log('Error writing recieved data.') } try { // recievedData: { version: "", id: "", subj: "" quiz: {} } - let d = recievedData + let data = recievedData // FIXME: if is for backwards compatibility, remove this sometime in the future - if (typeof d !== 'object') { - d = JSON.parse(recievedData) + if (typeof data !== 'object') { + data = JSON.parse(recievedData) } logger.DebugLog('recievedData JSON parsed', 'actions', 1) - logger.DebugLog(d, 'actions', 3) - let allQLength = d.quiz.length - let allQuestions = [] + logger.DebugLog(data, 'actions', 3) + let allQLength = data.quiz.length - d.quiz.forEach((question) => { - logger.DebugLog('Question:', 'actions', 2) - logger.DebugLog(question, 'actions', 2) - let q = new classes.Question(question.Q, question.A, question.data) - logger.DebugLog( - 'Searching for question in subj ' + d.subj, - 'actions', - 3 - ) - logger.DebugLog(q, 'actions', 3) + const worker = new Worker(processDataWorkerFile, { + workerData: { + data: data, + qdb: qdb, + }, + }) - let sames = qdb.Search(q, d.subj) - logger.DebugLog('Same questions:', 'actions', 2) - logger.DebugLog('Length: ' + sames.length, 'actions', 2) - logger.DebugLog(sames, 'actions', 3) - // if it didnt find any question, or every found questions match is lower thatn 80 - let isNew = - sames.length === 0 || - sames.every((searchResItem) => { - return searchResItem.match < minMatchAmmountToAdd - }) - logger.DebugLog('isNew: ' + isNew, 'actions', 2) - if (isNew) { - allQuestions.push(q) + worker.on('error', (err) => { + logger.Log('Process Data Worker error!', logger.GetColor('redbg')) + console.error(err) + reject(err) + }) + + worker.on('exit', (code) => { + logger.DebugLog('ProcessData exit, code: ' + code, 'actions', 1) + if (code !== 0) { + logger.Log( + 'Process Data Worker error! Exit code is not 0', + logger.GetColor('redbg') + ) + reject(new Error('Process Data Worker exit code is not 0!')) } }) - let color = logger.GetColor('green') - let msg = '' - if (allQuestions.length > 0) { - color = logger.GetColor('blue') - msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` - allQuestions.forEach((q) => { - const sName = classes.SUtils.GetSubjNameWithoutYear(d.subj) - logger.DebugLog( - 'Adding question with subjName: ' + sName + ' :', - 'actions', - 3 + worker.on('message', (workerMsg) => { + logger.DebugLog('Message from processData', 'actions', 1) + logger.DebugLog(workerMsg, 'actions', 1) + + const allQuestions = workerMsg.map((resultQuestion) => { + return new classes.Question( + resultQuestion.Q, + resultQuestion.A, + resultQuestion.data ) - logger.DebugLog(q, 'actions', 3) - qdb.AddQuestion(sName, q) }) - currWrites++ - logger.DebugLog('currWrites for data.json: ' + currWrites, 'actions', 1) - if (currWrites >= writeAfter && !dryRun) { - currWrites = 0 - try { - qdb.version = infos.version - qdb.motd = infos.motd - logger.DebugLog('version and motd set for data.json', 'actions', 3) - } catch (e) { - logger.Log('MOTD/Version writing/reading error!') + try { + let color = logger.GetColor('green') + let msg = '' + if (allQuestions.length > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` + allQuestions.forEach((currentQuestion) => { + const sName = classes.SUtils.GetSubjNameWithoutYear(data.subj) + logger.DebugLog( + 'Adding question with subjName: ' + sName + ' :', + 'actions', + 3 + ) + logger.DebugLog(currentQuestion, 'actions', 3) + qdb.AddQuestion(sName, currentQuestion) + }) + + currWrites++ + logger.DebugLog( + 'currWrites for data.json: ' + currWrites, + 'actions', + 1 + ) + if (currWrites >= writeAfter && !dryRun) { + currWrites = 0 + try { + qdb.version = infos.version + qdb.motd = infos.motd + logger.DebugLog( + 'version and motd set for data.json', + 'actions', + 3 + ) + } catch (err) { + logger.Log('MOTD/Version writing/reading error!') + } + logger.DebugLog('Writing data.json', 'actions', 1) + utils.WriteFile(JSON.stringify(qdb), dataFile) + logger.Log('\tData file written', color) + } else if (dryRun) { + logger.Log('\tDry run') + } + } else { + msg += `No new data ( ${allQLength} )` } - logger.DebugLog('Writing data.json', 'actions', 1) - utils.WriteFile(JSON.stringify(qdb), dataFile) - logger.Log('\tData file written', color) - } else if (dryRun) { - logger.Log('\tDry run') + + let subjRow = '\t' + data.subj + if (data.id) { + subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')' + idStats.LogId(data.id, data.subj) + } + logger.Log(subjRow) + if (data.version !== undefined) { + msg += '. Version: ' + data.version + } + + logger.Log('\t' + msg, color) + logger.DebugLog('New Questions:', 'actions', 2) + logger.DebugLog(allQuestions, 'actions', 2) + + logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) + resolve(allQLength.length) + } catch (error) { + console.log(error) + logger.Log( + 'Error while processing processData worker result!', + logger.GetColor('redbg') + ) + reject(new Error('Error while processing processData worker result!')) } - } else { - msg += `No new data ( ${allQLength} )` - } - - let subjRow = '\t' + d.subj - if (d.id) { - subjRow += ' ( CID: ' + logger.logHashed(d.id) + ')' - idStats.LogId(d.id, d.subj) - } - logger.Log(subjRow) - if (d.version !== undefined) { - msg += '. Version: ' + d.version - } - - logger.Log('\t' + msg, color) - logger.DebugLog('New Questions:', 'actions', 2) - logger.DebugLog(allQuestions, 'actions', 2) - - logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) - resolve(allQLength.length) - } catch (e) { - console.log(e) + }) + } catch (err) { + console.log(err) logger.Log('Couldnt parse JSON data', logger.GetColor('redbg')) reject(new Error('Couldnt parse JSON data')) } @@ -166,27 +194,31 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun) { // loading stuff function LoadJSON(dataFile) { + var data = JSON.parse(utils.ReadFile(dataFile)) + return LoadJSONFromObject(data) +} + +function LoadJSONFromObject(data) { try { - var d = JSON.parse(utils.ReadFile(dataFile)) - var r = new classes.QuestionDB() + var result = new classes.QuestionDB() var rt = [] - for (var i = 0; i < d.Subjects.length; i++) { - let s = new classes.Subject(d.Subjects[i].Name) + for (var i = 0; i < data.Subjects.length; i++) { + let subject = new classes.Subject(data.Subjects[i].Name) var j = 0 - for (j = 0; j < d.Subjects[i].Questions.length; j++) { - var currQ = d.Subjects[i].Questions[j] - s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data)) + for (j = 0; j < data.Subjects[i].Questions.length; j++) { + var currQ = data.Subjects[i].Questions[j] + subject.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data)) } rt.push({ - name: d.Subjects[i].Name, + name: data.Subjects[i].Name, count: j, }) - r.AddSubject(s) + result.AddSubject(subject) } - return r - } catch (e) { + return result + } catch (err) { logger.Log('Error loading sutff', logger.GetColor('redbg'), true) - console.log(e) + console.log(err) } } diff --git a/src/utils/dbSetup.js b/src/utils/dbSetup.js index c879f25..6026597 100644 --- a/src/utils/dbSetup.js +++ b/src/utils/dbSetup.js @@ -3,7 +3,7 @@ const logger = require('../utils/logger.js') const dbtools = require('../utils/dbtools.js') const dbStructPath = '../modules/api/apiDBStruct.json' const usersDBPath = '../data/dbs/users.db' -const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported +const { v4: uuidv4 } = require('uuid') let authDB diff --git a/src/utils/processData.js b/src/utils/processData.js new file mode 100644 index 0000000..fc8db78 --- /dev/null +++ b/src/utils/processData.js @@ -0,0 +1,62 @@ +const dataFile = './qminingPublic/data.json' +const recDataFile = './stats/recdata' + +const { + Worker, + isMainThread, + parentPort, + workerData, +} = require('worker_threads') +const logger = require('../utils/logger.js') +const actions = require('../utils/actions.js') +const classes = require('./classes.js') +classes.initLogger(logger.DebugLog) + +const minMatchAmmountToAdd = 90 // FIXME: test this value + +if (!isMainThread) { + logger.DebugLog('Starting worker thread', 'processdata', 1) + console.log(workerData) + + parentPort.postMessage( + ProcessData(workerData.data, actions.LoadJSONFromObject(workerData.qdb)) + ) +} else { + logger.Log( + 'Porcess data should not run on main thread!', + logger.GetColor('redbg') + ) +} + +function ProcessData(data, qdb) { + let allQuestions = [] + + data.quiz.forEach((question) => { + logger.DebugLog('Question:', 'actions', 2) + logger.DebugLog(question, 'actions', 2) + let currentQuestion = new classes.Question( + question.Q, + question.A, + question.data + ) + logger.DebugLog('Searching for question in subj ' + data.subj, 'actions', 3) + logger.DebugLog(currentQuestion, 'actions', 3) + + let sames = qdb.Search(currentQuestion, data.subj) + logger.DebugLog('Same questions:', 'actions', 2) + logger.DebugLog('Length: ' + sames.length, 'actions', 2) + logger.DebugLog(sames, 'actions', 3) + // if it didnt find any question, or every found questions match is lower thatn 80 + let isNew = + sames.length === 0 || + sames.every((searchResItem) => { + return searchResItem.match < minMatchAmmountToAdd + }) + logger.DebugLog('isNew: ' + isNew, 'actions', 2) + if (isNew) { + allQuestions.push(currentQuestion) + } + }) + + return allQuestions +}