/* ---------------------------------------------------------------------------- Question Server GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ module.exports = { ProcessIncomingRequest: ProcessIncomingRequest, LoadJSON: LoadJSON, backupData: backupData, } const recDataFile = './stats/recdata' const dataLockFile = './data/lockData' const logger = require('../utils/logger.js') const { searchData, createQuestion } = require('../utils/classes.js') const idStats = require('../utils/ids.js') const utils = require('../utils/utils.js') const { addQuestion, getSubjNameWithoutYear } = require('./classes.js') // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db const minMatchToAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate var currWrites = 0 interface QuestionData { type: String images?: Array hashedImages?: Array } interface Question { Q: String A: String data: QuestionData } interface Subject { Name: String Questions: Array } interface Db { name: String path: String data: Array } function ProcessIncomingRequest( recievedData, questionDbs: Array, dryRun, user ) { return Promise.all( questionDbs.map((qdb) => { return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) }) ) } function ProcessIncomingRequestUsingDb(recievedData, qdb: Db, dryRun, user) { return new Promise((resolve, reject) => { logger.DebugLog('Processing incoming request', 'actions', 1) if (recievedData === undefined) { logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) reject(new Error('Recieved data is undefined!')) } try { let towrite = logger.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' if (typeof recievedData === 'object') { towrite += JSON.stringify(recievedData) } else { towrite += recievedData } towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) logger.DebugLog('recDataFile written', 'actions', 1) } catch (err) { logger.log('Error writing recieved data.') } if (utils.FileExists(dataLockFile)) { logger.Log( 'Data lock file exists, skipping recieved data processing', logger.GetColor('red') ) resolve(-1) return } try { // recievedData: { version: "", id: "", subj: "" quiz: {} } let data = recievedData const recievedQuestions = [] // FIXME: if is for backwards compatibility, remove this sometime in the future if (typeof data !== 'object') { data = JSON.parse(recievedData) } logger.DebugLog('recievedData JSON parsed', 'actions', 1) logger.DebugLog(data, 'actions', 3) let allQLength = data.quiz.length const questionSearchPromises = [] data.quiz.forEach((question) => { logger.DebugLog('Question:', 'actions', 2) logger.DebugLog(question, 'actions', 2) let currentQuestion = createQuestion( question.Q, question.A, question.data ) logger.DebugLog( 'Searching for question in subj ' + data.subj, 'actions', 3 ) logger.DebugLog(currentQuestion, 'actions', 3) recievedQuestions.push(currentQuestion) questionSearchPromises.push( searchData(qdb.data, currentQuestion, data.subj) ) }) Promise.all(questionSearchPromises) .then((results) => { const allQuestions = [] // all new questions here that do not have result results.forEach((result, i) => { let add = result.every((res) => { return res.match < minMatchToAmmountToAdd }) if (add) { allQuestions.push(recievedQuestions[i]) } }) try { let color = logger.GetColor('green') let msg = '' if (allQuestions.length > 0) { color = logger.GetColor('blue') msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` allQuestions.forEach((currentQuestion) => { const sName = getSubjNameWithoutYear(data.subj) logger.DebugLog( 'Adding question with subjName: ' + sName + ' :', 'actions', 3 ) logger.DebugLog(currentQuestion, 'actions', 3) addQuestion(qdb.data, sName, currentQuestion) }) currWrites++ logger.DebugLog( 'currWrites for data.json: ' + currWrites, 'actions', 1 ) if (currWrites >= writeAfter && !dryRun) { currWrites = 0 logger.DebugLog('Writing data.json', 'actions', 1) utils.WriteFile(JSON.stringify(qdb.data), qdb.path) logger.Log('\tData file written', color) } else if (dryRun) { logger.Log('\tDry run') } } else { msg += `No new data ( ${allQLength} )` } let subjRow = '\t' + data.subj if (data.id) { subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')' } idStats.LogId(user.id, data.subj, allQuestions.length, allQLength) logger.Log(subjRow) if (data.version !== undefined) { msg += '. Version: ' + data.version } logger.Log('\t' + msg, color) logger.DebugLog('New Questions:', 'actions', 2) logger.DebugLog(allQuestions, 'actions', 2) logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) resolve(allQuestions.length) } catch (error) { console.error(error) logger.Log( 'Error while processing processData worker result!', logger.GetColor('redbg') ) reject( new Error('Error while processing processData worker result!') ) } }) .catch((err) => { logger.Log( 'Error while searching for questions in ProcessIncomingRequest!', logger.GetColor('redbg') ) console.error(err) }) } catch (err) { console.error(err) logger.Log( 'There was en error handling incoming quiz data, see stderr', logger.GetColor('redbg') ) reject(new Error('Couldnt parse JSON data')) } }) } function LoadJSON(dataFiles) { return dataFiles.reduce((acc, dataFile) => { if (!utils.FileExists(dataFile.path)) { utils.WriteFile(JSON.stringify([]), dataFile.path) } try { acc.push({ ...dataFile, data: JSON.parse(utils.ReadFile(dataFile.path)), }) } catch (err) { console.error(err) logger.Log( "data is undefined! Couldn't load data!", logger.GetColor('redbg') ) } return acc }, []) } function backupData(questionDbs) { questionDbs.forEach((data) => { const path = './publicDirs/qminingPublic/backs/' utils.CreatePath(path) try { logger.Log(`Backing up ${data.name}...`) utils.WriteFile( JSON.stringify(data.data), `${path}${data.name}_${utils.GetDateString(true)}.json` ) logger.Log('Done') } catch (err) { logger.Log( `Error backing up data file ${data.name}!`, logger.GetColor('redbg') ) console.error(err) } }) }