/* ---------------------------------------------------------------------------- Question Server GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ const recDataFile = './stats/recdata' const dataLockFile = './data/lockData' import logger from '../utils/logger' import { searchData, createQuestion } from '../utils/classes' import idStats from '../utils/ids' import utils from '../utils/utils' import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes' // types import { QuestionDb, Question, User, DataFile } from '../types/basicTypes' // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db const minMatchToAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate let currWrites = 0 export interface RecievedData { quiz: Array subj: string id: string version: string scriptVersion: string } interface Result { qdbName: string newQuestions: number } export function logResult( recievedData: RecievedData, result: Array, userId: number, dryRun?: boolean ): void { logger.Log('\t' + recievedData.subj) if (dryRun) { logger.Log('\tDry run') } let idRow = '\t' if (recievedData.version) { idRow += 'Version: ' + logger.logHashed(recievedData.version) } if (recievedData.id) { idRow += ', CID: ' + logger.logHashed(recievedData.id) } idRow += `, User #${logger.logHashed(userId.toString())}` logger.Log(idRow) if (result.length > 0) { result.forEach((res: Result) => { const allQLength = recievedData.quiz.length let msg = `${res.qdbName}: ` let color = logger.GetColor('green') if (res.newQuestions > 0) { color = logger.GetColor('blue') msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` } else { msg += `No new data ( ${allQLength} )` } logger.Log('\t' + msg, color) }) } else { logger.Log('\tNo db-s passed shouldSave!', logger.GetColor('red')) } } export function processIncomingRequest( recievedData: RecievedData, questionDbs: Array, dryRun: boolean, user: User ): Promise> { logger.DebugLog('Processing incoming request', 'actions', 1) if (recievedData === undefined) { logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) throw new Error('Recieved data is undefined!') } try { let towrite = utils.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' if (typeof recievedData === 'object') { towrite += JSON.stringify(recievedData) } else { towrite += recievedData } towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) logger.DebugLog('recDataFile written', 'actions', 1) } catch (err) { logger.Log('Error writing recieved data.') } if (utils.FileExists(dataLockFile)) { logger.Log( 'Data lock file exists, skipping recieved data processing', logger.GetColor('red') ) return new Promise((resolve) => resolve([])) } const promises: Array> = questionDbs.reduce((acc, qdb) => { if (qdb.shouldSave(recievedData)) { acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) } return acc }, []) return Promise.all(promises) } function processIncomingRequestUsingDb( recievedData: RecievedData, qdb: QuestionDb, dryRun: boolean, user: User ): Promise { return new Promise((resolve, reject) => { try { const recievedQuestions = [] logger.DebugLog('recievedData JSON parsed', 'actions', 1) logger.DebugLog(recievedData, 'actions', 3) const allQLength = recievedData.quiz.length const questionSearchPromises = [] recievedData.quiz.forEach((question) => { logger.DebugLog('Question:', 'actions', 2) logger.DebugLog(question, 'actions', 2) const currentQuestion = createQuestion( question.Q, question.A, question.data ) logger.DebugLog( 'Searching for question in subj ' + recievedData.subj, 'actions', 3 ) logger.DebugLog(currentQuestion, 'actions', 3) recievedQuestions.push(currentQuestion) questionSearchPromises.push( searchData(qdb, currentQuestion, recievedData.subj) ) }) Promise.all(questionSearchPromises) .then((results: Array) => { const allQuestions = [] // all new questions here that do not have result results.forEach((result, i) => { const add = result.result.every((res) => { return res.match < minMatchToAmmountToAdd }) if (add) { allQuestions.push(recievedQuestions[i]) } }) try { if (allQuestions.length > 0) { allQuestions.forEach((currentQuestion) => { const sName = getSubjNameWithoutYear(recievedData.subj) logger.DebugLog( 'Adding question with subjName: ' + sName + ' :', 'actions', 3 ) logger.DebugLog(currentQuestion, 'actions', 3) addQuestion(qdb.data, sName, currentQuestion) }) currWrites++ logger.DebugLog( 'currWrites for data.json: ' + currWrites, 'actions', 1 ) if (currWrites >= writeAfter && !dryRun) { currWrites = 0 logger.DebugLog('Writing data.json', 'actions', 1) utils.WriteFile(JSON.stringify(qdb.data), qdb.path) } } idStats.LogId( user.id, recievedData.subj, allQuestions.length, allQLength ) logger.DebugLog('New Questions:', 'actions', 2) logger.DebugLog(allQuestions, 'actions', 2) logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) resolve({ newQuestions: allQuestions.length, qdbName: qdb.name, }) } catch (error) { console.error(error) logger.Log( 'Error while processing processData worker result!', logger.GetColor('redbg') ) reject( new Error('Error while processing processData worker result!') ) } }) .catch((err) => { logger.Log( 'Error while searching for questions in ProcessIncomingRequest!', logger.GetColor('redbg') ) console.error(err) }) } catch (err) { console.error(err) logger.Log( 'There was en error handling incoming quiz data, see stderr', logger.GetColor('redbg') ) reject(new Error('Couldnt parse JSON data')) } }) } export function loadJSON(dataFiles: Array): Array { return dataFiles.reduce((acc, dataFile) => { if (!utils.FileExists(dataFile.path)) { utils.WriteFile(JSON.stringify([]), dataFile.path) } try { acc.push({ ...dataFile, data: JSON.parse(utils.ReadFile(dataFile.path)), }) } catch (err) { console.error(err) logger.Log( "data is undefined! Couldn't load data!", logger.GetColor('redbg') ) } return acc }, []) } export function backupData(questionDbs: Array): void { questionDbs.forEach((data) => { const path = './publicDirs/qminingPublic/backs/' utils.CreatePath(path) try { logger.Log(`Backing up ${data.name}...`) utils.WriteFile( JSON.stringify(data.data), `${path}${data.name}_${utils.GetDateString(true)}.json` ) logger.Log('Done') } catch (err) { logger.Log( `Error backing up data file ${data.name}!`, logger.GetColor('redbg') ) console.error(err) } }) }