/* ---------------------------------------------------------------------------- Question Server GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ const recDataFile = './stats/recdata' const dataLockFile = './data/lockData' import logger from '../utils/logger' import { createQuestion } from '../utils/classes' import { doALongTask } from './workerPool' import idStats from '../utils/ids' import utils from '../utils/utils' import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes' // types import { QuestionDb, Question, User, DataFile } from '../types/basicTypes' // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db const minMatchAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate let currWrites = 0 export interface RecievedData { quiz: Array subj: string id: string version: string scriptVersion: string location: string } export interface Result { qdbName: string newQuestions: number } export function logResult( recievedData: RecievedData, result: Array, userId: number, dryRun?: boolean ): void { logger.Log('\t' + recievedData.subj) if (dryRun) { logger.Log('\tDry run') } let idRow = '\t' if (recievedData.version) { idRow += 'Version: ' + logger.logHashed(recievedData.version) } if (recievedData.id) { idRow += ', CID: ' + logger.logHashed(recievedData.id) } idRow += `, User #${logger.logHashed(userId.toString())}` logger.Log(idRow) if (result.length > 0) { result.forEach((res: Result) => { const allQLength = recievedData.quiz.length let msg = `${res.qdbName}: ` let color = logger.GetColor('green') if (res.newQuestions > 0) { color = logger.GetColor('blue') msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` } else { msg += `No new data ( ${allQLength} )` } logger.Log('\t' + msg, color) }) } else { logger.Log('\tResults length is zero!', logger.GetColor('red')) } } export function processIncomingRequest( recievedData: RecievedData, questionDbs: Array, dryRun: boolean, user: User ): Promise> { logger.DebugLog('Processing incoming request', 'isadding', 1) if (recievedData === undefined) { logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) throw new Error('Recieved data is undefined!') } try { let towrite = utils.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' if (typeof recievedData === 'object') { towrite += JSON.stringify(recievedData) } else { towrite += recievedData } towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) logger.DebugLog('recDataFile written', 'isadding', 1) } catch (err) { logger.Log('Error writing recieved data.') } if (utils.FileExists(dataLockFile)) { logger.Log( 'Data lock file exists, skipping recieved data processing', logger.GetColor('red') ) return new Promise((resolve) => resolve([])) } // FIXME: this many promises and stuff might be unnecesarry const promises: Array> = questionDbs.reduce((acc, qdb) => { acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) return acc }, []) return Promise.all(promises) } function processIncomingRequestUsingDb( recievedData: RecievedData, qdb: QuestionDb, dryRun: boolean, user: User ): Promise { return new Promise((resolve, reject) => { try { const recievedQuestions = [] logger.DebugLog('recievedData JSON parsed', 'isadding', 1) logger.DebugLog(recievedData, 'isadding', 3) const allQLength = recievedData.quiz.length const questionSearchPromises = [] recievedData.quiz.forEach((question) => { logger.DebugLog('Question:', 'isadding', 2) logger.DebugLog(question, 'isadding', 2) const currentQuestion = createQuestion( question.Q, question.A, question.data ) logger.DebugLog( 'Searching for question in subj ' + recievedData.subj, 'isadding', 3 ) logger.DebugLog(currentQuestion, 'isadding', 3) if (isQuestionValid(currentQuestion)) { recievedQuestions.push(currentQuestion) // This here searches only in relevant subjects, and not all subjects questionSearchPromises.push( doALongTask({ type: 'work', data: { searchIn: [qdb.index], qdb: qdb.data, question: currentQuestion, subjName: recievedData.subj, searchTillMatchPercent: minMatchAmmountToAdd, }, }) ) } else { logger.DebugLog('Question isnt valid', 'isadding', 1) logger.DebugLog(currentQuestion, 'isadding', 1) } }) Promise.all(questionSearchPromises) .then((results: Array) => { const allQuestions = [] // all new questions here that do not have result results.forEach((result, i) => { const add = result.result.every((res) => { return res.match < minMatchAmmountToAdd }) if (add) { allQuestions.push(recievedQuestions[i]) } }) try { if (allQuestions.length > 0) { allQuestions.forEach((currentQuestion) => { const sName = getSubjNameWithoutYear(recievedData.subj) logger.DebugLog( 'Adding question with subjName: ' + sName + ' :', 'isadding', 3 ) logger.DebugLog(currentQuestion, 'isadding', 3) addQuestion(qdb.data, sName, currentQuestion) }) currWrites++ logger.DebugLog( 'currWrites for data.json: ' + currWrites, 'isadding', 1 ) if (currWrites >= writeAfter && !dryRun) { currWrites = 0 logger.DebugLog('Writing data.json', 'isadding', 1) utils.WriteFile(JSON.stringify(qdb.data), qdb.path) } } idStats.LogId( user.id, recievedData.subj, allQuestions.length, allQLength ) logger.DebugLog('New Questions:', 'isadding', 2) logger.DebugLog(allQuestions, 'isadding', 2) logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1) resolve({ newQuestions: allQuestions.length, qdbName: qdb.name, }) } catch (error) { console.error(error) logger.Log( 'Error while processing processData worker result!', logger.GetColor('redbg') ) reject( new Error('Error while processing processData worker result!') ) } }) .catch((err) => { logger.Log( 'Error while searching for questions in ProcessIncomingRequest!', logger.GetColor('redbg') ) console.error(err) }) } catch (err) { console.error(err) logger.Log( 'There was en error handling incoming quiz data, see stderr', logger.GetColor('redbg') ) reject(new Error('Couldnt parse JSON data')) } }) } function isQuestionValid(question: Question) { if (!question.Q) { return false } if (!question.A && question.data.type === 'simple') { return false } return true } export function shouldSearchDataFile( df: DataFile, testUrl: string, trueIfAlways?: Boolean ): Boolean { if ( typeof df.shouldSearch === 'string' && df.shouldSearch === 'always' && trueIfAlways ) { return true } if (typeof df.shouldSearch === 'object') { if (df.shouldSearch.location) { const { val } = df.shouldSearch.location return testUrl.includes(val) } } logger.DebugLog(`no suitable dbs for ${testUrl}`, 'shouldSearchDataFile', 1) return false } export function shouldSaveDataFile( df: DataFile, recievedData: RecievedData ): Boolean { if (df.shouldSave.version) { const { compare, val } = df.shouldSave.version if (compare === 'equals') { return recievedData.version === val } else if (compare === 'startsWith') { return recievedData.version.startsWith(val) } } if (df.shouldSave.version) { const { compare, val } = df.shouldSave.version if (compare === 'equals') { return recievedData.version === val } else if (compare === 'startsWith') { return recievedData.version.startsWith(val) } } if (df.shouldSave.location) { const { val } = df.shouldSave.location return recievedData.location.includes(val) } return false } export function loadJSON( dataFiles: Array, dataDir: string ): Array { return dataFiles.reduce((acc, dataFile, index) => { const dataPath = dataDir + dataFile.path if (!utils.FileExists(dataPath)) { utils.WriteFile(JSON.stringify([]), dataPath) } try { acc.push({ ...dataFile, path: dataPath, index: index, data: JSON.parse(utils.ReadFile(dataPath)), }) } catch (err) { console.error(err) logger.Log( "data is undefined! Couldn't load data!", logger.GetColor('redbg') ) } return acc }, []) } export function backupData(questionDbs: Array): void { questionDbs.forEach((data) => { const path = './publicDirs/qminingPublic/backs/' utils.CreatePath(path) try { logger.Log(`Backing up ${data.name}...`) utils.WriteFile( JSON.stringify(data.data), `${path}${data.name}_${utils.GetDateString(true)}.json` ) logger.Log('Done') } catch (err) { logger.Log( `Error backing up data file ${data.name}!`, logger.GetColor('redbg') ) console.error(err) } }) }