diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index 4dfe4af..77a21e8 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -29,13 +29,19 @@ import fs from 'fs' import logger from '../../utils/logger' import utils from '../../utils/utils' -import actions from '../../utils/actions' +import { + processIncomingRequest, + logResult, + backupData, + loadJSON, + RecievedData, +} from '../../utils/actions' import dbtools from '../../utils/dbtools' import auth from '../../middlewares/auth.middleware' -import { dataToString, searchData } from '../../utils/classes' +import { dataToString, searchDatas } from '../../utils/classes' import { SetupData } from '../../server' -import { ModuleType, User } from '../../types/basicTypes' +import { ModuleType, User, DataFile } from '../../types/basicTypes' // files const msgFile = 'stats/msgs' @@ -62,11 +68,6 @@ let userDB let url // eslint-disable-line let publicdirs = [] -export interface DataFile { - path: string - name: string -} - function GetApp(): ModuleType { const app = express() @@ -79,10 +80,21 @@ function GetApp(): ModuleType { const recivedFiles = publicDir + 'recivedfiles' const uloadFiles = publicDir + 'f' const dataFiles: Array = [ - { path: `${publicDir}oldData.json`, name: 'oldData' }, - { path: `${publicDir}data.json`, name: 'newData' }, + { + path: `${publicDir}oldData.json`, + name: 'oldData', + shouldSave: (recData: RecievedData): boolean => { + return recData.version.includes('2.0.') + }, + }, + { + path: `${publicDir}data.json`, + name: 'newData', + shouldSave: (recData: RecievedData): boolean => { + return recData.version.includes('2.1.') + }, + }, ] - const data: any = {} // TODO: remove const motdFile = publicDir + 'motd' const userSpecificMotdFile = publicDir + 'userSpecificMotd.json' const versionFile = publicDir + 'version' @@ -127,7 +139,7 @@ function GetApp(): ModuleType { }) ) - const questionDbs = actions.LoadJSON(dataFiles) + const questionDbs = loadJSON(dataFiles) let version = '' let motd = '' let userSpecificMotd = {} @@ -196,13 +208,16 @@ function GetApp(): ModuleType { } if (write) { - utils.WriteFile(JSON.stringify(userSpecificMotd), userSpecificMotdFile) + utils.WriteFile( + JSON.stringify(userSpecificMotd, null, 2), + userSpecificMotdFile + ) } return shouldSee } function Load() { - actions.backupData(questionDbs) + backupData(questionDbs) utils.WatchFile(userSpecificMotdFile, () => { logger.Log(`User Specific Motd updated`, logger.GetColor('green')) @@ -939,16 +954,9 @@ function GetApp(): ModuleType { const dryRun = testUsers.includes(user.id) - // automatically saves to dataFile every n write - // FIXME: req.body.datatoadd is for backwards compatibility, remove this sometime in the future - actions - .ProcessIncomingRequest( - req.body.datatoadd || req.body, - questionDbs, - dryRun, - user - ) + processIncomingRequest(req.body, questionDbs, dryRun, user) .then((resultArray) => { + logResult(req.body, resultArray) res.json({ success: resultArray.length > 0, // TODO check for -1s newQuestions: resultArray, @@ -986,8 +994,10 @@ function GetApp(): ModuleType { ) } - searchData(data, question, subj, recData) + searchDatas(questionDbs, question, subj, recData) .then((result) => { + // TODO: test + console.log(result) res.json({ result: result, success: true, @@ -1021,21 +1031,38 @@ function GetApp(): ModuleType { } }) + function getSubjCount(qdbs) { + return qdbs.reduce((acc, qdb) => { + return acc + qdb.data.length + }, 0) + } + + function getQuestionCount(qdbs) { + return qdbs.reduce((acc, qdb) => { + return ( + acc + + qdb.data.reduce((qacc, subject) => { + return qacc + subject.Questions.length + }, 0) + ) + }, 0) + } + function getSimplreRes() { return { - subjects: data.length, - questions: data.reduce((acc, subj) => { - return acc + subj.Questions.length - }, 0), + subjects: getSubjCount(questionDbs), + questions: getQuestionCount(questionDbs), } } function getDetailedRes() { - return data.map((subj) => { - return { - name: subj.Name, - count: subj.Questions.length, - } + return questionDbs.map((qdb) => { + return qdb.data.map((subj) => { + return { + name: subj.Name, + count: subj.Questions.length, + } + }) }) } @@ -1091,10 +1118,8 @@ function GetApp(): ModuleType { utils.AppendToFile( JSON.stringify({ date: utils.GetDateString(), - subjectCount: data.length, - questionCount: data.reduce((acc, subj) => { - return acc + subj.Questions.length - }, 0), + subjectCount: getSubjCount(questionDbs), + questionCount: getQuestionCount(questionDbs), userCount: dbtools.TableInfo(userDB, 'users').dataCount, }), dailyDataCountFile @@ -1165,7 +1190,7 @@ function GetApp(): ModuleType { } function DailyAction() { - actions.backupData(data) + backupData(questionDbs) BackupDB() ExportDailyDataCount() IncrementAvaiblePWs() diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index e34cca9..044d9dd 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -17,9 +17,13 @@ export interface Subject { Questions: Array } -export interface QuestionDb { - name: string +export interface DataFile { path: string + name: string + shouldSave: (recData: any) => boolean +} + +export interface QuestionDb extends DataFile { data: Array } diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 7a9129d..df25997 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -17,12 +17,6 @@ Question Server ------------------------------------------------------------------------- */ -export default { - ProcessIncomingRequest: ProcessIncomingRequest, - LoadJSON: LoadJSON, - backupData: backupData, -} - const recDataFile = './stats/recdata' const dataLockFile = './data/lockData' @@ -33,8 +27,7 @@ import utils from '../utils/utils' import { addQuestion, getSubjNameWithoutYear } from './classes' // types -import { QuestionDb, Question, User } from '../types/basicTypes' -import { DataFile } from '../modules/api/api' +import { QuestionDb, Question, User, DataFile } from '../types/basicTypes' // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db @@ -43,7 +36,7 @@ const minMatchToAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate let currWrites = 0 -interface RecievedData { +export interface RecievedData { quiz: Array subj: string id: string @@ -51,61 +44,93 @@ interface RecievedData { scriptVersion: string } -function ProcessIncomingRequest( +interface Result { + qdbName: string + newQuestions: number +} + +export function logResult( + recievedData: RecievedData, + result: Array +): void { + let subjRow = '\t' + recievedData.subj + if (recievedData.id) { + subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')' + } + logger.Log(subjRow) + + result.forEach((res: Result) => { + const allQLength = recievedData.quiz.length + let msg = `${res.qdbName}: ` + let color = logger.GetColor('green') + if (res.newQuestions > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` + } else { + msg += `No new data ( ${allQLength} )` + } + if (recievedData.version !== undefined) { + msg += '. Version: ' + recievedData.version + } + logger.Log('\t' + msg, color) + }) +} + +export function processIncomingRequest( recievedData: RecievedData, questionDbs: Array, dryRun: boolean, user: User -): Promise> { - return Promise.all( - questionDbs.map((qdb) => { - return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) - }) - ) +): Promise> { + logger.DebugLog('Processing incoming request', 'actions', 1) + + if (recievedData === undefined) { + logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) + throw new Error('Recieved data is undefined!') + } + + try { + let towrite = logger.GetDateString() + '\n' + towrite += + '------------------------------------------------------------------------------\n' + if (typeof recievedData === 'object') { + towrite += JSON.stringify(recievedData) + } else { + towrite += recievedData + } + towrite += + '\n------------------------------------------------------------------------------\n' + utils.AppendToFile(towrite, recDataFile) + logger.DebugLog('recDataFile written', 'actions', 1) + } catch (err) { + logger.Log('Error writing recieved data.') + } + + if (utils.FileExists(dataLockFile)) { + logger.Log( + 'Data lock file exists, skipping recieved data processing', + logger.GetColor('red') + ) + return new Promise((resolve) => resolve([])) + } + + const promises: Array> = questionDbs.reduce((acc, qdb) => { + if (qdb.shouldSave(recievedData)) { + acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) + } + return acc + }, []) + return Promise.all(promises) } -function ProcessIncomingRequestUsingDb( +function processIncomingRequestUsingDb( recievedData: RecievedData, qdb: QuestionDb, dryRun: boolean, user: User -): Promise { +): Promise { return new Promise((resolve, reject) => { - logger.DebugLog('Processing incoming request', 'actions', 1) - - if (recievedData === undefined) { - logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) - reject(new Error('Recieved data is undefined!')) - } - try { - let towrite = logger.GetDateString() + '\n' - towrite += - '------------------------------------------------------------------------------\n' - if (typeof recievedData === 'object') { - towrite += JSON.stringify(recievedData) - } else { - towrite += recievedData - } - towrite += - '\n------------------------------------------------------------------------------\n' - utils.AppendToFile(towrite, recDataFile) - logger.DebugLog('recDataFile written', 'actions', 1) - } catch (err) { - logger.Log('Error writing recieved data.') - } - - if (utils.FileExists(dataLockFile)) { - logger.Log( - 'Data lock file exists, skipping recieved data processing', - logger.GetColor('red') - ) - resolve(-1) - return - } - - try { - // recievedData: { version: "", id: "", subj: "" quiz: {} } const recievedQuestions = [] logger.DebugLog('recievedData JSON parsed', 'actions', 1) @@ -145,11 +170,7 @@ function ProcessIncomingRequestUsingDb( }) try { - let color = logger.GetColor('green') - let msg = '' if (allQuestions.length > 0) { - color = logger.GetColor('blue') - msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` allQuestions.forEach((currentQuestion) => { const sName = getSubjNameWithoutYear(recievedData.subj) logger.DebugLog( @@ -171,35 +192,26 @@ function ProcessIncomingRequestUsingDb( currWrites = 0 logger.DebugLog('Writing data.json', 'actions', 1) utils.WriteFile(JSON.stringify(qdb.data), qdb.path) - logger.Log('\tData file written', color) } else if (dryRun) { logger.Log('\tDry run') } - } else { - msg += `No new data ( ${allQLength} )` } - let subjRow = '\t' + recievedData.subj - if (recievedData.id) { - subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')' - } idStats.LogId( user.id, recievedData.subj, allQuestions.length, allQLength ) - logger.Log(subjRow) - if (recievedData.version !== undefined) { - msg += '. Version: ' + recievedData.version - } - logger.Log('\t' + msg, color) logger.DebugLog('New Questions:', 'actions', 2) logger.DebugLog(allQuestions, 'actions', 2) logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) - resolve(allQuestions.length) + resolve({ + newQuestions: allQuestions.length, + qdbName: qdb.name, + }) } catch (error) { console.error(error) logger.Log( @@ -229,7 +241,7 @@ function ProcessIncomingRequestUsingDb( }) } -function LoadJSON(dataFiles: Array) { +export function loadJSON(dataFiles: Array): Array { return dataFiles.reduce((acc, dataFile) => { if (!utils.FileExists(dataFile.path)) { utils.WriteFile(JSON.stringify([]), dataFile.path) @@ -251,7 +263,7 @@ function LoadJSON(dataFiles: Array) { }, []) } -function backupData(questionDbs: Array) { +export function backupData(questionDbs: Array): void { questionDbs.forEach((data) => { const path = './publicDirs/qminingPublic/backs/' utils.CreatePath(path) diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 3b7bc9e..ca3aa23 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -1,6 +1,11 @@ import { Worker, isMainThread, parentPort, workerData } from 'worker_threads' import logger from './logger' -import { Question, QuestionData, Subject } from '../types/basicTypes' +import { + Question, + QuestionDb, + QuestionData, + Subject, +} from '../types/basicTypes' // TODO interface SearchResultQuestion extends Question { @@ -402,6 +407,19 @@ function addQuestion( } } +function searchDatas( + data: Array, + question: any, + subjName: string, + questionData?: QuestionData +): Promise>> { + return Promise.all( + data.map((db) => { + return searchData(db.data, question, subjName, questionData) + }) + ) +} + // TODO: remove questionData, make question only Question type function searchData( data: Array, @@ -568,5 +586,6 @@ export { createQuestion, addQuestion, searchData, + searchDatas, dataToString, }