/* ---------------------------------------------------------------------------- Question Server GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ import fs from 'fs' import { Response } from 'express' import { fork, ChildProcess } from 'child_process' import type { Database } from 'better-sqlite3' import logger from '../../../utils/logger' import utils from '../../../utils/utils' import { User, DataFile, Request, QuestionDb, SubmoduleData, Question, QuestionFromScript, DbSearchResult, RegisteredUserEntry, Submodule, } from '../../../types/basicTypes' import { processIncomingRequest, logResult, shouldSaveDataFile, Result, backupData, shouldSearchDataFile, writeData, editDb, RecievedData, } from '../../../utils/actions' import { WorkerResult, // compareQuestionObj, } from '../../../utils/classes' import { doALongTask, msgAllWorker } from '../../../utils/workerPool' import dbtools from '../../../utils/dbtools' import { dataToString, getSubjNameWithoutYear, SearchResultQuestion, } from '../../../utils/qdbUtils' import { paths } from '../../../utils/files' import constants from '../../../constants' import { isJsonValidAndLogError, TestUsersSchema, } from '../../../types/typeSchemas' interface SavedQuestionData { fname: string subj: string userid: number testUrl: string date: string | Date } // interface SavedQuestion { // Questions: Question[] // subj: string // userid: number // testUrl: string // date: string // } const line = '====================================================' // lol function getSubjCount(qdbs: QuestionDb[]): number { return qdbs.reduce((acc, qdb) => { return acc + qdb.data.length }, 0) } function getQuestionCount(qdbs: QuestionDb[]): number { return qdbs.reduce((acc, qdb) => { return ( acc + qdb.data.reduce((qacc, subject) => { return qacc + subject.Questions.length }, 0) ) }, 0) } function ExportDailyDataCount(questionDbs: QuestionDb[], userDB: Database) { logger.Log('Saving daily data count ...') utils.AppendToFile( JSON.stringify({ date: utils.GetDateString(), subjectCount: getSubjCount(questionDbs), questionCount: getQuestionCount(questionDbs), questionDbsCount: questionDbs.length, userCount: dbtools.TableInfo(userDB, 'users').dataCount, }), paths.dailyDataCountFile ) } function getDbIndexesToSearchIn( testUrl: string, questionDbs: Array, trueIfAlways?: boolean ): number[] { return testUrl ? questionDbs.reduce((acc, qdb, i) => { if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) { acc.push(i) } return acc }, []) : [] } function getSimplreRes(questionDbs: QuestionDb[]): { subjects: number questions: number } { return { subjects: getSubjCount(questionDbs), questions: getQuestionCount(questionDbs), } } function getDetailedRes(questionDbs: QuestionDb[]) { return questionDbs.map((qdb) => { return { dbName: qdb.name, subjs: qdb.data.map((subj) => { return { name: subj.Name, count: subj.Questions.length, } }), } }) } function searchInDbs( question: Question, subj: string, searchIn: number[], testUrl?: string ): Promise { // searchIn could be [0], [1], ... to search every db in different thread. Put this into a // forEach(qdbs) to achieve this return new Promise((resolve) => { doALongTask({ type: 'work', data: { searchIn: searchIn, testUrl: testUrl, question: question, subjName: subj, searchInAllIfNoResult: true, }, }) .then((taskResult: WorkerResult) => { try { logger.DebugLog(taskResult, 'ask', 2) resolve({ question: question, result: taskResult.result as SearchResultQuestion[], success: true, }) } catch (err) { console.error(err) logger.Log( 'Error while sending ask results', logger.GetColor('redbg') ) } }) .catch((err) => { logger.Log('Search Data error!', logger.GetColor('redbg')) console.error(err) resolve({ question: question, message: `There was an error processing the question: ${err.message}`, result: [], success: false, }) }) }) } function getResult(data: { question: Question subj: string questionDbs: Array testUrl: string }): Promise { const { question, subj, questionDbs, testUrl } = data return new Promise((resolve) => { const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false) searchInDbs(question, subj, searchIn, testUrl).then( (res: DbSearchResult) => { if (res.result.length === 0) { logger.DebugLog( `No result while searching specific question db ${testUrl}`, 'ask', 1 ) const searchInMore = getDbIndexesToSearchIn( testUrl, questionDbs, true ).filter((x) => { return !searchIn.includes(x) }) searchInDbs(question, subj, searchInMore, testUrl).then( (res) => { resolve(res) } ) } else { resolve(res) } } ) }) } function dbExists(location: string, qdbs: Array) { return qdbs.some((qdb) => { return qdb.name === location }) } function writeAskData(body: QuestionFromScript) { try { let towrite = utils.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' towrite += JSON.stringify(body) towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, paths.askedQuestionFile) } catch (err) { logger.Log('Error writing revieved /ask POST data') console.error(err) } } function writeIsAddingData(body: RecievedData) { try { let towrite = utils.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' towrite += JSON.stringify(body) towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, paths.recievedQuestionFile) } catch (err) { logger.Log('Error writing revieved /ask POST data') console.error(err) } } function saveQuestion( questions: Question[], subj: string, testUrl: string, userid: number, savedQuestionsDir: string ) { // TODO: clear folder every now and then, check if saved questions exist if (!subj) { logger.Log('No subj name to save test question') return } const questionsToSave = { questions: questions, subj: subj, userid: userid, testUrl: testUrl, date: new Date(), } const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json` const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-') const subjPath = `${savedQuestionsDir}/${subject}` const savedSubjQuestionsFilePath = `${subjPath}/${constants.savedQuestionsFileName}` utils.CreatePath(subjPath, true) if (!utils.FileExists(savedSubjQuestionsFilePath)) { utils.WriteFile('[]', savedSubjQuestionsFilePath) } const savedQuestions: SavedQuestionData[] = utils.ReadJSON( savedSubjQuestionsFilePath ) const testExists = false // TODO: do this on another thread? // const testExists = savedQuestions.some((savedQuestion) => { // const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`) // return data.questions.some((dQuestion) => { // return questions.some((question) => { // const percent = compareQuestionObj( // createQuestion(question), // '', // createQuestion(dQuestion), // '' // ) // return percent.avg === 100 // }) // }) // }) if (testExists) { return } savedQuestions.push({ fname: fname, subj: subj, userid: userid, testUrl: testUrl, date: new Date(), }) utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath) utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`) } function loadSupportedSites() { const script = utils.ReadFile(paths.moodleTestUserscriptPath).split('\n') let i = 0 let stayIn = true let inHeader = false let inMatch = false const sites = [] while (i < script.length && stayIn) { if (inHeader) { if (script[i].includes('@match')) { inMatch = true } if (inMatch && !script[i].includes('match')) { stayIn = false inMatch = false } if (inMatch) { sites.push(script[i].split(' ').pop()) } } else { inHeader = script[i].includes('==UserScript==') } i++ } return sites } function LoadMOTD(motdFile: string) { return utils.ReadFile(motdFile) } function LoadTestUsers() { if (!utils.FileExists(paths.testUsersFile)) { utils.WriteFile('{}', paths.testUsersFile) } const testUsers = utils.ReadJSON<{ userIds: number[] }>(paths.testUsersFile) if ( !isJsonValidAndLogError(testUsers, TestUsersSchema, paths.testUsersFile) ) { return [] } else { return testUsers.userIds } } function getNewQdb( location: string, maxIndex: number, dbsFile: string, publicDir: string, questionDbs: QuestionDb[] ) { logger.Log( `No suitable questiondbs found for ${location}, creating a new one...` ) const newDb: DataFile = { path: `questionDbs/${location}.json`, name: location, shouldSearch: { location: { val: location, }, }, shouldSave: { location: { val: location, }, }, } utils.WriteFile( JSON.stringify( [ ...utils.ReadJSON(dbsFile), newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime ], null, 2 ), dbsFile ) // "loading" new db const loadedNewDb: QuestionDb = { ...newDb, data: [], path: publicDir + newDb.path, index: maxIndex, } utils.WriteFile('[]', loadedNewDb.path) questionDbs.push(loadedNewDb) msgAllWorker({ data: loadedNewDb, type: 'newdb', }) return loadedNewDb } function setup(data: SubmoduleData): Submodule { const { app, userDB, /* url */ publicdirs, moduleSpecificData: { getQuestionDbs, setQuestionDbs, dbsFile }, } = data const publicDir = publicdirs[0] const motdFile = publicDir + 'motd' const savedQuestionsDir = publicDir + 'savedQuestions' let version = utils.getScriptVersion() let supportedSites = loadSupportedSites() let motd = LoadMOTD(motdFile) let testUsers: number[] = LoadTestUsers() const filesToWatch = [ { fname: motdFile, logMsg: 'Motd updated', action: () => { motd = LoadMOTD(motdFile) }, }, { fname: paths.testUsersFile, logMsg: 'Test Users file changed', action: () => { testUsers = LoadTestUsers() }, }, { fname: paths.moodleTestUserscriptPath, logMsg: 'User script file changed', action: () => { version = utils.getScriptVersion() supportedSites = loadSupportedSites() }, }, ] app.get('/getDbs', (req: Request, res: Response) => { logger.LogReq(req) res.json( getQuestionDbs().map((qdb) => { return { path: qdb.path.replace(publicDir, ''), name: qdb.name, locked: qdb.locked, } }) ) }) app.get('/allqr.txt', function (req: Request, res: Response) { logger.LogReq(req) const db: string = req.query.db let stringifiedData = '' res.setHeader('content-type', 'text/plain; charset=utf-8') if (db) { const requestedDb = getQuestionDbs().find((qdb) => { return qdb.name === db }) if (!requestedDb) { res.end(`No such db ${db}`) return } stringifiedData = '\n' + line stringifiedData += ` Questions in ${requestedDb.name}: ` stringifiedData += line + '\n' stringifiedData += dataToString(requestedDb.data) stringifiedData += '\n' + line + line + '\n' } else { stringifiedData = getQuestionDbs() .map((qdb) => { let result = '' result += '\n' + line result += ` Questions in ${qdb.name}: ` result += line + '\n' result += dataToString(qdb.data) result += '\n' + line + line + '\n' return result }) .join('\n\n') } res.end(stringifiedData) }) app.post('/isAdding', function (req: Request, res: Response) { logger.LogReq(req) const user: User = req.session.user const dryRun = testUsers.includes(user.id) if (!req.body.location) { logger.Log( '\tbody.location is missing, client version:' + req.body.version ) res.json({ msg: 'body.location is missing' }) return } writeIsAddingData(req.body) const location = req.body.location.split('/')[2] try { let maxIndex = -1 const suitedQuestionDbs = getQuestionDbs().filter((qdb) => { if (maxIndex < qdb.index) { maxIndex = qdb.index } return shouldSaveDataFile(qdb, req.body) }, []) if (suitedQuestionDbs.length === 0) { if (!dbExists(location, getQuestionDbs())) { suitedQuestionDbs.push( getNewQdb( location, maxIndex, dbsFile, publicDir, getQuestionDbs() ) ) } else { logger.Log( `Tried to add existing db named ${location}!`, logger.GetColor('red') ) } } if (suitedQuestionDbs.length === 0) { res.json({ status: 'fail', msg: 'No suitable dbs to add questions to', }) return } processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user) .then((resultArray: Array) => { logResult(req.body, resultArray, user.id, dryRun) const totalNewQuestions = resultArray.reduce( (acc, sres) => { return acc + sres.newQuestions.length }, 0 ) res.json({ success: resultArray.length > 0, newQuestions: resultArray, totalNewQuestions: totalNewQuestions, }) if (totalNewQuestions > 0) { resultArray.forEach((result) => { msgAllWorker({ // TODO: recognize base64 image type: 'newQuestions', data: result, }) }) } }) .catch((err) => { logger.Log( 'Error during processing incoming request', logger.GetColor('redbg') ) console.error(err) res.json({ success: false, }) }) } catch (err) { logger.Log( 'Error during getting incoming request processor promises ', logger.GetColor('redbg') ) console.error(err) res.json({ success: false, }) } }) app.post('/ask', function (req: Request, res) { const user: User = req.session.user if (!req.body.questions) { res.json({ message: `ask something! { questions:'' ,subject:'', location:'' }`, result: [], success: false, }) return } const subj: string = req.body.subj || '' // TODO: test if testUrl is undefined (it shouldnt) const testUrl: string = req.body.testUrl ? req.body.testUrl.split('/')[2] : undefined writeAskData(req.body) // every question in a different thread const resultPromises: Promise[] = req.body.questions.map((question: Question) => { return getResult({ question: question, subj: subj, testUrl: testUrl, questionDbs: getQuestionDbs(), }) }) Promise.all(resultPromises).then((results: DbSearchResult[]) => { const response = results.map((result: DbSearchResult) => { return { answers: result.result, question: result.question, } }) res.json(response) const saveableQuestions = response.reduce((acc, res) => { const save = res.answers.every((answer) => { return answer.match < 90 }) if (save) { acc.push(res.question) } return acc }, []) if (saveableQuestions.length > 0) { saveQuestion( saveableQuestions, subj, testUrl, user.id, savedQuestionsDir ) } }) }) app.get('/supportedSites', function (req: Request, res: Response) { logger.LogReq(req) res.json(supportedSites) }) app.get('/datacount', function (req: Request, res: Response) { logger.LogReq(req) if (req.query.detailed === 'all') { res.json({ detailed: getDetailedRes(getQuestionDbs()), simple: getSimplreRes(getQuestionDbs()), }) } else if (req.query.detailed) { res.json(getDetailedRes(getQuestionDbs())) } else { res.json(getSimplreRes(getQuestionDbs())) } }) app.get('/infos', function (req: Request, res) { const user: User = req.session.user const result: { result: string uid: number version?: string subjinfo?: { subjects: number questions: number } motd?: string } = { result: 'success', uid: user.id, } if (req.query.subjinfo) { result.subjinfo = getSimplreRes(getQuestionDbs()) } if (req.query.version) { result.version = version } if (req.query.motd) { result.motd = motd } res.json(result) }) app.post('/registerscript', function (req: Request, res) { logger.LogReq(req) if (!utils.FileExists(paths.registeredScriptsFile)) { utils.WriteFile('[]', paths.registeredScriptsFile) } const ua: string = req.headers['user-agent'] const registeredScripts: RegisteredUserEntry[] = utils.ReadJSON( paths.registeredScriptsFile ) const { cid, uid, version, installSource, date } = req.body const index = registeredScripts.findIndex((registration) => { return registration.cid === cid }) if (index === -1) { const x: RegisteredUserEntry = { cid: cid, version: version, installSource: installSource, date: date, userAgent: ua, } if (uid) { x.uid = uid x.loginDate = date } registeredScripts.push(x) } else { const currRegistration = registeredScripts[index] if (!currRegistration.uid && uid) { registeredScripts[index] = { ...registeredScripts[index], uid: uid, loginDate: date, } } else { logger.DebugLog( `cid: ${cid}, uid: ${uid} tried to register multiple times`, 'register', 1 ) } } utils.WriteFile( JSON.stringify(registeredScripts, null, 2), paths.registeredScriptsFile ) res.json({ msg: 'done' }) }) app.get('/possibleAnswers', (req: Request, res: Response) => { logger.LogReq(req) const files = utils.ReadDir(savedQuestionsDir) files.sort(function (a, b) { return ( fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() - fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime() ) }) res.json({ savedQuestionsFileName: constants.savedQuestionsFileName, subjects: files.map((subj) => { return { name: subj, path: `savedQuestions/${subj}/`, } }), }) }) app.post('/rmPossibleAnswer', (req: Request, res: Response) => { logger.LogReq(req) const user: User = req.session.user const subj = req.body.subj const file = req.body.file const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${constants.savedQuestionsFileName}` const savedQuestions: SavedQuestionData[] = utils.ReadJSON(savedQuestionsPath) let path = `${savedQuestionsDir}/${subj}/${file}` while (path.includes('..')) { path = path.replace(/\.\./g, '.') } if (utils.FileExists(path)) { utils.deleteFile(path) utils.WriteFile( JSON.stringify( savedQuestions.filter((sq) => { return sq.fname !== file }) ), savedQuestionsPath ) logger.Log( `User #${user.id} deleted '${file}' from subject '${subj}'`, logger.GetColor('cyan') ) res.json({ res: 'ok', }) } else { logger.Log( `User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`, logger.GetColor('red') ) res.json({ res: 'fail', }) } }) app.post('/updateQuestion', (req: Request, res) => { logger.LogReq(req) const user: User = req.session.user const date = utils.GetDateString() const editType = req.body.type const selectedDb = req.body.selectedDb if (!editType || !selectedDb) { res.json({ status: 'fail', msg: 'No .editType or .selectedDb !', }) return } const dbIndex = getQuestionDbs().findIndex((qdb) => { return qdb.name === selectedDb.name }) const currDb = getQuestionDbs()[dbIndex] if (dbIndex === -1) { res.json({ status: 'fail', msg: `No question db named like ${selectedDb.name}!`, }) return } // ----------------- const { success, msg, resultDb, deletedQuestion, newVal, oldVal, deletedQuestions, changedQuestions, } = editDb(currDb, req.body) if (!success) { res.json({ success: success, msg: msg }) return } if (resultDb) { setQuestionDbs( getQuestionDbs().map((qdb, i) => { if (i === dbIndex) return resultDb return qdb }) ) } if (editType === 'delete') { const { index, subjName } = req.body logger.Log( `User #${user.id} deleted a question from '${subjName}'`, logger.GetColor('cyan') ) utils.AppendToFile( `${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`, paths.dataEditsLog ) utils.AppendToFile( JSON.stringify(deletedQuestion, null, 2), paths.dataEditsLog ) } if (editType === 'edit') { const { index, subjName } = req.body logger.Log( `User #${user.id} edited a question in '${subjName}'`, logger.GetColor('cyan') ) utils.AppendToFile( `${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`, paths.dataEditsLog ) utils.AppendToFile( JSON.stringify( { newVal: newVal, oldVal: oldVal, }, null, 2 ), paths.dataEditsLog ) } if (editType === 'subjEdit') { const { subjName } = req.body logger.Log( `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, logger.GetColor('cyan') ) utils.AppendToFile( `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, paths.dataEditsLog ) utils.AppendToFile( JSON.stringify( { deletedQuestions: deletedQuestions, changedQuestions: changedQuestions, }, null, 2 ), paths.dataEditsLog ) } // ------------------ if (success) { writeData(currDb.data, currDb.path) msgAllWorker({ type: 'dbEdit', data: { dbIndex: dbIndex, edits: req.body, }, }) } res.json({ success: true, msg: 'OK', }) }) let questionCleaner: ChildProcess = null app.get('/clearQuestions', (req: Request, res) => { // TODO: dont allow multiple instances // TODO: get status of it cleaning logger.LogReq(req) res.json({ error: 'Not implemented / tested!', }) return const user: User = req.session.user const status: string = req.query.status if (status) { if (!questionCleaner) { res.json({ msg: 'question cleaner not running', success: false, }) return } questionCleaner.once('message', function (response) { res.json({ msg: response, success: true, }) }) questionCleaner.send({ data: 'asd' }) return } if (questionCleaner) { res.json({ msg: 'question cleaner already running', success: false, }) return } questionCleaner = fork( `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, ['-s', `${process.cwd()}/${getQuestionDbs()[0].path}`] // TODO: this only cleans index // #0? ) questionCleaner.on('exit', function (code: number) { console.log('EXIT', code) questionCleaner = null }) res.json({ user: user, success: true, msg: 'OK', }) }) return { dailyAction: () => { backupData(getQuestionDbs()) ExportDailyDataCount(getQuestionDbs(), userDB) }, load: () => { backupData(getQuestionDbs()) filesToWatch.forEach((ftw) => { if (utils.FileExists(ftw.fname)) { utils.WatchFile(ftw.fname, () => { logger.Log(ftw.logMsg) ftw.action() }) ftw.action() } else { logger.Log( `File ${ftw.fname} does not exists to watch!`, logger.GetColor('redbg') ) } }) }, } } export default { setup: setup, }