import fs from 'fs' import logger from '../../../utils/logger' import utils from '../../../utils/utils' import { User, DataFile, Request, QuestionDb, SubmoduleData, } from '../../../types/basicTypes' import { processIncomingRequest, logResult, shouldSaveDataFile, Result, backupData, shouldSearchDataFile, loadJSON, writeData, editDb, } from '../../../utils/actions' import { dataToString, getSubjNameWithoutYear, // compareQuestionObj, } from '../../../utils/classes' import { doALongTask, msgAllWorker, initWorkerPool, } from '../../../utils/workerPool' import dbtools from '../../../utils/dbtools' const line = '====================================================' // lol const registeredScriptsFile = 'stats/registeredScripts.json' const testUsersFile = 'data/testUsers.json' const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js' const recievedQuestionFile = 'stats/recievedQuestions' const savedQuestionsFileName = 'savedQuestions.json' const oldMotdFile = 'publicDirs/qminingPublic/oldMotd' const dailyDataCountFile = 'stats/dailyDataCount' const dataEditsLog = 'stats/dataEdits' function getSubjCount(qdbs) { return qdbs.reduce((acc, qdb) => { return acc + qdb.data.length }, 0) } function getQuestionCount(qdbs) { return qdbs.reduce((acc, qdb) => { return ( acc + qdb.data.reduce((qacc, subject) => { return qacc + subject.Questions.length }, 0) ) }, 0) } function ExportDailyDataCount(questionDbs, userDB) { logger.Log('Saving daily data count ...') utils.AppendToFile( JSON.stringify({ date: utils.GetDateString(), subjectCount: getSubjCount(questionDbs), questionCount: getQuestionCount(questionDbs), userCount: dbtools.TableInfo(userDB, 'users').dataCount, }), dailyDataCountFile ) } function getDbIndexesToSearchIn( testUrl: string, questionDbs, trueIfAlways?: boolean ) { return testUrl ? questionDbs.reduce((acc, qdb, i) => { if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) { acc.push(i) } return acc }, []) : [] } function getSimplreRes(questionDbs) { return { subjects: getSubjCount(questionDbs), questions: getQuestionCount(questionDbs), } } function getDetailedRes(questionDbs) { return questionDbs.map((qdb) => { return { dbName: qdb.name, subjs: qdb.data.map((subj) => { return { name: subj.Name, count: subj.Questions.length, } }), } }) } function getMotd(version, motd) { if (version) { if (version.startsWith('2.0.')) { if (utils.FileExists(oldMotdFile)) { return utils.ReadFile(oldMotdFile) } } } return motd } function searchInDbs( question, subj, recData, recievedData, searchIn, testUrl? ) { // searchIn could be [0], [1], ... to search every db in different thread. Put this into a // forEach(qdbs) to achieve this return new Promise((resolve) => { doALongTask({ type: 'work', data: { searchIn: searchIn, testUrl: testUrl, question: question, subjName: subj, questionData: recData, searchInAllIfNoResult: true, }, }) .then((taskResult) => { try { logger.DebugLog( `Question result length: ${taskResult.length}`, 'ask', 1 ) logger.DebugLog(taskResult, 'ask', 2) resolve({ question: question, result: taskResult.result, success: true, }) } catch (err) { console.error(err) logger.Log( 'Error while sending ask results', logger.GetColor('redbg') ) } }) .catch((err) => { logger.Log('Search Data error!', logger.GetColor('redbg')) console.error(err) resolve({ mesage: `There was an error processing the question: ${err.message}`, result: [], recievedData: JSON.stringify(recievedData), success: false, }) }) }) } function getResult(data) { const { question, subj, recData, recievedData, questionDbs, testUrl } = data return new Promise((resolve) => { const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false) searchInDbs(question, subj, recData, recievedData, searchIn, testUrl).then( (res: any) => { if (res.result.length === 0) { logger.DebugLog( `No result while searching specific question db ${testUrl}`, 'ask', 1 ) const searchInMore = getDbIndexesToSearchIn( testUrl, questionDbs, true ).filter((x) => { return !searchIn.includes(x) }) searchInDbs( question, subj, recData, recievedData, searchInMore, testUrl ).then((res) => { resolve(res) }) } else { resolve(res) } } ) }) } function dbExists(location, qdbs: Array) { return qdbs.some((qdb) => { return qdb.name === location }) } function writeAskData(body) { try { let towrite = utils.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' towrite += JSON.stringify(body) towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recievedQuestionFile) } catch (err) { logger.Log('Error writing revieved /ask POST data') console.error(err) } } function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) { // TODO: clear folder every now and then, check if saved questions exist if (!subj) { logger.Log('No subj name to save test question') return } const questionsToSave = { questions: questions, subj: subj, userid: userid, testUrl: testUrl, date: new Date(), } const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json` const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-') const subjPath = `${savedQuestionsDir}/${subject}` const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}` utils.CreatePath(subjPath, true) if (!utils.FileExists(savedSubjQuestionsFilePath)) { utils.WriteFile('[]', savedSubjQuestionsFilePath) } const savedQuestions = utils.ReadJSON(savedSubjQuestionsFilePath) const testExists = false // TODO: do this on another thread? // const testExists = savedQuestions.some((savedQuestion) => { // const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`) // return data.questions.some((dQuestion) => { // return questions.some((question) => { // const percent = compareQuestionObj( // createQuestion(question), // '', // createQuestion(dQuestion), // '' // ) // return percent.avg === 100 // }) // }) // }) if (testExists) { return } savedQuestions.push({ fname: fname, subj: subj, userid: userid, testUrl: testUrl, date: new Date(), }) utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath) utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`) } function loadSupportedSites() { const script = utils.ReadFile(userScriptFile).split('\n') let i = 0 let stayIn = true let inHeader = false let inMatch = false const sites = [] while (i < script.length && stayIn) { if (inHeader) { if (script[i].includes('@match')) { inMatch = true } if (inMatch && !script[i].includes('match')) { stayIn = false inMatch = false } if (inMatch) { sites.push(script[i].split(' ').pop()) } } else { inHeader = script[i].includes('==UserScript==') } i++ } return sites } function LoadVersion() { const scriptContent = utils.ReadFile(userScriptFile) let temp: any = scriptContent.split('\n').find((x) => { return x.includes('@version') }) temp = temp.split(' ') temp = temp[temp.length - 1] return temp } function LoadMOTD(motdFile) { return utils.ReadFile(motdFile) } function LoadUserSpecificMOTD(userSpecificMotdFile) { try { return utils.ReadJSON(userSpecificMotdFile) } catch (err) { logger.Log('Couldnt parse user specific motd!', logger.GetColor('redbg')) console.error(err) } } function LoadTestUsers() { let testUsers = utils.ReadJSON(testUsersFile) if (testUsers) { testUsers = testUsers.userIds } return testUsers } function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) { logger.Log( `No suitable questiondbs found for ${location}, creating a new one...` ) const newDb: DataFile = { path: `questionDbs/${location}.json`, name: location, shouldSearch: { location: { val: location, }, }, shouldSave: { location: { val: location, }, }, } utils.WriteFile( JSON.stringify( [ ...utils.ReadJSON(dbsFile), newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime ], null, 2 ), dbsFile ) // "loading" new db const loadedNewDb: QuestionDb = { ...newDb, data: [], path: publicDir + newDb.path, index: maxIndex, } utils.WriteFile('[]', loadedNewDb.path) questionDbs.push(loadedNewDb) msgAllWorker({ newdb: loadedNewDb, type: 'newdb', }) return loadedNewDb } function setup(data: SubmoduleData): any { const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data const publicDir = publicdirs[0] const motdFile = publicDir + 'motd' const userSpecificMotdFile = publicDir + 'userSpecificMotd.json' const dbsFile = publicDir + 'questionDbs.json' const savedQuestionsDir = publicDir + 'savedQuestions' let version = LoadVersion() let supportedSites = loadSupportedSites() let motd = LoadMOTD(motdFile) let userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile) let testUsers: any = LoadTestUsers() const dataFiles: Array = utils.ReadJSON(dbsFile) const questionDbs: Array = loadJSON(dataFiles, publicDir) initWorkerPool(questionDbs) const filesToWatch = [ { fname: userSpecificMotdFile, logMsg: 'User Specific Motd updated', action: () => { userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile) }, }, { fname: motdFile, logMsg: 'Motd updated', action: () => { motd = LoadMOTD(motdFile) }, }, { fname: testUsersFile, logMsg: 'Test Users file changed', action: () => { testUsers = LoadTestUsers() }, }, { fname: userScriptFile, logMsg: 'User script file changed', action: () => { version = LoadVersion() supportedSites = loadSupportedSites() }, }, ] app.get('/getDbs', (req: Request, res: any) => { logger.LogReq(req) res.json( questionDbs.map((qdb) => { return { path: qdb.path.replace(publicDir, ''), name: qdb.name, locked: qdb.locked, } }) ) }) app.get('/allqr.txt', function (req: Request, res: any) { logger.LogReq(req) const db: any = req.query.db let stringifiedData = '' res.setHeader('content-type', 'text/plain; charset=utf-8') if (db) { const requestedDb = questionDbs.find((qdb) => { return qdb.name === db }) if (!requestedDb) { res.end(`No such db ${db}`) return } stringifiedData = '\n' + line stringifiedData += ` Questions in ${requestedDb.name}: ` stringifiedData += line + '\n' stringifiedData += dataToString(requestedDb.data) stringifiedData += '\n' + line + line + '\n' } else { stringifiedData = questionDbs .map((qdb) => { let result = '' result += '\n' + line result += ` Questions in ${qdb.name}: ` result += line + '\n' result += dataToString(qdb.data) result += '\n' + line + line + '\n' return result }) .join('\n\n') } res.end(stringifiedData) }) app.post('/isAdding', function (req: Request, res: any) { logger.LogReq(req) const user: User = req.session.user const dryRun = testUsers.includes(user.id) if (!req.body.location) { logger.Log( '\tbody.location is missing, client version:' + req.body.version ) res.json({ msg: 'body.location is missing' }) return } const location = req.body.location.split('/')[2] try { let maxIndex = -1 const suitedQuestionDbs = questionDbs.filter((qdb) => { if (maxIndex < qdb.index) { maxIndex = qdb.index } return shouldSaveDataFile(qdb, req.body) }, []) if (suitedQuestionDbs.length === 0) { if (!dbExists(location, questionDbs)) { suitedQuestionDbs.push( getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) ) } else { logger.Log( `Tried to add existing db named ${location}!`, logger.GetColor('red') ) } } if (suitedQuestionDbs.length === 0) { res.json({ status: 'fail', msg: 'No suitable dbs to add questions to', }) return } processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user) .then((resultArray: Array) => { logResult(req.body, resultArray, user.id, dryRun) const totalNewQuestions = resultArray.reduce((acc, sres) => { return acc + sres.newQuestions.length }, 0) res.json({ success: resultArray.length > 0, newQuestions: resultArray, totalNewQuestions: totalNewQuestions, }) if (totalNewQuestions > 0) { resultArray.forEach((result) => { msgAllWorker({ type: 'newQuestions', data: result, }) }) } }) .catch((err) => { logger.Log( 'Error during processing incoming request', logger.GetColor('redbg') ) console.error(err) res.json({ success: false, }) }) } catch (err) { logger.Log( 'Error during getting incoming request processor promises ', logger.GetColor('redbg') ) console.error(err) res.json({ success: false, }) } }) app.post('/ask', function (req: Request, res) { const user: User = req.session.user if (!req.body.questions) { res.json({ message: `ask something! { question:'' ,subject:'', location:'' }`, result: [], recievedData: JSON.stringify(req.body), success: false, }) return } const subj: any = req.body.subj || '' // TODO: test if testUrl is undefined (it shouldnt) const testUrl = req.body.testUrl ? req.body.testUrl.split('/')[2] : undefined writeAskData(req.body) // every question in a different thread const resultPromises = req.body.questions.map((question) => { return getResult({ question: question, subj: subj, recData: req.body, testUrl: testUrl, questionDbs: questionDbs, }) }) Promise.all(resultPromises).then((results) => { const response = results.map((result: any) => { return { answers: result.result, question: result.question, } }) res.json(response) const saveableQuestions = response.reduce((acc, res) => { const save = res.answers.every((answer) => { return answer.match < 90 }) if (save) { acc.push(res.question) } return acc }, []) if (saveableQuestions.length > 0) { saveQuestion( saveableQuestions, subj, testUrl, user.id, savedQuestionsDir ) } }) }) app.get('/ask', function (req: Request, res) { if (Object.keys(req.query).length === 0) { logger.DebugLog(`No query params /ask GET`, 'ask', 1) res.json({ message: 'ask something! ?q=[question]&subj=[subject]&data=[question data]. "subj" is optimal for faster result', result: [], recievedData: JSON.stringify(req.query), success: false, }) return } if (req.query.q && req.query.data) { const subj: any = req.query.subj || '' const question = req.query.q const recData: any = req.query.data getResult({ question: question, subj: subj, recData: recData, recievedData: req.query, questionDbs: questionDbs, }).then((result) => { res.json(result) }) } else { logger.DebugLog(`Invalid question`, 'ask', 1) res.json({ message: `Invalid question :(`, result: [], recievedData: JSON.stringify(req.query), success: false, }) } }) app.get('/supportedSites', function (req: Request, res: any) { logger.LogReq(req) res.json(supportedSites) }) app.get('/datacount', function (req: Request, res: any) { logger.LogReq(req) if (req.query.detailed === 'all') { res.json({ detailed: getDetailedRes(questionDbs), simple: getSimplreRes(questionDbs), }) } else if (req.query.detailed) { res.json(getDetailedRes(questionDbs)) } else { res.json(getSimplreRes(questionDbs)) } }) app.get('/infos', function (req: Request, res) { const user: User = req.session.user const result: any = { result: 'success', uid: user.id, } if (req.query.subjinfo) { result.subjinfo = getSimplreRes(questionDbs) } if (req.query.version) { result.version = version } if (req.query.motd) { result.motd = getMotd(req.query.cversion, motd) if (userSpecificMotd[user.id]) { result.userSpecificMotd = { msg: userSpecificMotd[user.id].msg, seen: userSpecificMotd[user.id].seen, } } } res.json(result) }) app.post('/infos', (req: Request, res) => { const user: User = req.session.user if (req.body.userSpecificMotdSeen && !userSpecificMotd[user.id].seen) { userSpecificMotd[user.id].seen = true logger.Log( `User #${user.id}'s user specific motd is now seen:`, logger.GetColor('bluebg') ) logger.Log(userSpecificMotd[user.id].msg) utils.WriteFile( JSON.stringify(userSpecificMotd, null, 2), userSpecificMotdFile ) } res.json({ msg: 'done' }) }) app.post('/registerscript', function (req: Request, res) { logger.LogReq(req) if (!utils.FileExists(registeredScriptsFile)) { utils.WriteFile('[]', registeredScriptsFile) } const ua: any = req.headers['user-agent'] const registeredScripts = utils.ReadJSON(registeredScriptsFile) const { cid, uid, version, installSource, date } = req.body const index = registeredScripts.findIndex((registration) => { return registration.cid === cid }) if (index === -1) { const x: any = { cid: cid, version: version, installSource: installSource, date: date, userAgent: ua, } if (uid) { x.uid = uid x.loginDate = date } registeredScripts.push(x) } else { const currRegistration = registeredScripts[index] if (!currRegistration.uid && uid) { registeredScripts[index] = { ...registeredScripts[index], uid: uid, loginDate: date, } } else { logger.DebugLog( `cid: ${cid}, uid: ${uid} tried to register multiple times`, 'register', 1 ) } } utils.WriteFile( JSON.stringify(registeredScripts, null, 2), registeredScriptsFile ) res.json({ msg: 'done' }) }) app.get('/possibleAnswers', (req: Request, res: any) => { logger.LogReq(req) const files = utils.ReadDir(savedQuestionsDir) files.sort(function (a, b) { return ( fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() - fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime() ) }) res.json({ savedQuestionsFileName: savedQuestionsFileName, subjects: files.map((subj) => { return { name: subj, path: `savedQuestions/${subj}/`, } }), }) }) app.post('/rmPossibleAnswer', (req: Request, res: any) => { logger.LogReq(req) const user: User = req.session.user const subj = req.body.subj const file = req.body.file const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}` const savedQuestions = utils.ReadJSON(savedQuestionsPath) let path = `${savedQuestionsDir}/${subj}/${file}` // to prevent deleting ../../../ ... /etc/shadow while (path.includes('..')) { path = path.replace(/\.\./g, '.') } if (utils.FileExists(path)) { utils.deleteFile(path) utils.WriteFile( JSON.stringify( savedQuestions.filter((sq) => { return sq.fname !== file }) ), savedQuestionsPath ) logger.Log( `User #${user.id} deleted '${file}' from subject '${subj}'`, logger.GetColor('cyan') ) res.json({ res: 'ok', }) } else { logger.Log( `User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`, logger.GetColor('red') ) res.json({ res: 'fail', }) } }) app.post('/updateQuestion', (req: Request, res) => { logger.LogReq(req) const user: User = req.session.user const date = utils.GetDateString() const editType = req.body.type const selectedDb = req.body.selectedDb if (!editType || !selectedDb) { res.json({ status: 'fail', msg: 'No .editType or .selectedDb !', }) return } const dbIndex = questionDbs.findIndex((qdb) => { return qdb.name === selectedDb.name }) const currDb = questionDbs[dbIndex] if (dbIndex === -1) { res.json({ status: 'fail', msg: `No question db named like ${selectedDb.name}!`, }) return } // ----------------- const { success, msg, resultDb, deletedQuestion, newVal, oldVal, deletedQuestions, changedQuestions, } = editDb(currDb, req.body) if (!success) { res.json({ success: success, msg: msg }) return } if (resultDb) { questionDbs[dbIndex] = resultDb } if (editType === 'delete') { const { index, subjName } = req.body logger.Log( `User #${user.id} deleted a question from '${subjName}'`, logger.GetColor('cyan') ) utils.AppendToFile( `${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`, dataEditsLog ) utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog) } if (editType === 'edit') { const { index, subjName } = req.body logger.Log( `User #${user.id} edited a question in '${subjName}'`, logger.GetColor('cyan') ) utils.AppendToFile( `${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`, dataEditsLog ) utils.AppendToFile( JSON.stringify( { newVal: newVal, oldVal: oldVal, }, null, 2 ), dataEditsLog ) } if (editType === 'subjEdit') { const { subjName } = req.body logger.Log( `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, logger.GetColor('cyan') ) utils.AppendToFile( `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, dataEditsLog ) utils.AppendToFile( JSON.stringify( { deletedQuestions: deletedQuestions, changedQuestions: changedQuestions, }, null, 2 ), dataEditsLog ) } // ------------------ if (success) { writeData(currDb.data, currDb.path) msgAllWorker({ type: 'dbEdit', data: { dbIndex: dbIndex, edits: req.body, }, }) } res.json({ success: true, msg: 'OK', }) }) return { dailyAction: () => { backupData(questionDbs) ExportDailyDataCount(questionDbs, userDB) }, load: () => { backupData(questionDbs) filesToWatch.forEach((ftw) => { if (utils.FileExists(ftw.fname)) { utils.WatchFile(ftw.fname, () => { logger.Log(ftw.logMsg) ftw.action() }) ftw.action() } else { logger.Log( `File ${ftw.fname} does not exists to watch!`, logger.GetColor('redbg') ) } }) }, } } export default { setup: setup, }