Files
mrfrys-node-server/src/modules/api/submodules/qminingapi.ts
T

1001 lines
25 KiB
TypeScript

import fs from 'fs'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import {
User,
DataFile,
Request,
QuestionDb,
SubmoduleData,
} from '../../../types/basicTypes'
import {
processIncomingRequest,
logResult,
shouldSaveDataFile,
Result,
backupData,
shouldSearchDataFile,
loadJSON,
writeData,
editDb,
} from '../../../utils/actions'
import {
dataToString,
getSubjNameWithoutYear,
// compareQuestionObj,
} from '../../../utils/classes'
import {
doALongTask,
msgAllWorker,
initWorkerPool,
} from '../../../utils/workerPool'
import dbtools from '../../../utils/dbtools'
const line = '====================================================' // lol
const registeredScriptsFile = 'stats/registeredScripts.json'
const testUsersFile = 'data/testUsers.json'
const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js'
const recievedQuestionFile = 'stats/recievedQuestions'
const savedQuestionsFileName = 'savedQuestions.json'
const oldMotdFile = 'publicDirs/qminingPublic/oldMotd'
const dailyDataCountFile = 'stats/dailyDataCount'
const dataEditsLog = 'stats/dataEdits'
function getSubjCount(qdbs) {
return qdbs.reduce((acc, qdb) => {
return acc + qdb.data.length
}, 0)
}
function getQuestionCount(qdbs) {
return qdbs.reduce((acc, qdb) => {
return (
acc +
qdb.data.reduce((qacc, subject) => {
return qacc + subject.Questions.length
}, 0)
)
}, 0)
}
function ExportDailyDataCount(questionDbs, userDB) {
logger.Log('Saving daily data count ...')
utils.AppendToFile(
JSON.stringify({
date: utils.GetDateString(),
subjectCount: getSubjCount(questionDbs),
questionCount: getQuestionCount(questionDbs),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
}),
dailyDataCountFile
)
}
function getDbIndexesToSearchIn(
testUrl: string,
questionDbs,
trueIfAlways?: boolean
) {
return testUrl
? questionDbs.reduce((acc, qdb, i) => {
if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) {
acc.push(i)
}
return acc
}, [])
: []
}
function getSimplreRes(questionDbs) {
return {
subjects: getSubjCount(questionDbs),
questions: getQuestionCount(questionDbs),
}
}
function getDetailedRes(questionDbs) {
return questionDbs.map((qdb) => {
return {
dbName: qdb.name,
subjs: qdb.data.map((subj) => {
return {
name: subj.Name,
count: subj.Questions.length,
}
}),
}
})
}
function getMotd(version, motd) {
if (version) {
if (version.startsWith('2.0.')) {
if (utils.FileExists(oldMotdFile)) {
return utils.ReadFile(oldMotdFile)
}
}
}
return motd
}
function searchInDbs(
question,
subj,
recData,
recievedData,
searchIn,
testUrl?
) {
// searchIn could be [0], [1], ... to search every db in different thread. Put this into a
// forEach(qdbs) to achieve this
return new Promise((resolve) => {
doALongTask({
type: 'work',
data: {
searchIn: searchIn,
testUrl: testUrl,
question: question,
subjName: subj,
questionData: recData,
searchInAllIfNoResult: true,
},
})
.then((taskResult) => {
try {
logger.DebugLog(
`Question result length: ${taskResult.length}`,
'ask',
1
)
logger.DebugLog(taskResult, 'ask', 2)
resolve({
question: question,
result: taskResult.result,
success: true,
})
} catch (err) {
console.error(err)
logger.Log(
'Error while sending ask results',
logger.GetColor('redbg')
)
}
})
.catch((err) => {
logger.Log('Search Data error!', logger.GetColor('redbg'))
console.error(err)
resolve({
mesage: `There was an error processing the question: ${err.message}`,
result: [],
recievedData: JSON.stringify(recievedData),
success: false,
})
})
})
}
function getResult(data) {
const { question, subj, recData, recievedData, questionDbs, testUrl } = data
return new Promise((resolve) => {
const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false)
searchInDbs(question, subj, recData, recievedData, searchIn, testUrl).then(
(res: any) => {
if (res.result.length === 0) {
logger.DebugLog(
`No result while searching specific question db ${testUrl}`,
'ask',
1
)
const searchInMore = getDbIndexesToSearchIn(
testUrl,
questionDbs,
true
).filter((x) => {
return !searchIn.includes(x)
})
searchInDbs(
question,
subj,
recData,
recievedData,
searchInMore,
testUrl
).then((res) => {
resolve(res)
})
} else {
resolve(res)
}
}
)
})
}
function dbExists(location, qdbs: Array<QuestionDb>) {
return qdbs.some((qdb) => {
return qdb.name === location
})
}
function writeAskData(body) {
try {
let towrite = utils.GetDateString() + '\n'
towrite +=
'------------------------------------------------------------------------------\n'
towrite += JSON.stringify(body)
towrite +=
'\n------------------------------------------------------------------------------\n'
utils.AppendToFile(towrite, recievedQuestionFile)
} catch (err) {
logger.Log('Error writing revieved /ask POST data')
console.error(err)
}
}
function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) {
// TODO: clear folder every now and then, check if saved questions exist
if (!subj) {
logger.Log('No subj name to save test question')
return
}
const questionsToSave = {
questions: questions,
subj: subj,
userid: userid,
testUrl: testUrl,
date: new Date(),
}
const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json`
const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-')
const subjPath = `${savedQuestionsDir}/${subject}`
const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}`
utils.CreatePath(subjPath, true)
if (!utils.FileExists(savedSubjQuestionsFilePath)) {
utils.WriteFile('[]', savedSubjQuestionsFilePath)
}
const savedQuestions = utils.ReadJSON(savedSubjQuestionsFilePath)
const testExists = false
// TODO: do this on another thread?
// const testExists = savedQuestions.some((savedQuestion) => {
// const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`)
// return data.questions.some((dQuestion) => {
// return questions.some((question) => {
// const percent = compareQuestionObj(
// createQuestion(question),
// '',
// createQuestion(dQuestion),
// ''
// )
// return percent.avg === 100
// })
// })
// })
if (testExists) {
return
}
savedQuestions.push({
fname: fname,
subj: subj,
userid: userid,
testUrl: testUrl,
date: new Date(),
})
utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath)
utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`)
}
function LoadVersion() {
const scriptContent = utils.ReadFile(userScriptFile)
let temp: any = scriptContent.split('\n').find((x) => {
return x.includes('@version')
})
temp = temp.split(' ')
temp = temp[temp.length - 1]
return temp
}
function LoadMOTD(motdFile) {
return utils.ReadFile(motdFile)
}
function LoadUserSpecificMOTD(userSpecificMotdFile) {
try {
return utils.ReadJSON(userSpecificMotdFile)
} catch (err) {
logger.Log('Couldnt parse user specific motd!', logger.GetColor('redbg'))
console.error(err)
}
}
function LoadTestUsers() {
let testUsers = utils.ReadJSON(testUsersFile)
if (testUsers) {
testUsers = testUsers.userIds
}
return testUsers
}
function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) {
logger.Log(
`No suitable questiondbs found for ${location}, creating a new one...`
)
const newDb: DataFile = {
path: `questionDbs/${location}.json`,
name: location,
shouldSearch: {
location: {
val: location,
},
},
shouldSave: {
location: {
val: location,
},
},
}
utils.WriteFile(
JSON.stringify(
[
...utils.ReadJSON(dbsFile),
newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime
],
null,
2
),
dbsFile
)
// "loading" new db
const loadedNewDb: QuestionDb = {
...newDb,
data: [],
path: publicDir + newDb.path,
index: maxIndex,
}
utils.WriteFile('[]', loadedNewDb.path)
questionDbs.push(loadedNewDb)
msgAllWorker({
newdb: loadedNewDb,
type: 'newdb',
})
return loadedNewDb
}
function setup(data: SubmoduleData): any {
const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data
const publicDir = publicdirs[0]
const motdFile = publicDir + 'motd'
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
const dbsFile = publicDir + 'questionDbs.json'
const savedQuestionsDir = publicDir + 'savedQuestions'
let version = LoadVersion()
let motd = LoadMOTD(motdFile)
let userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
let testUsers: any = LoadTestUsers()
const dataFiles: Array<DataFile> = utils.ReadJSON(dbsFile)
const questionDbs: Array<QuestionDb> = loadJSON(dataFiles, publicDir)
initWorkerPool(questionDbs)
const filesToWatch = [
{
fname: userSpecificMotdFile,
logMsg: 'User Specific Motd updated',
action: () => {
userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
},
},
{
fname: motdFile,
logMsg: 'Motd updated',
action: () => {
motd = LoadMOTD(motdFile)
},
},
{
fname: testUsersFile,
logMsg: 'Test Users file changed',
action: () => {
testUsers = LoadTestUsers()
},
},
{
fname: userScriptFile,
logMsg: 'User script file changed',
action: () => {
version = LoadVersion()
},
},
]
app.get('/getDbs', (req: Request, res: any) => {
logger.LogReq(req)
res.json(
questionDbs.map((qdb) => {
return {
path: qdb.path.replace(publicDir, ''),
name: qdb.name,
locked: qdb.locked,
}
})
)
})
app.get('/allqr.txt', function (req: Request, res: any) {
logger.LogReq(req)
const db: any = req.query.db
let stringifiedData = ''
res.setHeader('content-type', 'text/plain; charset=utf-8')
if (db) {
const requestedDb = questionDbs.find((qdb) => {
return qdb.name === db
})
if (!requestedDb) {
res.end(`No such db ${db}`)
return
}
stringifiedData = '\n' + line
stringifiedData += ` Questions in ${requestedDb.name}: `
stringifiedData += line + '\n'
stringifiedData += dataToString(requestedDb.data)
stringifiedData += '\n' + line + line + '\n'
} else {
stringifiedData = questionDbs
.map((qdb) => {
let result = ''
result += '\n' + line
result += ` Questions in ${qdb.name}: `
result += line + '\n'
result += dataToString(qdb.data)
result += '\n' + line + line + '\n'
return result
})
.join('\n\n')
}
res.end(stringifiedData)
})
app.post('/isAdding', function (req: Request, res: any) {
logger.LogReq(req)
const user: User = req.session.user
const dryRun = testUsers.includes(user.id)
if (!req.body.location) {
logger.Log(
'\tbody.location is missing, client version:' + req.body.version
)
res.json({ msg: 'body.location is missing' })
return
}
const location = req.body.location.split('/')[2]
try {
let maxIndex = -1
const suitedQuestionDbs = questionDbs.filter((qdb) => {
if (maxIndex < qdb.index) {
maxIndex = qdb.index
}
return shouldSaveDataFile(qdb, req.body)
}, [])
if (suitedQuestionDbs.length === 0) {
if (!dbExists(location, questionDbs)) {
suitedQuestionDbs.push(
getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs)
)
} else {
logger.Log(
`Tried to add existing db named ${location}!`,
logger.GetColor('red')
)
}
}
if (suitedQuestionDbs.length === 0) {
res.json({
status: 'fail',
msg: 'No suitable dbs to add questions to',
})
return
}
processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user)
.then((resultArray: Array<Result>) => {
logResult(req.body, resultArray, user.id, dryRun)
const totalNewQuestions = resultArray.reduce((acc, sres) => {
return acc + sres.newQuestions.length
}, 0)
res.json({
success: resultArray.length > 0,
newQuestions: resultArray,
totalNewQuestions: totalNewQuestions,
})
if (totalNewQuestions > 0) {
resultArray.forEach((result) => {
msgAllWorker({
type: 'newQuestions',
data: result,
})
})
}
})
.catch((err) => {
logger.Log(
'Error during processing incoming request',
logger.GetColor('redbg')
)
console.error(err)
res.json({
success: false,
})
})
} catch (err) {
logger.Log(
'Error during getting incoming request processor promises ',
logger.GetColor('redbg')
)
console.error(err)
res.json({
success: false,
})
}
})
app.post('/ask', function (req: Request, res) {
const user: User = req.session.user
if (!req.body.questions) {
res.json({
message: `ask something! { question:'' ,subject:'', location:'' }`,
result: [],
recievedData: JSON.stringify(req.body),
success: false,
})
return
}
const subj: any = req.body.subj || ''
// TODO: test if testUrl is undefined (it shouldnt)
const testUrl = req.body.testUrl
? req.body.testUrl.split('/')[2]
: undefined
writeAskData(req.body)
// every question in a different thread
const resultPromises = req.body.questions.map((question) => {
return getResult({
question: question,
subj: subj,
recData: req.body,
testUrl: testUrl,
questionDbs: questionDbs,
})
})
Promise.all(resultPromises).then((results) => {
const response = results.map((result: any) => {
return {
answers: result.result,
question: result.question,
}
})
res.json(response)
const saveableQuestions = response.reduce((acc, res) => {
const save = res.answers.every((answer) => {
return answer.match < 90
})
if (save) {
acc.push(res.question)
}
return acc
}, [])
if (saveableQuestions.length > 0) {
saveQuestion(
saveableQuestions,
subj,
testUrl,
user.id,
savedQuestionsDir
)
}
})
})
app.get('/ask', function (req: Request, res) {
if (Object.keys(req.query).length === 0) {
logger.DebugLog(`No query params /ask GET`, 'ask', 1)
res.json({
message:
'ask something! ?q=[question]&subj=[subject]&data=[question data]. "subj" is optimal for faster result',
result: [],
recievedData: JSON.stringify(req.query),
success: false,
})
return
}
if (req.query.q && req.query.data) {
const subj: any = req.query.subj || ''
const question = req.query.q
const recData: any = req.query.data
getResult({
question: question,
subj: subj,
recData: recData,
recievedData: req.query,
questionDbs: questionDbs,
}).then((result) => {
res.json(result)
})
} else {
logger.DebugLog(`Invalid question`, 'ask', 1)
res.json({
message: `Invalid question :(`,
result: [],
recievedData: JSON.stringify(req.query),
success: false,
})
}
})
app.get('/datacount', function (req: Request, res: any) {
logger.LogReq(req)
if (req.query.detailed === 'all') {
res.json({
detailed: getDetailedRes(questionDbs),
simple: getSimplreRes(questionDbs),
})
} else if (req.query.detailed) {
res.json(getDetailedRes(questionDbs))
} else {
res.json(getSimplreRes(questionDbs))
}
})
app.get('/infos', function (req: Request, res) {
const user: User = req.session.user
const result: any = {
result: 'success',
uid: user.id,
}
if (req.query.subjinfo) {
result.subjinfo = getSimplreRes(questionDbs)
}
if (req.query.version) {
result.version = version
}
if (req.query.motd) {
result.motd = getMotd(req.query.cversion, motd)
if (userSpecificMotd[user.id]) {
result.userSpecificMotd = {
msg: userSpecificMotd[user.id].msg,
seen: userSpecificMotd[user.id].seen,
}
}
}
res.json(result)
})
app.post('/infos', (req: Request, res) => {
const user: User = req.session.user
if (req.body.userSpecificMotdSeen && !userSpecificMotd[user.id].seen) {
userSpecificMotd[user.id].seen = true
logger.Log(
`User #${user.id}'s user specific motd is now seen:`,
logger.GetColor('bluebg')
)
logger.Log(userSpecificMotd[user.id].msg)
utils.WriteFile(
JSON.stringify(userSpecificMotd, null, 2),
userSpecificMotdFile
)
}
res.json({ msg: 'done' })
})
app.post('/registerscript', function (req: Request, res) {
logger.LogReq(req)
if (!utils.FileExists(registeredScriptsFile)) {
utils.WriteFile('[]', registeredScriptsFile)
}
const ip: any =
req.headers['cf-connecting-ip'] || req.connection.remoteAddress
const ua: any = req.headers['user-agent']
const registeredScripts = utils.ReadJSON(registeredScriptsFile)
const { cid, uid, version, installSource, date } = req.body
const index = registeredScripts.findIndex((registration) => {
return registration.cid === cid
})
if (index === -1) {
const x: any = {
cid: cid,
version: version,
installSource: installSource,
date: date,
ip: ip,
userAgent: ua,
}
if (uid) {
x.uid = uid
x.loginDate = date
}
registeredScripts.push(x)
} else {
const currRegistration = registeredScripts[index]
if (!currRegistration.uid && uid) {
registeredScripts[index] = {
...registeredScripts[index],
uid: uid,
loginDate: date,
}
} else {
logger.DebugLog(
`cid: ${cid}, uid: ${uid} tried to register multiple times`,
'register',
1
)
}
}
utils.WriteFile(
JSON.stringify(registeredScripts, null, 2),
registeredScriptsFile
)
res.json({ msg: 'done' })
})
app.get('/possibleAnswers', (req: Request, res: any) => {
logger.LogReq(req)
const files = utils.ReadDir(savedQuestionsDir)
files.sort(function (a, b) {
return (
fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() -
fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime()
)
})
res.json({
savedQuestionsFileName: savedQuestionsFileName,
subjects: files.map((subj) => {
return {
name: subj,
path: `savedQuestions/${subj}/`,
}
}),
})
})
app.post('/rmPossibleAnswer', (req: Request, res: any) => {
logger.LogReq(req)
const user: User = req.session.user
const subj = req.body.subj
const file = req.body.file
const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}`
const savedQuestions = utils.ReadJSON(savedQuestionsPath)
let path = `${savedQuestionsDir}/${subj}/${file}`
// to prevent deleting ../../../ ... /etc/shadow
while (path.includes('..')) {
path = path.replace(/\.\./g, '.')
}
if (utils.FileExists(path)) {
utils.deleteFile(path)
utils.WriteFile(
JSON.stringify(
savedQuestions.filter((sq) => {
return sq.fname !== file
})
),
savedQuestionsPath
)
logger.Log(
`User #${user.id} deleted '${file}' from subject '${subj}'`,
logger.GetColor('cyan')
)
res.json({
res: 'ok',
})
} else {
logger.Log(
`User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`,
logger.GetColor('red')
)
res.json({
res: 'fail',
})
}
})
app.post('/updateQuestion', (req: Request, res) => {
logger.LogReq(req)
const user: User = req.session.user
const date = utils.GetDateString()
const editType = req.body.type
const selectedDb = req.body.selectedDb
if (!editType || !selectedDb) {
res.json({
status: 'fail',
msg: 'No .editType or .selectedDb !',
})
return
}
const dbIndex = questionDbs.findIndex((qdb) => {
return qdb.name === selectedDb.name
})
const currDb = questionDbs[dbIndex]
if (dbIndex === -1) {
res.json({
status: 'fail',
msg: `No question db named like ${selectedDb.name}!`,
})
return
}
// -----------------
const {
success,
msg,
resultDb,
deletedQuestion,
newVal,
oldVal,
deletedQuestions,
changedQuestions,
} = editDb(currDb, req.body)
if (!success) {
res.json({ success: success, msg: msg })
return
}
if (resultDb) {
questionDbs[dbIndex] = resultDb
}
if (editType === 'delete') {
const { index, subjName } = req.body
logger.Log(
`User #${user.id} deleted a question from '${subjName}'`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`,
dataEditsLog
)
utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog)
}
if (editType === 'edit') {
const { index, subjName } = req.body
logger.Log(
`User #${user.id} edited a question in '${subjName}'`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`,
dataEditsLog
)
utils.AppendToFile(
JSON.stringify(
{
newVal: newVal,
oldVal: oldVal,
},
null,
2
),
dataEditsLog
)
}
if (editType === 'subjEdit') {
const { subjName } = req.body
logger.Log(
`User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
dataEditsLog
)
utils.AppendToFile(
JSON.stringify(
{
deletedQuestions: deletedQuestions,
changedQuestions: changedQuestions,
},
null,
2
),
dataEditsLog
)
}
// ------------------
if (success) {
writeData(currDb.data, currDb.path)
msgAllWorker({
type: 'dbEdit',
data: {
dbIndex: dbIndex,
edits: req.body,
},
})
}
res.json({
success: true,
msg: 'OK',
})
})
return {
dailyAction: () => {
backupData(questionDbs)
ExportDailyDataCount(questionDbs, userDB)
},
load: () => {
backupData(questionDbs)
filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) {
utils.WatchFile(ftw.fname, () => {
logger.Log(ftw.logMsg)
ftw.action()
})
ftw.action()
} else {
logger.Log(
`File ${ftw.fname} does not exists to watch!`,
logger.GetColor('redbg')
)
}
})
},
}
}
export default {
setup: setup,
}