mrfrys-node-server/src/modules/api/submodules/qminingapi.ts

1093 lines
32 KiB
TypeScript

/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
import fs from 'fs'
import { Response } from 'express'
import { fork, ChildProcess } from 'child_process'
import type { Database } from 'better-sqlite3'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import {
User,
DataFile,
Request,
QuestionDb,
SubmoduleData,
Question,
QuestionFromScript,
DbSearchResult,
RegisteredUserEntry,
Submodule,
} from '../../../types/basicTypes'
import {
processIncomingRequest,
logResult,
shouldSaveDataFile,
Result,
backupData,
shouldSearchDataFile,
writeData,
editDb,
RecievedData,
} from '../../../utils/actions'
import {
WorkerResult,
// compareQuestionObj,
} from '../../../utils/classes'
import { doALongTask, msgAllWorker } from '../../../utils/workerPool'
import dbtools from '../../../utils/dbtools'
import {
dataToString,
getSubjNameWithoutYear,
SearchResultQuestion,
} from '../../../utils/qdbUtils'
import { paths } from '../../../utils/files'
import constants from '../../../constants'
import {
isJsonValidAndLogError,
TestUsersSchema,
} from '../../../types/typeSchemas'
interface SavedQuestionData {
fname: string
subj: string
userid: number
testUrl: string
date: string | Date
}
// interface SavedQuestion {
// Questions: Question[]
// subj: string
// userid: number
// testUrl: string
// date: string
// }
const line = '====================================================' // lol
function getSubjCount(qdbs: QuestionDb[]): number {
return qdbs.reduce((acc, qdb) => {
return acc + qdb.data.length
}, 0)
}
function getQuestionCount(qdbs: QuestionDb[]): number {
return qdbs.reduce((acc, qdb) => {
return (
acc +
qdb.data.reduce((qacc, subject) => {
return qacc + subject.Questions.length
}, 0)
)
}, 0)
}
function ExportDailyDataCount(questionDbs: QuestionDb[], userDB: Database) {
logger.Log('Saving daily data count ...')
utils.AppendToFile(
JSON.stringify({
date: utils.GetDateString(),
subjectCount: getSubjCount(questionDbs),
questionCount: getQuestionCount(questionDbs),
questionDbsCount: questionDbs.length,
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
}),
paths.dailyDataCountFile
)
}
function getDbIndexesToSearchIn(
testUrl: string,
questionDbs: Array<QuestionDb>,
trueIfAlways?: boolean
): number[] {
return testUrl
? questionDbs.reduce((acc, qdb, i) => {
if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) {
acc.push(i)
}
return acc
}, [])
: []
}
function getSimplreRes(questionDbs: QuestionDb[]): {
subjects: number
questions: number
} {
return {
subjects: getSubjCount(questionDbs),
questions: getQuestionCount(questionDbs),
}
}
function getDetailedRes(questionDbs: QuestionDb[]) {
return questionDbs.map((qdb) => {
return {
dbName: qdb.name,
subjs: qdb.data.map((subj) => {
return {
name: subj.Name,
count: subj.Questions.length,
}
}),
}
})
}
function searchInDbs(
question: Question,
subj: string,
searchIn: number[],
testUrl?: string
): Promise<DbSearchResult> {
// searchIn could be [0], [1], ... to search every db in different thread. Put this into a
// forEach(qdbs) to achieve this
return new Promise((resolve) => {
doALongTask({
type: 'work',
data: {
searchIn: searchIn,
testUrl: testUrl,
question: question,
subjName: subj,
searchInAllIfNoResult: true,
},
})
.then((taskResult: WorkerResult) => {
try {
logger.DebugLog(taskResult, 'ask', 2)
resolve({
question: question,
result: taskResult.result as SearchResultQuestion[],
success: true,
})
} catch (err) {
console.error(err)
logger.Log(
'Error while sending ask results',
logger.GetColor('redbg')
)
}
})
.catch((err) => {
logger.Log('Search Data error!', logger.GetColor('redbg'))
console.error(err)
resolve({
question: question,
message: `There was an error processing the question: ${err.message}`,
result: [],
success: false,
})
})
})
}
function getResult(data: {
question: Question
subj: string
questionDbs: Array<QuestionDb>
testUrl: string
}): Promise<DbSearchResult> {
const { question, subj, questionDbs, testUrl } = data
return new Promise((resolve) => {
const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false)
searchInDbs(question, subj, searchIn, testUrl).then(
(res: DbSearchResult) => {
if (res.result.length === 0) {
logger.DebugLog(
`No result while searching specific question db ${testUrl}`,
'ask',
1
)
const searchInMore = getDbIndexesToSearchIn(
testUrl,
questionDbs,
true
).filter((x) => {
return !searchIn.includes(x)
})
searchInDbs(question, subj, searchInMore, testUrl).then(
(res) => {
resolve(res)
}
)
} else {
resolve(res)
}
}
)
})
}
function dbExists(location: string, qdbs: Array<QuestionDb>) {
return qdbs.some((qdb) => {
return qdb.name === location
})
}
function writeAskData(body: QuestionFromScript) {
try {
let towrite = utils.GetDateString() + '\n'
towrite +=
'------------------------------------------------------------------------------\n'
towrite += JSON.stringify(body)
towrite +=
'\n------------------------------------------------------------------------------\n'
utils.AppendToFile(towrite, paths.askedQuestionFile)
} catch (err) {
logger.Log('Error writing revieved /ask POST data')
console.error(err)
}
}
function writeIsAddingData(body: RecievedData) {
try {
let towrite = utils.GetDateString() + '\n'
towrite +=
'------------------------------------------------------------------------------\n'
towrite += JSON.stringify(body)
towrite +=
'\n------------------------------------------------------------------------------\n'
utils.AppendToFile(towrite, paths.recievedQuestionFile)
} catch (err) {
logger.Log('Error writing revieved /ask POST data')
console.error(err)
}
}
function saveQuestion(
questions: Question[],
subj: string,
testUrl: string,
userid: number,
savedQuestionsDir: string
) {
// TODO: clear folder every now and then, check if saved questions exist
if (!subj) {
logger.Log('No subj name to save test question')
return
}
const questionsToSave = {
questions: questions,
subj: subj,
userid: userid,
testUrl: testUrl,
date: new Date(),
}
const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json`
const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-')
const subjPath = `${savedQuestionsDir}/${subject}`
const savedSubjQuestionsFilePath = `${subjPath}/${constants.savedQuestionsFileName}`
utils.CreatePath(subjPath, true)
if (!utils.FileExists(savedSubjQuestionsFilePath)) {
utils.WriteFile('[]', savedSubjQuestionsFilePath)
}
const savedQuestions: SavedQuestionData[] = utils.ReadJSON(
savedSubjQuestionsFilePath
)
const testExists = false
// TODO: do this on another thread?
// const testExists = savedQuestions.some((savedQuestion) => {
// const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`)
// return data.questions.some((dQuestion) => {
// return questions.some((question) => {
// const percent = compareQuestionObj(
// createQuestion(question),
// '',
// createQuestion(dQuestion),
// ''
// )
// return percent.avg === 100
// })
// })
// })
if (testExists) {
return
}
savedQuestions.push({
fname: fname,
subj: subj,
userid: userid,
testUrl: testUrl,
date: new Date(),
})
utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath)
utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`)
}
function loadSupportedSites() {
const script = utils.ReadFile(paths.moodleTestUserscriptPath).split('\n')
let i = 0
let stayIn = true
let inHeader = false
let inMatch = false
const sites = []
while (i < script.length && stayIn) {
if (inHeader) {
if (script[i].includes('@match')) {
inMatch = true
}
if (inMatch && !script[i].includes('match')) {
stayIn = false
inMatch = false
}
if (inMatch) {
sites.push(script[i].split(' ').pop())
}
} else {
inHeader = script[i].includes('==UserScript==')
}
i++
}
return sites
}
function LoadMOTD(motdFile: string) {
return utils.ReadFile(motdFile)
}
function LoadTestUsers() {
if (!utils.FileExists(paths.testUsersFile)) {
utils.WriteFile('{}', paths.testUsersFile)
}
const testUsers = utils.ReadJSON<{ userIds: number[] }>(paths.testUsersFile)
if (
!isJsonValidAndLogError(testUsers, TestUsersSchema, paths.testUsersFile)
) {
return []
} else {
return testUsers.userIds
}
}
function getNewQdb(
location: string,
maxIndex: number,
dbsFile: string,
publicDir: string,
questionDbs: QuestionDb[]
) {
logger.Log(
`No suitable questiondbs found for ${location}, creating a new one...`
)
const newDb: DataFile = {
path: `questionDbs/${location}.json`,
name: location,
shouldSearch: {
location: {
val: location,
},
},
shouldSave: {
location: {
val: location,
},
},
}
utils.WriteFile(
JSON.stringify(
[
...utils.ReadJSON(dbsFile),
newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime
],
null,
2
),
dbsFile
)
// "loading" new db
const loadedNewDb: QuestionDb = {
...newDb,
data: [],
path: publicDir + newDb.path,
index: maxIndex,
}
utils.WriteFile('[]', loadedNewDb.path)
questionDbs.push(loadedNewDb)
msgAllWorker({
data: loadedNewDb,
type: 'newdb',
})
return loadedNewDb
}
function setup(data: SubmoduleData): Submodule {
const {
app,
userDB,
/* url */ publicdirs,
moduleSpecificData: { getQuestionDbs, setQuestionDbs, dbsFile },
} = data
const publicDir = publicdirs[0]
const motdFile = publicDir + 'motd'
const savedQuestionsDir = publicDir + 'savedQuestions'
let version = utils.getScriptVersion()
let supportedSites = loadSupportedSites()
let motd = LoadMOTD(motdFile)
let testUsers: number[] = LoadTestUsers()
const filesToWatch = [
{
fname: motdFile,
logMsg: 'Motd updated',
action: () => {
motd = LoadMOTD(motdFile)
},
},
{
fname: paths.testUsersFile,
logMsg: 'Test Users file changed',
action: () => {
testUsers = LoadTestUsers()
},
},
{
fname: paths.moodleTestUserscriptPath,
logMsg: 'User script file changed',
action: () => {
version = utils.getScriptVersion()
supportedSites = loadSupportedSites()
},
},
]
app.get('/getDbs', (req: Request, res: Response) => {
logger.LogReq(req)
res.json(
getQuestionDbs().map((qdb) => {
return {
path: qdb.path.replace(publicDir, ''),
name: qdb.name,
locked: qdb.locked,
}
})
)
})
app.get('/allqr.txt', function (req: Request, res: Response) {
logger.LogReq(req)
const db: string = req.query.db
let stringifiedData = ''
res.setHeader('content-type', 'text/plain; charset=utf-8')
if (db) {
const requestedDb = getQuestionDbs().find((qdb) => {
return qdb.name === db
})
if (!requestedDb) {
res.end(`No such db ${db}`)
return
}
stringifiedData = '\n' + line
stringifiedData += ` Questions in ${requestedDb.name}: `
stringifiedData += line + '\n'
stringifiedData += dataToString(requestedDb.data)
stringifiedData += '\n' + line + line + '\n'
} else {
stringifiedData = getQuestionDbs()
.map((qdb) => {
let result = ''
result += '\n' + line
result += ` Questions in ${qdb.name}: `
result += line + '\n'
result += dataToString(qdb.data)
result += '\n' + line + line + '\n'
return result
})
.join('\n\n')
}
res.end(stringifiedData)
})
app.post('/isAdding', function (req: Request<RecievedData>, res: Response) {
logger.LogReq(req)
const user: User = req.session.user
const dryRun = testUsers.includes(user.id)
if (!req.body.location) {
logger.Log(
'\tbody.location is missing, client version:' + req.body.version
)
res.json({ msg: 'body.location is missing' })
return
}
writeIsAddingData(req.body)
const location = req.body.location.split('/')[2]
try {
let maxIndex = -1
const suitedQuestionDbs = getQuestionDbs().filter((qdb) => {
if (maxIndex < qdb.index) {
maxIndex = qdb.index
}
return shouldSaveDataFile(qdb, req.body)
}, [])
if (suitedQuestionDbs.length === 0) {
if (!dbExists(location, getQuestionDbs())) {
suitedQuestionDbs.push(
getNewQdb(
location,
maxIndex,
dbsFile,
publicDir,
getQuestionDbs()
)
)
} else {
logger.Log(
`Tried to add existing db named ${location}!`,
logger.GetColor('red')
)
}
}
if (suitedQuestionDbs.length === 0) {
res.json({
status: 'fail',
msg: 'No suitable dbs to add questions to',
})
return
}
processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user)
.then((resultArray: Array<Result>) => {
logResult(req.body, resultArray, user.id, dryRun)
const totalNewQuestions = resultArray.reduce(
(acc, sres) => {
return acc + sres.newQuestions.length
},
0
)
res.json({
success: resultArray.length > 0,
newQuestions: resultArray,
totalNewQuestions: totalNewQuestions,
})
if (totalNewQuestions > 0) {
resultArray.forEach((result) => {
msgAllWorker({
// TODO: recognize base64 image
type: 'newQuestions',
data: result,
})
})
}
})
.catch((err) => {
logger.Log(
'Error during processing incoming request',
logger.GetColor('redbg')
)
console.error(err)
res.json({
success: false,
})
})
} catch (err) {
logger.Log(
'Error during getting incoming request processor promises ',
logger.GetColor('redbg')
)
console.error(err)
res.json({
success: false,
})
}
})
app.post('/ask', function (req: Request<QuestionFromScript>, res) {
const user: User = req.session.user
if (!req.body.questions) {
res.json({
message: `ask something! { questions:'' ,subject:'', location:'' }`,
result: [],
success: false,
})
return
}
const subj: string = req.body.subj || ''
// TODO: test if testUrl is undefined (it shouldnt)
const testUrl: string = req.body.testUrl
? req.body.testUrl.split('/')[2]
: undefined
writeAskData(req.body)
// every question in a different thread
const resultPromises: Promise<DbSearchResult>[] =
req.body.questions.map((question: Question) => {
return getResult({
question: question,
subj: subj,
testUrl: testUrl,
questionDbs: getQuestionDbs(),
})
})
Promise.all(resultPromises).then((results: DbSearchResult[]) => {
const response = results.map((result: DbSearchResult) => {
return {
answers: result.result,
question: result.question,
}
})
res.json(response)
const saveableQuestions = response.reduce((acc, res) => {
const save = res.answers.every((answer) => {
return answer.match < 90
})
if (save) {
acc.push(res.question)
}
return acc
}, [])
if (saveableQuestions.length > 0) {
saveQuestion(
saveableQuestions,
subj,
testUrl,
user.id,
savedQuestionsDir
)
}
})
})
app.get('/supportedSites', function (req: Request, res: Response) {
logger.LogReq(req)
res.json(supportedSites)
})
app.get('/datacount', function (req: Request, res: Response) {
logger.LogReq(req)
if (req.query.detailed === 'all') {
res.json({
detailed: getDetailedRes(getQuestionDbs()),
simple: getSimplreRes(getQuestionDbs()),
})
} else if (req.query.detailed) {
res.json(getDetailedRes(getQuestionDbs()))
} else {
res.json(getSimplreRes(getQuestionDbs()))
}
})
app.get('/infos', function (req: Request, res) {
const user: User = req.session.user
const result: {
result: string
uid: number
version?: string
subjinfo?: {
subjects: number
questions: number
}
motd?: string
} = {
result: 'success',
uid: user.id,
}
if (req.query.subjinfo) {
result.subjinfo = getSimplreRes(getQuestionDbs())
}
if (req.query.version) {
result.version = version
}
if (req.query.motd) {
result.motd = motd
}
res.json(result)
})
app.post('/registerscript', function (req: Request, res) {
logger.LogReq(req)
if (!utils.FileExists(paths.registeredScriptsFile)) {
utils.WriteFile('[]', paths.registeredScriptsFile)
}
const ua: string = req.headers['user-agent']
const registeredScripts: RegisteredUserEntry[] = utils.ReadJSON(
paths.registeredScriptsFile
)
const { cid, uid, version, installSource, date } = req.body
const index = registeredScripts.findIndex((registration) => {
return registration.cid === cid
})
if (index === -1) {
const x: RegisteredUserEntry = {
cid: cid,
version: version,
installSource: installSource,
date: date,
userAgent: ua,
}
if (uid) {
x.uid = uid
x.loginDate = date
}
registeredScripts.push(x)
} else {
const currRegistration = registeredScripts[index]
if (!currRegistration.uid && uid) {
registeredScripts[index] = {
...registeredScripts[index],
uid: uid,
loginDate: date,
}
} else {
logger.DebugLog(
`cid: ${cid}, uid: ${uid} tried to register multiple times`,
'register',
1
)
}
}
utils.WriteFile(
JSON.stringify(registeredScripts, null, 2),
paths.registeredScriptsFile
)
res.json({ msg: 'done' })
})
app.get('/possibleAnswers', (req: Request, res: Response) => {
logger.LogReq(req)
const files = utils.ReadDir(savedQuestionsDir)
files.sort(function (a, b) {
return (
fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() -
fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime()
)
})
res.json({
savedQuestionsFileName: constants.savedQuestionsFileName,
subjects: files.map((subj) => {
return {
name: subj,
path: `savedQuestions/${subj}/`,
}
}),
})
})
app.post('/rmPossibleAnswer', (req: Request, res: Response) => {
logger.LogReq(req)
const user: User = req.session.user
const subj = req.body.subj
const file = req.body.file
const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${constants.savedQuestionsFileName}`
const savedQuestions: SavedQuestionData[] =
utils.ReadJSON(savedQuestionsPath)
let path = `${savedQuestionsDir}/${subj}/${file}`
while (path.includes('..')) {
path = path.replace(/\.\./g, '.')
}
if (utils.FileExists(path)) {
utils.deleteFile(path)
utils.WriteFile(
JSON.stringify(
savedQuestions.filter((sq) => {
return sq.fname !== file
})
),
savedQuestionsPath
)
logger.Log(
`User #${user.id} deleted '${file}' from subject '${subj}'`,
logger.GetColor('cyan')
)
res.json({
res: 'ok',
})
} else {
logger.Log(
`User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`,
logger.GetColor('red')
)
res.json({
res: 'fail',
})
}
})
app.post('/updateQuestion', (req: Request, res) => {
logger.LogReq(req)
const user: User = req.session.user
const date = utils.GetDateString()
const editType = req.body.type
const selectedDb = req.body.selectedDb
if (!editType || !selectedDb) {
res.json({
status: 'fail',
msg: 'No .editType or .selectedDb !',
})
return
}
const dbIndex = getQuestionDbs().findIndex((qdb) => {
return qdb.name === selectedDb.name
})
const currDb = getQuestionDbs()[dbIndex]
if (dbIndex === -1) {
res.json({
status: 'fail',
msg: `No question db named like ${selectedDb.name}!`,
})
return
}
// -----------------
const {
success,
msg,
resultDb,
deletedQuestion,
newVal,
oldVal,
deletedQuestions,
changedQuestions,
} = editDb(currDb, req.body)
if (!success) {
res.json({ success: success, msg: msg })
return
}
if (resultDb) {
setQuestionDbs(
getQuestionDbs().map((qdb, i) => {
if (i === dbIndex) return resultDb
return qdb
})
)
}
if (editType === 'delete') {
const { index, subjName } = req.body
logger.Log(
`User #${user.id} deleted a question from '${subjName}'`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`,
paths.dataEditsLog
)
utils.AppendToFile(
JSON.stringify(deletedQuestion, null, 2),
paths.dataEditsLog
)
}
if (editType === 'edit') {
const { index, subjName } = req.body
logger.Log(
`User #${user.id} edited a question in '${subjName}'`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`,
paths.dataEditsLog
)
utils.AppendToFile(
JSON.stringify(
{
newVal: newVal,
oldVal: oldVal,
},
null,
2
),
paths.dataEditsLog
)
}
if (editType === 'subjEdit') {
const { subjName } = req.body
logger.Log(
`User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
paths.dataEditsLog
)
utils.AppendToFile(
JSON.stringify(
{
deletedQuestions: deletedQuestions,
changedQuestions: changedQuestions,
},
null,
2
),
paths.dataEditsLog
)
}
// ------------------
if (success) {
writeData(currDb.data, currDb.path)
msgAllWorker({
type: 'dbEdit',
data: {
dbIndex: dbIndex,
edits: req.body,
},
})
}
res.json({
success: true,
msg: 'OK',
})
})
let questionCleaner: ChildProcess = null
app.get('/clearQuestions', (req: Request, res) => {
// TODO: dont allow multiple instances
// TODO: get status of it cleaning
logger.LogReq(req)
res.json({
error: 'Not implemented / tested!',
})
return
const user: User = req.session.user
const status: string = req.query.status
if (status) {
if (!questionCleaner) {
res.json({
msg: 'question cleaner not running',
success: false,
})
return
}
questionCleaner.once('message', function (response) {
res.json({
msg: response,
success: true,
})
})
questionCleaner.send({ data: 'asd' })
return
}
if (questionCleaner) {
res.json({
msg: 'question cleaner already running',
success: false,
})
return
}
questionCleaner = fork(
`${process.cwd()}/src/standaloneUtils/rmDuplicates.js`,
['-s', `${process.cwd()}/${getQuestionDbs()[0].path}`] // TODO: this only cleans index
// #0?
)
questionCleaner.on('exit', function (code: number) {
console.log('EXIT', code)
questionCleaner = null
})
res.json({
user: user,
success: true,
msg: 'OK',
})
})
return {
dailyAction: () => {
backupData(getQuestionDbs())
ExportDailyDataCount(getQuestionDbs(), userDB)
},
load: () => {
backupData(getQuestionDbs())
filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) {
utils.WatchFile(ftw.fname, () => {
logger.Log(ftw.logMsg)
ftw.action()
})
ftw.action()
} else {
logger.Log(
`File ${ftw.fname} does not exists to watch!`,
logger.GetColor('redbg')
)
}
})
},
}
}
export default {
setup: setup,
}