mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
added / fixed some types
This commit is contained in:
mrfry
parent
5f12284bb8
commit
bc5c293539
41 changed files with 4378 additions and 8304 deletions
|
|
@ -1,5 +1,7 @@
|
|||
import fs from 'fs'
|
||||
import { fork } from 'child_process'
|
||||
import { Response } from 'express'
|
||||
import { fork, ChildProcess } from 'child_process'
|
||||
import type { Database } from 'better-sqlite3'
|
||||
|
||||
import logger from '../../../utils/logger'
|
||||
import utils from '../../../utils/utils'
|
||||
|
|
@ -9,6 +11,11 @@ import {
|
|||
Request,
|
||||
QuestionDb,
|
||||
SubmoduleData,
|
||||
Question,
|
||||
QuestionFromScript,
|
||||
DbSearchResult,
|
||||
RegisteredUserEntry,
|
||||
Submodule,
|
||||
} from '../../../types/basicTypes'
|
||||
import {
|
||||
processIncomingRequest,
|
||||
|
|
@ -24,6 +31,7 @@ import {
|
|||
import {
|
||||
dataToString,
|
||||
getSubjNameWithoutYear,
|
||||
WorkerResult,
|
||||
// compareQuestionObj,
|
||||
} from '../../../utils/classes'
|
||||
import {
|
||||
|
|
@ -33,6 +41,22 @@ import {
|
|||
} from '../../../utils/workerPool'
|
||||
import dbtools from '../../../utils/dbtools'
|
||||
|
||||
interface SavedQuestionData {
|
||||
fname: string
|
||||
subj: string
|
||||
userid: number
|
||||
testUrl: string
|
||||
date: string | Date
|
||||
}
|
||||
|
||||
// interface SavedQuestion {
|
||||
// Questions: Question[]
|
||||
// subj: string
|
||||
// userid: number
|
||||
// testUrl: string
|
||||
// date: string
|
||||
// }
|
||||
|
||||
const line = '====================================================' // lol
|
||||
const registeredScriptsFile = 'stats/registeredScripts.json'
|
||||
const testUsersFile = 'data/testUsers.json'
|
||||
|
|
@ -43,13 +67,13 @@ const oldMotdFile = 'publicDirs/qminingPublic/oldMotd'
|
|||
const dailyDataCountFile = 'stats/dailyDataCount'
|
||||
const dataEditsLog = 'stats/dataEdits'
|
||||
|
||||
function getSubjCount(qdbs) {
|
||||
function getSubjCount(qdbs: QuestionDb[]): number {
|
||||
return qdbs.reduce((acc, qdb) => {
|
||||
return acc + qdb.data.length
|
||||
}, 0)
|
||||
}
|
||||
|
||||
function getQuestionCount(qdbs) {
|
||||
function getQuestionCount(qdbs: QuestionDb[]): number {
|
||||
return qdbs.reduce((acc, qdb) => {
|
||||
return (
|
||||
acc +
|
||||
|
|
@ -60,7 +84,7 @@ function getQuestionCount(qdbs) {
|
|||
}, 0)
|
||||
}
|
||||
|
||||
function ExportDailyDataCount(questionDbs, userDB) {
|
||||
function ExportDailyDataCount(questionDbs: QuestionDb[], userDB: Database) {
|
||||
logger.Log('Saving daily data count ...')
|
||||
utils.AppendToFile(
|
||||
JSON.stringify({
|
||||
|
|
@ -76,9 +100,9 @@ function ExportDailyDataCount(questionDbs, userDB) {
|
|||
|
||||
function getDbIndexesToSearchIn(
|
||||
testUrl: string,
|
||||
questionDbs,
|
||||
questionDbs: Array<QuestionDb>,
|
||||
trueIfAlways?: boolean
|
||||
) {
|
||||
): number[] {
|
||||
return testUrl
|
||||
? questionDbs.reduce((acc, qdb, i) => {
|
||||
if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) {
|
||||
|
|
@ -89,14 +113,17 @@ function getDbIndexesToSearchIn(
|
|||
: []
|
||||
}
|
||||
|
||||
function getSimplreRes(questionDbs) {
|
||||
function getSimplreRes(questionDbs: QuestionDb[]): {
|
||||
subjects: number
|
||||
questions: number
|
||||
} {
|
||||
return {
|
||||
subjects: getSubjCount(questionDbs),
|
||||
questions: getQuestionCount(questionDbs),
|
||||
}
|
||||
}
|
||||
|
||||
function getDetailedRes(questionDbs) {
|
||||
function getDetailedRes(questionDbs: QuestionDb[]) {
|
||||
return questionDbs.map((qdb) => {
|
||||
return {
|
||||
dbName: qdb.name,
|
||||
|
|
@ -110,7 +137,7 @@ function getDetailedRes(questionDbs) {
|
|||
})
|
||||
}
|
||||
|
||||
function getMotd(version, motd) {
|
||||
function getMotd(version: any, motd: string) {
|
||||
if (version) {
|
||||
if (version.startsWith('2.0.')) {
|
||||
if (utils.FileExists(oldMotdFile)) {
|
||||
|
|
@ -122,13 +149,11 @@ function getMotd(version, motd) {
|
|||
}
|
||||
|
||||
function searchInDbs(
|
||||
question,
|
||||
subj,
|
||||
recData,
|
||||
recievedData,
|
||||
searchIn,
|
||||
testUrl?
|
||||
) {
|
||||
question: Question,
|
||||
subj: string,
|
||||
searchIn: number[],
|
||||
testUrl?: string
|
||||
): Promise<DbSearchResult> {
|
||||
// searchIn could be [0], [1], ... to search every db in different thread. Put this into a
|
||||
// forEach(qdbs) to achieve this
|
||||
return new Promise((resolve) => {
|
||||
|
|
@ -139,17 +164,11 @@ function searchInDbs(
|
|||
testUrl: testUrl,
|
||||
question: question,
|
||||
subjName: subj,
|
||||
questionData: recData,
|
||||
searchInAllIfNoResult: true,
|
||||
},
|
||||
})
|
||||
.then((taskResult) => {
|
||||
.then((taskResult: WorkerResult) => {
|
||||
try {
|
||||
logger.DebugLog(
|
||||
`Question result length: ${taskResult.length}`,
|
||||
'ask',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(taskResult, 'ask', 2)
|
||||
resolve({
|
||||
question: question,
|
||||
|
|
@ -168,22 +187,27 @@ function searchInDbs(
|
|||
logger.Log('Search Data error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
resolve({
|
||||
mesage: `There was an error processing the question: ${err.message}`,
|
||||
question: question,
|
||||
message: `There was an error processing the question: ${err.message}`,
|
||||
result: [],
|
||||
recievedData: JSON.stringify(recievedData),
|
||||
success: false,
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function getResult(data) {
|
||||
const { question, subj, recData, recievedData, questionDbs, testUrl } = data
|
||||
function getResult(data: {
|
||||
question: Question
|
||||
subj: string
|
||||
questionDbs: Array<QuestionDb>
|
||||
testUrl: string
|
||||
}): Promise<DbSearchResult> {
|
||||
const { question, subj, questionDbs, testUrl } = data
|
||||
return new Promise((resolve) => {
|
||||
const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false)
|
||||
|
||||
searchInDbs(question, subj, recData, recievedData, searchIn, testUrl).then(
|
||||
(res: any) => {
|
||||
searchInDbs(question, subj, searchIn, testUrl).then(
|
||||
(res: DbSearchResult) => {
|
||||
if (res.result.length === 0) {
|
||||
logger.DebugLog(
|
||||
`No result while searching specific question db ${testUrl}`,
|
||||
|
|
@ -197,14 +221,7 @@ function getResult(data) {
|
|||
).filter((x) => {
|
||||
return !searchIn.includes(x)
|
||||
})
|
||||
searchInDbs(
|
||||
question,
|
||||
subj,
|
||||
recData,
|
||||
recievedData,
|
||||
searchInMore,
|
||||
testUrl
|
||||
).then((res) => {
|
||||
searchInDbs(question, subj, searchInMore, testUrl).then((res) => {
|
||||
resolve(res)
|
||||
})
|
||||
} else {
|
||||
|
|
@ -215,13 +232,13 @@ function getResult(data) {
|
|||
})
|
||||
}
|
||||
|
||||
function dbExists(location, qdbs: Array<QuestionDb>) {
|
||||
function dbExists(location: string, qdbs: Array<QuestionDb>) {
|
||||
return qdbs.some((qdb) => {
|
||||
return qdb.name === location
|
||||
})
|
||||
}
|
||||
|
||||
function writeAskData(body) {
|
||||
function writeAskData(body: QuestionFromScript) {
|
||||
try {
|
||||
let towrite = utils.GetDateString() + '\n'
|
||||
towrite +=
|
||||
|
|
@ -236,7 +253,13 @@ function writeAskData(body) {
|
|||
}
|
||||
}
|
||||
|
||||
function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) {
|
||||
function saveQuestion(
|
||||
questions: Question[],
|
||||
subj: string,
|
||||
testUrl: string,
|
||||
userid: number,
|
||||
savedQuestionsDir: string
|
||||
) {
|
||||
// TODO: clear folder every now and then, check if saved questions exist
|
||||
if (!subj) {
|
||||
logger.Log('No subj name to save test question')
|
||||
|
|
@ -259,7 +282,9 @@ function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) {
|
|||
utils.WriteFile('[]', savedSubjQuestionsFilePath)
|
||||
}
|
||||
|
||||
const savedQuestions = utils.ReadJSON(savedSubjQuestionsFilePath)
|
||||
const savedQuestions: SavedQuestionData[] = utils.ReadJSON(
|
||||
savedSubjQuestionsFilePath
|
||||
)
|
||||
|
||||
const testExists = false
|
||||
// TODO: do this on another thread?
|
||||
|
|
@ -327,7 +352,7 @@ function loadSupportedSites() {
|
|||
function LoadVersion() {
|
||||
const scriptContent = utils.ReadFile(userScriptFile)
|
||||
|
||||
let temp: any = scriptContent.split('\n').find((x) => {
|
||||
let temp: string | string[] = scriptContent.split('\n').find((x) => {
|
||||
return x.includes('@version')
|
||||
})
|
||||
temp = temp.split(' ')
|
||||
|
|
@ -336,19 +361,10 @@ function LoadVersion() {
|
|||
return temp
|
||||
}
|
||||
|
||||
function LoadMOTD(motdFile) {
|
||||
function LoadMOTD(motdFile: string) {
|
||||
return utils.ReadFile(motdFile)
|
||||
}
|
||||
|
||||
function LoadUserSpecificMOTD(userSpecificMotdFile) {
|
||||
try {
|
||||
return utils.ReadJSON(userSpecificMotdFile)
|
||||
} catch (err) {
|
||||
logger.Log('Couldnt parse user specific motd!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
function LoadTestUsers() {
|
||||
let testUsers = utils.ReadJSON(testUsersFile)
|
||||
if (testUsers) {
|
||||
|
|
@ -357,7 +373,13 @@ function LoadTestUsers() {
|
|||
return testUsers
|
||||
}
|
||||
|
||||
function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) {
|
||||
function getNewQdb(
|
||||
location: string,
|
||||
maxIndex: number,
|
||||
dbsFile: string,
|
||||
publicDir: string,
|
||||
questionDbs: QuestionDb[]
|
||||
) {
|
||||
logger.Log(
|
||||
`No suitable questiondbs found for ${location}, creating a new one...`
|
||||
)
|
||||
|
|
@ -399,40 +421,31 @@ function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) {
|
|||
|
||||
questionDbs.push(loadedNewDb)
|
||||
msgAllWorker({
|
||||
newdb: loadedNewDb,
|
||||
data: loadedNewDb,
|
||||
type: 'newdb',
|
||||
})
|
||||
|
||||
return loadedNewDb
|
||||
}
|
||||
|
||||
function setup(data: SubmoduleData): any {
|
||||
function setup(data: SubmoduleData): Submodule {
|
||||
const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data
|
||||
|
||||
const publicDir = publicdirs[0]
|
||||
const motdFile = publicDir + 'motd'
|
||||
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
|
||||
const dbsFile = publicDir + 'questionDbs.json'
|
||||
const savedQuestionsDir = publicDir + 'savedQuestions'
|
||||
|
||||
let version = LoadVersion()
|
||||
let supportedSites = loadSupportedSites()
|
||||
let motd = LoadMOTD(motdFile)
|
||||
let userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
|
||||
let testUsers: any = LoadTestUsers()
|
||||
let testUsers: number[] = LoadTestUsers()
|
||||
|
||||
const dataFiles: Array<DataFile> = utils.ReadJSON(dbsFile)
|
||||
const questionDbs: Array<QuestionDb> = loadJSON(dataFiles, publicDir)
|
||||
initWorkerPool(questionDbs)
|
||||
|
||||
const filesToWatch = [
|
||||
{
|
||||
fname: userSpecificMotdFile,
|
||||
logMsg: 'User Specific Motd updated',
|
||||
action: () => {
|
||||
userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
|
||||
},
|
||||
},
|
||||
{
|
||||
fname: motdFile,
|
||||
logMsg: 'Motd updated',
|
||||
|
|
@ -457,7 +470,7 @@ function setup(data: SubmoduleData): any {
|
|||
},
|
||||
]
|
||||
|
||||
app.get('/getDbs', (req: Request, res: any) => {
|
||||
app.get('/getDbs', (req: Request, res: Response) => {
|
||||
logger.LogReq(req)
|
||||
res.json(
|
||||
questionDbs.map((qdb) => {
|
||||
|
|
@ -470,7 +483,7 @@ function setup(data: SubmoduleData): any {
|
|||
)
|
||||
})
|
||||
|
||||
app.get('/allqr.txt', function (req: Request, res: any) {
|
||||
app.get('/allqr.txt', function (req: Request, res: Response) {
|
||||
logger.LogReq(req)
|
||||
const db: any = req.query.db
|
||||
let stringifiedData = ''
|
||||
|
|
@ -508,7 +521,7 @@ function setup(data: SubmoduleData): any {
|
|||
res.end(stringifiedData)
|
||||
})
|
||||
|
||||
app.post('/isAdding', function (req: Request, res: any) {
|
||||
app.post('/isAdding', function (req: Request, res: Response) {
|
||||
logger.LogReq(req)
|
||||
const user: User = req.session.user
|
||||
const dryRun = testUsers.includes(user.id)
|
||||
|
|
@ -596,39 +609,39 @@ function setup(data: SubmoduleData): any {
|
|||
}
|
||||
})
|
||||
|
||||
app.post('/ask', function (req: Request, res) {
|
||||
app.post('/ask', function (req: Request<QuestionFromScript>, res) {
|
||||
const user: User = req.session.user
|
||||
|
||||
if (!req.body.questions) {
|
||||
res.json({
|
||||
message: `ask something! { questions:'' ,subject:'', location:'' }`,
|
||||
result: [],
|
||||
recievedData: JSON.stringify(req.body),
|
||||
success: false,
|
||||
})
|
||||
return
|
||||
}
|
||||
const subj: any = req.body.subj || ''
|
||||
const subj: string = req.body.subj || ''
|
||||
// TODO: test if testUrl is undefined (it shouldnt)
|
||||
const testUrl = req.body.testUrl
|
||||
const testUrl: string = req.body.testUrl
|
||||
? req.body.testUrl.split('/')[2]
|
||||
: undefined
|
||||
|
||||
writeAskData(req.body)
|
||||
|
||||
// every question in a different thread
|
||||
const resultPromises = req.body.questions.map((question) => {
|
||||
return getResult({
|
||||
question: question,
|
||||
subj: subj,
|
||||
recData: req.body,
|
||||
testUrl: testUrl,
|
||||
questionDbs: questionDbs,
|
||||
})
|
||||
})
|
||||
const resultPromises: Promise<DbSearchResult>[] = req.body.questions.map(
|
||||
(question: Question) => {
|
||||
return getResult({
|
||||
question: question,
|
||||
subj: subj,
|
||||
testUrl: testUrl,
|
||||
questionDbs: questionDbs,
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
Promise.all(resultPromises).then((results) => {
|
||||
const response = results.map((result: any) => {
|
||||
Promise.all(resultPromises).then((results: DbSearchResult[]) => {
|
||||
const response = results.map((result: DbSearchResult) => {
|
||||
return {
|
||||
answers: result.result,
|
||||
question: result.question,
|
||||
|
|
@ -659,50 +672,13 @@ function setup(data: SubmoduleData): any {
|
|||
})
|
||||
})
|
||||
|
||||
app.get('/ask', function (req: Request, res) {
|
||||
if (Object.keys(req.query).length === 0) {
|
||||
logger.DebugLog(`No query params /ask GET`, 'ask', 1)
|
||||
res.json({
|
||||
message:
|
||||
'ask something! ?q=[question]&subj=[subject]&data=[question data]. "subj" is optimal for faster result',
|
||||
result: [],
|
||||
recievedData: JSON.stringify(req.query),
|
||||
success: false,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (req.query.q && req.query.data) {
|
||||
const subj: any = req.query.subj || ''
|
||||
const question = req.query.q
|
||||
const recData: any = req.query.data
|
||||
getResult({
|
||||
question: question,
|
||||
subj: subj,
|
||||
recData: recData,
|
||||
recievedData: req.query,
|
||||
questionDbs: questionDbs,
|
||||
}).then((result) => {
|
||||
res.json(result)
|
||||
})
|
||||
} else {
|
||||
logger.DebugLog(`Invalid question`, 'ask', 1)
|
||||
res.json({
|
||||
message: `Invalid question :(`,
|
||||
result: [],
|
||||
recievedData: JSON.stringify(req.query),
|
||||
success: false,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
app.get('/supportedSites', function (req: Request, res: any) {
|
||||
app.get('/supportedSites', function (req: Request, res: Response) {
|
||||
logger.LogReq(req)
|
||||
|
||||
res.json(supportedSites)
|
||||
})
|
||||
|
||||
app.get('/datacount', function (req: Request, res: any) {
|
||||
app.get('/datacount', function (req: Request, res: Response) {
|
||||
logger.LogReq(req)
|
||||
if (req.query.detailed === 'all') {
|
||||
res.json({
|
||||
|
|
@ -719,7 +695,16 @@ function setup(data: SubmoduleData): any {
|
|||
app.get('/infos', function (req: Request, res) {
|
||||
const user: User = req.session.user
|
||||
|
||||
const result: any = {
|
||||
const result: {
|
||||
result: string
|
||||
uid: number
|
||||
version?: string
|
||||
subjinfo?: {
|
||||
subjects: number
|
||||
questions: number
|
||||
}
|
||||
motd?: string
|
||||
} = {
|
||||
result: 'success',
|
||||
uid: user.id,
|
||||
}
|
||||
|
|
@ -732,34 +717,9 @@ function setup(data: SubmoduleData): any {
|
|||
}
|
||||
if (req.query.motd) {
|
||||
result.motd = getMotd(req.query.cversion, motd)
|
||||
if (userSpecificMotd[user.id]) {
|
||||
result.userSpecificMotd = {
|
||||
msg: userSpecificMotd[user.id].msg,
|
||||
seen: userSpecificMotd[user.id].seen,
|
||||
}
|
||||
}
|
||||
}
|
||||
res.json(result)
|
||||
})
|
||||
app.post('/infos', (req: Request, res) => {
|
||||
const user: User = req.session.user
|
||||
|
||||
if (req.body.userSpecificMotdSeen && !userSpecificMotd[user.id].seen) {
|
||||
userSpecificMotd[user.id].seen = true
|
||||
|
||||
logger.Log(
|
||||
`User #${user.id}'s user specific motd is now seen:`,
|
||||
logger.GetColor('bluebg')
|
||||
)
|
||||
logger.Log(userSpecificMotd[user.id].msg)
|
||||
utils.WriteFile(
|
||||
JSON.stringify(userSpecificMotd, null, 2),
|
||||
userSpecificMotdFile
|
||||
)
|
||||
}
|
||||
|
||||
res.json({ msg: 'done' })
|
||||
})
|
||||
|
||||
app.post('/registerscript', function (req: Request, res) {
|
||||
logger.LogReq(req)
|
||||
|
|
@ -769,7 +729,9 @@ function setup(data: SubmoduleData): any {
|
|||
}
|
||||
|
||||
const ua: any = req.headers['user-agent']
|
||||
const registeredScripts = utils.ReadJSON(registeredScriptsFile)
|
||||
const registeredScripts: RegisteredUserEntry[] = utils.ReadJSON(
|
||||
registeredScriptsFile
|
||||
)
|
||||
const { cid, uid, version, installSource, date } = req.body
|
||||
|
||||
const index = registeredScripts.findIndex((registration) => {
|
||||
|
|
@ -816,7 +778,7 @@ function setup(data: SubmoduleData): any {
|
|||
res.json({ msg: 'done' })
|
||||
})
|
||||
|
||||
app.get('/possibleAnswers', (req: Request, res: any) => {
|
||||
app.get('/possibleAnswers', (req: Request, res: Response) => {
|
||||
logger.LogReq(req)
|
||||
const files = utils.ReadDir(savedQuestionsDir)
|
||||
|
||||
|
|
@ -838,16 +800,16 @@ function setup(data: SubmoduleData): any {
|
|||
})
|
||||
})
|
||||
|
||||
app.post('/rmPossibleAnswer', (req: Request, res: any) => {
|
||||
app.post('/rmPossibleAnswer', (req: Request, res: Response) => {
|
||||
logger.LogReq(req)
|
||||
const user: User = req.session.user
|
||||
|
||||
const subj = req.body.subj
|
||||
const file = req.body.file
|
||||
const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}`
|
||||
const savedQuestions = utils.ReadJSON(savedQuestionsPath)
|
||||
const savedQuestions: SavedQuestionData[] =
|
||||
utils.ReadJSON(savedQuestionsPath)
|
||||
let path = `${savedQuestionsDir}/${subj}/${file}`
|
||||
// to prevent deleting ../../../ ... /etc/shadow
|
||||
while (path.includes('..')) {
|
||||
path = path.replace(/\.\./g, '.')
|
||||
}
|
||||
|
|
@ -1007,7 +969,7 @@ function setup(data: SubmoduleData): any {
|
|||
})
|
||||
})
|
||||
|
||||
let questionCleaner = null
|
||||
let questionCleaner: ChildProcess = null
|
||||
app.get('/clearQuestions', (req: Request, res) => {
|
||||
// TODO: dont allow multiple instances
|
||||
// TODO: get status of it cleaning
|
||||
|
|
@ -1046,7 +1008,7 @@ function setup(data: SubmoduleData): any {
|
|||
`${process.cwd()}/src/standaloneUtils/rmDuplicates.js`,
|
||||
['-s', `${process.cwd()}/${questionDbs[0].path}`]
|
||||
)
|
||||
questionCleaner.on('exit', function (code) {
|
||||
questionCleaner.on('exit', function (code: number) {
|
||||
console.log('EXIT', code)
|
||||
questionCleaner = null
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue