mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Handling multiple data files
This commit is contained in:
mrfry
parent
d9b424cbd1
commit
728931d56e
4 changed files with 172 additions and 112 deletions
|
|
@ -29,13 +29,19 @@ import fs from 'fs'
|
|||
|
||||
import logger from '../../utils/logger'
|
||||
import utils from '../../utils/utils'
|
||||
import actions from '../../utils/actions'
|
||||
import {
|
||||
processIncomingRequest,
|
||||
logResult,
|
||||
backupData,
|
||||
loadJSON,
|
||||
RecievedData,
|
||||
} from '../../utils/actions'
|
||||
import dbtools from '../../utils/dbtools'
|
||||
import auth from '../../middlewares/auth.middleware'
|
||||
import { dataToString, searchData } from '../../utils/classes'
|
||||
import { dataToString, searchDatas } from '../../utils/classes'
|
||||
|
||||
import { SetupData } from '../../server'
|
||||
import { ModuleType, User } from '../../types/basicTypes'
|
||||
import { ModuleType, User, DataFile } from '../../types/basicTypes'
|
||||
|
||||
// files
|
||||
const msgFile = 'stats/msgs'
|
||||
|
|
@ -62,11 +68,6 @@ let userDB
|
|||
let url // eslint-disable-line
|
||||
let publicdirs = []
|
||||
|
||||
export interface DataFile {
|
||||
path: string
|
||||
name: string
|
||||
}
|
||||
|
||||
function GetApp(): ModuleType {
|
||||
const app = express()
|
||||
|
||||
|
|
@ -79,10 +80,21 @@ function GetApp(): ModuleType {
|
|||
const recivedFiles = publicDir + 'recivedfiles'
|
||||
const uloadFiles = publicDir + 'f'
|
||||
const dataFiles: Array<DataFile> = [
|
||||
{ path: `${publicDir}oldData.json`, name: 'oldData' },
|
||||
{ path: `${publicDir}data.json`, name: 'newData' },
|
||||
{
|
||||
path: `${publicDir}oldData.json`,
|
||||
name: 'oldData',
|
||||
shouldSave: (recData: RecievedData): boolean => {
|
||||
return recData.version.includes('2.0.')
|
||||
},
|
||||
},
|
||||
{
|
||||
path: `${publicDir}data.json`,
|
||||
name: 'newData',
|
||||
shouldSave: (recData: RecievedData): boolean => {
|
||||
return recData.version.includes('2.1.')
|
||||
},
|
||||
},
|
||||
]
|
||||
const data: any = {} // TODO: remove
|
||||
const motdFile = publicDir + 'motd'
|
||||
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
|
||||
const versionFile = publicDir + 'version'
|
||||
|
|
@ -127,7 +139,7 @@ function GetApp(): ModuleType {
|
|||
})
|
||||
)
|
||||
|
||||
const questionDbs = actions.LoadJSON(dataFiles)
|
||||
const questionDbs = loadJSON(dataFiles)
|
||||
let version = ''
|
||||
let motd = ''
|
||||
let userSpecificMotd = {}
|
||||
|
|
@ -196,13 +208,16 @@ function GetApp(): ModuleType {
|
|||
}
|
||||
|
||||
if (write) {
|
||||
utils.WriteFile(JSON.stringify(userSpecificMotd), userSpecificMotdFile)
|
||||
utils.WriteFile(
|
||||
JSON.stringify(userSpecificMotd, null, 2),
|
||||
userSpecificMotdFile
|
||||
)
|
||||
}
|
||||
return shouldSee
|
||||
}
|
||||
|
||||
function Load() {
|
||||
actions.backupData(questionDbs)
|
||||
backupData(questionDbs)
|
||||
|
||||
utils.WatchFile(userSpecificMotdFile, () => {
|
||||
logger.Log(`User Specific Motd updated`, logger.GetColor('green'))
|
||||
|
|
@ -939,16 +954,9 @@ function GetApp(): ModuleType {
|
|||
|
||||
const dryRun = testUsers.includes(user.id)
|
||||
|
||||
// automatically saves to dataFile every n write
|
||||
// FIXME: req.body.datatoadd is for backwards compatibility, remove this sometime in the future
|
||||
actions
|
||||
.ProcessIncomingRequest(
|
||||
req.body.datatoadd || req.body,
|
||||
questionDbs,
|
||||
dryRun,
|
||||
user
|
||||
)
|
||||
processIncomingRequest(req.body, questionDbs, dryRun, user)
|
||||
.then((resultArray) => {
|
||||
logResult(req.body, resultArray)
|
||||
res.json({
|
||||
success: resultArray.length > 0, // TODO check for -1s
|
||||
newQuestions: resultArray,
|
||||
|
|
@ -986,8 +994,10 @@ function GetApp(): ModuleType {
|
|||
)
|
||||
}
|
||||
|
||||
searchData(data, question, subj, recData)
|
||||
searchDatas(questionDbs, question, subj, recData)
|
||||
.then((result) => {
|
||||
// TODO: test
|
||||
console.log(result)
|
||||
res.json({
|
||||
result: result,
|
||||
success: true,
|
||||
|
|
@ -1021,21 +1031,38 @@ function GetApp(): ModuleType {
|
|||
}
|
||||
})
|
||||
|
||||
function getSubjCount(qdbs) {
|
||||
return qdbs.reduce((acc, qdb) => {
|
||||
return acc + qdb.data.length
|
||||
}, 0)
|
||||
}
|
||||
|
||||
function getQuestionCount(qdbs) {
|
||||
return qdbs.reduce((acc, qdb) => {
|
||||
return (
|
||||
acc +
|
||||
qdb.data.reduce((qacc, subject) => {
|
||||
return qacc + subject.Questions.length
|
||||
}, 0)
|
||||
)
|
||||
}, 0)
|
||||
}
|
||||
|
||||
function getSimplreRes() {
|
||||
return {
|
||||
subjects: data.length,
|
||||
questions: data.reduce((acc, subj) => {
|
||||
return acc + subj.Questions.length
|
||||
}, 0),
|
||||
subjects: getSubjCount(questionDbs),
|
||||
questions: getQuestionCount(questionDbs),
|
||||
}
|
||||
}
|
||||
|
||||
function getDetailedRes() {
|
||||
return data.map((subj) => {
|
||||
return {
|
||||
name: subj.Name,
|
||||
count: subj.Questions.length,
|
||||
}
|
||||
return questionDbs.map((qdb) => {
|
||||
return qdb.data.map((subj) => {
|
||||
return {
|
||||
name: subj.Name,
|
||||
count: subj.Questions.length,
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1091,10 +1118,8 @@ function GetApp(): ModuleType {
|
|||
utils.AppendToFile(
|
||||
JSON.stringify({
|
||||
date: utils.GetDateString(),
|
||||
subjectCount: data.length,
|
||||
questionCount: data.reduce((acc, subj) => {
|
||||
return acc + subj.Questions.length
|
||||
}, 0),
|
||||
subjectCount: getSubjCount(questionDbs),
|
||||
questionCount: getQuestionCount(questionDbs),
|
||||
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
|
||||
}),
|
||||
dailyDataCountFile
|
||||
|
|
@ -1165,7 +1190,7 @@ function GetApp(): ModuleType {
|
|||
}
|
||||
|
||||
function DailyAction() {
|
||||
actions.backupData(data)
|
||||
backupData(questionDbs)
|
||||
BackupDB()
|
||||
ExportDailyDataCount()
|
||||
IncrementAvaiblePWs()
|
||||
|
|
|
|||
|
|
@ -17,9 +17,13 @@ export interface Subject {
|
|||
Questions: Array<Question>
|
||||
}
|
||||
|
||||
export interface QuestionDb {
|
||||
name: string
|
||||
export interface DataFile {
|
||||
path: string
|
||||
name: string
|
||||
shouldSave: (recData: any) => boolean
|
||||
}
|
||||
|
||||
export interface QuestionDb extends DataFile {
|
||||
data: Array<Subject>
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,12 +17,6 @@ Question Server
|
|||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
export default {
|
||||
ProcessIncomingRequest: ProcessIncomingRequest,
|
||||
LoadJSON: LoadJSON,
|
||||
backupData: backupData,
|
||||
}
|
||||
|
||||
const recDataFile = './stats/recdata'
|
||||
const dataLockFile = './data/lockData'
|
||||
|
||||
|
|
@ -33,8 +27,7 @@ import utils from '../utils/utils'
|
|||
import { addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
|
||||
// types
|
||||
import { QuestionDb, Question, User } from '../types/basicTypes'
|
||||
import { DataFile } from '../modules/api/api'
|
||||
import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
|
||||
|
||||
// if a recievend question doesnt match at least this % to any other question in the db it gets
|
||||
// added to db
|
||||
|
|
@ -43,7 +36,7 @@ const minMatchToAmmountToAdd = 90
|
|||
const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
|
||||
let currWrites = 0
|
||||
|
||||
interface RecievedData {
|
||||
export interface RecievedData {
|
||||
quiz: Array<Question>
|
||||
subj: string
|
||||
id: string
|
||||
|
|
@ -51,61 +44,93 @@ interface RecievedData {
|
|||
scriptVersion: string
|
||||
}
|
||||
|
||||
function ProcessIncomingRequest(
|
||||
interface Result {
|
||||
qdbName: string
|
||||
newQuestions: number
|
||||
}
|
||||
|
||||
export function logResult(
|
||||
recievedData: RecievedData,
|
||||
result: Array<Result>
|
||||
): void {
|
||||
let subjRow = '\t' + recievedData.subj
|
||||
if (recievedData.id) {
|
||||
subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
|
||||
}
|
||||
logger.Log(subjRow)
|
||||
|
||||
result.forEach((res: Result) => {
|
||||
const allQLength = recievedData.quiz.length
|
||||
let msg = `${res.qdbName}: `
|
||||
let color = logger.GetColor('green')
|
||||
if (res.newQuestions > 0) {
|
||||
color = logger.GetColor('blue')
|
||||
msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
|
||||
} else {
|
||||
msg += `No new data ( ${allQLength} )`
|
||||
}
|
||||
if (recievedData.version !== undefined) {
|
||||
msg += '. Version: ' + recievedData.version
|
||||
}
|
||||
logger.Log('\t' + msg, color)
|
||||
})
|
||||
}
|
||||
|
||||
export function processIncomingRequest(
|
||||
recievedData: RecievedData,
|
||||
questionDbs: Array<QuestionDb>,
|
||||
dryRun: boolean,
|
||||
user: User
|
||||
): Promise<Array<number>> {
|
||||
return Promise.all(
|
||||
questionDbs.map((qdb) => {
|
||||
return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user)
|
||||
})
|
||||
)
|
||||
): Promise<Array<Result>> {
|
||||
logger.DebugLog('Processing incoming request', 'actions', 1)
|
||||
|
||||
if (recievedData === undefined) {
|
||||
logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
|
||||
throw new Error('Recieved data is undefined!')
|
||||
}
|
||||
|
||||
try {
|
||||
let towrite = logger.GetDateString() + '\n'
|
||||
towrite +=
|
||||
'------------------------------------------------------------------------------\n'
|
||||
if (typeof recievedData === 'object') {
|
||||
towrite += JSON.stringify(recievedData)
|
||||
} else {
|
||||
towrite += recievedData
|
||||
}
|
||||
towrite +=
|
||||
'\n------------------------------------------------------------------------------\n'
|
||||
utils.AppendToFile(towrite, recDataFile)
|
||||
logger.DebugLog('recDataFile written', 'actions', 1)
|
||||
} catch (err) {
|
||||
logger.Log('Error writing recieved data.')
|
||||
}
|
||||
|
||||
if (utils.FileExists(dataLockFile)) {
|
||||
logger.Log(
|
||||
'Data lock file exists, skipping recieved data processing',
|
||||
logger.GetColor('red')
|
||||
)
|
||||
return new Promise((resolve) => resolve([]))
|
||||
}
|
||||
|
||||
const promises: Array<Promise<Result>> = questionDbs.reduce((acc, qdb) => {
|
||||
if (qdb.shouldSave(recievedData)) {
|
||||
acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user))
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
return Promise.all(promises)
|
||||
}
|
||||
|
||||
function ProcessIncomingRequestUsingDb(
|
||||
function processIncomingRequestUsingDb(
|
||||
recievedData: RecievedData,
|
||||
qdb: QuestionDb,
|
||||
dryRun: boolean,
|
||||
user: User
|
||||
): Promise<number> {
|
||||
): Promise<Result> {
|
||||
return new Promise((resolve, reject) => {
|
||||
logger.DebugLog('Processing incoming request', 'actions', 1)
|
||||
|
||||
if (recievedData === undefined) {
|
||||
logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
|
||||
reject(new Error('Recieved data is undefined!'))
|
||||
}
|
||||
|
||||
try {
|
||||
let towrite = logger.GetDateString() + '\n'
|
||||
towrite +=
|
||||
'------------------------------------------------------------------------------\n'
|
||||
if (typeof recievedData === 'object') {
|
||||
towrite += JSON.stringify(recievedData)
|
||||
} else {
|
||||
towrite += recievedData
|
||||
}
|
||||
towrite +=
|
||||
'\n------------------------------------------------------------------------------\n'
|
||||
utils.AppendToFile(towrite, recDataFile)
|
||||
logger.DebugLog('recDataFile written', 'actions', 1)
|
||||
} catch (err) {
|
||||
logger.Log('Error writing recieved data.')
|
||||
}
|
||||
|
||||
if (utils.FileExists(dataLockFile)) {
|
||||
logger.Log(
|
||||
'Data lock file exists, skipping recieved data processing',
|
||||
logger.GetColor('red')
|
||||
)
|
||||
resolve(-1)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// recievedData: { version: "", id: "", subj: "" quiz: {} }
|
||||
const recievedQuestions = []
|
||||
|
||||
logger.DebugLog('recievedData JSON parsed', 'actions', 1)
|
||||
|
|
@ -145,11 +170,7 @@ function ProcessIncomingRequestUsingDb(
|
|||
})
|
||||
|
||||
try {
|
||||
let color = logger.GetColor('green')
|
||||
let msg = ''
|
||||
if (allQuestions.length > 0) {
|
||||
color = logger.GetColor('blue')
|
||||
msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
|
||||
allQuestions.forEach((currentQuestion) => {
|
||||
const sName = getSubjNameWithoutYear(recievedData.subj)
|
||||
logger.DebugLog(
|
||||
|
|
@ -171,35 +192,26 @@ function ProcessIncomingRequestUsingDb(
|
|||
currWrites = 0
|
||||
logger.DebugLog('Writing data.json', 'actions', 1)
|
||||
utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
|
||||
logger.Log('\tData file written', color)
|
||||
} else if (dryRun) {
|
||||
logger.Log('\tDry run')
|
||||
}
|
||||
} else {
|
||||
msg += `No new data ( ${allQLength} )`
|
||||
}
|
||||
|
||||
let subjRow = '\t' + recievedData.subj
|
||||
if (recievedData.id) {
|
||||
subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
|
||||
}
|
||||
idStats.LogId(
|
||||
user.id,
|
||||
recievedData.subj,
|
||||
allQuestions.length,
|
||||
allQLength
|
||||
)
|
||||
logger.Log(subjRow)
|
||||
if (recievedData.version !== undefined) {
|
||||
msg += '. Version: ' + recievedData.version
|
||||
}
|
||||
|
||||
logger.Log('\t' + msg, color)
|
||||
logger.DebugLog('New Questions:', 'actions', 2)
|
||||
logger.DebugLog(allQuestions, 'actions', 2)
|
||||
|
||||
logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
|
||||
resolve(allQuestions.length)
|
||||
resolve({
|
||||
newQuestions: allQuestions.length,
|
||||
qdbName: qdb.name,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.Log(
|
||||
|
|
@ -229,7 +241,7 @@ function ProcessIncomingRequestUsingDb(
|
|||
})
|
||||
}
|
||||
|
||||
function LoadJSON(dataFiles: Array<DataFile>) {
|
||||
export function loadJSON(dataFiles: Array<DataFile>): Array<QuestionDb> {
|
||||
return dataFiles.reduce((acc, dataFile) => {
|
||||
if (!utils.FileExists(dataFile.path)) {
|
||||
utils.WriteFile(JSON.stringify([]), dataFile.path)
|
||||
|
|
@ -251,7 +263,7 @@ function LoadJSON(dataFiles: Array<DataFile>) {
|
|||
}, [])
|
||||
}
|
||||
|
||||
function backupData(questionDbs: Array<QuestionDb>) {
|
||||
export function backupData(questionDbs: Array<QuestionDb>): void {
|
||||
questionDbs.forEach((data) => {
|
||||
const path = './publicDirs/qminingPublic/backs/'
|
||||
utils.CreatePath(path)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'
|
||||
import logger from './logger'
|
||||
import { Question, QuestionData, Subject } from '../types/basicTypes'
|
||||
import {
|
||||
Question,
|
||||
QuestionDb,
|
||||
QuestionData,
|
||||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
|
||||
// TODO
|
||||
interface SearchResultQuestion extends Question {
|
||||
|
|
@ -402,6 +407,19 @@ function addQuestion(
|
|||
}
|
||||
}
|
||||
|
||||
function searchDatas(
|
||||
data: Array<QuestionDb>,
|
||||
question: any,
|
||||
subjName: string,
|
||||
questionData?: QuestionData
|
||||
): Promise<Array<Array<SearchResultQuestion>>> {
|
||||
return Promise.all(
|
||||
data.map((db) => {
|
||||
return searchData(db.data, question, subjName, questionData)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// TODO: remove questionData, make question only Question type
|
||||
function searchData(
|
||||
data: Array<Subject>,
|
||||
|
|
@ -568,5 +586,6 @@ export {
|
|||
createQuestion,
|
||||
addQuestion,
|
||||
searchData,
|
||||
searchDatas,
|
||||
dataToString,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue