mrfrys-node-server/src/utils/actions.ts
2022-12-10 15:34:54 +01:00

869 lines
26 KiB
TypeScript
Executable file

/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
const recDataFile = './stats/recdata'
const dataLockFile = './data/lockData'
import logger from '../utils/logger'
import {
createQuestion,
WorkerResult,
SearchResultQuestion,
} from '../utils/classes'
import { doALongTask, msgAllWorker } from './workerPool'
import idStats from '../utils/ids'
import utils from '../utils/utils'
import { addQuestion, getSubjNameWithoutYear } from './classes'
// types
import {
QuestionDb,
Subject,
Question,
User,
DataFile,
} from '../types/basicTypes'
// if a recievend question doesnt match at least this % to any other question in the db it gets
// added to db
const minMatchAmmountToAdd = 90
const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
let currWrites = 0
export interface RecievedData {
quiz: Array<Question>
subj: string
id: string
version: string
location: string
}
export interface Result {
qdbIndex: number
qdbName: string
subjName: string
newQuestions: Array<Question>
}
export function logResult(
recievedData: RecievedData,
result: Array<Result>,
userId: number,
dryRun?: boolean
): void {
logger.Log('\t' + recievedData.subj)
if (dryRun) {
logger.Log('\tDry run')
}
let idRow = '\t'
if (recievedData.version) {
idRow += 'Version: ' + logger.logHashed(recievedData.version)
}
if (recievedData.id) {
idRow += ', CID: ' + logger.logHashed(recievedData.id)
}
idRow += `, User #${logger.logHashed(userId.toString())}`
logger.Log(idRow)
if (result.length > 0) {
result.forEach((res: Result) => {
const allQLength = recievedData.quiz.length
let msg = `${res.qdbName}: `
let color = logger.GetColor('green')
if (res.newQuestions.length > 0) {
color = logger.GetColor('blue')
msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )`
} else {
msg += `No new data ( ${allQLength} )`
}
logger.Log('\t' + msg, color)
})
} else {
logger.Log('\tResults length is zero!', logger.GetColor('red'))
}
}
export function processIncomingRequest(
recievedData: RecievedData,
questionDbs: Array<QuestionDb>,
dryRun: boolean,
user: User
): Promise<Array<Result>> {
logger.DebugLog('Processing incoming request', 'isadding', 1)
if (recievedData === undefined) {
logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
throw new Error('Recieved data is undefined!')
}
try {
let towrite = utils.GetDateString() + '\n'
towrite +=
'------------------------------------------------------------------------------\n'
if (typeof recievedData === 'object') {
towrite += JSON.stringify(recievedData)
} else {
towrite += recievedData
}
towrite +=
'\n------------------------------------------------------------------------------\n'
utils.AppendToFile(towrite, recDataFile)
logger.DebugLog('recDataFile written', 'isadding', 1)
} catch (err) {
logger.Log('Error writing recieved data.')
}
if (utils.FileExists(dataLockFile)) {
logger.Log(
'Data lock file exists, skipping recieved data processing',
logger.GetColor('red')
)
return new Promise((resolve) => resolve([]))
}
// FIXME: this many promises and stuff might be unnecesarry
const promises: Array<Promise<Result>> = questionDbs.reduce((acc, qdb) => {
acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user))
return acc
}, [])
return Promise.all(promises)
}
function processIncomingRequestUsingDb(
recievedData: RecievedData,
qdb: QuestionDb,
dryRun: boolean,
user: User
): Promise<Result> {
return new Promise((resolve, reject) => {
try {
const recievedQuestions: Question[] = []
logger.DebugLog('recievedData JSON parsed', 'isadding', 1)
logger.DebugLog(recievedData, 'isadding', 3)
const allQLength = recievedData.quiz.length
const questionSearchPromises: Promise<WorkerResult>[] = []
recievedData.quiz.forEach((question) => {
logger.DebugLog('Question:', 'isadding', 2)
logger.DebugLog(question, 'isadding', 2)
const currentQuestion = createQuestion(
question.Q,
question.A,
question.data
)
logger.DebugLog(
'Searching for question in subj ' + recievedData.subj,
'isadding',
3
)
logger.DebugLog(currentQuestion, 'isadding', 3)
if (isQuestionValid(currentQuestion)) {
recievedQuestions.push(currentQuestion)
// This here searches only in relevant subjects, and not all subjects
questionSearchPromises.push(
doALongTask({
type: 'work',
data: {
searchIn: [qdb.index],
question: currentQuestion,
subjName: recievedData.subj,
searchTillMatchPercent: minMatchAmmountToAdd,
},
})
)
} else {
logger.DebugLog('Question isnt valid', 'isadding', 1)
logger.DebugLog(currentQuestion, 'isadding', 1)
}
})
Promise.all(questionSearchPromises)
.then((results: Array<WorkerResult>) => {
const allQuestions: Question[] = [] // all new questions here that do not have result
results.forEach((result: WorkerResult, i) => {
const add = (
result.result as SearchResultQuestion[]
).every((res: SearchResultQuestion) => {
return res.match < minMatchAmmountToAdd
})
if (add && !result.error) {
allQuestions.push(recievedQuestions[i])
}
})
try {
const subjName = getSubjNameWithoutYear(
recievedData.subj
)
if (allQuestions.length > 0) {
addQuestionsToDb(allQuestions, subjName, qdb)
currWrites++
logger.DebugLog(
'currWrites for data.json: ' + currWrites,
'isadding',
1
)
if (currWrites >= writeAfter && !dryRun) {
currWrites = 0
logger.DebugLog(
'Writing data.json',
'isadding',
1
)
writeData(qdb.data, qdb.path)
}
}
idStats.LogId(
user.id,
recievedData.subj,
allQuestions.length,
allQLength
)
logger.DebugLog('New Questions:', 'isadding', 2)
logger.DebugLog(allQuestions, 'isadding', 2)
logger.DebugLog(
'ProcessIncomingRequest done',
'isadding',
1
)
resolve({
newQuestions: allQuestions,
subjName: recievedData.subj,
qdbIndex: qdb.index,
qdbName: qdb.name,
})
runCleanWorker(recievedData.quiz, subjName, qdb)
} catch (error) {
console.error(error)
logger.Log(
'Error while processing processData worker result!',
logger.GetColor('redbg')
)
reject(
new Error(
'Error while processing processData worker result!'
)
)
}
})
.catch((err) => {
logger.Log(
'Error while searching for questions in ProcessIncomingRequest!',
logger.GetColor('redbg')
)
console.error(err)
})
} catch (err) {
console.error(err)
logger.Log(
'There was en error handling incoming quiz data, see stderr',
logger.GetColor('redbg')
)
reject(new Error('Couldnt parse JSON data'))
}
})
}
function addQuestionsToDb(
allQuestions: Question[],
subjName: string,
qdb: QuestionDb
) {
allQuestions.forEach((currentQuestion) => {
logger.DebugLog(
'Adding question with subjName: ' + subjName + ' :',
'isadding',
3
)
logger.DebugLog(currentQuestion, 'isadding', 3)
addQuestion(qdb.data, subjName, {
...currentQuestion,
data: {
...currentQuestion.data,
date: new Date().getTime(),
},
})
})
}
function runCleanWorker(
recievedQuesitons: Question[],
subjName: string,
qdb: QuestionDb
) {
if (qdb.overwrites && qdb.overwrites.length) {
// check if subject needs to be updated, and qdb has overwriteFromDate
const overwrite = qdb.overwrites.find((x) => {
return subjName.toLowerCase().includes(x.subjName.toLowerCase())
})
if (!overwrite) {
return
}
// logger.Log(
// `\tStarting cleaning in subject "${logger.C(
// 'green'
// )}${subjName}${logger.C('')}" (matched: "${logger.C('green')}${
// overwrite.subjName
// }${logger.C('')}")`
// )
// pass recieved questions to a worker
doALongTask({
type: 'dbClean',
data: {
questions: recievedQuesitons,
subjToClean: subjName,
overwriteFromDate: overwrite.overwriteFromDate,
qdbIndex: qdb.index,
},
}).then(({ result: questionIndexesToRemove }) => {
const subjIndex = qdb.data.findIndex((x) => {
return x.Name.toLowerCase().includes(subjName.toLowerCase())
})
// sends msgs to all workers to remove it too
msgAllWorker({
type: 'rmQuestions',
data: {
questionIndexesToRemove:
questionIndexesToRemove as number[][],
subjIndex: subjIndex,
qdbIndex: qdb.index,
recievedQuestions: recievedQuesitons,
},
})
// it adds the recieved question WITH DATE!
// recievedQuestions doesnt have date-s
qdb.data[subjIndex].Questions = updateQuestionsInArray(
questionIndexesToRemove as number[][],
qdb.data[subjIndex].Questions,
recievedQuesitons
)
// saves the file
writeData(qdb.data, qdb.path)
logger.Log(
`\tRemoved ${logger.C('green')}${
(questionIndexesToRemove as number[][]).filter(
(x: number[]) => x.length > 1
).length
}${logger.C()} old questions from ${logger.C(
'green'
)}${subjName}${logger.C()}`
)
})
}
}
export function updateQuestionsInArray(
questionIndexesToRemove: number[][],
questions: Question[],
newQuestions: Question[]
): Question[] {
const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => {
if (x.length > 1) {
return [...acc, ...x]
}
return acc
}, [])
const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => {
return questionIndexesToRemove[i].length > 1
})
return [
...questions.filter((_x, i) => {
return !indexesToRemove.includes(i)
}),
...newQuestionsToAdd.map((x) => {
x.data.date = new Date()
return x
}),
]
}
export function isQuestionValid(question: Question): boolean {
if (!question.Q) {
return false
}
if (!question.A && question.data.type === 'simple') {
return false
}
return true
}
export function shouldSearchDataFile(
df: DataFile,
testUrl: string,
trueIfAlways?: boolean
): boolean {
if (
typeof df.shouldSearch === 'string' &&
df.shouldSearch === 'always' &&
trueIfAlways
) {
return true
}
if (typeof df.shouldSearch === 'object') {
if (df.shouldSearch.location) {
const { val } = df.shouldSearch.location
return testUrl.includes(val)
}
}
logger.DebugLog(`no suitable dbs for ${testUrl}`, 'shouldSearchDataFile', 1)
return false
}
export function shouldSaveDataFile(
df: DataFile,
recievedData: RecievedData
): boolean {
if (df.locked) {
return false
}
if (df.shouldSave.version) {
const { compare, val } = df.shouldSave.version
if (compare === 'equals') {
return recievedData.version === val
} else if (compare === 'startsWith') {
return recievedData.version.startsWith(val)
}
}
if (df.shouldSave.version) {
const { compare, val } = df.shouldSave.version
if (compare === 'equals') {
return recievedData.version === val
} else if (compare === 'startsWith') {
return recievedData.version.startsWith(val)
}
}
if (df.shouldSave.location) {
const { val } = df.shouldSave.location
return recievedData.location.includes(val)
}
return false
}
export function loadData(path: string): Array<Subject> {
return JSON.parse(utils.ReadFile(path)).map((subj: Subject) => {
return {
Name: subj.Name,
Questions: subj.Questions.map((question: Question) => {
return createQuestion(question)
}),
}
})
}
export function loadJSON(
dataFiles: Array<DataFile>,
dataDir: string
): Array<QuestionDb> {
const res: Array<QuestionDb> = dataFiles.reduce((acc, dataFile, index) => {
const dataPath = dataDir + dataFile.path
if (!utils.FileExists(dataPath)) {
utils.WriteFile(JSON.stringify([]), dataPath)
}
try {
acc.push({
...dataFile,
path: dataPath,
index: index,
data: loadData(dataPath),
})
} catch (err) {
console.error(err)
logger.Log(
"data is undefined! Couldn't load data!",
logger.GetColor('redbg')
)
}
return acc
}, [])
let subjCount = 0
let questionCount = 0
res.forEach((qdb) => {
subjCount += qdb.data.length
qdb.data.forEach((subj) => {
questionCount += subj.Questions.length
})
})
logger.Log(
`Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`,
logger.GetColor('green')
)
return res
}
export function writeData(data: Array<Subject>, path: string): void {
utils.WriteFile(
JSON.stringify(
data.map((subj) => {
return {
Name: subj.Name,
Questions: subj.Questions.map((question) => {
return {
Q: question.Q,
A: question.A,
data: question.data,
}
}),
}
})
),
path
)
}
export function backupData(questionDbs: Array<QuestionDb>): void {
questionDbs.forEach((data) => {
const path = './publicDirs/qminingPublic/backs/'
utils.CreatePath(path)
try {
// logger.Log(`Backing up ${data.name}...`)
writeData(
data.data,
`${path}${data.name}_${utils.GetDateString(
undefined,
true
)}.json`
)
// logger.Log('Done')
} catch (err) {
logger.Log(
`Error backing up data file ${data.name}!`,
logger.GetColor('redbg')
)
console.error(err)
}
})
}
function deleteFromDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
type: string
selectedDb: { path: string; name: string }
}
): {
success: boolean
msg: string
deletedQuestion?: Question
resultDb?: QuestionDb
} {
// {
// "index": 0,
// "subjName": "VHDL programozás",
// "type": "delete",
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
const { index, subjName } = edits
let deletedQuestion: Question
if (isNaN(index) || !subjName) {
return {
success: false,
msg: 'No .index or .subjName !',
}
}
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.filter((question, i) => {
if (index === i) {
deletedQuestion = question
return false
} else {
return true
}
}),
}
}
})
return {
success: true,
msg: 'Delete successfull',
deletedQuestion: deletedQuestion,
resultDb: questionDb,
}
}
function editQuestionInDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
type: string
selectedDb: { path: string; name: string }
newVal: Question
}
): {
success: boolean
msg: string
newVal?: Question
oldVal?: Question
resultDb?: QuestionDb
} {
// {
// "index": 0,
// "subjName": "Elektronika",
// "type": "edit",
// "newVal": {
// "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.",
// "A": "Igaz",
// "data": {
// "type": "simple",
// "possibleAnswers": [
// "Igaz"
// ]
// },
// "possibleAnswers": [
// "Igaz"
// ]
// },
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
const { index, subjName, newVal } = edits
let oldVal: Question
if (isNaN(index) || !subjName) {
return {
success: false,
msg: 'No .index or .subjName !',
}
}
if (!isQuestionValid(newVal)) {
return {
success: false,
msg: 'edited question is not valid',
}
}
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.map((question, i) => {
if (index === i) {
oldVal = question
return createQuestion(newVal)
} else {
return question
}
}),
}
}
})
return {
success: true,
msg: 'Edit successfull',
oldVal: oldVal,
newVal: newVal,
resultDb: questionDb,
}
}
function editSubjInDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
type: string
selectedDb: { path: string; name: string }
deletedQuestions?: Array<number>
changedQuestions?: Array<{
index: number
value: Question
}>
}
): {
success: boolean
msg: string
deletedQuestions?: Array<Question>
changedQuestions?: { oldVal: Question; newVal: Question }[]
resultDb?: QuestionDb
} {
// {
// "subjName": "Elektronika",
// "changedQuestions": [
// {
// "index": 1,
// "value": {
// "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.",
// "A": "Hamis",
// "data": {
// "type": "simple",
// "possibleAnswers": [
// "Igaz",
// "Hamis"
// ]
// }
// }
// }
// ],
// "deletedQuestions": [
// 0
// ],
// "type": "subjEdit",
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
const { subjName, changedQuestions, deletedQuestions } = edits
const deletedQuestionsToWrite: Question[] = []
const changedQuestionsToWrite: { oldVal: Question; newVal: Question }[] = []
if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) {
return {
success: false,
msg: 'no changedQuestions or deletedQuestions!',
}
}
// processing changed questions
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.map((question, i) => {
const changedTo = changedQuestions.find((cq) => {
return cq.index === i
})
if (changedTo) {
changedQuestionsToWrite.push({
oldVal: question,
newVal: changedTo.value,
})
return createQuestion(changedTo.value)
} else {
return question
}
}),
}
}
})
// processing deletedQuestions
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.filter((question, i) => {
const isDeleted = deletedQuestions.includes(i)
if (isDeleted) {
deletedQuestionsToWrite.push(question)
return false
} else {
return true
}
}),
}
}
})
return {
success: true,
msg: 'subj edit successfull',
deletedQuestions: deletedQuestionsToWrite,
changedQuestions: changedQuestionsToWrite,
resultDb: questionDb,
}
}
// FIXME: newVal is optional in some places but not in others
export interface Edits {
index: number
subjName: string
selectedDb: { path: string; name: string }
type: string
newVal: Question
deletedQuestion?: Array<number>
changedQuestions?: Array<{
index: number
value: Question
}>
}
export function editDb(
questionDb: QuestionDb,
edits: Edits
): {
success: boolean
msg: string
resultDb?: QuestionDb
deletedQuestion?: Question
newVal?: Question
oldVal?: Question
deletedQuestions?: Array<Question>
changedQuestions?: { oldVal: Question; newVal: Question }[]
} {
if (edits.type === 'delete') {
return deleteFromDb(questionDb, edits)
}
if (edits.type === 'edit') {
return editQuestionInDb(questionDb, edits)
}
if (edits.type === 'subjEdit') {
return editSubjInDb(questionDb, edits)
}
return {
success: false,
msg: 'DB edit error, no matched type',
}
}