From a6ee415c5b2c38a5e15db6b078bd491ef3355f41 Mon Sep 17 00:00:00 2001 From: mrfry Date: Fri, 16 Apr 2021 11:06:17 +0200 Subject: [PATCH] worker db edit/update refactor --- src/modules/api/submodules/qminingapi.ts | 242 ++++-------------- src/utils/actions.ts | 297 ++++++++++++++++++++++- src/utils/classes.ts | 71 +++++- src/utils/workerPool.ts | 33 ++- 4 files changed, 428 insertions(+), 215 deletions(-) diff --git a/src/modules/api/submodules/qminingapi.ts b/src/modules/api/submodules/qminingapi.ts index 4c3c07d..52b44ce 100644 --- a/src/modules/api/submodules/qminingapi.ts +++ b/src/modules/api/submodules/qminingapi.ts @@ -14,14 +14,13 @@ import { logResult, shouldSaveDataFile, Result, - isQuestionValid, backupData, shouldSearchDataFile, loadJSON, writeData, + editDb, } from '../../../utils/actions' import { - createQuestion, dataToString, getSubjNameWithoutYear, // compareQuestionObj, @@ -524,7 +523,7 @@ function setup(data: SubmoduleData): any { logResult(req.body, resultArray, user.id, dryRun) const totalNewQuestions = resultArray.reduce((acc, sres) => { - return acc + sres.newQuestions + return acc + sres.newQuestions.length }, 0) res.json({ @@ -534,9 +533,11 @@ function setup(data: SubmoduleData): any { }) if (totalNewQuestions > 0) { - msgAllWorker({ - qdbs: questionDbs, - type: 'update', + resultArray.forEach((result) => { + msgAllWorker({ + type: 'newQuestions', + data: result, + }) }) } }) @@ -842,7 +843,6 @@ function setup(data: SubmoduleData): any { logger.LogReq(req) const user: User = req.session.user const date = utils.GetDateString() - let saveDb = false const editType = req.body.type const selectedDb = req.body.selectedDb @@ -867,44 +867,28 @@ function setup(data: SubmoduleData): any { return } - // { - // "index": 0, - // "subjName": "VHDL programozás", - // "type": "delete", - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } + // ----------------- + const { + success, + msg, + resultDb, + deletedQuestion, + newVal, + oldVal, + deletedQuestions, + changedQuestions, + } = editDb(currDb, req.body) + + if (!success) { + res.json({ success: success, msg: msg }) + return + } + if (resultDb) { + questionDbs[dbIndex] = resultDb + } + if (editType === 'delete') { const { index, subjName } = req.body - let deletedQuestion = {} - if (isNaN(index) || !subjName) { - res.json({ - status: 'fail', - msg: 'No .index or .subjName !', - }) - return - } - - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.filter((question, i) => { - if (index === i) { - deletedQuestion = question - return false - } else { - return true - } - }), - } - } - }) - logger.Log( `User #${user.id} deleted a question from '${subjName}'`, logger.GetColor('cyan') @@ -914,68 +898,10 @@ function setup(data: SubmoduleData): any { dataEditsLog ) utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog) - saveDb = true } - // { - // "index": 0, - // "subjName": "Elektronika", - // "type": "edit", - // "newVal": { - // "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.", - // "A": "Igaz", - // "data": { - // "type": "simple", - // "possibleAnswers": [ - // "Igaz" - // ] - // }, - // "possibleAnswers": [ - // "Igaz" - // ] - // }, - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } if (editType === 'edit') { - const { index, subjName, newVal } = req.body - let oldVal = {} - if (isNaN(index) || !subjName) { - res.json({ - status: 'fail', - msg: 'No .index or .subjName !', - }) - return - } - if (!isQuestionValid(newVal)) { - res.json({ - status: 'fail', - msg: 'edited question is not valid', - question: newVal, - }) - return - } - - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.map((question, i) => { - if (index === i) { - oldVal = question - return createQuestion(newVal) - } else { - return question - } - }), - } - } - }) - + const { index, subjName } = req.body logger.Log( `User #${user.id} edited a question in '${subjName}'`, logger.GetColor('cyan') @@ -995,128 +921,48 @@ function setup(data: SubmoduleData): any { ), dataEditsLog ) - saveDb = true } - // { - // "subjName": "Elektronika", - // "changedQuestions": [ - // { - // "index": 1, - // "value": { - // "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.", - // "A": "Hamis", - // "data": { - // "type": "simple", - // "possibleAnswers": [ - // "Igaz", - // "Hamis" - // ] - // } - // } - // } - // ], - // "deletedQuestions": [ - // 0 - // ], - // "type": "subjEdit", - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } if (editType === 'subjEdit') { - const { subjName, changedQuestions, deletedQuestions } = req.body - const deletedQuestionsToWrite = [] - const changedQuestionsToWrite = [] - if ( - !Array.isArray(changedQuestions) || - !Array.isArray(deletedQuestions) - ) { - res.json({ - status: 'fail', - msg: 'no changedQuestions or deletedQuestions!', - }) - return - } - - // processing changed questions - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.map((question, i) => { - const changedTo = changedQuestions.find((cq) => { - return cq.index === i - }) - if (changedTo) { - changedQuestionsToWrite.push({ - oldVal: question, - newVal: changedTo.value, - }) - return createQuestion(changedTo.value) - } else { - return question - } - }), - } - } - }) - - // processing deletedQuestions - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.filter((question, i) => { - const isDeleted = deletedQuestions.includes(i) - if (isDeleted) { - deletedQuestionsToWrite.push(question) - return false - } else { - return true - } - }), - } - } - }) - + const { subjName } = req.body logger.Log( - `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`, + `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, logger.GetColor('cyan') ) utils.AppendToFile( - `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`, + `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, dataEditsLog ) utils.AppendToFile( JSON.stringify( { - deletedQuestions: deletedQuestionsToWrite, - changedQuestions: changedQuestionsToWrite, + deletedQuestions: deletedQuestions, + changedQuestions: changedQuestions, }, null, 2 ), dataEditsLog ) - saveDb = true } + // ------------------ - if (saveDb) { + if (success) { writeData(currDb.data, currDb.path) msgAllWorker({ - qdbs: questionDbs, - type: 'update', + type: 'dbEdit', + data: { + dbIndex: dbIndex, + edits: req.body, + }, + }).then((res) => { + console.log('AAAAAAAAAAA', res) }) } res.json({ - status: 'OK', + success: true, + msg: 'OK', }) }) diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 2c08e89..317fe7d 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -53,8 +53,10 @@ export interface RecievedData { } export interface Result { + qdbIndex: number qdbName: string - newQuestions: number + subjName: string + newQuestions: Array } export function logResult( @@ -84,9 +86,9 @@ export function logResult( const allQLength = recievedData.quiz.length let msg = `${res.qdbName}: ` let color = logger.GetColor('green') - if (res.newQuestions > 0) { + if (res.newQuestions.length > 0) { color = logger.GetColor('blue') - msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` + msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )` } else { msg += `No new data ( ${allQLength} )` } @@ -241,7 +243,9 @@ function processIncomingRequestUsingDb( logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1) resolve({ - newQuestions: allQuestions.length, + newQuestions: allQuestions, + subjName: recievedData.subj, + qdbIndex: qdb.index, qdbName: qdb.name, }) } catch (error) { @@ -439,3 +443,288 @@ export function backupData(questionDbs: Array): void { } }) } + +function deleteFromDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + } +): { + success: Boolean + msg: string + deletedQuestion?: Question + resultDb?: QuestionDb +} { + // { + // "index": 0, + // "subjName": "VHDL programozás", + // "type": "delete", + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { index, subjName } = edits + let deletedQuestion + if (isNaN(index) || !subjName) { + return { + success: false, + msg: 'No .index or .subjName !', + } + } + + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.filter((question, i) => { + if (index === i) { + deletedQuestion = question + return false + } else { + return true + } + }), + } + } + }) + + return { + success: true, + msg: 'Delete successfull', + deletedQuestion: deletedQuestion, + resultDb: questionDb, + } +} + +function editQuestionInDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + newVal?: Question + } +): { + success: Boolean + msg: string + newVal?: Question + oldVal?: Question + resultDb?: QuestionDb +} { + // { + // "index": 0, + // "subjName": "Elektronika", + // "type": "edit", + // "newVal": { + // "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.", + // "A": "Igaz", + // "data": { + // "type": "simple", + // "possibleAnswers": [ + // "Igaz" + // ] + // }, + // "possibleAnswers": [ + // "Igaz" + // ] + // }, + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { index, subjName, newVal } = edits + let oldVal + if (isNaN(index) || !subjName) { + return { + success: false, + msg: 'No .index or .subjName !', + } + } + if (!isQuestionValid(newVal)) { + return { + success: false, + msg: 'edited question is not valid', + } + } + + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.map((question, i) => { + if (index === i) { + oldVal = question + return createQuestion(newVal) + } else { + return question + } + }), + } + } + }) + + return { + success: true, + msg: 'Edit successfull', + oldVal: oldVal, + newVal: newVal, + resultDb: questionDb, + } +} + +function editSubjInDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + deletedQuestions?: Array + changedQuestions?: Array<{ + index: number + value: Question + }> + } +): { + success: Boolean + msg: string + deletedQuestions?: Array + changedQuestions?: Array + resultDb?: QuestionDb +} { + // { + // "subjName": "Elektronika", + // "changedQuestions": [ + // { + // "index": 1, + // "value": { + // "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.", + // "A": "Hamis", + // "data": { + // "type": "simple", + // "possibleAnswers": [ + // "Igaz", + // "Hamis" + // ] + // } + // } + // } + // ], + // "deletedQuestions": [ + // 0 + // ], + // "type": "subjEdit", + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { subjName, changedQuestions, deletedQuestions } = edits + const deletedQuestionsToWrite = [] + const changedQuestionsToWrite = [] + if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) { + return { + success: false, + msg: 'no changedQuestions or deletedQuestions!', + } + } + + // processing changed questions + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.map((question, i) => { + const changedTo = changedQuestions.find((cq) => { + return cq.index === i + }) + if (changedTo) { + changedQuestionsToWrite.push({ + oldVal: question, + newVal: changedTo.value, + }) + return createQuestion(changedTo.value) + } else { + return question + } + }), + } + } + }) + + // processing deletedQuestions + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.filter((question, i) => { + const isDeleted = deletedQuestions.includes(i) + if (isDeleted) { + deletedQuestionsToWrite.push(question) + return false + } else { + return true + } + }), + } + } + }) + + return { + success: true, + msg: 'subj edit successfull', + deletedQuestions: deletedQuestionsToWrite, + changedQuestions: changedQuestionsToWrite, + resultDb: questionDb, + } +} + +export function editDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + selectedDb: { path: string; name: string } + type: string + newVal?: Question + deletedQuestion?: Array + changedQuestions?: Array<{ + index: number + value: Question + }> + } +): { + success: Boolean + msg: string + resultDb?: QuestionDb + deletedQuestion?: Question + newVal?: Question + oldVal?: Question + deletedQuestions?: Array + changedQuestions?: Array +} { + if (edits.type === 'delete') { + return deleteFromDb(questionDb, edits) + } + if (edits.type === 'edit') { + return editQuestionInDb(questionDb, edits) + } + + if (edits.type === 'subjEdit') { + return editSubjInDb(questionDb, edits) + } +} diff --git a/src/utils/classes.ts b/src/utils/classes.ts index bfaa879..495e635 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -1,6 +1,12 @@ import { isMainThread, parentPort, workerData } from 'worker_threads' import logger from './logger' -import { Question, QuestionData, Subject } from '../types/basicTypes' +import { + Question, + QuestionData, + QuestionDb, + Subject, +} from '../types/basicTypes' +import { editDb } from './actions' interface SearchResultQuestion extends Question { match: number @@ -581,7 +587,7 @@ function doSearch( if (!isMainThread) { const { workerIndex } = workerData - let qdbs: Array = workerData.initData + let qdbs: Array = workerData.initData logger.Log( `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty` @@ -670,9 +676,64 @@ if (!isMainThread) { // !isNaN(index) ? `#${index}` : '' // }done!` // ) - } else if (msg.type === 'update') { - qdbs = msg.qdbs - logger.DebugLog(`Worker update ${workerIndex}`, 'worker update', 1) + } else if (msg.type === 'dbEdit') { + const { dbIndex, edits } = msg.data + const { resultDb } = editDb(qdbs[dbIndex], edits) + qdbs[dbIndex] = resultDb + logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1) + + parentPort.postMessage({ + msg: `From thread #${workerIndex}: db edit`, + workerIndex: workerIndex, + }) + } else if (msg.type === 'newQuestions') { + const { subjName, qdbIndex, newQuestions } = msg.data + let added = false + qdbs = qdbs.map((qdb) => { + if (qdb.index === qdbIndex) { + return { + ...qdb, + data: qdb.data.map((subj) => { + if (subj.Name === subjName) { + added = true + return { + Name: subj.Name, + Questions: [...subj.Questions, ...newQuestions], + } + } else { + return subj + } + }), + } + } else { + return qdb + } + }) + + if (!added) { + qdbs = qdbs.map((qdb) => { + if (qdb.index === qdbIndex) { + return { + ...qdb, + data: [ + ...qdb.data, + { + Name: subjName, + Questions: [...newQuestions], + }, + ], + } + } else { + return qdb + } + }) + } + logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1) + + // parentPort.postMessage({ + // msg: `From thread #${workerIndex}: update done`, + // workerIndex: workerIndex, + // }) // console.log(`[THREAD #${workerIndex}]: update`) } else if (msg.type === 'newdb') { diff --git a/src/utils/workerPool.ts b/src/utils/workerPool.ts index ab51f14..07d8e1b 100644 --- a/src/utils/workerPool.ts +++ b/src/utils/workerPool.ts @@ -20,6 +20,30 @@ function handleWorkerError(client, err) { pool.destroy(client) } +// TODO: accuire all workers here, and handle errors so they can be removed if threads exit +export function msgAllWorker(data: any): Promise { + console.log('MSGING ALL WORKER') + return new Promise((resolve) => { + const promises = [] + workers.forEach((worker) => { + worker.postMessage(data) + console.log('MSGD') + promises.push( + new Promise((resolve) => { + worker.once('message', (msg) => { + console.log(worker.index, 'ONCE MESSASGE RESOLVE') + resolve(msg) + }) + }) + ) + }) + Promise.all(promises).then((res) => { + console.log('MSG ALL DONE', res) + resolve(res) + }) + }) +} + export function doALongTask(obj: any): Promise { if (pool.pending > alertOnPendingCount) { logger.Log( @@ -91,13 +115,6 @@ export function initWorkerPool(initData: any): void { pool = genericPool.createPool(factory, opts) } -// TODO: accuire all workers here, and handle errors so they can be removed if threads exit -export function msgAllWorker(data: any): void { - workers.forEach((worker) => { - worker.postMessage(data) - }) -} - // --------------------------------------------------------------------------- function getAWorker(i, initData) { @@ -140,10 +157,10 @@ function doSomething(client, obj) { const { /* index, */ worker } = client return new Promise((resolve) => { // console.log('[ACCUIRE]: #' + index) - worker.postMessage(obj) worker.once('message', (msg) => { resolve(msg) }) + worker.postMessage(obj) }) }