albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

worker db edit/update refactor

Browse source
This commit is contained in:
mrfry 2021-04-16 11:06:17 +02:00
parent e2c15f8445
commit a6ee415c5b
4 changed files with 428 additions and 215 deletions
Download patch file Download diff file Expand all files Collapse all files

242
src/modules/api/submodules/qminingapi.ts
View file

@ -14,14 +14,13 @@ import {
logResult,
shouldSaveDataFile,
Result,
isQuestionValid,
backupData,
shouldSearchDataFile,
loadJSON,
writeData,
editDb,
} from '../../../utils/actions'
import {
createQuestion,
dataToString,
getSubjNameWithoutYear,
// compareQuestionObj,
@ -524,7 +523,7 @@ function setup(data: SubmoduleData): any {
logResult(req.body, resultArray, user.id, dryRun)
const totalNewQuestions = resultArray.reduce((acc, sres) => {
return acc + sres.newQuestions
return acc + sres.newQuestions.length
}, 0)
res.json({
@ -534,9 +533,11 @@ function setup(data: SubmoduleData): any {
})
if (totalNewQuestions > 0) {
msgAllWorker({
qdbs: questionDbs,
type: 'update',
resultArray.forEach((result) => {
msgAllWorker({
type: 'newQuestions',
data: result,
})
})
}
})
@ -842,7 +843,6 @@ function setup(data: SubmoduleData): any {
logger.LogReq(req)
const user: User = req.session.user
const date = utils.GetDateString()
let saveDb = false
const editType = req.body.type
const selectedDb = req.body.selectedDb
@ -867,44 +867,28 @@ function setup(data: SubmoduleData): any {
return
}
// {
// "index": 0,
// "subjName": "VHDL programozás",
// "type": "delete",
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
// -----------------
const {
success,
msg,
resultDb,
deletedQuestion,
newVal,
oldVal,
deletedQuestions,
changedQuestions,
} = editDb(currDb, req.body)
if (!success) {
res.json({ success: success, msg: msg })
return
}
if (resultDb) {
questionDbs[dbIndex] = resultDb
}
if (editType === 'delete') {
const { index, subjName } = req.body
let deletedQuestion = {}
if (isNaN(index) || !subjName) {
res.json({
status: 'fail',
msg: 'No .index or .subjName !',
})
return
}
questionDbs[dbIndex].data = currDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.filter((question, i) => {
if (index === i) {
deletedQuestion = question
return false
} else {
return true
}
}),
}
}
})
logger.Log(
`User #${user.id} deleted a question from '${subjName}'`,
logger.GetColor('cyan')
@ -914,68 +898,10 @@ function setup(data: SubmoduleData): any {
dataEditsLog
)
utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog)
saveDb = true
}
// {
// "index": 0,
// "subjName": "Elektronika",
// "type": "edit",
// "newVal": {
// "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.",
// "A": "Igaz",
// "data": {
// "type": "simple",
// "possibleAnswers": [
// "Igaz"
// ]
// },
// "possibleAnswers": [
// "Igaz"
// ]
// },
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
if (editType === 'edit') {
const { index, subjName, newVal } = req.body
let oldVal = {}
if (isNaN(index) || !subjName) {
res.json({
status: 'fail',
msg: 'No .index or .subjName !',
})
return
}
if (!isQuestionValid(newVal)) {
res.json({
status: 'fail',
msg: 'edited question is not valid',
question: newVal,
})
return
}
questionDbs[dbIndex].data = currDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.map((question, i) => {
if (index === i) {
oldVal = question
return createQuestion(newVal)
} else {
return question
}
}),
}
}
})
const { index, subjName } = req.body
logger.Log(
`User #${user.id} edited a question in '${subjName}'`,
logger.GetColor('cyan')
@ -995,128 +921,48 @@ function setup(data: SubmoduleData): any {
),
dataEditsLog
)
saveDb = true
}
// {
// "subjName": "Elektronika",
// "changedQuestions": [
// {
// "index": 1,
// "value": {
// "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.",
// "A": "Hamis",
// "data": {
// "type": "simple",
// "possibleAnswers": [
// "Igaz",
// "Hamis"
// ]
// }
// }
// }
// ],
// "deletedQuestions": [
// 0
// ],
// "type": "subjEdit",
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
if (editType === 'subjEdit') {
const { subjName, changedQuestions, deletedQuestions } = req.body
const deletedQuestionsToWrite = []
const changedQuestionsToWrite = []
if (
!Array.isArray(changedQuestions) ||
!Array.isArray(deletedQuestions)
) {
res.json({
status: 'fail',
msg: 'no changedQuestions or deletedQuestions!',
})
return
}
// processing changed questions
questionDbs[dbIndex].data = currDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.map((question, i) => {
const changedTo = changedQuestions.find((cq) => {
return cq.index === i
})
if (changedTo) {
changedQuestionsToWrite.push({
oldVal: question,
newVal: changedTo.value,
})
return createQuestion(changedTo.value)
} else {
return question
}
}),
}
}
})
// processing deletedQuestions
questionDbs[dbIndex].data = currDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.filter((question, i) => {
const isDeleted = deletedQuestions.includes(i)
if (isDeleted) {
deletedQuestionsToWrite.push(question)
return false
} else {
return true
}
}),
}
}
})
const { subjName } = req.body
logger.Log(
`User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`,
`User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
logger.GetColor('cyan')
)
utils.AppendToFile(
`${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`,
`${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
dataEditsLog
)
utils.AppendToFile(
JSON.stringify(
{
deletedQuestions: deletedQuestionsToWrite,
changedQuestions: changedQuestionsToWrite,
deletedQuestions: deletedQuestions,
changedQuestions: changedQuestions,
},
null,
2
),
dataEditsLog
)
saveDb = true
}
// ------------------
if (saveDb) {
if (success) {
writeData(currDb.data, currDb.path)
msgAllWorker({
qdbs: questionDbs,
type: 'update',
type: 'dbEdit',
data: {
dbIndex: dbIndex,
edits: req.body,
},
}).then((res) => {
console.log('AAAAAAAAAAA', res)
})
}
res.json({
status: 'OK',
success: true,
msg: 'OK',
})
})

297
src/utils/actions.ts
View file

@ -53,8 +53,10 @@ export interface RecievedData {
}
export interface Result {
qdbIndex: number
qdbName: string
newQuestions: number
subjName: string
newQuestions: Array<Question>
}
export function logResult(
@ -84,9 +86,9 @@ export function logResult(
const allQLength = recievedData.quiz.length
let msg = `${res.qdbName}: `
let color = logger.GetColor('green')
if (res.newQuestions > 0) {
if (res.newQuestions.length > 0) {
color = logger.GetColor('blue')
msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )`
} else {
msg += `No new data ( ${allQLength} )`
}
@ -241,7 +243,9 @@ function processIncomingRequestUsingDb(
logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1)
resolve({
newQuestions: allQuestions.length,
newQuestions: allQuestions,
subjName: recievedData.subj,
qdbIndex: qdb.index,
qdbName: qdb.name,
})
} catch (error) {
@ -439,3 +443,288 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
}
})
}
function deleteFromDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
type: string
selectedDb: { path: string; name: string }
}
): {
success: Boolean
msg: string
deletedQuestion?: Question
resultDb?: QuestionDb
} {
// {
// "index": 0,
// "subjName": "VHDL programozás",
// "type": "delete",
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
const { index, subjName } = edits
let deletedQuestion
if (isNaN(index) || !subjName) {
return {
success: false,
msg: 'No .index or .subjName !',
}
}
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.filter((question, i) => {
if (index === i) {
deletedQuestion = question
return false
} else {
return true
}
}),
}
}
})
return {
success: true,
msg: 'Delete successfull',
deletedQuestion: deletedQuestion,
resultDb: questionDb,
}
}
function editQuestionInDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
type: string
selectedDb: { path: string; name: string }
newVal?: Question
}
): {
success: Boolean
msg: string
newVal?: Question
oldVal?: Question
resultDb?: QuestionDb
} {
// {
// "index": 0,
// "subjName": "Elektronika",
// "type": "edit",
// "newVal": {
// "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.",
// "A": "Igaz",
// "data": {
// "type": "simple",
// "possibleAnswers": [
// "Igaz"
// ]
// },
// "possibleAnswers": [
// "Igaz"
// ]
// },
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
const { index, subjName, newVal } = edits
let oldVal
if (isNaN(index) || !subjName) {
return {
success: false,
msg: 'No .index or .subjName !',
}
}
if (!isQuestionValid(newVal)) {
return {
success: false,
msg: 'edited question is not valid',
}
}
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.map((question, i) => {
if (index === i) {
oldVal = question
return createQuestion(newVal)
} else {
return question
}
}),
}
}
})
return {
success: true,
msg: 'Edit successfull',
oldVal: oldVal,
newVal: newVal,
resultDb: questionDb,
}
}
function editSubjInDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
type: string
selectedDb: { path: string; name: string }
deletedQuestions?: Array<number>
changedQuestions?: Array<{
index: number
value: Question
}>
}
): {
success: Boolean
msg: string
deletedQuestions?: Array<Question>
changedQuestions?: Array<Question>
resultDb?: QuestionDb
} {
// {
// "subjName": "Elektronika",
// "changedQuestions": [
// {
// "index": 1,
// "value": {
// "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.",
// "A": "Hamis",
// "data": {
// "type": "simple",
// "possibleAnswers": [
// "Igaz",
// "Hamis"
// ]
// }
// }
// }
// ],
// "deletedQuestions": [
// 0
// ],
// "type": "subjEdit",
// "selectedDb": {
// "path": "questionDbs/elearning.uni-obuda.hu.json",
// "name": "elearning.uni-obuda.hu"
// }
// }
const { subjName, changedQuestions, deletedQuestions } = edits
const deletedQuestionsToWrite = []
const changedQuestionsToWrite = []
if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) {
return {
success: false,
msg: 'no changedQuestions or deletedQuestions!',
}
}
// processing changed questions
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.map((question, i) => {
const changedTo = changedQuestions.find((cq) => {
return cq.index === i
})
if (changedTo) {
changedQuestionsToWrite.push({
oldVal: question,
newVal: changedTo.value,
})
return createQuestion(changedTo.value)
} else {
return question
}
}),
}
}
})
// processing deletedQuestions
questionDb.data = questionDb.data.map((subj) => {
if (subj.Name !== subjName) {
return subj
} else {
return {
...subj,
Questions: subj.Questions.filter((question, i) => {
const isDeleted = deletedQuestions.includes(i)
if (isDeleted) {
deletedQuestionsToWrite.push(question)
return false
} else {
return true
}
}),
}
}
})
return {
success: true,
msg: 'subj edit successfull',
deletedQuestions: deletedQuestionsToWrite,
changedQuestions: changedQuestionsToWrite,
resultDb: questionDb,
}
}
export function editDb(
questionDb: QuestionDb,
edits: {
index: number
subjName: string
selectedDb: { path: string; name: string }
type: string
newVal?: Question
deletedQuestion?: Array<number>
changedQuestions?: Array<{
index: number
value: Question
}>
}
): {
success: Boolean
msg: string
resultDb?: QuestionDb
deletedQuestion?: Question
newVal?: Question
oldVal?: Question
deletedQuestions?: Array<Question>
changedQuestions?: Array<Question>
} {
if (edits.type === 'delete') {
return deleteFromDb(questionDb, edits)
}
if (edits.type === 'edit') {
return editQuestionInDb(questionDb, edits)
}
if (edits.type === 'subjEdit') {
return editSubjInDb(questionDb, edits)
}
}

71
src/utils/classes.ts
View file

@ -1,6 +1,12 @@
import { isMainThread, parentPort, workerData } from 'worker_threads'
import logger from './logger'
import { Question, QuestionData, Subject } from '../types/basicTypes'
import {
Question,
QuestionData,
QuestionDb,
Subject,
} from '../types/basicTypes'
import { editDb } from './actions'
interface SearchResultQuestion extends Question {
match: number
@ -581,7 +587,7 @@ function doSearch(
if (!isMainThread) {
const { workerIndex } = workerData
let qdbs: Array<any> = workerData.initData
let qdbs: Array<QuestionDb> = workerData.initData
logger.Log(
`[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
@ -670,9 +676,64 @@ if (!isMainThread) {
// !isNaN(index) ? `#${index}` : ''
// }done!`
// )
} else if (msg.type === 'update') {
qdbs = msg.qdbs
logger.DebugLog(`Worker update ${workerIndex}`, 'worker update', 1)
} else if (msg.type === 'dbEdit') {
const { dbIndex, edits } = msg.data
const { resultDb } = editDb(qdbs[dbIndex], edits)
qdbs[dbIndex] = resultDb
logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1)
parentPort.postMessage({
msg: `From thread #${workerIndex}: db edit`,
workerIndex: workerIndex,
})
} else if (msg.type === 'newQuestions') {
const { subjName, qdbIndex, newQuestions } = msg.data
let added = false
qdbs = qdbs.map((qdb) => {
if (qdb.index === qdbIndex) {
return {
...qdb,
data: qdb.data.map((subj) => {
if (subj.Name === subjName) {
added = true
return {
Name: subj.Name,
Questions: [...subj.Questions, ...newQuestions],
}
} else {
return subj
}
}),
}
} else {
return qdb
}
})
if (!added) {
qdbs = qdbs.map((qdb) => {
if (qdb.index === qdbIndex) {
return {
...qdb,
data: [
...qdb.data,
{
Name: subjName,
Questions: [...newQuestions],
},
],
}
} else {
return qdb
}
})
}
logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1)
// parentPort.postMessage({
// msg: `From thread #${workerIndex}: update done`,
// workerIndex: workerIndex,
// })
// console.log(`[THREAD #${workerIndex}]: update`)
} else if (msg.type === 'newdb') {

33
src/utils/workerPool.ts
View file

@ -20,6 +20,30 @@ function handleWorkerError(client, err) {
pool.destroy(client)
}
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: any): Promise<any> {
console.log('MSGING ALL WORKER')
return new Promise((resolve) => {
const promises = []
workers.forEach((worker) => {
worker.postMessage(data)
console.log('MSGD')
promises.push(
new Promise((resolve) => {
worker.once('message', (msg) => {
console.log(worker.index, 'ONCE MESSASGE RESOLVE')
resolve(msg)
})
})
)
})
Promise.all(promises).then((res) => {
console.log('MSG ALL DONE', res)
resolve(res)
})
})
}
export function doALongTask(obj: any): Promise<any> {
if (pool.pending > alertOnPendingCount) {
logger.Log(
@ -91,13 +115,6 @@ export function initWorkerPool(initData: any): void {
pool = genericPool.createPool(factory, opts)
}
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: any): void {
workers.forEach((worker) => {
worker.postMessage(data)
})
}
// ---------------------------------------------------------------------------
function getAWorker(i, initData) {
@ -140,10 +157,10 @@ function doSomething(client, obj) {
const { /* index, */ worker } = client
return new Promise((resolve) => {
// console.log('[ACCUIRE]: #' + index)
worker.postMessage(obj)
worker.once('message', (msg) => {
resolve(msg)
})
worker.postMessage(obj)
})
}