worker db edit/update refactor

src/modules/api/submodules/qminingapi.ts

@@ -14,14 +14,13 @@ import {
   logResult,
   shouldSaveDataFile,
   Result,
-  isQuestionValid,
   backupData,
   shouldSearchDataFile,
   loadJSON,
   writeData,
+  editDb,
 } from '../../../utils/actions'
 import {
-  createQuestion,
   dataToString,
   getSubjNameWithoutYear,
   // compareQuestionObj,
@@ -524,7 +523,7 @@ function setup(data: SubmoduleData): any {
           logResult(req.body, resultArray, user.id, dryRun)
 
           const totalNewQuestions = resultArray.reduce((acc, sres) => {
-            return acc + sres.newQuestions
+            return acc + sres.newQuestions.length
           }, 0)
 
           res.json({
@@ -534,9 +533,11 @@ function setup(data: SubmoduleData): any {
           })
 
           if (totalNewQuestions > 0) {
+            resultArray.forEach((result) => {
               msgAllWorker({
-              qdbs: questionDbs,
+                type: 'newQuestions',
-              type: 'update',
+                data: result,
+              })
             })
           }
         })
@@ -842,7 +843,6 @@ function setup(data: SubmoduleData): any {
     logger.LogReq(req)
     const user: User = req.session.user
     const date = utils.GetDateString()
-    let saveDb = false
 
     const editType = req.body.type
     const selectedDb = req.body.selectedDb
@@ -867,44 +867,28 @@ function setup(data: SubmoduleData): any {
       return
     }
 
-    // {
+    // -----------------
-    //   "index": 0,
+    const {
-    //   "subjName": "VHDL programozás",
+      success,
-    //   "type": "delete",
+      msg,
-    //   "selectedDb": {
+      resultDb,
-    //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+      deletedQuestion,
-    //     "name": "elearning.uni-obuda.hu"
+      newVal,
-    //   }
+      oldVal,
-    // }
+      deletedQuestions,
-    if (editType === 'delete') {
+      changedQuestions,
-      const { index, subjName } = req.body
+    } = editDb(currDb, req.body)
-      let deletedQuestion = {}
+
-      if (isNaN(index) || !subjName) {
+    if (!success) {
-        res.json({
+      res.json({ success: success, msg: msg })
-          status: 'fail',
-          msg: 'No .index or .subjName !',
-        })
       return
     }
+    if (resultDb) {
+      questionDbs[dbIndex] = resultDb
+    }
 
-      questionDbs[dbIndex].data = currDb.data.map((subj) => {
+    if (editType === 'delete') {
-        if (subj.Name !== subjName) {
+      const { index, subjName } = req.body
-          return subj
-        } else {
-          return {
-            ...subj,
-            Questions: subj.Questions.filter((question, i) => {
-              if (index === i) {
-                deletedQuestion = question
-                return false
-              } else {
-                return true
-              }
-            }),
-          }
-        }
-      })
-
       logger.Log(
         `User #${user.id} deleted a question from '${subjName}'`,
         logger.GetColor('cyan')
@@ -914,68 +898,10 @@ function setup(data: SubmoduleData): any {
         dataEditsLog
       )
       utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog)
-      saveDb = true
     }
 
-    // {
-    //   "index": 0,
-    //   "subjName": "Elektronika",
-    //   "type": "edit",
-    //   "newVal": {
-    //     "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.",
-    //     "A": "Igaz",
-    //     "data": {
-    //       "type": "simple",
-    //       "possibleAnswers": [
-    //         "Igaz"
-    //       ]
-    //     },
-    //     "possibleAnswers": [
-    //       "Igaz"
-    //     ]
-    //   },
-    //   "selectedDb": {
-    //     "path": "questionDbs/elearning.uni-obuda.hu.json",
-    //     "name": "elearning.uni-obuda.hu"
-    //   }
-    // }
     if (editType === 'edit') {
-      const { index, subjName, newVal } = req.body
+      const { index, subjName } = req.body
-      let oldVal = {}
-      if (isNaN(index) || !subjName) {
-        res.json({
-          status: 'fail',
-          msg: 'No .index or .subjName !',
-        })
-        return
-      }
-      if (!isQuestionValid(newVal)) {
-        res.json({
-          status: 'fail',
-          msg: 'edited question is not valid',
-          question: newVal,
-        })
-        return
-      }
-
-      questionDbs[dbIndex].data = currDb.data.map((subj) => {
-        if (subj.Name !== subjName) {
-          return subj
-        } else {
-          return {
-            ...subj,
-            Questions: subj.Questions.map((question, i) => {
-              if (index === i) {
-                oldVal = question
-                return createQuestion(newVal)
-              } else {
-                return question
-              }
-            }),
-          }
-        }
-      })
-
       logger.Log(
         `User #${user.id} edited a question in '${subjName}'`,
         logger.GetColor('cyan')
@@ -995,128 +921,48 @@ function setup(data: SubmoduleData): any {
         ),
         dataEditsLog
       )
-      saveDb = true
     }
 
-    // {
-    //   "subjName": "Elektronika",
-    //   "changedQuestions": [
-    //     {
-    //       "index": 1,
-    //       "value": {
-    //         "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.",
-    //         "A": "Hamis",
-    //         "data": {
-    //           "type": "simple",
-    //           "possibleAnswers": [
-    //             "Igaz",
-    //             "Hamis"
-    //           ]
-    //         }
-    //       }
-    //     }
-    //   ],
-    //   "deletedQuestions": [
-    //     0
-    //   ],
-    //   "type": "subjEdit",
-    //   "selectedDb": {
-    //     "path": "questionDbs/elearning.uni-obuda.hu.json",
-    //     "name": "elearning.uni-obuda.hu"
-    //   }
-    // }
     if (editType === 'subjEdit') {
-      const { subjName, changedQuestions, deletedQuestions } = req.body
+      const { subjName } = req.body
-      const deletedQuestionsToWrite = []
-      const changedQuestionsToWrite = []
-      if (
-        !Array.isArray(changedQuestions) ||
-        !Array.isArray(deletedQuestions)
-      ) {
-        res.json({
-          status: 'fail',
-          msg: 'no changedQuestions or deletedQuestions!',
-        })
-        return
-      }
-
-      // processing changed questions
-      questionDbs[dbIndex].data = currDb.data.map((subj) => {
-        if (subj.Name !== subjName) {
-          return subj
-        } else {
-          return {
-            ...subj,
-            Questions: subj.Questions.map((question, i) => {
-              const changedTo = changedQuestions.find((cq) => {
-                return cq.index === i
-              })
-              if (changedTo) {
-                changedQuestionsToWrite.push({
-                  oldVal: question,
-                  newVal: changedTo.value,
-                })
-                return createQuestion(changedTo.value)
-              } else {
-                return question
-              }
-            }),
-          }
-        }
-      })
-
-      // processing deletedQuestions
-      questionDbs[dbIndex].data = currDb.data.map((subj) => {
-        if (subj.Name !== subjName) {
-          return subj
-        } else {
-          return {
-            ...subj,
-            Questions: subj.Questions.filter((question, i) => {
-              const isDeleted = deletedQuestions.includes(i)
-              if (isDeleted) {
-                deletedQuestionsToWrite.push(question)
-                return false
-              } else {
-                return true
-              }
-            }),
-          }
-        }
-      })
-
       logger.Log(
-        `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`,
+        `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
         logger.GetColor('cyan')
       )
       utils.AppendToFile(
-        `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`,
+        `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
         dataEditsLog
       )
       utils.AppendToFile(
         JSON.stringify(
           {
-            deletedQuestions: deletedQuestionsToWrite,
+            deletedQuestions: deletedQuestions,
-            changedQuestions: changedQuestionsToWrite,
+            changedQuestions: changedQuestions,
           },
           null,
           2
         ),
         dataEditsLog
       )
-      saveDb = true
     }
+    // ------------------
 
-    if (saveDb) {
+    if (success) {
       writeData(currDb.data, currDb.path)
       msgAllWorker({
-        qdbs: questionDbs,
+        type: 'dbEdit',
-        type: 'update',
+        data: {
+          dbIndex: dbIndex,
+          edits: req.body,
+        },
+      }).then((res) => {
+        console.log('AAAAAAAAAAA', res)
       })
     }
 
     res.json({
-      status: 'OK',
+      success: true,
+      msg: 'OK',
     })
   })
 

src/utils/actions.ts

@@ -53,8 +53,10 @@ export interface RecievedData {
 }
 
 export interface Result {
+  qdbIndex: number
   qdbName: string
-  newQuestions: number
+  subjName: string
+  newQuestions: Array<Question>
 }
 
 export function logResult(
@@ -84,9 +86,9 @@ export function logResult(
       const allQLength = recievedData.quiz.length
       let msg = `${res.qdbName}: `
       let color = logger.GetColor('green')
-      if (res.newQuestions > 0) {
+      if (res.newQuestions.length > 0) {
         color = logger.GetColor('blue')
-        msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
+        msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )`
       } else {
         msg += `No new data ( ${allQLength} )`
       }
@@ -241,7 +243,9 @@ function processIncomingRequestUsingDb(
 
             logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1)
             resolve({
-              newQuestions: allQuestions.length,
+              newQuestions: allQuestions,
+              subjName: recievedData.subj,
+              qdbIndex: qdb.index,
               qdbName: qdb.name,
             })
           } catch (error) {
@@ -439,3 +443,288 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
     }
   })
 }
+
+function deleteFromDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    type: string
+    selectedDb: { path: string; name: string }
+  }
+): {
+  success: Boolean
+  msg: string
+  deletedQuestion?: Question
+  resultDb?: QuestionDb
+} {
+  // {
+  //   "index": 0,
+  //   "subjName": "VHDL programozás",
+  //   "type": "delete",
+  //   "selectedDb": {
+  //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+  //     "name": "elearning.uni-obuda.hu"
+  //   }
+  // }
+  const { index, subjName } = edits
+  let deletedQuestion
+  if (isNaN(index) || !subjName) {
+    return {
+      success: false,
+      msg: 'No .index or .subjName !',
+    }
+  }
+
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.filter((question, i) => {
+          if (index === i) {
+            deletedQuestion = question
+            return false
+          } else {
+            return true
+          }
+        }),
+      }
+    }
+  })
+
+  return {
+    success: true,
+    msg: 'Delete successfull',
+    deletedQuestion: deletedQuestion,
+    resultDb: questionDb,
+  }
+}
+
+function editQuestionInDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    type: string
+    selectedDb: { path: string; name: string }
+    newVal?: Question
+  }
+): {
+  success: Boolean
+  msg: string
+  newVal?: Question
+  oldVal?: Question
+  resultDb?: QuestionDb
+} {
+  // {
+  //   "index": 0,
+  //   "subjName": "Elektronika",
+  //   "type": "edit",
+  //   "newVal": {
+  //     "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.",
+  //     "A": "Igaz",
+  //     "data": {
+  //       "type": "simple",
+  //       "possibleAnswers": [
+  //         "Igaz"
+  //       ]
+  //     },
+  //     "possibleAnswers": [
+  //       "Igaz"
+  //     ]
+  //   },
+  //   "selectedDb": {
+  //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+  //     "name": "elearning.uni-obuda.hu"
+  //   }
+  // }
+  const { index, subjName, newVal } = edits
+  let oldVal
+  if (isNaN(index) || !subjName) {
+    return {
+      success: false,
+      msg: 'No .index or .subjName !',
+    }
+  }
+  if (!isQuestionValid(newVal)) {
+    return {
+      success: false,
+      msg: 'edited question is not valid',
+    }
+  }
+
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.map((question, i) => {
+          if (index === i) {
+            oldVal = question
+            return createQuestion(newVal)
+          } else {
+            return question
+          }
+        }),
+      }
+    }
+  })
+
+  return {
+    success: true,
+    msg: 'Edit successfull',
+    oldVal: oldVal,
+    newVal: newVal,
+    resultDb: questionDb,
+  }
+}
+
+function editSubjInDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    type: string
+    selectedDb: { path: string; name: string }
+    deletedQuestions?: Array<number>
+    changedQuestions?: Array<{
+      index: number
+      value: Question
+    }>
+  }
+): {
+  success: Boolean
+  msg: string
+  deletedQuestions?: Array<Question>
+  changedQuestions?: Array<Question>
+  resultDb?: QuestionDb
+} {
+  // {
+  //   "subjName": "Elektronika",
+  //   "changedQuestions": [
+  //     {
+  //       "index": 1,
+  //       "value": {
+  //         "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.",
+  //         "A": "Hamis",
+  //         "data": {
+  //           "type": "simple",
+  //           "possibleAnswers": [
+  //             "Igaz",
+  //             "Hamis"
+  //           ]
+  //         }
+  //       }
+  //     }
+  //   ],
+  //   "deletedQuestions": [
+  //     0
+  //   ],
+  //   "type": "subjEdit",
+  //   "selectedDb": {
+  //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+  //     "name": "elearning.uni-obuda.hu"
+  //   }
+  // }
+  const { subjName, changedQuestions, deletedQuestions } = edits
+  const deletedQuestionsToWrite = []
+  const changedQuestionsToWrite = []
+  if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) {
+    return {
+      success: false,
+      msg: 'no changedQuestions or deletedQuestions!',
+    }
+  }
+
+  // processing changed questions
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.map((question, i) => {
+          const changedTo = changedQuestions.find((cq) => {
+            return cq.index === i
+          })
+          if (changedTo) {
+            changedQuestionsToWrite.push({
+              oldVal: question,
+              newVal: changedTo.value,
+            })
+            return createQuestion(changedTo.value)
+          } else {
+            return question
+          }
+        }),
+      }
+    }
+  })
+
+  // processing deletedQuestions
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.filter((question, i) => {
+          const isDeleted = deletedQuestions.includes(i)
+          if (isDeleted) {
+            deletedQuestionsToWrite.push(question)
+            return false
+          } else {
+            return true
+          }
+        }),
+      }
+    }
+  })
+
+  return {
+    success: true,
+    msg: 'subj edit successfull',
+    deletedQuestions: deletedQuestionsToWrite,
+    changedQuestions: changedQuestionsToWrite,
+    resultDb: questionDb,
+  }
+}
+
+export function editDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    selectedDb: { path: string; name: string }
+    type: string
+    newVal?: Question
+    deletedQuestion?: Array<number>
+    changedQuestions?: Array<{
+      index: number
+      value: Question
+    }>
+  }
+): {
+  success: Boolean
+  msg: string
+  resultDb?: QuestionDb
+  deletedQuestion?: Question
+  newVal?: Question
+  oldVal?: Question
+  deletedQuestions?: Array<Question>
+  changedQuestions?: Array<Question>
+} {
+  if (edits.type === 'delete') {
+    return deleteFromDb(questionDb, edits)
+  }
+  if (edits.type === 'edit') {
+    return editQuestionInDb(questionDb, edits)
+  }
+
+  if (edits.type === 'subjEdit') {
+    return editSubjInDb(questionDb, edits)
+  }
+}

src/utils/classes.ts

@@ -1,6 +1,12 @@
 import { isMainThread, parentPort, workerData } from 'worker_threads'
 import logger from './logger'
-import { Question, QuestionData, Subject } from '../types/basicTypes'
+import {
+  Question,
+  QuestionData,
+  QuestionDb,
+  Subject,
+} from '../types/basicTypes'
+import { editDb } from './actions'
 
 interface SearchResultQuestion extends Question {
   match: number
@@ -581,7 +587,7 @@ function doSearch(
 
 if (!isMainThread) {
   const { workerIndex } = workerData
-  let qdbs: Array<any> = workerData.initData
+  let qdbs: Array<QuestionDb> = workerData.initData
 
   logger.Log(
     `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
@@ -670,9 +676,64 @@ if (!isMainThread) {
       //     !isNaN(index) ? `#${index}` : ''
       //   }done!`
       // )
-    } else if (msg.type === 'update') {
+    } else if (msg.type === 'dbEdit') {
-      qdbs = msg.qdbs
+      const { dbIndex, edits } = msg.data
-      logger.DebugLog(`Worker update ${workerIndex}`, 'worker update', 1)
+      const { resultDb } = editDb(qdbs[dbIndex], edits)
+      qdbs[dbIndex] = resultDb
+      logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1)
+
+      parentPort.postMessage({
+        msg: `From thread #${workerIndex}: db edit`,
+        workerIndex: workerIndex,
+      })
+    } else if (msg.type === 'newQuestions') {
+      const { subjName, qdbIndex, newQuestions } = msg.data
+      let added = false
+      qdbs = qdbs.map((qdb) => {
+        if (qdb.index === qdbIndex) {
+          return {
+            ...qdb,
+            data: qdb.data.map((subj) => {
+              if (subj.Name === subjName) {
+                added = true
+                return {
+                  Name: subj.Name,
+                  Questions: [...subj.Questions, ...newQuestions],
+                }
+              } else {
+                return subj
+              }
+            }),
+          }
+        } else {
+          return qdb
+        }
+      })
+
+      if (!added) {
+        qdbs = qdbs.map((qdb) => {
+          if (qdb.index === qdbIndex) {
+            return {
+              ...qdb,
+              data: [
+                ...qdb.data,
+                {
+                  Name: subjName,
+                  Questions: [...newQuestions],
+                },
+              ],
+            }
+          } else {
+            return qdb
+          }
+        })
+      }
+      logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1)
+
+      // parentPort.postMessage({
+      //   msg: `From thread #${workerIndex}: update done`,
+      //   workerIndex: workerIndex,
+      // })
 
       // console.log(`[THREAD #${workerIndex}]: update`)
     } else if (msg.type === 'newdb') {

src/utils/workerPool.ts

@@ -20,6 +20,30 @@ function handleWorkerError(client, err) {
   pool.destroy(client)
 }
 
+// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
+export function msgAllWorker(data: any): Promise<any> {
+  console.log('MSGING ALL WORKER')
+  return new Promise((resolve) => {
+    const promises = []
+    workers.forEach((worker) => {
+      worker.postMessage(data)
+      console.log('MSGD')
+      promises.push(
+        new Promise((resolve) => {
+          worker.once('message', (msg) => {
+            console.log(worker.index, 'ONCE MESSASGE RESOLVE')
+            resolve(msg)
+          })
+        })
+      )
+    })
+    Promise.all(promises).then((res) => {
+      console.log('MSG ALL DONE', res)
+      resolve(res)
+    })
+  })
+}
+
 export function doALongTask(obj: any): Promise<any> {
   if (pool.pending > alertOnPendingCount) {
     logger.Log(
@@ -91,13 +115,6 @@ export function initWorkerPool(initData: any): void {
   pool = genericPool.createPool(factory, opts)
 }
 
-// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
-export function msgAllWorker(data: any): void {
-  workers.forEach((worker) => {
-    worker.postMessage(data)
-  })
-}
-
 // ---------------------------------------------------------------------------
 
 function getAWorker(i, initData) {
@@ -140,10 +157,10 @@ function doSomething(client, obj) {
   const { /* index, */ worker } = client
   return new Promise((resolve) => {
     // console.log('[ACCUIRE]: #' + index)
-    worker.postMessage(obj)
     worker.once('message', (msg) => {
       resolve(msg)
     })
+    worker.postMessage(obj)
   })
 }