Actions rewrite to handle multiple data

package-lock.json

@@ -119,9 +119,9 @@
       }
     },
     "@eslint/eslintrc": {
-      "version": "0.1.3",
+      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.1.tgz",
-      "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==",
+      "integrity": "sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA==",
       "requires": {
         "ajv": "^6.12.4",
         "debug": "^4.1.1",
@@ -136,9 +136,9 @@
       },
       "dependencies": {
         "ajv": {
-          "version": "6.12.5",
+          "version": "6.12.6",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-          "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
+          "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
           "requires": {
             "fast-deep-equal": "^3.1.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -147,9 +147,9 @@
           }
         },
         "debug": {
-          "version": "4.2.0",
+          "version": "4.3.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
-          "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
+          "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
           "requires": {
             "ms": "2.1.2"
           }
@@ -174,11 +174,6 @@
         }
       }
     },
-    "@types/color-name": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
-      "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
-    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -194,9 +189,9 @@
       }
     },
     "acorn": {
-      "version": "7.4.0",
+      "version": "7.4.1",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
-      "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w=="
+      "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="
     },
     "acorn-jsx": {
       "version": "5.3.1",
@@ -673,12 +668,12 @@
       "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
     },
     "eslint": {
-      "version": "7.10.0",
+      "version": "7.14.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.14.0.tgz",
-      "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==",
+      "integrity": "sha512-5YubdnPXrlrYAFCKybPuHIAH++PINe1pmKNc5wQRB9HSbqIK1ywAnntE3Wwua4giKu0bjligf1gLF6qxMGOYRA==",
       "requires": {
         "@babel/code-frame": "^7.0.0",
-        "@eslint/eslintrc": "^0.1.3",
+        "@eslint/eslintrc": "^0.2.1",
         "ajv": "^6.10.0",
         "chalk": "^4.0.0",
         "cross-spawn": "^7.0.2",
@@ -687,7 +682,7 @@
         "enquirer": "^2.3.5",
         "eslint-scope": "^5.1.1",
         "eslint-utils": "^2.1.0",
-        "eslint-visitor-keys": "^1.3.0",
+        "eslint-visitor-keys": "^2.0.0",
         "espree": "^7.3.0",
         "esquery": "^1.2.0",
         "esutils": "^2.0.2",
@@ -722,11 +717,10 @@
           "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
         },
         "ansi-styles": {
-          "version": "4.2.1",
+          "version": "4.3.0",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
-          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
           "requires": {
-            "@types/color-name": "^1.1.1",
             "color-convert": "^2.0.1"
           }
         },
@@ -753,13 +747,18 @@
           "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
         },
         "debug": {
-          "version": "4.2.0",
+          "version": "4.3.1",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
-          "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
+          "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
           "requires": {
             "ms": "2.1.2"
           }
         },
+        "eslint-visitor-keys": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
+          "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ=="
+        },
         "globals": {
           "version": "12.4.0",
           "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
@@ -1187,9 +1186,9 @@
       }
     },
     "import-fresh": {
-      "version": "3.2.1",
+      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz",
-      "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==",
+      "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==",
       "requires": {
         "parent-module": "^1.0.0",
         "resolve-from": "^4.0.0"
@@ -2226,9 +2225,9 @@
       "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg=="
     },
     "v8-compile-cache": {
-      "version": "2.1.1",
+      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz",
-      "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ=="
+      "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q=="
     },
     "vary": {
       "version": "1.1.2",

package.json

@@ -8,7 +8,7 @@
     "cookie-parser": "^1.4.5",
     "cors": "^2.8.5",
     "ejs": "^1.0.0",
-    "eslint": "^7.10.0",
+    "eslint": "^7.14.0",
     "express": "^4.6.1",
     "express-ejs-layouts": "^1.1.0",
     "sqlite3": "^4.1.1",

src/modules/api/api.js

@@ -68,7 +68,10 @@ function GetApp() {
   // files in public dirs
   const recivedFiles = publicDir + 'recivedfiles'
   const uloadFiles = publicDir + 'f'
-  const dataFile = publicDir + 'data.json'
+  const dataFiles = [
+    { path: `${publicDir}oldData.json`, name: 'oldData' },
+    { path: `${publicDir}data.json`, name: 'newData' },
+  ]
   const motdFile = publicDir + 'motd'
   const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
   const versionFile = publicDir + 'version'
@@ -113,7 +116,7 @@ function GetApp() {
     })
   )
 
-  var data = actions.LoadJSON(dataFile)
+  var questionDbs = actions.LoadJSON(dataFiles)
   var version = ''
   var motd = ''
   var userSpecificMotd = {}
@@ -187,7 +190,7 @@ function GetApp() {
   }
 
   function Load() {
-    actions.backupData(data)
+    actions.backupData(questionDbs)
 
     utils.WatchFile(userSpecificMotdFile, () => {
       logger.Log(`User Specific Motd updated`, logger.GetColor('green'))
@@ -800,7 +803,11 @@ function GetApp() {
 
   app.get('/allqr.txt', function(req, res) {
     res.set('Content-Type', 'text/plain')
-    res.send(dataToString(data))
+    const stringifiedData = questionDbs.map((qdb) => {
+      return dataToString(qdb.data)
+    })
+    // TODO: test this
+    res.send(stringifiedData.join('\n\n'))
     res.end()
     logger.LogReq(req)
   })
@@ -881,18 +888,19 @@ function GetApp() {
       )
 
       // making backup
-      utils.CopyFile(
+      // TODO
-        './' + dataFile,
+      // utils.CopyFile(
-        `./publicDirs/qminingPublic/backs/data_before_${
+      //   './' + dataFile,
-          user.name
+      //   `./publicDirs/qminingPublic/backs/data_before_${
-        }_${utils.GetDateString().replace(/ /g, '_')}`
+      //     user.name
-      ) // TODO: rewrite to dinamyc public!!!
+      //   }_${utils.GetDateString().replace(/ /g, '_')}`
-      logger.Log('Backup made')
+      // ) // TODO: rewrite to dinamyc public!!!
-      // writing data
+      // logger.Log('Backup made')
-      utils.WriteFile(JSON.stringify(newData), dataFile)
+      // // writing data
-      logger.Log('New data file written')
+      // utils.WriteFile(JSON.stringify(newData), dataFile)
-      // reloading data file
+      // logger.Log('New data file written')
-      data = [...newData]
+      // // reloading data file
+      // data = [...newData]
       // data = newData
       logger.Log('Data set to newData')
 
@@ -920,7 +928,7 @@ function GetApp() {
     actions
       .ProcessIncomingRequest(
         req.body.datatoadd || req.body,
-        data,
+        questionDbs,
         dryRun,
         user
       )

src/utils/actions.js

@@ -23,7 +23,6 @@ module.exports = {
   backupData: backupData,
 }
 
-const dataFile = './publicDirs/qminingPublic/data.json'
 const recDataFile = './stats/recdata'
 const dataLockFile = './data/lockData'
 
@@ -39,7 +38,17 @@ const minMatchToAmmountToAdd = 90
 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
 var currWrites = 0
 
-function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
+// TODO: function that runs ProcessINcReq for every questiondb
+
+function ProcessIncomingRequest(recievedData, questionDbs, dryRun, user) {
+  return Promise.all(
+    questionDbs.map((qdb) => {
+      return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user)
+    })
+  )
+}
+
+function ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) {
   return new Promise((resolve, reject) => {
     logger.DebugLog('Processing incoming request', 'actions', 1)
 
@@ -102,7 +111,9 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
         )
         logger.DebugLog(currentQuestion, 'actions', 3)
         recievedQuestions.push(currentQuestion)
-        questionSearchPromises.push(searchData(qdb, currentQuestion, data.subj))
+        questionSearchPromises.push(
+          searchData(qdb.data, currentQuestion, data.subj)
+        )
       })
 
       Promise.all(questionSearchPromises)
@@ -131,7 +142,7 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
                   3
                 )
                 logger.DebugLog(currentQuestion, 'actions', 3)
-                addQuestion(qdb, sName, currentQuestion)
+                addQuestion(qdb.data, sName, currentQuestion)
               })
 
               currWrites++
@@ -143,7 +154,7 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
               if (currWrites >= writeAfter && !dryRun) {
                 currWrites = 0
                 logger.DebugLog('Writing data.json', 'actions', 1)
-                utils.WriteFile(JSON.stringify(qdb), dataFile)
+                utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
                 logger.Log('\tData file written', color)
               } else if (dryRun) {
                 logger.Log('\tDry run')
@@ -197,30 +208,45 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
   })
 }
 
-// loading stuff
+function LoadJSON(dataFiles) {
-function LoadJSON(dataFile) {
+  return dataFiles.reduce((acc, dataFile) => {
-  var data = JSON.parse(utils.ReadFile(dataFile))
+    if (!utils.FileExists(dataFile.path)) {
-  if (!data) {
+      utils.WriteFile(JSON.stringify([]), dataFile.path)
+    }
+
+    try {
+      acc.push({
+        ...dataFile,
+        data: JSON.parse(utils.ReadFile(dataFile.path)),
+      })
+    } catch (err) {
+      console.error(err)
       logger.Log(
         "data is undefined! Couldn't load data!",
         logger.GetColor('redbg')
       )
     }
-  return data
+    return acc
+  }, [])
 }
 
-function backupData(data) {
+function backupData(questionDbs) {
+  questionDbs.forEach((data) => {
     const path = './publicDirs/qminingPublic/backs/'
     utils.CreatePath(path)
     try {
-    logger.Log('Backing up data...')
+      logger.Log(`Backing up ${data.name}...`)
       utils.WriteFile(
-      JSON.stringify(data),
+        JSON.stringify(data.data),
-      `${path}data_${utils.GetDateString(true)}.json`
+        `${path}${data.name}_${utils.GetDateString(true)}.json`
       )
       logger.Log('Done')
     } catch (err) {
-    logger.Log('Error backing up data file!', logger.GetColor('redbg'))
+      logger.Log(
+        `Error backing up data file ${data.name}!`,
+        logger.GetColor('redbg')
+      )
       console.error(err)
     }
+  })
 }

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 12affac1a279b20fe57dc5a7eb44fb799b8cc6e9
+Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3