diff --git a/package-lock.json b/package-lock.json index 9440e1a..f7bc726 100644 --- a/package-lock.json +++ b/package-lock.json @@ -119,9 +119,9 @@ } }, "@eslint/eslintrc": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz", - "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==", + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.1.tgz", + "integrity": "sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA==", "requires": { "ajv": "^6.12.4", "debug": "^4.1.1", @@ -136,9 +136,9 @@ }, "dependencies": { "ajv": { - "version": "6.12.5", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz", - "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==", + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -147,9 +147,9 @@ } }, "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "requires": { "ms": "2.1.2" } @@ -174,11 +174,6 @@ } } }, - "@types/color-name": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", - "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" - }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -194,9 +189,9 @@ } }, "acorn": { - "version": "7.4.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", - "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==" + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==" }, "acorn-jsx": { "version": "5.3.1", @@ -673,12 +668,12 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, "eslint": { - "version": "7.10.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz", - "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==", + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.14.0.tgz", + "integrity": "sha512-5YubdnPXrlrYAFCKybPuHIAH++PINe1pmKNc5wQRB9HSbqIK1ywAnntE3Wwua4giKu0bjligf1gLF6qxMGOYRA==", "requires": { "@babel/code-frame": "^7.0.0", - "@eslint/eslintrc": "^0.1.3", + "@eslint/eslintrc": "^0.2.1", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", @@ -687,7 +682,7 @@ "enquirer": "^2.3.5", "eslint-scope": "^5.1.1", "eslint-utils": "^2.1.0", - "eslint-visitor-keys": "^1.3.0", + "eslint-visitor-keys": "^2.0.0", "espree": "^7.3.0", "esquery": "^1.2.0", "esutils": "^2.0.2", @@ -722,11 +717,10 @@ "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" }, "ansi-styles": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", - "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "requires": { - "@types/color-name": "^1.1.1", "color-convert": "^2.0.1" } }, @@ -753,13 +747,18 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "debug": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", - "integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", "requires": { "ms": "2.1.2" } }, + "eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==" + }, "globals": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", @@ -1187,9 +1186,9 @@ } }, "import-fresh": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", - "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", "requires": { "parent-module": "^1.0.0", "resolve-from": "^4.0.0" @@ -2226,9 +2225,9 @@ "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==" }, "v8-compile-cache": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", - "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==" + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz", + "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==" }, "vary": { "version": "1.1.2", diff --git a/package.json b/package.json index 89ff841..bd9559d 100755 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "cookie-parser": "^1.4.5", "cors": "^2.8.5", "ejs": "^1.0.0", - "eslint": "^7.10.0", + "eslint": "^7.14.0", "express": "^4.6.1", "express-ejs-layouts": "^1.1.0", "sqlite3": "^4.1.1", diff --git a/src/modules/api/api.js b/src/modules/api/api.js index 7581bda..517ffcf 100644 --- a/src/modules/api/api.js +++ b/src/modules/api/api.js @@ -68,7 +68,10 @@ function GetApp() { // files in public dirs const recivedFiles = publicDir + 'recivedfiles' const uloadFiles = publicDir + 'f' - const dataFile = publicDir + 'data.json' + const dataFiles = [ + { path: `${publicDir}oldData.json`, name: 'oldData' }, + { path: `${publicDir}data.json`, name: 'newData' }, + ] const motdFile = publicDir + 'motd' const userSpecificMotdFile = publicDir + 'userSpecificMotd.json' const versionFile = publicDir + 'version' @@ -113,7 +116,7 @@ function GetApp() { }) ) - var data = actions.LoadJSON(dataFile) + var questionDbs = actions.LoadJSON(dataFiles) var version = '' var motd = '' var userSpecificMotd = {} @@ -187,7 +190,7 @@ function GetApp() { } function Load() { - actions.backupData(data) + actions.backupData(questionDbs) utils.WatchFile(userSpecificMotdFile, () => { logger.Log(`User Specific Motd updated`, logger.GetColor('green')) @@ -800,7 +803,11 @@ function GetApp() { app.get('/allqr.txt', function(req, res) { res.set('Content-Type', 'text/plain') - res.send(dataToString(data)) + const stringifiedData = questionDbs.map((qdb) => { + return dataToString(qdb.data) + }) + // TODO: test this + res.send(stringifiedData.join('\n\n')) res.end() logger.LogReq(req) }) @@ -881,18 +888,19 @@ function GetApp() { ) // making backup - utils.CopyFile( - './' + dataFile, - `./publicDirs/qminingPublic/backs/data_before_${ - user.name - }_${utils.GetDateString().replace(/ /g, '_')}` - ) // TODO: rewrite to dinamyc public!!! - logger.Log('Backup made') - // writing data - utils.WriteFile(JSON.stringify(newData), dataFile) - logger.Log('New data file written') - // reloading data file - data = [...newData] + // TODO + // utils.CopyFile( + // './' + dataFile, + // `./publicDirs/qminingPublic/backs/data_before_${ + // user.name + // }_${utils.GetDateString().replace(/ /g, '_')}` + // ) // TODO: rewrite to dinamyc public!!! + // logger.Log('Backup made') + // // writing data + // utils.WriteFile(JSON.stringify(newData), dataFile) + // logger.Log('New data file written') + // // reloading data file + // data = [...newData] // data = newData logger.Log('Data set to newData') @@ -920,7 +928,7 @@ function GetApp() { actions .ProcessIncomingRequest( req.body.datatoadd || req.body, - data, + questionDbs, dryRun, user ) diff --git a/src/utils/actions.js b/src/utils/actions.js index 08913f0..713552a 100755 --- a/src/utils/actions.js +++ b/src/utils/actions.js @@ -23,7 +23,6 @@ module.exports = { backupData: backupData, } -const dataFile = './publicDirs/qminingPublic/data.json' const recDataFile = './stats/recdata' const dataLockFile = './data/lockData' @@ -39,7 +38,17 @@ const minMatchToAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate var currWrites = 0 -function ProcessIncomingRequest(recievedData, qdb, dryRun, user) { +// TODO: function that runs ProcessINcReq for every questiondb + +function ProcessIncomingRequest(recievedData, questionDbs, dryRun, user) { + return Promise.all( + questionDbs.map((qdb) => { + return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) + }) + ) +} + +function ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) { return new Promise((resolve, reject) => { logger.DebugLog('Processing incoming request', 'actions', 1) @@ -102,7 +111,9 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) { ) logger.DebugLog(currentQuestion, 'actions', 3) recievedQuestions.push(currentQuestion) - questionSearchPromises.push(searchData(qdb, currentQuestion, data.subj)) + questionSearchPromises.push( + searchData(qdb.data, currentQuestion, data.subj) + ) }) Promise.all(questionSearchPromises) @@ -131,7 +142,7 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) { 3 ) logger.DebugLog(currentQuestion, 'actions', 3) - addQuestion(qdb, sName, currentQuestion) + addQuestion(qdb.data, sName, currentQuestion) }) currWrites++ @@ -143,7 +154,7 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) { if (currWrites >= writeAfter && !dryRun) { currWrites = 0 logger.DebugLog('Writing data.json', 'actions', 1) - utils.WriteFile(JSON.stringify(qdb), dataFile) + utils.WriteFile(JSON.stringify(qdb.data), qdb.path) logger.Log('\tData file written', color) } else if (dryRun) { logger.Log('\tDry run') @@ -197,30 +208,45 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) { }) } -// loading stuff -function LoadJSON(dataFile) { - var data = JSON.parse(utils.ReadFile(dataFile)) - if (!data) { - logger.Log( - "data is undefined! Couldn't load data!", - logger.GetColor('redbg') - ) - } - return data +function LoadJSON(dataFiles) { + return dataFiles.reduce((acc, dataFile) => { + if (!utils.FileExists(dataFile.path)) { + utils.WriteFile(JSON.stringify([]), dataFile.path) + } + + try { + acc.push({ + ...dataFile, + data: JSON.parse(utils.ReadFile(dataFile.path)), + }) + } catch (err) { + console.error(err) + logger.Log( + "data is undefined! Couldn't load data!", + logger.GetColor('redbg') + ) + } + return acc + }, []) } -function backupData(data) { - const path = './publicDirs/qminingPublic/backs/' - utils.CreatePath(path) - try { - logger.Log('Backing up data...') - utils.WriteFile( - JSON.stringify(data), - `${path}data_${utils.GetDateString(true)}.json` - ) - logger.Log('Done') - } catch (err) { - logger.Log('Error backing up data file!', logger.GetColor('redbg')) - console.error(err) - } +function backupData(questionDbs) { + questionDbs.forEach((data) => { + const path = './publicDirs/qminingPublic/backs/' + utils.CreatePath(path) + try { + logger.Log(`Backing up ${data.name}...`) + utils.WriteFile( + JSON.stringify(data.data), + `${path}${data.name}_${utils.GetDateString(true)}.json` + ) + logger.Log('Done') + } catch (err) { + logger.Log( + `Error backing up data file ${data.name}!`, + logger.GetColor('redbg') + ) + console.error(err) + } + }) } diff --git a/submodules/qmining-page b/submodules/qmining-page index 12affac..4a7ad2d 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 12affac1a279b20fe57dc5a7eb44fb799b8cc6e9 +Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3