albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Actions rewrite to handle multiple data

Browse source
This commit is contained in:
mrfry 2020-11-23 10:08:05 +01:00
parent 677c320e90
commit 1acc122e26
5 changed files with 116 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

69
package-lock.json generated
View file

@ -119,9 +119,9 @@
} }
}, },
"@eslint/eslintrc": { "@eslint/eslintrc": {
"version": "0.1.3", "version": "0.2.1",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.1.tgz",
"integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==", "integrity": "sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA==",
"requires": { "requires": {
"ajv": "^6.12.4", "ajv": "^6.12.4",
"debug": "^4.1.1", "debug": "^4.1.1",
@ -136,9 +136,9 @@
}, },
"dependencies": { "dependencies": {
"ajv": { "ajv": {
"version": "6.12.5", "version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
"integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==", "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
"requires": { "requires": {
"fast-deep-equal": "^3.1.1", "fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0", "fast-json-stable-stringify": "^2.0.0",
@ -147,9 +147,9 @@
} }
}, },
"debug": { "debug": {
"version": "4.2.0", "version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": { "requires": {
"ms": "2.1.2" "ms": "2.1.2"
} }
@ -174,11 +174,6 @@
} }
} }
}, },
"@types/color-name": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
"integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
},
"abbrev": { "abbrev": {
"version": "1.1.1", "version": "1.1.1",
"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@ -194,9 +189,9 @@
} }
}, },
"acorn": { "acorn": {
"version": "7.4.0", "version": "7.4.1",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz",
"integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==" "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A=="
}, },
"acorn-jsx": { "acorn-jsx": {
"version": "5.3.1", "version": "5.3.1",
@ -673,12 +668,12 @@
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
}, },
"eslint": { "eslint": {
"version": "7.10.0", "version": "7.14.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz", "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.14.0.tgz",
"integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==", "integrity": "sha512-5YubdnPXrlrYAFCKybPuHIAH++PINe1pmKNc5wQRB9HSbqIK1ywAnntE3Wwua4giKu0bjligf1gLF6qxMGOYRA==",
"requires": { "requires": {
"@babel/code-frame": "^7.0.0", "@babel/code-frame": "^7.0.0",
"@eslint/eslintrc": "^0.1.3", "@eslint/eslintrc": "^0.2.1",
"ajv": "^6.10.0", "ajv": "^6.10.0",
"chalk": "^4.0.0", "chalk": "^4.0.0",
"cross-spawn": "^7.0.2", "cross-spawn": "^7.0.2",
@ -687,7 +682,7 @@
"enquirer": "^2.3.5", "enquirer": "^2.3.5",
"eslint-scope": "^5.1.1", "eslint-scope": "^5.1.1",
"eslint-utils": "^2.1.0", "eslint-utils": "^2.1.0",
"eslint-visitor-keys": "^1.3.0", "eslint-visitor-keys": "^2.0.0",
"espree": "^7.3.0", "espree": "^7.3.0",
"esquery": "^1.2.0", "esquery": "^1.2.0",
"esutils": "^2.0.2", "esutils": "^2.0.2",
@ -722,11 +717,10 @@
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==" "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
}, },
"ansi-styles": { "ansi-styles": {
"version": "4.2.1", "version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"requires": { "requires": {
"@types/color-name": "^1.1.1",
"color-convert": "^2.0.1" "color-convert": "^2.0.1"
} }
}, },
@ -753,13 +747,18 @@
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
}, },
"debug": { "debug": {
"version": "4.2.0", "version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==", "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": { "requires": {
"ms": "2.1.2" "ms": "2.1.2"
} }
}, },
"eslint-visitor-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
"integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ=="
},
"globals": { "globals": {
"version": "12.4.0", "version": "12.4.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
@ -1187,9 +1186,9 @@
} }
}, },
"import-fresh": { "import-fresh": {
"version": "3.2.1", "version": "3.2.2",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz",
"integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==",
"requires": { "requires": {
"parent-module": "^1.0.0", "parent-module": "^1.0.0",
"resolve-from": "^4.0.0" "resolve-from": "^4.0.0"
@ -2226,9 +2225,9 @@
"integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==" "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg=="
}, },
"v8-compile-cache": { "v8-compile-cache": {
"version": "2.1.1", "version": "2.2.0",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz",
"integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==" "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q=="
}, },
"vary": { "vary": {
"version": "1.1.2", "version": "1.1.2",

2
package.json
View file

@ -8,7 +8,7 @@
"cookie-parser": "^1.4.5", "cookie-parser": "^1.4.5",
"cors": "^2.8.5", "cors": "^2.8.5",
"ejs": "^1.0.0", "ejs": "^1.0.0",
"eslint": "^7.10.0", "eslint": "^7.14.0",
"express": "^4.6.1", "express": "^4.6.1",
"express-ejs-layouts": "^1.1.0", "express-ejs-layouts": "^1.1.0",
"sqlite3": "^4.1.1", "sqlite3": "^4.1.1",

42
src/modules/api/api.js
View file

@ -68,7 +68,10 @@ function GetApp() {
// files in public dirs // files in public dirs
const recivedFiles = publicDir + 'recivedfiles' const recivedFiles = publicDir + 'recivedfiles'
const uloadFiles = publicDir + 'f' const uloadFiles = publicDir + 'f'
const dataFile = publicDir + 'data.json' const dataFiles = [
{ path: `${publicDir}oldData.json`, name: 'oldData' },
{ path: `${publicDir}data.json`, name: 'newData' },
]
const motdFile = publicDir + 'motd' const motdFile = publicDir + 'motd'
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json' const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
const versionFile = publicDir + 'version' const versionFile = publicDir + 'version'
@ -113,7 +116,7 @@ function GetApp() {
}) })
) )
var data = actions.LoadJSON(dataFile) var questionDbs = actions.LoadJSON(dataFiles)
var version = '' var version = ''
var motd = '' var motd = ''
var userSpecificMotd = {} var userSpecificMotd = {}
@ -187,7 +190,7 @@ function GetApp() {
} }
function Load() { function Load() {
actions.backupData(data) actions.backupData(questionDbs)
utils.WatchFile(userSpecificMotdFile, () => { utils.WatchFile(userSpecificMotdFile, () => {
logger.Log(`User Specific Motd updated`, logger.GetColor('green')) logger.Log(`User Specific Motd updated`, logger.GetColor('green'))
@ -800,7 +803,11 @@ function GetApp() {
app.get('/allqr.txt', function(req, res) { app.get('/allqr.txt', function(req, res) {
res.set('Content-Type', 'text/plain') res.set('Content-Type', 'text/plain')
res.send(dataToString(data)) const stringifiedData = questionDbs.map((qdb) => {
return dataToString(qdb.data)
})
// TODO: test this
res.send(stringifiedData.join('\n\n'))
res.end() res.end()
logger.LogReq(req) logger.LogReq(req)
}) })
@ -881,18 +888,19 @@ function GetApp() {
) )
// making backup // making backup
utils.CopyFile( // TODO
'./' + dataFile, // utils.CopyFile(
`./publicDirs/qminingPublic/backs/data_before_${ // './' + dataFile,
user.name // `./publicDirs/qminingPublic/backs/data_before_${
}_${utils.GetDateString().replace(/ /g, '_')}` // user.name
) // TODO: rewrite to dinamyc public!!! // }_${utils.GetDateString().replace(/ /g, '_')}`
logger.Log('Backup made') // ) // TODO: rewrite to dinamyc public!!!
// writing data // logger.Log('Backup made')
utils.WriteFile(JSON.stringify(newData), dataFile) // // writing data
logger.Log('New data file written') // utils.WriteFile(JSON.stringify(newData), dataFile)
// reloading data file // logger.Log('New data file written')
data = [...newData] // // reloading data file
// data = [...newData]
// data = newData // data = newData
logger.Log('Data set to newData') logger.Log('Data set to newData')
@ -920,7 +928,7 @@ function GetApp() {
actions actions
.ProcessIncomingRequest( .ProcessIncomingRequest(
req.body.datatoadd || req.body, req.body.datatoadd || req.body,
data, questionDbs,
dryRun, dryRun,
user user
) )

84
src/utils/actions.js
View file

@ -23,7 +23,6 @@ module.exports = {
backupData: backupData, backupData: backupData,
} }
const dataFile = './publicDirs/qminingPublic/data.json'
const recDataFile = './stats/recdata' const recDataFile = './stats/recdata'
const dataLockFile = './data/lockData' const dataLockFile = './data/lockData'
@ -39,7 +38,17 @@ const minMatchToAmmountToAdd = 90
const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
var currWrites = 0 var currWrites = 0
function ProcessIncomingRequest(recievedData, qdb, dryRun, user) { // TODO: function that runs ProcessINcReq for every questiondb
function ProcessIncomingRequest(recievedData, questionDbs, dryRun, user) {
return Promise.all(
questionDbs.map((qdb) => {
return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user)
})
)
}
function ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
logger.DebugLog('Processing incoming request', 'actions', 1) logger.DebugLog('Processing incoming request', 'actions', 1)
@ -102,7 +111,9 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
) )
logger.DebugLog(currentQuestion, 'actions', 3) logger.DebugLog(currentQuestion, 'actions', 3)
recievedQuestions.push(currentQuestion) recievedQuestions.push(currentQuestion)
questionSearchPromises.push(searchData(qdb, currentQuestion, data.subj)) questionSearchPromises.push(
searchData(qdb.data, currentQuestion, data.subj)
)
}) })
Promise.all(questionSearchPromises) Promise.all(questionSearchPromises)
@ -131,7 +142,7 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
3 3
) )
logger.DebugLog(currentQuestion, 'actions', 3) logger.DebugLog(currentQuestion, 'actions', 3)
addQuestion(qdb, sName, currentQuestion) addQuestion(qdb.data, sName, currentQuestion)
}) })
currWrites++ currWrites++
@ -143,7 +154,7 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
if (currWrites >= writeAfter && !dryRun) { if (currWrites >= writeAfter && !dryRun) {
currWrites = 0 currWrites = 0
logger.DebugLog('Writing data.json', 'actions', 1) logger.DebugLog('Writing data.json', 'actions', 1)
utils.WriteFile(JSON.stringify(qdb), dataFile) utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
logger.Log('\tData file written', color) logger.Log('\tData file written', color)
} else if (dryRun) { } else if (dryRun) {
logger.Log('\tDry run') logger.Log('\tDry run')
@ -197,30 +208,45 @@ function ProcessIncomingRequest(recievedData, qdb, dryRun, user) {
}) })
} }
// loading stuff function LoadJSON(dataFiles) {
function LoadJSON(dataFile) { return dataFiles.reduce((acc, dataFile) => {
var data = JSON.parse(utils.ReadFile(dataFile)) if (!utils.FileExists(dataFile.path)) {
if (!data) { utils.WriteFile(JSON.stringify([]), dataFile.path)
logger.Log( }
"data is undefined! Couldn't load data!",
logger.GetColor('redbg') try {
) acc.push({
} ...dataFile,
return data data: JSON.parse(utils.ReadFile(dataFile.path)),
})
} catch (err) {
console.error(err)
logger.Log(
"data is undefined! Couldn't load data!",
logger.GetColor('redbg')
)
}
return acc
}, [])
} }
function backupData(data) { function backupData(questionDbs) {
const path = './publicDirs/qminingPublic/backs/' questionDbs.forEach((data) => {
utils.CreatePath(path) const path = './publicDirs/qminingPublic/backs/'
try { utils.CreatePath(path)
logger.Log('Backing up data...') try {
utils.WriteFile( logger.Log(`Backing up ${data.name}...`)
JSON.stringify(data), utils.WriteFile(
`${path}data_${utils.GetDateString(true)}.json` JSON.stringify(data.data),
) `${path}${data.name}_${utils.GetDateString(true)}.json`
logger.Log('Done') )
} catch (err) { logger.Log('Done')
logger.Log('Error backing up data file!', logger.GetColor('redbg')) } catch (err) {
console.error(err) logger.Log(
} `Error backing up data file ${data.name}!`,
logger.GetColor('redbg')
)
console.error(err)
}
})
} }

2
submodules/qmining-page

@ -1 +1 @@
Subproject commit 12affac1a279b20fe57dc5a7eb44fb799b8cc6e9 Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3