diff --git a/package-lock.json b/package-lock.json index 36672ca..b8c8450 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "express": "^4.6.1", "express-ejs-layouts": "^1.1.0", "express-fileupload": "^1.2.1", - "generic-pool": "^3.7.1", + "queue-microtask": "^1.2.3", "sqlite3": "^4.1.1", "ts-node": "^9.0.0", "typescript": "^4.1.2", @@ -29,7 +29,7 @@ }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^4.8.1", - "@typescript-eslint/parser": "^4.8.1", + "@typescript-eslint/parser": "^4.22.0", "eslint": "^7.14.0" } }, @@ -410,18 +410,104 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-4.8.1.tgz", - "integrity": "sha512-QND8XSVetATHK9y2Ltc/XBl5Ro7Y62YuZKnPEwnNPB8E379fDsvzJ1dMJ46fg/VOmk0hXhatc+GXs5MaXuL5Uw==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-4.22.0.tgz", + "integrity": "sha512-z/bGdBJJZJN76nvAY9DkJANYgK3nlRstRRi74WHm3jjgf2I8AglrSY+6l7ogxOmn55YJ6oKZCLLy+6PW70z15Q==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "4.8.1", - "@typescript-eslint/types": "4.8.1", - "@typescript-eslint/typescript-estree": "4.8.1", + "@typescript-eslint/scope-manager": "4.22.0", + "@typescript-eslint/types": "4.22.0", + "@typescript-eslint/typescript-estree": "4.22.0", "debug": "^4.1.1" }, "engines": { "node": "^10.12.0 || >=12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^5.0.0 || ^6.0.0 || ^7.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/scope-manager": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.22.0.tgz", + "integrity": "sha512-OcCO7LTdk6ukawUM40wo61WdeoA7NM/zaoq1/2cs13M7GyiF+T4rxuA4xM+6LeHWjWbss7hkGXjFDRcKD4O04Q==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "4.22.0", + "@typescript-eslint/visitor-keys": "4.22.0" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/types": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.22.0.tgz", + "integrity": "sha512-sW/BiXmmyMqDPO2kpOhSy2Py5w6KvRRsKZnV0c4+0nr4GIcedJwXAq+RHNK4lLVEZAJYFltnnk1tJSlbeS9lYA==", + "dev": true, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/typescript-estree": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.22.0.tgz", + "integrity": "sha512-TkIFeu5JEeSs5ze/4NID+PIcVjgoU3cUQUIZnH3Sb1cEn1lBo7StSV5bwPuJQuoxKXlzAObjYTilOEKRuhR5yg==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "4.22.0", + "@typescript-eslint/visitor-keys": "4.22.0", + "debug": "^4.1.1", + "globby": "^11.0.1", + "is-glob": "^4.0.1", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/@typescript-eslint/parser/node_modules/@typescript-eslint/visitor-keys": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.22.0.tgz", + "integrity": "sha512-nnMu4F+s4o0sll6cBSsTeVsT4cwxB7zECK3dFxzEjPBii9xLpq4yqqsy/FU5zMfan6G60DKZSCXAa3sHJZrcYw==", + "dev": true, + "dependencies": { + "@typescript-eslint/types": "4.22.0", + "eslint-visitor-keys": "^2.0.0" + }, + "engines": { + "node": "^8.10.0 || ^10.13.0 || >=11.10.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" } }, "node_modules/@typescript-eslint/parser/node_modules/debug": { @@ -436,12 +522,36 @@ "node": ">=6.0" } }, + "node_modules/@typescript-eslint/parser/node_modules/eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/@typescript-eslint/parser/node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true }, + "node_modules/@typescript-eslint/parser/node_modules/semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@typescript-eslint/scope-manager": { "version": "4.8.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.8.1.tgz", @@ -1822,14 +1932,6 @@ "wide-align": "^1.1.0" } }, - "node_modules/generic-pool": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.7.1.tgz", - "integrity": "sha512-ug6DAZoNgWm6q5KhPFA+hzXfBLFQu5sTXxPpv44DmE0A2g+CiHoq9LTVdkXpZMkYVMoGw83F6W+WT0h0MFMK/w==", - "engines": { - "node": ">= 4" - } - }, "node_modules/getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -2198,6 +2300,24 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/lru-cache/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + }, "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -2726,6 +2846,25 @@ "node": ">=0.4.x" } }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -3917,17 +4056,58 @@ } }, "@typescript-eslint/parser": { - "version": "4.8.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-4.8.1.tgz", - "integrity": "sha512-QND8XSVetATHK9y2Ltc/XBl5Ro7Y62YuZKnPEwnNPB8E379fDsvzJ1dMJ46fg/VOmk0hXhatc+GXs5MaXuL5Uw==", + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-4.22.0.tgz", + "integrity": "sha512-z/bGdBJJZJN76nvAY9DkJANYgK3nlRstRRi74WHm3jjgf2I8AglrSY+6l7ogxOmn55YJ6oKZCLLy+6PW70z15Q==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "4.8.1", - "@typescript-eslint/types": "4.8.1", - "@typescript-eslint/typescript-estree": "4.8.1", + "@typescript-eslint/scope-manager": "4.22.0", + "@typescript-eslint/types": "4.22.0", + "@typescript-eslint/typescript-estree": "4.22.0", "debug": "^4.1.1" }, "dependencies": { + "@typescript-eslint/scope-manager": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.22.0.tgz", + "integrity": "sha512-OcCO7LTdk6ukawUM40wo61WdeoA7NM/zaoq1/2cs13M7GyiF+T4rxuA4xM+6LeHWjWbss7hkGXjFDRcKD4O04Q==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.22.0", + "@typescript-eslint/visitor-keys": "4.22.0" + } + }, + "@typescript-eslint/types": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.22.0.tgz", + "integrity": "sha512-sW/BiXmmyMqDPO2kpOhSy2Py5w6KvRRsKZnV0c4+0nr4GIcedJwXAq+RHNK4lLVEZAJYFltnnk1tJSlbeS9lYA==", + "dev": true + }, + "@typescript-eslint/typescript-estree": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.22.0.tgz", + "integrity": "sha512-TkIFeu5JEeSs5ze/4NID+PIcVjgoU3cUQUIZnH3Sb1cEn1lBo7StSV5bwPuJQuoxKXlzAObjYTilOEKRuhR5yg==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.22.0", + "@typescript-eslint/visitor-keys": "4.22.0", + "debug": "^4.1.1", + "globby": "^11.0.1", + "is-glob": "^4.0.1", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "@typescript-eslint/visitor-keys": { + "version": "4.22.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.22.0.tgz", + "integrity": "sha512-nnMu4F+s4o0sll6cBSsTeVsT4cwxB7zECK3dFxzEjPBii9xLpq4yqqsy/FU5zMfan6G60DKZSCXAa3sHJZrcYw==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.22.0", + "eslint-visitor-keys": "^2.0.0" + } + }, "debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -3937,11 +4117,26 @@ "ms": "2.1.2" } }, + "eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", + "dev": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "dev": true + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } } } }, @@ -5053,11 +5248,6 @@ "wide-align": "^1.1.0" } }, - "generic-pool": { - "version": "3.7.1", - "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.7.1.tgz", - "integrity": "sha512-ug6DAZoNgWm6q5KhPFA+hzXfBLFQu5sTXxPpv44DmE0A2g+CiHoq9LTVdkXpZMkYVMoGw83F6W+WT0h0MFMK/w==" - }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -5356,6 +5546,23 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + }, + "dependencies": { + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -5775,6 +5982,11 @@ "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==" + }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", diff --git a/package.json b/package.json index cfdbc57..fd87afc 100755 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "express": "^4.6.1", "express-ejs-layouts": "^1.1.0", "express-fileupload": "^1.2.1", - "generic-pool": "^3.7.1", + "queue-microtask": "^1.2.3", "sqlite3": "^4.1.1", "ts-node": "^9.0.0", "typescript": "^4.1.2", @@ -31,7 +31,7 @@ }, "devDependencies": { "@typescript-eslint/eslint-plugin": "^4.8.1", - "@typescript-eslint/parser": "^4.8.1", + "@typescript-eslint/parser": "^4.22.0", "eslint": "^7.14.0" } } diff --git a/src/modules/api/submodules/qminingapi.ts b/src/modules/api/submodules/qminingapi.ts index 4c3c07d..ac2d52f 100644 --- a/src/modules/api/submodules/qminingapi.ts +++ b/src/modules/api/submodules/qminingapi.ts @@ -14,14 +14,13 @@ import { logResult, shouldSaveDataFile, Result, - isQuestionValid, backupData, shouldSearchDataFile, loadJSON, writeData, + editDb, } from '../../../utils/actions' import { - createQuestion, dataToString, getSubjNameWithoutYear, // compareQuestionObj, @@ -524,7 +523,7 @@ function setup(data: SubmoduleData): any { logResult(req.body, resultArray, user.id, dryRun) const totalNewQuestions = resultArray.reduce((acc, sres) => { - return acc + sres.newQuestions + return acc + sres.newQuestions.length }, 0) res.json({ @@ -534,9 +533,11 @@ function setup(data: SubmoduleData): any { }) if (totalNewQuestions > 0) { - msgAllWorker({ - qdbs: questionDbs, - type: 'update', + resultArray.forEach((result) => { + msgAllWorker({ + type: 'newQuestions', + data: result, + }) }) } }) @@ -842,7 +843,6 @@ function setup(data: SubmoduleData): any { logger.LogReq(req) const user: User = req.session.user const date = utils.GetDateString() - let saveDb = false const editType = req.body.type const selectedDb = req.body.selectedDb @@ -867,44 +867,28 @@ function setup(data: SubmoduleData): any { return } - // { - // "index": 0, - // "subjName": "VHDL programozás", - // "type": "delete", - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } + // ----------------- + const { + success, + msg, + resultDb, + deletedQuestion, + newVal, + oldVal, + deletedQuestions, + changedQuestions, + } = editDb(currDb, req.body) + + if (!success) { + res.json({ success: success, msg: msg }) + return + } + if (resultDb) { + questionDbs[dbIndex] = resultDb + } + if (editType === 'delete') { const { index, subjName } = req.body - let deletedQuestion = {} - if (isNaN(index) || !subjName) { - res.json({ - status: 'fail', - msg: 'No .index or .subjName !', - }) - return - } - - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.filter((question, i) => { - if (index === i) { - deletedQuestion = question - return false - } else { - return true - } - }), - } - } - }) - logger.Log( `User #${user.id} deleted a question from '${subjName}'`, logger.GetColor('cyan') @@ -914,68 +898,10 @@ function setup(data: SubmoduleData): any { dataEditsLog ) utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog) - saveDb = true } - // { - // "index": 0, - // "subjName": "Elektronika", - // "type": "edit", - // "newVal": { - // "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.", - // "A": "Igaz", - // "data": { - // "type": "simple", - // "possibleAnswers": [ - // "Igaz" - // ] - // }, - // "possibleAnswers": [ - // "Igaz" - // ] - // }, - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } if (editType === 'edit') { - const { index, subjName, newVal } = req.body - let oldVal = {} - if (isNaN(index) || !subjName) { - res.json({ - status: 'fail', - msg: 'No .index or .subjName !', - }) - return - } - if (!isQuestionValid(newVal)) { - res.json({ - status: 'fail', - msg: 'edited question is not valid', - question: newVal, - }) - return - } - - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.map((question, i) => { - if (index === i) { - oldVal = question - return createQuestion(newVal) - } else { - return question - } - }), - } - } - }) - + const { index, subjName } = req.body logger.Log( `User #${user.id} edited a question in '${subjName}'`, logger.GetColor('cyan') @@ -995,128 +921,46 @@ function setup(data: SubmoduleData): any { ), dataEditsLog ) - saveDb = true } - // { - // "subjName": "Elektronika", - // "changedQuestions": [ - // { - // "index": 1, - // "value": { - // "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.", - // "A": "Hamis", - // "data": { - // "type": "simple", - // "possibleAnswers": [ - // "Igaz", - // "Hamis" - // ] - // } - // } - // } - // ], - // "deletedQuestions": [ - // 0 - // ], - // "type": "subjEdit", - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } if (editType === 'subjEdit') { - const { subjName, changedQuestions, deletedQuestions } = req.body - const deletedQuestionsToWrite = [] - const changedQuestionsToWrite = [] - if ( - !Array.isArray(changedQuestions) || - !Array.isArray(deletedQuestions) - ) { - res.json({ - status: 'fail', - msg: 'no changedQuestions or deletedQuestions!', - }) - return - } - - // processing changed questions - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.map((question, i) => { - const changedTo = changedQuestions.find((cq) => { - return cq.index === i - }) - if (changedTo) { - changedQuestionsToWrite.push({ - oldVal: question, - newVal: changedTo.value, - }) - return createQuestion(changedTo.value) - } else { - return question - } - }), - } - } - }) - - // processing deletedQuestions - questionDbs[dbIndex].data = currDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.filter((question, i) => { - const isDeleted = deletedQuestions.includes(i) - if (isDeleted) { - deletedQuestionsToWrite.push(question) - return false - } else { - return true - } - }), - } - } - }) - + const { subjName } = req.body logger.Log( - `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`, + `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, logger.GetColor('cyan') ) utils.AppendToFile( - `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestionsToWrite.length}, deleted: ${deletedQuestionsToWrite.length}`, + `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, dataEditsLog ) utils.AppendToFile( JSON.stringify( { - deletedQuestions: deletedQuestionsToWrite, - changedQuestions: changedQuestionsToWrite, + deletedQuestions: deletedQuestions, + changedQuestions: changedQuestions, }, null, 2 ), dataEditsLog ) - saveDb = true } + // ------------------ - if (saveDb) { + if (success) { writeData(currDb.data, currDb.path) msgAllWorker({ - qdbs: questionDbs, - type: 'update', + type: 'dbEdit', + data: { + dbIndex: dbIndex, + edits: req.body, + }, }) } res.json({ - status: 'OK', + success: true, + msg: 'OK', }) }) diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 2c08e89..317fe7d 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -53,8 +53,10 @@ export interface RecievedData { } export interface Result { + qdbIndex: number qdbName: string - newQuestions: number + subjName: string + newQuestions: Array } export function logResult( @@ -84,9 +86,9 @@ export function logResult( const allQLength = recievedData.quiz.length let msg = `${res.qdbName}: ` let color = logger.GetColor('green') - if (res.newQuestions > 0) { + if (res.newQuestions.length > 0) { color = logger.GetColor('blue') - msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )` + msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )` } else { msg += `No new data ( ${allQLength} )` } @@ -241,7 +243,9 @@ function processIncomingRequestUsingDb( logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1) resolve({ - newQuestions: allQuestions.length, + newQuestions: allQuestions, + subjName: recievedData.subj, + qdbIndex: qdb.index, qdbName: qdb.name, }) } catch (error) { @@ -439,3 +443,288 @@ export function backupData(questionDbs: Array): void { } }) } + +function deleteFromDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + } +): { + success: Boolean + msg: string + deletedQuestion?: Question + resultDb?: QuestionDb +} { + // { + // "index": 0, + // "subjName": "VHDL programozás", + // "type": "delete", + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { index, subjName } = edits + let deletedQuestion + if (isNaN(index) || !subjName) { + return { + success: false, + msg: 'No .index or .subjName !', + } + } + + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.filter((question, i) => { + if (index === i) { + deletedQuestion = question + return false + } else { + return true + } + }), + } + } + }) + + return { + success: true, + msg: 'Delete successfull', + deletedQuestion: deletedQuestion, + resultDb: questionDb, + } +} + +function editQuestionInDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + newVal?: Question + } +): { + success: Boolean + msg: string + newVal?: Question + oldVal?: Question + resultDb?: QuestionDb +} { + // { + // "index": 0, + // "subjName": "Elektronika", + // "type": "edit", + // "newVal": { + // "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.", + // "A": "Igaz", + // "data": { + // "type": "simple", + // "possibleAnswers": [ + // "Igaz" + // ] + // }, + // "possibleAnswers": [ + // "Igaz" + // ] + // }, + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { index, subjName, newVal } = edits + let oldVal + if (isNaN(index) || !subjName) { + return { + success: false, + msg: 'No .index or .subjName !', + } + } + if (!isQuestionValid(newVal)) { + return { + success: false, + msg: 'edited question is not valid', + } + } + + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.map((question, i) => { + if (index === i) { + oldVal = question + return createQuestion(newVal) + } else { + return question + } + }), + } + } + }) + + return { + success: true, + msg: 'Edit successfull', + oldVal: oldVal, + newVal: newVal, + resultDb: questionDb, + } +} + +function editSubjInDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + deletedQuestions?: Array + changedQuestions?: Array<{ + index: number + value: Question + }> + } +): { + success: Boolean + msg: string + deletedQuestions?: Array + changedQuestions?: Array + resultDb?: QuestionDb +} { + // { + // "subjName": "Elektronika", + // "changedQuestions": [ + // { + // "index": 1, + // "value": { + // "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.", + // "A": "Hamis", + // "data": { + // "type": "simple", + // "possibleAnswers": [ + // "Igaz", + // "Hamis" + // ] + // } + // } + // } + // ], + // "deletedQuestions": [ + // 0 + // ], + // "type": "subjEdit", + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { subjName, changedQuestions, deletedQuestions } = edits + const deletedQuestionsToWrite = [] + const changedQuestionsToWrite = [] + if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) { + return { + success: false, + msg: 'no changedQuestions or deletedQuestions!', + } + } + + // processing changed questions + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.map((question, i) => { + const changedTo = changedQuestions.find((cq) => { + return cq.index === i + }) + if (changedTo) { + changedQuestionsToWrite.push({ + oldVal: question, + newVal: changedTo.value, + }) + return createQuestion(changedTo.value) + } else { + return question + } + }), + } + } + }) + + // processing deletedQuestions + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.filter((question, i) => { + const isDeleted = deletedQuestions.includes(i) + if (isDeleted) { + deletedQuestionsToWrite.push(question) + return false + } else { + return true + } + }), + } + } + }) + + return { + success: true, + msg: 'subj edit successfull', + deletedQuestions: deletedQuestionsToWrite, + changedQuestions: changedQuestionsToWrite, + resultDb: questionDb, + } +} + +export function editDb( + questionDb: QuestionDb, + edits: { + index: number + subjName: string + selectedDb: { path: string; name: string } + type: string + newVal?: Question + deletedQuestion?: Array + changedQuestions?: Array<{ + index: number + value: Question + }> + } +): { + success: Boolean + msg: string + resultDb?: QuestionDb + deletedQuestion?: Question + newVal?: Question + oldVal?: Question + deletedQuestions?: Array + changedQuestions?: Array +} { + if (edits.type === 'delete') { + return deleteFromDb(questionDb, edits) + } + if (edits.type === 'edit') { + return editQuestionInDb(questionDb, edits) + } + + if (edits.type === 'subjEdit') { + return editSubjInDb(questionDb, edits) + } +} diff --git a/src/utils/classes.ts b/src/utils/classes.ts index bfaa879..aa6455a 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -1,6 +1,12 @@ import { isMainThread, parentPort, workerData } from 'worker_threads' import logger from './logger' -import { Question, QuestionData, Subject } from '../types/basicTypes' +import { + Question, + QuestionData, + QuestionDb, + Subject, +} from '../types/basicTypes' +import { editDb } from './actions' interface SearchResultQuestion extends Question { match: number @@ -581,7 +587,7 @@ function doSearch( if (!isMainThread) { const { workerIndex } = workerData - let qdbs: Array = workerData.initData + let qdbs: Array = workerData.initData logger.Log( `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty` @@ -670,14 +676,72 @@ if (!isMainThread) { // !isNaN(index) ? `#${index}` : '' // }done!` // ) - } else if (msg.type === 'update') { - qdbs = msg.qdbs - logger.DebugLog(`Worker update ${workerIndex}`, 'worker update', 1) + } else if (msg.type === 'dbEdit') { + const { dbIndex, edits } = msg.data + const { resultDb } = editDb(qdbs[dbIndex], edits) + qdbs[dbIndex] = resultDb + logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1) + + parentPort.postMessage({ + msg: `From thread #${workerIndex}: db edit`, + workerIndex: workerIndex, + }) + } else if (msg.type === 'newQuestions') { + const { subjName, qdbIndex, newQuestions } = msg.data + let added = false + qdbs = qdbs.map((qdb) => { + if (qdb.index === qdbIndex) { + return { + ...qdb, + data: qdb.data.map((subj) => { + if (subj.Name === subjName) { + added = true + return { + Name: subj.Name, + Questions: [...subj.Questions, ...newQuestions], + } + } else { + return subj + } + }), + } + } else { + return qdb + } + }) + + if (!added) { + qdbs = qdbs.map((qdb) => { + if (qdb.index === qdbIndex) { + return { + ...qdb, + data: [ + ...qdb.data, + { + Name: subjName, + Questions: [...newQuestions], + }, + ], + } + } else { + return qdb + } + }) + } + logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1) + + // parentPort.postMessage({ + // msg: `From thread #${workerIndex}: update done`, + // workerIndex: workerIndex, + // }) // console.log(`[THREAD #${workerIndex}]: update`) } else if (msg.type === 'newdb') { qdbs.push(msg.newdb) // console.log(`[THREAD #${workerIndex}]: newdb`) + } else { + logger.Log(`Invalid msg type!`, logger.GetColor('redbg')) + console.error(msg) } }) } else { diff --git a/src/utils/workerPool.ts b/src/utils/workerPool.ts index aa4df3c..ebd00ef 100644 --- a/src/utils/workerPool.ts +++ b/src/utils/workerPool.ts @@ -1,105 +1,173 @@ import { Worker } from 'worker_threads' -import genericPool from 'generic-pool' +import { v4 as uuidv4 } from 'uuid' +import { EventEmitter } from 'events' import os from 'os' import logger from './logger' -// import { QuestionDb } from '../types/basicTypes' -// --------------------------------------------------------------------------- +interface WorkerObj { + worker: any + index: number + free: Boolean +} + +interface PendingJob { + workData: any + doneEvent: any + targetWorkerIndex?: number +} const alertOnPendingCount = 10 const workerFile = './src/utils/classes.ts' -let pool: any = null -let workers: any = null +let workers: Array +const pendingJobs: { + [id: string]: PendingJob +} = {} + +const jobEvents = new EventEmitter() + +jobEvents.on('jobDone', () => { + processJob() +}) + +jobEvents.on('newJob', () => { + processJob() +}) // --------------------------------------------------------------------------- -function handleWorkerError(client, err) { +function handleWorkerError(worker: WorkerObj, err) { + // TODO: restart worker if exited or things like that logger.Log('resourcePromise error', logger.GetColor('redbg')) console.error(err) - pool.destroy(client) } -export function doALongTask(obj: any): Promise { - if (pool.pending > alertOnPendingCount) { +// TODO: accuire all workers here, and handle errors so they can be removed if threads exit +export function msgAllWorker(data: any): Promise { + console.log('MSGING ALL WORKER') + return new Promise((resolve) => { + const promises = [] + workers.forEach((worker) => { + promises.push(doALongTask(data, worker.index)) + }) + Promise.all(promises).then((res) => { + resolve(res) + }) + }) +} + +export function doALongTask( + obj: any, + targetWorkerIndex?: number +): Promise { + if (Object.keys(pendingJobs).length > alertOnPendingCount) { logger.Log( - `More than ${alertOnPendingCount} callers waiting for free resource! (${pool.pending})`, + `More than ${alertOnPendingCount} callers waiting for free resource! (${ + Object.keys(pendingJobs).length + })`, logger.GetColor('redbg') ) } + const jobId = uuidv4() + // FIXME: delete doneEvent? + const doneEvent = new EventEmitter() + pendingJobs[jobId] = { + workData: obj, + targetWorkerIndex: targetWorkerIndex, + doneEvent: doneEvent, + } + jobEvents.emit('newJob') return new Promise((resolve) => { - let currClient - pool - .acquire() - .then(function(client) { - currClient = client - doSomething(client, obj) - .then((res) => { - resolve(res) - // TODO: check if result is really a result, and want to release port - pool.release(client) - // console.log('[RELEASE]: #' + client.index) - }) - .catch(function(err) { - handleWorkerError(currClient, err) - }) - }) - .catch(function(err) { - handleWorkerError(currClient, err) - }) + doneEvent.once('done', (result) => { + jobEvents.emit('jobDone') + resolve(result) + }) }) } export function initWorkerPool(initData: any): void { - if (workers && pool) { - logger.Log('WORKER AND POOL ALREADY EXISTS', logger.GetColor('redbg')) + if (workers) { + logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg')) return } workers = [] const factory = { - create: function() { - const currInd = workers.length - const worker = getAWorker(currInd, initData) - workers.push(worker) - return { - worker: worker, - index: currInd, - } + create: function(index) { + return getAWorker(index, initData) }, destroy: function(client) { - // console.log('[DESTROY]') client.worker.terminate() - // console.log('[DESTROYED] #' + client.index) }, } const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length if (process.env.NS_THREAD_COUNT) { logger.Log( - `Setting thread count from enviroment variable NS_THREAD_COUNT: '${threadCount}'`, + `Setting thread count from enviroment variable NS_WORKER_COUNT: '${workerCount}'`, logger.GetColor('red') ) } - const opts = { - min: threadCount, // minimum size of the pool - max: threadCount, // maximum size of the pool - maxWaitingClients: 999, + for (let i = 0; i < workerCount; i++) { + workers.push({ + worker: factory.create(i), + index: i, + free: true, + }) } - - pool = genericPool.createPool(factory, opts) -} - -// TODO: accuire all workers here, and handle errors so they can be removed if threads exit -export function msgAllWorker(data: any): void { - workers.forEach((worker) => { - worker.postMessage(data) - }) } // --------------------------------------------------------------------------- +function processJob() { + if (Object.keys(pendingJobs).length > 0) { + // FIXME: FIFO OR ANYTHING ELSE (JOB PROCESSING ORDER) + const keys = Object.keys(pendingJobs) + let jobKey, freeWorker + let i = 0 + while (!freeWorker && i < keys.length) { + jobKey = keys[i] + if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) { + if (workers[pendingJobs[jobKey].targetWorkerIndex].free) { + freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex] + console.log( + `RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}` + ) + } + } else { + freeWorker = workers.find((worker) => { + return worker.free + }) + if (freeWorker) { + console.log(`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`) + } + } + i++ + } + + if (!freeWorker) { + console.log('NO FREE WORKER') + return + } + + if (freeWorker.free) { + freeWorker.free = false + } + const job = pendingJobs[jobKey] + delete pendingJobs[jobKey] + + doSomething(freeWorker, job.workData) + .then((res) => { + freeWorker.free = true + job.doneEvent.emit('done', res) + }) + .catch(function(err) { + handleWorkerError(freeWorker, err) + }) + } +} + function getAWorker(i, initData) { const worker = workerTs(workerFile, { workerData: { @@ -110,15 +178,6 @@ function getAWorker(i, initData) { worker.setMaxListeners(50) - // worker.on('message', (msg) => { - // logger.Log(`[MAIN]: Msg from worker #${i}`) - // logger.Log(msg) - // }) - - // worker.on('online', () => { - // logger.Log(`[THREAD #${i}]: Worker ${i} online`) - // }) - worker.on('error', (err) => { logger.Log('Worker error!', logger.GetColor('redbg')) console.error(err) @@ -136,10 +195,9 @@ function getAWorker(i, initData) { // --------------------------------------------------------------------------- -function doSomething(client, obj) { - const { /* index, */ worker } = client +function doSomething(currWorker, obj) { + const { /* index, */ worker } = currWorker return new Promise((resolve) => { - // console.log('[ACCUIRE]: #' + index) worker.postMessage(obj) worker.once('message', (msg) => { resolve(msg)