diff --git a/.eslintrc.js b/.eslintrc.js index 5dda5f1..4dfaebb 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -6,7 +6,7 @@ module.exports = { jest: true, }, parser: 'babel-eslint', - extends: ['eslint:recommended'], + extends: ['eslint:recommended', 'plugin:typescript/recommended'], globals: { Atomics: 'readonly', SharedArrayBuffer: 'readonly', @@ -16,6 +16,9 @@ module.exports = { eqeqeq: ['warn', 'smart'], 'no-unused-vars': 'warn', 'no-prototype-builtins': 'off', - 'id-length': ['warn', { exceptions: ['i', 'j', 't', 'Q', 'A', 'C', 'q', 'a', 'b'] }], + 'id-length': [ + 'warn', + { exceptions: ['i', 'j', 't', 'Q', 'A', 'C', 'q', 'a', 'b'] }, + ], }, } diff --git a/package-lock.json b/package-lock.json index f7bc726..e56bfe5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -174,6 +174,11 @@ } } }, + "@types/node": { + "version": "14.14.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.9.tgz", + "integrity": "sha512-JsoLXFppG62tWTklIoO4knA+oDTYsmqWxHRvd4lpmfQRNhX6osheUOWETP2jMoV/2bEHuMra8Pp3Dmo/stBFcw==" + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -805,6 +810,14 @@ } } }, + "eslint-plugin-typescript": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-typescript/-/eslint-plugin-typescript-0.14.0.tgz", + "integrity": "sha512-2u1WnnDF2mkWWgU1lFQ2RjypUlmRoBEvQN02y9u+IL12mjWlkKFGEBnVsjs9Y8190bfPQCvWly1c2rYYUSOxWw==", + "requires": { + "requireindex": "~1.1.0" + } + }, "eslint-scope": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", @@ -1807,6 +1820,11 @@ } } }, + "requireindex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/requireindex/-/requireindex-1.1.0.tgz", + "integrity": "sha1-5UBLgVV+91225JxacgBIk/4D4WI=" + }, "resolve": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", @@ -2180,6 +2198,11 @@ "mime-types": "~2.1.24" } }, + "typescript": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.1.2.tgz", + "integrity": "sha512-thGloWsGH3SOxv1SoY7QojKi0tc+8FnOmiarEGMbd/lar7QOEd3hvlx3Fp5y6FlDUGl9L+pd4n2e+oToGMmhRQ==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index bd9559d..19c21c0 100755 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "node-ejs", "main": "src/server.js", "dependencies": { + "@types/node": "^14.14.9", "babel-eslint": "^10.1.0", "better-sqlite3": "^6.0.1", "connect-busboy": "0.0.2", @@ -9,15 +10,18 @@ "cors": "^2.8.5", "ejs": "^1.0.0", "eslint": "^7.14.0", + "eslint-plugin-typescript": "^0.14.0", "express": "^4.6.1", "express-ejs-layouts": "^1.1.0", "sqlite3": "^4.1.1", + "typescript": "^4.1.2", "url": "^0.11.0", "uuid": "^7.0.3", "vhost": "^3.0.2" }, "scripts": { - "start": "node ./src/server.js", - "dev": "NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node ./src/server.js" + "start": "node ./dist/server.js", + "dev": "NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node ./dist/server.js", + "build": "NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 tsc" } } diff --git a/src/utils/actions.ts b/src/utils/actions.ts new file mode 100755 index 0000000..c3d5c80 --- /dev/null +++ b/src/utils/actions.ts @@ -0,0 +1,278 @@ +/* ---------------------------------------------------------------------------- +Question Server + GitLab: <https://gitlab.com/MrFry/mrfrys-node-server> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + + ------------------------------------------------------------------------- */ + +module.exports = { + ProcessIncomingRequest: ProcessIncomingRequest, + LoadJSON: LoadJSON, + backupData: backupData, +} + +const recDataFile = './stats/recdata' +const dataLockFile = './data/lockData' + +const logger = require('../utils/logger.js') +const { searchData, createQuestion } = require('../utils/classes.js') +const idStats = require('../utils/ids.js') +const utils = require('../utils/utils.js') +const { addQuestion, getSubjNameWithoutYear } = require('./classes.js') +// if a recievend question doesnt match at least this % to any other question in the db it gets +// added to db +const minMatchToAmmountToAdd = 90 + +const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate +var currWrites = 0 + +interface QuestionData { + type: String + images?: Array<String> + hashedImages?: Array<String> +} + +interface Question { + Q: String + A: String + data: QuestionData +} + +interface Subject { + Name: String + Questions: Array<Question> +} + +interface Db { + name: String + path: String + data: Array<Subject> +} + +function ProcessIncomingRequest( + recievedData, + questionDbs: Array<Db>, + dryRun, + user +) { + return Promise.all( + questionDbs.map((qdb) => { + return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) + }) + ) +} + +function ProcessIncomingRequestUsingDb(recievedData, qdb: Db, dryRun, user) { + return new Promise((resolve, reject) => { + logger.DebugLog('Processing incoming request', 'actions', 1) + + if (recievedData === undefined) { + logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) + reject(new Error('Recieved data is undefined!')) + } + + try { + let towrite = logger.GetDateString() + '\n' + towrite += + '------------------------------------------------------------------------------\n' + if (typeof recievedData === 'object') { + towrite += JSON.stringify(recievedData) + } else { + towrite += recievedData + } + towrite += + '\n------------------------------------------------------------------------------\n' + utils.AppendToFile(towrite, recDataFile) + logger.DebugLog('recDataFile written', 'actions', 1) + } catch (err) { + logger.log('Error writing recieved data.') + } + + if (utils.FileExists(dataLockFile)) { + logger.Log( + 'Data lock file exists, skipping recieved data processing', + logger.GetColor('red') + ) + resolve(-1) + return + } + + try { + // recievedData: { version: "", id: "", subj: "" quiz: {} } + let data = recievedData + const recievedQuestions = [] + // FIXME: if is for backwards compatibility, remove this sometime in the future + if (typeof data !== 'object') { + data = JSON.parse(recievedData) + } + + logger.DebugLog('recievedData JSON parsed', 'actions', 1) + logger.DebugLog(data, 'actions', 3) + let allQLength = data.quiz.length + const questionSearchPromises = [] + data.quiz.forEach((question) => { + logger.DebugLog('Question:', 'actions', 2) + logger.DebugLog(question, 'actions', 2) + let currentQuestion = createQuestion( + question.Q, + question.A, + question.data + ) + logger.DebugLog( + 'Searching for question in subj ' + data.subj, + 'actions', + 3 + ) + logger.DebugLog(currentQuestion, 'actions', 3) + recievedQuestions.push(currentQuestion) + questionSearchPromises.push( + searchData(qdb.data, currentQuestion, data.subj) + ) + }) + + Promise.all(questionSearchPromises) + .then((results) => { + const allQuestions = [] // all new questions here that do not have result + results.forEach((result, i) => { + let add = result.every((res) => { + return res.match < minMatchToAmmountToAdd + }) + if (add) { + allQuestions.push(recievedQuestions[i]) + } + }) + + try { + let color = logger.GetColor('green') + let msg = '' + if (allQuestions.length > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` + allQuestions.forEach((currentQuestion) => { + const sName = getSubjNameWithoutYear(data.subj) + logger.DebugLog( + 'Adding question with subjName: ' + sName + ' :', + 'actions', + 3 + ) + logger.DebugLog(currentQuestion, 'actions', 3) + addQuestion(qdb.data, sName, currentQuestion) + }) + + currWrites++ + logger.DebugLog( + 'currWrites for data.json: ' + currWrites, + 'actions', + 1 + ) + if (currWrites >= writeAfter && !dryRun) { + currWrites = 0 + logger.DebugLog('Writing data.json', 'actions', 1) + utils.WriteFile(JSON.stringify(qdb.data), qdb.path) + logger.Log('\tData file written', color) + } else if (dryRun) { + logger.Log('\tDry run') + } + } else { + msg += `No new data ( ${allQLength} )` + } + + let subjRow = '\t' + data.subj + if (data.id) { + subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')' + } + idStats.LogId(user.id, data.subj, allQuestions.length, allQLength) + logger.Log(subjRow) + if (data.version !== undefined) { + msg += '. Version: ' + data.version + } + + logger.Log('\t' + msg, color) + logger.DebugLog('New Questions:', 'actions', 2) + logger.DebugLog(allQuestions, 'actions', 2) + + logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) + resolve(allQuestions.length) + } catch (error) { + console.error(error) + logger.Log( + 'Error while processing processData worker result!', + logger.GetColor('redbg') + ) + reject( + new Error('Error while processing processData worker result!') + ) + } + }) + .catch((err) => { + logger.Log( + 'Error while searching for questions in ProcessIncomingRequest!', + logger.GetColor('redbg') + ) + console.error(err) + }) + } catch (err) { + console.error(err) + logger.Log( + 'There was en error handling incoming quiz data, see stderr', + logger.GetColor('redbg') + ) + reject(new Error('Couldnt parse JSON data')) + } + }) +} + +function LoadJSON(dataFiles) { + return dataFiles.reduce((acc, dataFile) => { + if (!utils.FileExists(dataFile.path)) { + utils.WriteFile(JSON.stringify([]), dataFile.path) + } + + try { + acc.push({ + ...dataFile, + data: JSON.parse(utils.ReadFile(dataFile.path)), + }) + } catch (err) { + console.error(err) + logger.Log( + "data is undefined! Couldn't load data!", + logger.GetColor('redbg') + ) + } + return acc + }, []) +} + +function backupData(questionDbs) { + questionDbs.forEach((data) => { + const path = './publicDirs/qminingPublic/backs/' + utils.CreatePath(path) + try { + logger.Log(`Backing up ${data.name}...`) + utils.WriteFile( + JSON.stringify(data.data), + `${path}${data.name}_${utils.GetDateString(true)}.json` + ) + logger.Log('Done') + } catch (err) { + logger.Log( + `Error backing up data file ${data.name}!`, + logger.GetColor('redbg') + ) + console.error(err) + } + }) +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..80b665b --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "allowJs": true, + "module": "system", + "removeComments": true, + "preserveConstEnums": true, + "sourceMap": true, + "outDir": "dist", + "noImplicitAny": false, + "lib": [ + "ES2020" + ] + }, + "files": [ + "src/server.js" + ], + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "submodules", + "devel" + ] +}