From 86acdec2f52e4134ed4e1fefb86a18b22dd89715 Mon Sep 17 00:00:00 2001 From: mrfry Date: Tue, 19 Oct 2021 10:09:14 +0200 Subject: [PATCH 1/8] Added manualSearch script --- src/standaloneUtils/manualSearch.js | 93 +++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 src/standaloneUtils/manualSearch.js diff --git a/src/standaloneUtils/manualSearch.js b/src/standaloneUtils/manualSearch.js new file mode 100644 index 0000000..a3b6aef --- /dev/null +++ b/src/standaloneUtils/manualSearch.js @@ -0,0 +1,93 @@ +const utils = require('../../dist/utils/utils.js').default // eslint-disable-line +const logger = require('../../dist/utils/logger.js').default // eslint-disable-line +const { doSearch } = require('../../dist/utils/classes.js') // eslint-disable-line +const { loadData } = require('../../dist/utils/actions.js') // eslint-disable-line +const fs = require('fs') // eslint-disable-line + +const minpercent = 95 +const logPath = './duplicateRemovingLog/' +const globalLog = './duplicateRemovingLog/log' +utils.CreatePath(logPath) +utils.WriteFile('', globalLog) + +const params = process.argv.splice(2) +const path = params[0] +const question = params[1] +const answer = params[2] + +// --------------------------------------------------------------------------------- +// +// --------------------------------------------------------------------------------- + +console.time('SEARCH') +const searchRes = search({ + qdb: loadData(path), + subjName: 'Elektronika', + question: { + Q: question, + A: answer, + data: { + type: 'simple', + }, + }, + searchTillMatchPercent: 80, +}) +hr() +console.log('Search result') +hr() +showSearchResult(searchRes) +hr() +console.timeEnd('SEARCH') +log( + `Searched for question: "${C('green')}${question}${C()}" answer: "${C( + 'green' + )}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"` +) +hr() + +// --------------------------------------------------------------------------------- +// logging and tools +// --------------------------------------------------------------------------------- + +function showSearchResult(res) { + res.forEach((x) => { + console.log(`${C('green')}Q:${C()}`, x.q.Q) + console.log(`${C('green')}A:${C()}`, x.q.A) + console.log(`${C('green')}match:${C()}`, x.match) + console.log() + }) + console.log(`Result length: ${C('green')}${res.length}${C()}`) +} + +function search({ qdb, subjName, question, searchInAllIfNoResult }) { + return doSearch( + qdb, + subjName, + question, + null, + minpercent, + searchInAllIfNoResult + ) +} + +function hr() { + let res = '' + for (let i = 0; i < process.stdout.columns; i++) { + res += '=' + } + log(`${C('cyan')}${res}${C()}`) +} + +function log(text) { + utils.AppendToFile(text, globalLog) + if (process.stdout.isTTY) { + process.stdout.clearLine() + process.stdout.cursorTo(0) + } + + console.log(text) +} + +function C(color) { + return logger.C(color) +} From 44678f4a2ce542dfd8a97083702fbbf290495ab0 Mon Sep 17 00:00:00 2001 From: mrfry Date: Mon, 22 Nov 2021 20:14:45 +0100 Subject: [PATCH 2/8] package updates, server stats script update --- package-lock.json | 36 +++++++++++++++---------------- scripts/serverStats.js | 24 ++++++++++++++++----- submodules/moodle-test-userscript | 2 +- 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/package-lock.json b/package-lock.json index f219bec..5f7f924 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1444,9 +1444,9 @@ } }, "node_modules/ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true, "engines": { "node": ">=8" @@ -7656,9 +7656,9 @@ } }, "node_modules/tar": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz", - "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==", + "version": "6.1.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -7758,9 +7758,9 @@ "dev": true }, "node_modules/tmpl": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", - "integrity": "sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, "node_modules/to-fast-properties": { @@ -9644,9 +9644,9 @@ } }, "ansi-regex": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", - "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", "dev": true }, "ansi-styles": { @@ -14473,9 +14473,9 @@ } }, "tar": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz", - "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==", + "version": "6.1.11", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", + "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", "requires": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -14561,9 +14561,9 @@ "dev": true }, "tmpl": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.4.tgz", - "integrity": "sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE=", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", "dev": true }, "to-fast-properties": { diff --git a/scripts/serverStats.js b/scripts/serverStats.js index 671df7c..858e9c3 100644 --- a/scripts/serverStats.js +++ b/scripts/serverStats.js @@ -83,19 +83,29 @@ const filterFromDailyStats = [ const fs = require('fs') // eslint-disable-line const dir = process.argv[2] +const startDay = !isNaN(parseInt(process.argv[3])) && parseInt(process.argv[3]) > 0 ? 0 : parseInt(process.argv[3]) if (!dir) { console.log('No params') process.exit() } function getDayIndex(offset) { + let os = offset if (!offset) { - offset = 0 + os = 0 + } + + if (!isNaN(startDay)) { + if (!offset) { + os = startDay + } else { + os = startDay + offset + } } const date = new Date() - if (offset) { - date.setDate(date.getDate() + offset) + if (os) { + date.setDate(date.getDate() + os) } return ( date.getFullYear() + @@ -172,7 +182,11 @@ function countLinesMatching(text, toMatch) { } function getDayName(day) { - switch (day) { + let d = day + if (!isNaN(startDay)) { + d += startDay + } + switch (d) { case 0: case undefined: return 'Today' @@ -181,7 +195,7 @@ function getDayName(day) { case -2: return 'Before yesterday' default: - return `Day ${day.toString()}` + return `Day ${d.toString()}` } } diff --git a/submodules/moodle-test-userscript b/submodules/moodle-test-userscript index 27211c7..d06e990 160000 --- a/submodules/moodle-test-userscript +++ b/submodules/moodle-test-userscript @@ -1 +1 @@ -Subproject commit 27211c7bc83e0e930fabc430ffb5615b52523106 +Subproject commit d06e9900445ca8af4ee354244b056f24bb76bc0c From 9d3f14b4a8612db84834751a1ee5daa2472f61a4 Mon Sep 17 00:00:00 2001 From: mrfry Date: Sun, 9 Jan 2022 17:37:23 +0100 Subject: [PATCH 3/8] Added create JSON from txt script --- src/standaloneUtils/createJSON.js | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/standaloneUtils/createJSON.js diff --git a/src/standaloneUtils/createJSON.js b/src/standaloneUtils/createJSON.js new file mode 100644 index 0000000..a141409 --- /dev/null +++ b/src/standaloneUtils/createJSON.js @@ -0,0 +1,33 @@ +const fs = require('fs') + +const params = process.argv +const file = params[2] + +const data = fs.readFileSync(file,'utf8').split('\n') +console.log(data) + +let currVal = {} +const res = data.reduce((acc, val) => { + const formattedVal = val.replace(/\r/g, '').trim() + + if (formattedVal.startsWith('#')) return acc + if (formattedVal.startsWith('Q')) { + currVal = { + Q: formattedVal + } + return acc + } + if (formattedVal.startsWith('A')) { + currVal.A = formattedVal + return [ + ...acc, + currVal + ] + } + + return acc +}, []) + +console.log(res) +fs.writeFileSync('./res.json', JSON.stringify(res, null, 2)) +console.log('DONE') From 49704776a1377b6cc232461bcffc3777d21aa7d9 Mon Sep 17 00:00:00 2001 From: mrfry Date: Mon, 21 Feb 2022 20:40:08 +0100 Subject: [PATCH 4/8] Build script fix --- scripts/make.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/make.sh b/scripts/make.sh index 2b9b8a8..ce2af7d 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -58,7 +58,7 @@ log "Making moodle test userscript" mkdir -pv "$PWD/publicDirs/qminingPublic/moodle-test-userscript" checkFile "$PWD/submodules/moodle-test-userscript/stable.user.js" ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/" -ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic" +ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/" log "mkdir-ing/touching :3" From a8d4a35eb87a3f7454085d40c2a464ba89063d97 Mon Sep 17 00:00:00 2001 From: mrfry Date: Mon, 21 Feb 2022 20:41:29 +0100 Subject: [PATCH 5/8] Create json script update --- src/standaloneUtils/createJSON.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/standaloneUtils/createJSON.js b/src/standaloneUtils/createJSON.js index a141409..81329c0 100644 --- a/src/standaloneUtils/createJSON.js +++ b/src/standaloneUtils/createJSON.js @@ -6,6 +6,8 @@ const file = params[2] const data = fs.readFileSync(file,'utf8').split('\n') console.log(data) +console.log("TODO: remove 'Q: ' and 'A: '") + let currVal = {} const res = data.reduce((acc, val) => { const formattedVal = val.replace(/\r/g, '').trim() @@ -21,7 +23,12 @@ const res = data.reduce((acc, val) => { currVal.A = formattedVal return [ ...acc, - currVal + { + ...currVal, + data: { + type: 'simple' + } + } ] } From f65153635456fc39b2b51f8701805cb77272f8f7 Mon Sep 17 00:00:00 2001 From: mrfry Date: Mon, 21 Feb 2022 20:41:46 +0100 Subject: [PATCH 6/8] Added clear questions api --- src/modules/api/submodules/qminingapi.ts | 54 +++++++++++++++++++++++- src/standaloneUtils/rmDuplicates.js | 32 +++++++++++++- 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/src/modules/api/submodules/qminingapi.ts b/src/modules/api/submodules/qminingapi.ts index 5b954a4..273f98c 100644 --- a/src/modules/api/submodules/qminingapi.ts +++ b/src/modules/api/submodules/qminingapi.ts @@ -1,4 +1,5 @@ import fs from 'fs' +import { fork } from 'child_process' import logger from '../../../utils/logger' import utils from '../../../utils/utils' @@ -600,7 +601,7 @@ function setup(data: SubmoduleData): any { if (!req.body.questions) { res.json({ - message: `ask something! { question:'' ,subject:'', location:'' }`, + message: `ask something! { questions:'' ,subject:'', location:'' }`, result: [], recievedData: JSON.stringify(req.body), success: false, @@ -1006,6 +1007,57 @@ function setup(data: SubmoduleData): any { }) }) + let questionCleaner = null + app.get('/clearQuestions', (req: Request, res) => { + // TODO: dont allow multiple instances + // TODO: get status of it cleaning + logger.LogReq(req) + const user: User = req.session.user + const status: any = req.query.status + + if (status) { + if (!questionCleaner) { + res.json({ + msg: 'question cleaner not running', + success: false, + }) + return + } + questionCleaner.once('message', function (response) { + res.json({ + msg: response, + success: true, + }) + }) + + questionCleaner.send({ data: 'asd' }) + return + } + + if (questionCleaner) { + res.json({ + msg: 'question cleaner already running', + success: false, + }) + return + } + + questionCleaner = fork( + `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, + ['-s', `${process.cwd()}/${questionDbs[0].path}`] + ) + questionCleaner.on('exit', function (code) { + console.log('EXIT', code) + questionCleaner = null + }) + + res.json({ + user: user, + success: true, + msg: 'OK', + }) + }) + return { dailyAction: () => { backupData(questionDbs) diff --git a/src/standaloneUtils/rmDuplicates.js b/src/standaloneUtils/rmDuplicates.js index e09cdec..2526bde 100644 --- a/src/standaloneUtils/rmDuplicates.js +++ b/src/standaloneUtils/rmDuplicates.js @@ -41,7 +41,30 @@ const globalLog = './duplicateRemovingLog/log' utils.CreatePath(logPath) utils.WriteFile('', globalLog) -const params = process.argv.splice(2) +// ---------------------------------------------- +let currentMaxIndex = -1 +let currentIndex = -1 +process.on('message', function () { + process.send({ + currentMaxIndex: currentMaxIndex, + currentIndex: currentIndex, + }) +}) +// ---------------------------------------------- + +let params = process.argv.splice(2) +let silenced = false +if (params.includes('-s')) { + silenced = true +} +params = params.filter((x) => { + return !x.startsWith('-') +}) +console.log(params) +if (params.length === 0) { + console.log('At least 1 parameter required (path to DB)') + process.exit(1) +} const pathA = params[0] const pathB = params[1] @@ -268,6 +291,7 @@ function difference({ dbA, dbB }) { iterateSubjects(dbA, () => { dbLength++ }) + currentMaxIndex = dbLength const getResultDbLength = () => { let resultDbLength = 0 @@ -303,6 +327,7 @@ function difference({ dbA, dbB }) { printProgressBar(processedQuestions, dbLength) processedQuestions++ + currentIndex = processedQuestions const res = hasRequiredPercent(searchRes, minpercent) @@ -408,6 +433,7 @@ function hr() { function log(text) { utils.AppendToFile(text, globalLog) + if (silenced) return if (process.stdout.isTTY) { process.stdout.clearLine() process.stdout.cursorTo(0) @@ -431,7 +457,7 @@ function writeInSameLine(text, returnToLineStart) { } function printProgressBar(current, total) { - if (!process.stdout.isTTY) { + if (!process.stdout.isTTY || silenced) { return } const width = process.stdout.columns - 30 @@ -462,3 +488,5 @@ function printProgressBar(current, total) { function C(color) { return logger.C(color) } + +process.exit() From 799930b3e15c008b18cc5e65d800835c6c55e5fb Mon Sep 17 00:00:00 2001 From: mrfry Date: Mon, 21 Feb 2022 20:43:03 +0100 Subject: [PATCH 7/8] Extra modules fix --- src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.ts b/src/server.ts index 79f3aef..d01fe64 100755 --- a/src/server.ts +++ b/src/server.ts @@ -44,7 +44,7 @@ import utils from './utils/utils' import dbtools from './utils/dbtools' import reqlogger from './middlewares/reqlogger.middleware' import idStats from './utils/ids' -const extraModulesFile = './data/extraModules.json' +const extraModulesFile = './src/extraModules/extraModules.json' const statExcludeFile = './data/statExclude.json' const modulesFile = './src/modules.json' const usersDBPath = './data/dbs/users.db' From 3fe01eec9b522d5cb5958a2c62e1d9f405273ba5 Mon Sep 17 00:00:00 2001 From: mrfry Date: Sun, 20 Mar 2022 08:01:10 +0100 Subject: [PATCH 8/8] Sql injection fixes --- src/modules/api/submodules/chat.ts | 2 +- src/utils/dbtools.ts | 12 ++++++++++-- submodules/qmining-page | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/modules/api/submodules/chat.ts b/src/modules/api/submodules/chat.ts index 1bf64a6..0823e4f 100644 --- a/src/modules/api/submodules/chat.ts +++ b/src/modules/api/submodules/chat.ts @@ -131,7 +131,7 @@ function setup(data: SubmoduleData): void { const msgObj = { sender: userid, reciever: parseInt(reciever), - msg: msg.replace(/'/g, '').replace(/;/g, ''), + msg: dbtools.sanitizeQuery(msg), type: type || 'text', date: new Date().getTime(), unread: 1, diff --git a/src/utils/dbtools.ts b/src/utils/dbtools.ts index 13c1454..2fd00dc 100644 --- a/src/utils/dbtools.ts +++ b/src/utils/dbtools.ts @@ -13,6 +13,7 @@ export default { Insert: Insert, CloseDB: CloseDB, runStatement: runStatement, + sanitizeQuery: sanitizeQuery, } import Sqlite from 'better-sqlite3' @@ -21,14 +22,21 @@ import utils from '../utils/utils' const debugLog = process.env.NS_SQL_DEBUG_LOG +function sanitizeQuery(val: string): string { + return val.replace(/'/g, '').replace(/;/g, '') +} + // { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4 function GetSqlQuerry(conditions: any, type: string, joiner?: string) { const res = Object.keys(conditions).reduce((acc, key) => { const item = conditions[key] + const conditionKey = sanitizeQuery(key) + const condition = sanitizeQuery(conditions[key]) + if (typeof item === 'string') { - acc.push(`${key} = '${conditions[key]}'`) + acc.push(`${conditionKey} = '${condition}'`) } else { - acc.push(`${key} = ${conditions[key]}`) + acc.push(`${conditionKey} = ${condition}`) } return acc }, []) diff --git a/submodules/qmining-page b/submodules/qmining-page index 2b491e6..dd70010 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 2b491e6bec6eee78400e3e042418def12df3243f +Subproject commit dd7001023eaf9169863268325c2219ed0822895b