diff --git a/scripts/make.sh b/scripts/make.sh index 2b9b8a8..ce2af7d 100755 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -58,7 +58,7 @@ log "Making moodle test userscript" mkdir -pv "$PWD/publicDirs/qminingPublic/moodle-test-userscript" checkFile "$PWD/submodules/moodle-test-userscript/stable.user.js" ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/" -ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic" +ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/" log "mkdir-ing/touching :3" diff --git a/src/modules/api/submodules/chat.ts b/src/modules/api/submodules/chat.ts index 1bf64a6..0823e4f 100644 --- a/src/modules/api/submodules/chat.ts +++ b/src/modules/api/submodules/chat.ts @@ -131,7 +131,7 @@ function setup(data: SubmoduleData): void { const msgObj = { sender: userid, reciever: parseInt(reciever), - msg: msg.replace(/'/g, '').replace(/;/g, ''), + msg: dbtools.sanitizeQuery(msg), type: type || 'text', date: new Date().getTime(), unread: 1, diff --git a/src/modules/api/submodules/qminingapi.ts b/src/modules/api/submodules/qminingapi.ts index 5b954a4..273f98c 100644 --- a/src/modules/api/submodules/qminingapi.ts +++ b/src/modules/api/submodules/qminingapi.ts @@ -1,4 +1,5 @@ import fs from 'fs' +import { fork } from 'child_process' import logger from '../../../utils/logger' import utils from '../../../utils/utils' @@ -600,7 +601,7 @@ function setup(data: SubmoduleData): any { if (!req.body.questions) { res.json({ - message: `ask something! { question:'' ,subject:'', location:'' }`, + message: `ask something! { questions:'' ,subject:'', location:'' }`, result: [], recievedData: JSON.stringify(req.body), success: false, @@ -1006,6 +1007,57 @@ function setup(data: SubmoduleData): any { }) }) + let questionCleaner = null + app.get('/clearQuestions', (req: Request, res) => { + // TODO: dont allow multiple instances + // TODO: get status of it cleaning + logger.LogReq(req) + const user: User = req.session.user + const status: any = req.query.status + + if (status) { + if (!questionCleaner) { + res.json({ + msg: 'question cleaner not running', + success: false, + }) + return + } + questionCleaner.once('message', function (response) { + res.json({ + msg: response, + success: true, + }) + }) + + questionCleaner.send({ data: 'asd' }) + return + } + + if (questionCleaner) { + res.json({ + msg: 'question cleaner already running', + success: false, + }) + return + } + + questionCleaner = fork( + `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, + ['-s', `${process.cwd()}/${questionDbs[0].path}`] + ) + questionCleaner.on('exit', function (code) { + console.log('EXIT', code) + questionCleaner = null + }) + + res.json({ + user: user, + success: true, + msg: 'OK', + }) + }) + return { dailyAction: () => { backupData(questionDbs) diff --git a/src/server.ts b/src/server.ts index 79f3aef..d01fe64 100755 --- a/src/server.ts +++ b/src/server.ts @@ -44,7 +44,7 @@ import utils from './utils/utils' import dbtools from './utils/dbtools' import reqlogger from './middlewares/reqlogger.middleware' import idStats from './utils/ids' -const extraModulesFile = './data/extraModules.json' +const extraModulesFile = './src/extraModules/extraModules.json' const statExcludeFile = './data/statExclude.json' const modulesFile = './src/modules.json' const usersDBPath = './data/dbs/users.db' diff --git a/src/standaloneUtils/createJSON.js b/src/standaloneUtils/createJSON.js new file mode 100644 index 0000000..81329c0 --- /dev/null +++ b/src/standaloneUtils/createJSON.js @@ -0,0 +1,40 @@ +const fs = require('fs') + +const params = process.argv +const file = params[2] + +const data = fs.readFileSync(file,'utf8').split('\n') +console.log(data) + +console.log("TODO: remove 'Q: ' and 'A: '") + +let currVal = {} +const res = data.reduce((acc, val) => { + const formattedVal = val.replace(/\r/g, '').trim() + + if (formattedVal.startsWith('#')) return acc + if (formattedVal.startsWith('Q')) { + currVal = { + Q: formattedVal + } + return acc + } + if (formattedVal.startsWith('A')) { + currVal.A = formattedVal + return [ + ...acc, + { + ...currVal, + data: { + type: 'simple' + } + } + ] + } + + return acc +}, []) + +console.log(res) +fs.writeFileSync('./res.json', JSON.stringify(res, null, 2)) +console.log('DONE') diff --git a/src/standaloneUtils/manualSearch.js b/src/standaloneUtils/manualSearch.js new file mode 100644 index 0000000..a3b6aef --- /dev/null +++ b/src/standaloneUtils/manualSearch.js @@ -0,0 +1,93 @@ +const utils = require('../../dist/utils/utils.js').default // eslint-disable-line +const logger = require('../../dist/utils/logger.js').default // eslint-disable-line +const { doSearch } = require('../../dist/utils/classes.js') // eslint-disable-line +const { loadData } = require('../../dist/utils/actions.js') // eslint-disable-line +const fs = require('fs') // eslint-disable-line + +const minpercent = 95 +const logPath = './duplicateRemovingLog/' +const globalLog = './duplicateRemovingLog/log' +utils.CreatePath(logPath) +utils.WriteFile('', globalLog) + +const params = process.argv.splice(2) +const path = params[0] +const question = params[1] +const answer = params[2] + +// --------------------------------------------------------------------------------- +// +// --------------------------------------------------------------------------------- + +console.time('SEARCH') +const searchRes = search({ + qdb: loadData(path), + subjName: 'Elektronika', + question: { + Q: question, + A: answer, + data: { + type: 'simple', + }, + }, + searchTillMatchPercent: 80, +}) +hr() +console.log('Search result') +hr() +showSearchResult(searchRes) +hr() +console.timeEnd('SEARCH') +log( + `Searched for question: "${C('green')}${question}${C()}" answer: "${C( + 'green' + )}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"` +) +hr() + +// --------------------------------------------------------------------------------- +// logging and tools +// --------------------------------------------------------------------------------- + +function showSearchResult(res) { + res.forEach((x) => { + console.log(`${C('green')}Q:${C()}`, x.q.Q) + console.log(`${C('green')}A:${C()}`, x.q.A) + console.log(`${C('green')}match:${C()}`, x.match) + console.log() + }) + console.log(`Result length: ${C('green')}${res.length}${C()}`) +} + +function search({ qdb, subjName, question, searchInAllIfNoResult }) { + return doSearch( + qdb, + subjName, + question, + null, + minpercent, + searchInAllIfNoResult + ) +} + +function hr() { + let res = '' + for (let i = 0; i < process.stdout.columns; i++) { + res += '=' + } + log(`${C('cyan')}${res}${C()}`) +} + +function log(text) { + utils.AppendToFile(text, globalLog) + if (process.stdout.isTTY) { + process.stdout.clearLine() + process.stdout.cursorTo(0) + } + + console.log(text) +} + +function C(color) { + return logger.C(color) +} diff --git a/src/standaloneUtils/rmDuplicates.js b/src/standaloneUtils/rmDuplicates.js index e09cdec..2526bde 100644 --- a/src/standaloneUtils/rmDuplicates.js +++ b/src/standaloneUtils/rmDuplicates.js @@ -41,7 +41,30 @@ const globalLog = './duplicateRemovingLog/log' utils.CreatePath(logPath) utils.WriteFile('', globalLog) -const params = process.argv.splice(2) +// ---------------------------------------------- +let currentMaxIndex = -1 +let currentIndex = -1 +process.on('message', function () { + process.send({ + currentMaxIndex: currentMaxIndex, + currentIndex: currentIndex, + }) +}) +// ---------------------------------------------- + +let params = process.argv.splice(2) +let silenced = false +if (params.includes('-s')) { + silenced = true +} +params = params.filter((x) => { + return !x.startsWith('-') +}) +console.log(params) +if (params.length === 0) { + console.log('At least 1 parameter required (path to DB)') + process.exit(1) +} const pathA = params[0] const pathB = params[1] @@ -268,6 +291,7 @@ function difference({ dbA, dbB }) { iterateSubjects(dbA, () => { dbLength++ }) + currentMaxIndex = dbLength const getResultDbLength = () => { let resultDbLength = 0 @@ -303,6 +327,7 @@ function difference({ dbA, dbB }) { printProgressBar(processedQuestions, dbLength) processedQuestions++ + currentIndex = processedQuestions const res = hasRequiredPercent(searchRes, minpercent) @@ -408,6 +433,7 @@ function hr() { function log(text) { utils.AppendToFile(text, globalLog) + if (silenced) return if (process.stdout.isTTY) { process.stdout.clearLine() process.stdout.cursorTo(0) @@ -431,7 +457,7 @@ function writeInSameLine(text, returnToLineStart) { } function printProgressBar(current, total) { - if (!process.stdout.isTTY) { + if (!process.stdout.isTTY || silenced) { return } const width = process.stdout.columns - 30 @@ -462,3 +488,5 @@ function printProgressBar(current, total) { function C(color) { return logger.C(color) } + +process.exit() diff --git a/src/utils/dbtools.ts b/src/utils/dbtools.ts index 13c1454..2fd00dc 100644 --- a/src/utils/dbtools.ts +++ b/src/utils/dbtools.ts @@ -13,6 +13,7 @@ export default { Insert: Insert, CloseDB: CloseDB, runStatement: runStatement, + sanitizeQuery: sanitizeQuery, } import Sqlite from 'better-sqlite3' @@ -21,14 +22,21 @@ import utils from '../utils/utils' const debugLog = process.env.NS_SQL_DEBUG_LOG +function sanitizeQuery(val: string): string { + return val.replace(/'/g, '').replace(/;/g, '') +} + // { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4 function GetSqlQuerry(conditions: any, type: string, joiner?: string) { const res = Object.keys(conditions).reduce((acc, key) => { const item = conditions[key] + const conditionKey = sanitizeQuery(key) + const condition = sanitizeQuery(conditions[key]) + if (typeof item === 'string') { - acc.push(`${key} = '${conditions[key]}'`) + acc.push(`${conditionKey} = '${condition}'`) } else { - acc.push(`${key} = ${conditions[key]}`) + acc.push(`${conditionKey} = ${condition}`) } return acc }, []) diff --git a/submodules/qmining-page b/submodules/qmining-page index c92615e..1012880 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit c92615e32366b106f64e3145ddc706bddd42915b +Subproject commit 1012880e899ad328f7a637a6d1512dab7afd24d5