Merge

scripts/make.sh

@@ -58,7 +58,7 @@ log "Making moodle test userscript"
 mkdir -pv "$PWD/publicDirs/qminingPublic/moodle-test-userscript"
 checkFile "$PWD/submodules/moodle-test-userscript/stable.user.js"
 ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/"
-ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic"
+ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/"
 
 log "mkdir-ing/touching :3"
 

src/modules/api/submodules/chat.ts

@@ -131,7 +131,7 @@ function setup(data: SubmoduleData): void {
         const msgObj = {
           sender: userid,
           reciever: parseInt(reciever),
-          msg: msg.replace(/'/g, '').replace(/;/g, ''),
+          msg: dbtools.sanitizeQuery(msg),
           type: type || 'text',
           date: new Date().getTime(),
           unread: 1,

src/modules/api/submodules/qminingapi.ts

@@ -1,4 +1,5 @@
 import fs from 'fs'
+import { fork } from 'child_process'
 
 import logger from '../../../utils/logger'
 import utils from '../../../utils/utils'
@@ -600,7 +601,7 @@ function setup(data: SubmoduleData): any {
 
     if (!req.body.questions) {
       res.json({
-        message: `ask something! { question:'' ,subject:'', location:'' }`,
+        message: `ask something! { questions:'' ,subject:'', location:'' }`,
         result: [],
         recievedData: JSON.stringify(req.body),
         success: false,
@@ -1006,6 +1007,57 @@ function setup(data: SubmoduleData): any {
     })
   })
 
+  let questionCleaner = null
+  app.get('/clearQuestions', (req: Request, res) => {
+    // TODO: dont allow multiple instances
+    // TODO: get status of it cleaning
+    logger.LogReq(req)
+    const user: User = req.session.user
+    const status: any = req.query.status
+
+    if (status) {
+      if (!questionCleaner) {
+        res.json({
+          msg: 'question cleaner not running',
+          success: false,
+        })
+        return
+      }
+      questionCleaner.once('message', function (response) {
+        res.json({
+          msg: response,
+          success: true,
+        })
+      })
+
+      questionCleaner.send({ data: 'asd' })
+      return
+    }
+
+    if (questionCleaner) {
+      res.json({
+        msg: 'question cleaner already running',
+        success: false,
+      })
+      return
+    }
+
+    questionCleaner = fork(
+      `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`,
+      ['-s', `${process.cwd()}/${questionDbs[0].path}`]
+    )
+    questionCleaner.on('exit', function (code) {
+      console.log('EXIT', code)
+      questionCleaner = null
+    })
+
+    res.json({
+      user: user,
+      success: true,
+      msg: 'OK',
+    })
+  })
+
   return {
     dailyAction: () => {
       backupData(questionDbs)

src/server.ts

@@ -44,7 +44,7 @@ import utils from './utils/utils'
 import dbtools from './utils/dbtools'
 import reqlogger from './middlewares/reqlogger.middleware'
 import idStats from './utils/ids'
-const extraModulesFile = './data/extraModules.json'
+const extraModulesFile = './src/extraModules/extraModules.json'
 const statExcludeFile = './data/statExclude.json'
 const modulesFile = './src/modules.json'
 const usersDBPath = './data/dbs/users.db'

src/standaloneUtils/createJSON.js

@@ -0,0 +1,40 @@
+const fs = require('fs')
+
+const params = process.argv
+const file = params[2]
+
+const data = fs.readFileSync(file,'utf8').split('\n')
+console.log(data)
+
+console.log("TODO: remove 'Q: ' and 'A: '")
+
+let currVal = {}
+const res = data.reduce((acc, val) => {
+  const formattedVal = val.replace(/\r/g, '').trim()
+
+  if (formattedVal.startsWith('#')) return acc
+  if (formattedVal.startsWith('Q')) {
+    currVal = {
+      Q: formattedVal
+    }
+    return acc
+  } 
+  if (formattedVal.startsWith('A')) {
+    currVal.A = formattedVal
+    return [
+      ...acc,
+      {
+        ...currVal,
+        data: {
+          type: 'simple'
+        }
+      }
+    ]
+  } 
+
+  return acc
+}, [])
+
+console.log(res)
+fs.writeFileSync('./res.json', JSON.stringify(res, null, 2))
+console.log('DONE')

src/standaloneUtils/manualSearch.js

@@ -0,0 +1,93 @@
+const utils = require('../../dist/utils/utils.js').default // eslint-disable-line
+const logger = require('../../dist/utils/logger.js').default // eslint-disable-line
+const { doSearch } = require('../../dist/utils/classes.js') // eslint-disable-line
+const { loadData } = require('../../dist/utils/actions.js') // eslint-disable-line
+const fs = require('fs') // eslint-disable-line
+
+const minpercent = 95
+const logPath = './duplicateRemovingLog/'
+const globalLog = './duplicateRemovingLog/log'
+utils.CreatePath(logPath)
+utils.WriteFile('', globalLog)
+
+const params = process.argv.splice(2)
+const path = params[0]
+const question = params[1]
+const answer = params[2]
+
+// ---------------------------------------------------------------------------------
+//
+// ---------------------------------------------------------------------------------
+
+console.time('SEARCH')
+const searchRes = search({
+  qdb: loadData(path),
+  subjName: 'Elektronika',
+  question: {
+    Q: question,
+    A: answer,
+    data: {
+      type: 'simple',
+    },
+  },
+  searchTillMatchPercent: 80,
+})
+hr()
+console.log('Search result')
+hr()
+showSearchResult(searchRes)
+hr()
+console.timeEnd('SEARCH')
+log(
+  `Searched for question: "${C('green')}${question}${C()}" answer: "${C(
+    'green'
+  )}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"`
+)
+hr()
+
+// ---------------------------------------------------------------------------------
+// logging and tools
+// ---------------------------------------------------------------------------------
+
+function showSearchResult(res) {
+  res.forEach((x) => {
+    console.log(`${C('green')}Q:${C()}`, x.q.Q)
+    console.log(`${C('green')}A:${C()}`, x.q.A)
+    console.log(`${C('green')}match:${C()}`, x.match)
+    console.log()
+  })
+  console.log(`Result length: ${C('green')}${res.length}${C()}`)
+}
+
+function search({ qdb, subjName, question, searchInAllIfNoResult }) {
+  return doSearch(
+    qdb,
+    subjName,
+    question,
+    null,
+    minpercent,
+    searchInAllIfNoResult
+  )
+}
+
+function hr() {
+  let res = ''
+  for (let i = 0; i < process.stdout.columns; i++) {
+    res += '='
+  }
+  log(`${C('cyan')}${res}${C()}`)
+}
+
+function log(text) {
+  utils.AppendToFile(text, globalLog)
+  if (process.stdout.isTTY) {
+    process.stdout.clearLine()
+    process.stdout.cursorTo(0)
+  }
+
+  console.log(text)
+}
+
+function C(color) {
+  return logger.C(color)
+}

src/standaloneUtils/rmDuplicates.js

@@ -41,7 +41,30 @@ const globalLog = './duplicateRemovingLog/log'
 utils.CreatePath(logPath)
 utils.WriteFile('', globalLog)
 
-const params = process.argv.splice(2)
+// ----------------------------------------------
+let currentMaxIndex = -1
+let currentIndex = -1
+process.on('message', function () {
+  process.send({
+    currentMaxIndex: currentMaxIndex,
+    currentIndex: currentIndex,
+  })
+})
+// ----------------------------------------------
+
+let params = process.argv.splice(2)
+let silenced = false
+if (params.includes('-s')) {
+  silenced = true
+}
+params = params.filter((x) => {
+  return !x.startsWith('-')
+})
+console.log(params)
+if (params.length === 0) {
+  console.log('At least 1 parameter required (path to DB)')
+  process.exit(1)
+}
 
 const pathA = params[0]
 const pathB = params[1]
@@ -268,6 +291,7 @@ function difference({ dbA, dbB }) {
   iterateSubjects(dbA, () => {
     dbLength++
   })
+  currentMaxIndex = dbLength
 
   const getResultDbLength = () => {
     let resultDbLength = 0
@@ -303,6 +327,7 @@ function difference({ dbA, dbB }) {
 
       printProgressBar(processedQuestions, dbLength)
       processedQuestions++
+      currentIndex = processedQuestions
 
       const res = hasRequiredPercent(searchRes, minpercent)
 
@@ -408,6 +433,7 @@ function hr() {
 
 function log(text) {
   utils.AppendToFile(text, globalLog)
+  if (silenced) return
   if (process.stdout.isTTY) {
     process.stdout.clearLine()
     process.stdout.cursorTo(0)
@@ -431,7 +457,7 @@ function writeInSameLine(text, returnToLineStart) {
 }
 
 function printProgressBar(current, total) {
-  if (!process.stdout.isTTY) {
+  if (!process.stdout.isTTY || silenced) {
     return
   }
   const width = process.stdout.columns - 30
@@ -462,3 +488,5 @@ function printProgressBar(current, total) {
 function C(color) {
   return logger.C(color)
 }
+
+process.exit()

src/utils/dbtools.ts

@@ -13,6 +13,7 @@ export default {
   Insert: Insert,
   CloseDB: CloseDB,
   runStatement: runStatement,
+  sanitizeQuery: sanitizeQuery,
 }
 
 import Sqlite from 'better-sqlite3'
@@ -21,14 +22,21 @@ import utils from '../utils/utils'
 
 const debugLog = process.env.NS_SQL_DEBUG_LOG
 
+function sanitizeQuery(val: string): string {
+  return val.replace(/'/g, '').replace(/;/g, '')
+}
+
 // { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
 function GetSqlQuerry(conditions: any, type: string, joiner?: string) {
   const res = Object.keys(conditions).reduce((acc, key) => {
     const item = conditions[key]
+    const conditionKey = sanitizeQuery(key)
+    const condition = sanitizeQuery(conditions[key])
+
     if (typeof item === 'string') {
-      acc.push(`${key} = '${conditions[key]}'`)
+      acc.push(`${conditionKey} = '${condition}'`)
     } else {
-      acc.push(`${key} = ${conditions[key]}`)
+      acc.push(`${conditionKey} = ${condition}`)
     }
     return acc
   }, [])

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit c92615e32366b106f64e3145ddc706bddd42915b
+Subproject commit 1012880e899ad328f7a637a6d1512dab7afd24d5