Merge branch 'master' of gitlab.com:MrFry/mrfrys-node-server

src/modules/api/submodules/chat.ts

@@ -146,39 +146,42 @@ function setup(data: SubmoduleData): void {
         chatMessageRead({ sender: chatPartner, reciever: userid })
       })
 
-      socket.on('chat message', (message) => {
+      socket.on(
-        const { reciever, msg, type } = message
+        'chat message',
-        if (!reciever || !msg || !type) {
+        (message: { reciever: string; msg: string; type: string }) => {
-          return
+          const { reciever, msg, type } = message
-        }
+          if (!reciever || !msg || !msg.trim() || !type) {
-        const recieverUser = dbtools.Select(userDB, 'users', {
+            return
-          id: reciever,
+          }
-        })[0]
+          const recieverUser = dbtools.Select(userDB, 'users', {
-        if (!recieverUser) {
+            id: reciever,
-          socket.emit('chat message', {
+          })[0]
-            success: false,
+          if (!recieverUser) {
-            date: new Date().getTime(),
+            socket.emit('chat message', {
-            sender: reciever,
+              success: false,
-            reciever: userid,
+              date: new Date().getTime(),
-            type: 'text',
+              sender: reciever,
-            msg: `A #${reciever} számú felhasználó nem létezik`,
+              reciever: userid,
-          })
+              type: 'text',
-          return
+              msg: `A #${reciever} számú felhasználó nem létezik`,
-        }
+            })
+            return
+          }
 
-        const msgObj = {
+          const msgObj = {
-          sender: userid,
+            sender: userid,
-          reciever: parseInt(reciever),
+            reciever: parseInt(reciever),
-          msg: dbtools.sanitizeQuery(msg),
+            msg: dbtools.sanitizeQuery(msg),
-          type: type || 'text',
+            type: type || 'text',
-          date: new Date().getTime(),
+            date: new Date().getTime(),
-          unread: 1,
+            unread: 1,
+          }
+          dbtools.Insert(msgDB, 'msgs', msgObj)
+          if (userid !== parseInt(reciever)) {
+            io.sockets.in(reciever.toString()).emit('chat message', msgObj)
+          }
         }
-        dbtools.Insert(msgDB, 'msgs', msgObj)
+      )
-        if (userid !== reciever) {
-          io.sockets.in(reciever.toString()).emit('chat message', msgObj)
-        }
-      })
     })
 
     // socket.on('disconnect', () => {})
@@ -201,6 +204,59 @@ function setup(data: SubmoduleData): void {
       })
   })
 
+  app.post('/postfeedbackfile', function (req: Request, res) {
+    logger.LogReq(req)
+    const user: User = req.session.user
+
+    utils
+      .uploadFile(req, uloadFiles)
+      .then(({ filePath }) => {
+        const fileName = filePath.replace(publicDir, '')
+        const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => {
+          return fileName.toLowerCase().includes(ext)
+        })
+        const msgObj = {
+          sender: user.id,
+          reciever: 1,
+          msg: fileName,
+          type: isImage ? 'img' : 'file',
+          date: new Date().getTime(),
+          unread: 1,
+        }
+        dbtools.Insert(msgDB, 'msgs', msgObj)
+
+        res.json({ success: true })
+        io.sockets.in('1').emit('chat message', msgObj)
+      })
+      .catch(() => {
+        res.json({ success: false, msg: 'error during uploading' })
+        return
+      })
+  })
+
+  app.post('/postfeedback', function (req: Request<{ content: string }>, res) {
+    logger.LogReq(req)
+    const user: User = req.session.user
+    const { content } = req.body
+    if (!content || !content.trim()) {
+      res.json({ success: false })
+      return
+    }
+
+    const msgObj = {
+      sender: user.id,
+      reciever: 1,
+      msg: dbtools.sanitizeQuery(req.body.content),
+      type: 'text',
+      date: new Date().getTime(),
+      unread: 1,
+    }
+    dbtools.Insert(msgDB, 'msgs', msgObj)
+
+    res.json({ success: true })
+    io.sockets.in('1').emit('chat message', msgObj)
+  })
+
   app.get('/hasNewMsg', (req: Request, res) => {
     const user: User = req.session.user
     const userid: number = user.id

src/modules/api/submodules/feedback.ts

@@ -20,63 +20,14 @@
 
 import { Response } from 'express'
 
-import logger from '../../../utils/logger'
 import utils from '../../../utils/utils'
-import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+import { Request, SubmoduleData } from '../../../types/basicTypes'
 
-const msgFile = 'stats/msgs'
 const uloadFiles = 'data/f'
 
 function setup(data: SubmoduleData): void {
   const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
 
-  app.post('/postfeedbackfile', function (req: Request, res: Response) {
-    utils
-      .uploadFile(req, uloadFiles)
-      .then(() => {
-        res.json({ success: true })
-      })
-      .catch(() => {
-        res.json({ success: false, msg: 'error during uploading' })
-        return
-      })
-
-    logger.LogReq(req)
-    logger.Log('New feedback file', logger.GetColor('bluebg'))
-  })
-
-  app.post('/postfeedback', function (req: Request, res: Response) {
-    logger.LogReq(req)
-    if (req.body.fromLogin) {
-      logger.Log(
-        'New feedback message from Login page',
-        logger.GetColor('bluebg')
-      )
-    } else {
-      logger.Log(
-        'New feedback message from feedback page',
-        logger.GetColor('bluebg')
-      )
-    }
-
-    const user: User = req.session.user
-
-    utils.AppendToFile(
-      utils.GetDateString() +
-        ':\n' +
-        JSON.stringify(
-          {
-            ...req.body,
-            userID: user ? user.id : 'no user',
-          },
-          null,
-          2
-        ),
-      msgFile
-    )
-    res.json({ success: true })
-  })
-
   app.route('/fosuploader').post(function (req: Request, res: Response) {
     utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
       res.redirect('/f/' + fileName)

src/modules/api/submodules/qminingapi.ts

@@ -53,6 +53,7 @@ import {
   dataToString,
   getSubjNameWithoutYear,
   WorkerResult,
+  SearchResultQuestion,
   // compareQuestionObj,
 } from '../../../utils/classes'
 import {
@@ -194,7 +195,7 @@ function searchInDbs(
           logger.DebugLog(taskResult, 'ask', 2)
           resolve({
             question: question,
-            result: taskResult.result,
+            result: taskResult.result as SearchResultQuestion[],
             success: true,
           })
         } catch (err) {

src/modules/api/submodules/userManagement.ts

@@ -248,6 +248,7 @@ function setup(data: SubmoduleData): Submodule {
     logger.LogReq(req)
     const sessionID = req.cookies.sessionID
     const user: User = req.session.user
+    const { all } = req.query
 
     if (!user) {
       res.json({
@@ -262,10 +263,16 @@ function setup(data: SubmoduleData): Submodule {
       logger.GetColor('cyan')
     )
 
-    // removing session from db
+    if (all) {
-    dbtools.Delete(userDB, 'sessions', {
+      dbtools.Delete(userDB, 'sessions', {
-      id: sessionID,
+        userID: user.id,
-    })
+      })
+    } else {
+      dbtools.Delete(userDB, 'sessions', {
+        id: sessionID,
+      })
+    }
+
     res.clearCookie('sessionID').json({
       msg: 'Successfull logout',
       result: 'success',

src/tests/oldQuestionRemoving.test.ts

@@ -0,0 +1,197 @@
+import { updateQuestionsInArray } from '../utils/actions'
+import { createQuestion } from '../utils/classes'
+import { cleanDb } from '../utils/classes'
+import { QuestionDb, Subject, Question } from '../types/basicTypes'
+
+const date = (x?: number) => new Date().getTime() + (x || 0)
+
+const q1 = createQuestion(
+  'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
+  'piaci áruk eltérhet a névértéktől.',
+  {
+    type: 'simple',
+    date: date(-1000),
+  }
+)
+const q2 = createQuestion(
+  'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
+  'afjléa gféda gfdjs légf',
+  {
+    type: 'simple',
+    date: date(-1000),
+  }
+)
+const q3 = createQuestion(
+  'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
+  'afjlsd gfds dgfs gf  sdgf d',
+  {
+    type: 'simple',
+    date: date(-1000),
+  }
+)
+const q4 = createQuestion(
+  'A kötvény névértéke',
+  'A kötvényen feltüntetett meghatározott nagyságú összeg.',
+  {
+    type: 'simple',
+    date: date(-1000),
+  }
+)
+const q5 = createQuestion(
+  'Mi az osztalék? asd asd',
+  'A vállalati profit egy része..',
+  {
+    type: 'simple',
+    date: date(1000),
+  }
+)
+const q6 = createQuestion(
+  'valaim nagyon értelmes kérdés asd asd',
+  'A vállalati profit egy része..',
+  {
+    type: 'simple',
+    date: date(1000),
+  }
+)
+
+function setupTest({
+  newQuestions,
+  data,
+  subjToClean,
+}: {
+  newQuestions: Question[]
+  data: Subject[]
+  subjToClean?: string
+}) {
+  const recievedQuestions: Question[] = newQuestions.map((x) => {
+    return {
+      ...x,
+      data: {
+        ...x.data,
+        date: date(),
+      },
+    }
+  })
+  const subjName = subjToClean || 'subject'
+  const overwriteFromDate = date(-100)
+  const qdbIndex = 0
+  const qdbs: QuestionDb[] = [
+    {
+      name: 'test',
+      data: data,
+      index: 0,
+      path: '',
+      shouldSearch: 'asd',
+      shouldSave: {},
+    },
+  ]
+  const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
+    return x.Name.toLowerCase().includes(subjName.toLowerCase())
+  })
+
+  const questionIndexesToRemove = cleanDb(
+    {
+      questions: recievedQuestions,
+      subjToClean: subjName,
+      overwriteFromDate: overwriteFromDate,
+      qdbIndex: qdbIndex,
+    },
+    qdbs
+  )
+
+  const updatedQuestions = updateQuestionsInArray(
+    questionIndexesToRemove,
+    qdbs[qdbIndex].data[subjIndex].Questions,
+    recievedQuestions
+  )
+
+  return {
+    questionIndexesToRemove: questionIndexesToRemove,
+    updatedQuestions: updatedQuestions,
+    overwriteFromDate: overwriteFromDate,
+    subjIndex: subjIndex,
+  }
+}
+
+const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] }
+
+test('Old and duplicate questions should be removed from the database', () => {
+  const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
+    setupTest({ newQuestions: [q1, q4, q5], data: [s1] })
+
+  expect(questionIndexesToRemove.length).toBe(3)
+  expect(questionIndexesToRemove[0].length).toBe(2)
+
+  expect(updatedQuestions.length).toBe(3)
+  const toremoveCount = updatedQuestions.filter((question) => {
+    return question.Q.includes('TOREMOVE')
+  }).length
+  expect(toremoveCount).toBe(1)
+  const newQuestion = updatedQuestions.find((question) => {
+    return question.Q.includes('TOREMOVE')
+  })
+  expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
+})
+
+const s2: Subject = {
+  Name: 'test subject',
+  Questions: [q1, q2, q3, q4, q5, q6],
+}
+
+test('Old and duplicate questions should be removed from the database round 2', () => {
+  const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
+    setupTest({ newQuestions: [q1, q4, q5], data: [s2] })
+
+  expect(questionIndexesToRemove.length).toBe(3)
+  expect(questionIndexesToRemove[0].length).toBe(3)
+
+  expect(updatedQuestions.length).toBe(4)
+  const toremoveCount = updatedQuestions.filter((question) => {
+    return question.Q.includes('TOREMOVE')
+  }).length
+  expect(toremoveCount).toBe(1)
+  const newQuestion = updatedQuestions.find((question) => {
+    return question.Q.includes('TOREMOVE')
+  })
+  expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
+})
+
+const s3: Subject = {
+  Name: 'test subject',
+  Questions: [q5, q6].map((x) => ({
+    ...x,
+    data: {
+      ...x.data,
+      date: date(+50000),
+    },
+  })),
+}
+
+test('Old and duplicate questions should be removed from the database: questions should be left alone when they are newer', () => {
+  const { questionIndexesToRemove, updatedQuestions } = setupTest({
+    newQuestions: [q5, q6],
+    data: [s3],
+  })
+
+  expect(questionIndexesToRemove.length).toBe(2)
+  questionIndexesToRemove.forEach((x) => {
+    expect(x.length).toBe(0)
+  })
+
+  expect(updatedQuestions.length).toBe(2)
+})
+
+const s4: Subject = {
+  Name: 'something else',
+  Questions: [q5, q6],
+}
+
+test('Old and duplicate questions should be removed from the database:other subjects should be left alone', () => {
+  const { subjIndex } = setupTest({
+    newQuestions: [q5, q6],
+    data: [s2, s1, s4, s3],
+    subjToClean: 'else',
+  })
+
+  expect(subjIndex).toBe(2)
+})

src/types/basicTypes.ts

@@ -56,6 +56,10 @@ export interface DataFile {
   path: string
   name: string
   locked?: Boolean
+  overwrites?: Array<{
+    subjName: string
+    overwriteFromDate: number
+  }>
   shouldSearch:
     | string
     | {

src/utils/actions.ts

@@ -27,7 +27,7 @@ import {
   WorkerResult,
   SearchResultQuestion,
 } from '../utils/classes'
-import { doALongTask } from './workerPool'
+import { doALongTask, msgAllWorker } from './workerPool'
 import idStats from '../utils/ids'
 import utils from '../utils/utils'
 import { addQuestion, getSubjNameWithoutYear } from './classes'
@@ -201,32 +201,20 @@ function processIncomingRequestUsingDb(
         .then((results: Array<WorkerResult>) => {
           const allQuestions: Question[] = [] // all new questions here that do not have result
           results.forEach((result: WorkerResult, i) => {
-            const add = result.result.every((res: SearchResultQuestion) => {
+            const add = (result.result as SearchResultQuestion[]).every(
-              return res.match < minMatchAmmountToAdd
+              (res: SearchResultQuestion) => {
-            })
+                return res.match < minMatchAmmountToAdd
+              }
+            )
             if (add && !result.error) {
               allQuestions.push(recievedQuestions[i])
             }
           })
 
           try {
+            const subjName = getSubjNameWithoutYear(recievedData.subj)
             if (allQuestions.length > 0) {
-              allQuestions.forEach((currentQuestion) => {
+              addQuestionsToDb(allQuestions, subjName, qdb)
-                const sName = getSubjNameWithoutYear(recievedData.subj)
-                logger.DebugLog(
-                  'Adding question with subjName: ' + sName + ' :',
-                  'isadding',
-                  3
-                )
-                logger.DebugLog(currentQuestion, 'isadding', 3)
-                addQuestion(qdb.data, sName, {
-                  ...currentQuestion,
-                  data: {
-                    ...currentQuestion.data,
-                    date: new Date().getTime(),
-                  },
-                })
-              })
 
               currWrites++
               logger.DebugLog(
@@ -258,6 +246,7 @@ function processIncomingRequestUsingDb(
               qdbIndex: qdb.index,
               qdbName: qdb.name,
             })
+            runCleanWorker(recievedData.quiz, subjName, qdb)
           } catch (error) {
             console.error(error)
             logger.Log(
@@ -287,6 +276,124 @@ function processIncomingRequestUsingDb(
   })
 }
 
+function addQuestionsToDb(
+  allQuestions: Question[],
+  subjName: string,
+  qdb: QuestionDb
+) {
+  allQuestions.forEach((currentQuestion) => {
+    logger.DebugLog(
+      'Adding question with subjName: ' + subjName + ' :',
+      'isadding',
+      3
+    )
+    logger.DebugLog(currentQuestion, 'isadding', 3)
+    addQuestion(qdb.data, subjName, {
+      ...currentQuestion,
+      data: {
+        ...currentQuestion.data,
+        date: new Date().getTime(),
+      },
+    })
+  })
+}
+
+function runCleanWorker(
+  recievedQuesitons: Question[],
+  subjName: string,
+  qdb: QuestionDb
+) {
+  if (qdb.overwrites && qdb.overwrites.length) {
+    // check if subject needs to be updated, and qdb has overwriteFromDate
+    const overwrite = qdb.overwrites.find((x) => {
+      return subjName.toLowerCase().includes(x.subjName.toLowerCase())
+    })
+
+    if (!overwrite) {
+      return
+    }
+    // logger.Log(
+    //   `\tStarting cleaning in subject "${logger.C(
+    //     'green'
+    //   )}${subjName}${logger.C('')}" (matched: "${logger.C('green')}${
+    //     overwrite.subjName
+    //   }${logger.C('')}")`
+    // )
+    // pass recieved questions to a worker
+    doALongTask({
+      type: 'dbClean',
+      data: {
+        questions: recievedQuesitons,
+        subjToClean: subjName,
+        overwriteFromDate: overwrite.overwriteFromDate,
+        qdbIndex: qdb.index,
+      },
+    }).then(({ result: questionIndexesToRemove }) => {
+      const subjIndex = qdb.data.findIndex((x) => {
+        return x.Name.toLowerCase().includes(subjName.toLowerCase())
+      })
+      //   sends msgs to all workers to remove it too
+
+      msgAllWorker({
+        type: 'rmQuestions',
+        data: {
+          questionIndexesToRemove: questionIndexesToRemove as number[][],
+          subjIndex: subjIndex,
+          qdbIndex: qdb.index,
+          recievedQuestions: recievedQuesitons,
+        },
+      })
+
+      //   it adds the recieved question WITH DATE!
+      //     recievedQuestions doesnt have date-s
+      qdb.data[subjIndex].Questions = updateQuestionsInArray(
+        questionIndexesToRemove as number[][],
+        qdb.data[subjIndex].Questions,
+        recievedQuesitons
+      )
+
+      //   saves the file
+      writeData(qdb.data, qdb.path)
+      logger.Log(
+        `\tRemoved ${logger.C('green')}${
+          (questionIndexesToRemove as number[][]).filter(
+            (x: number[]) => x.length > 1
+          ).length
+        }${logger.C()} old questions from ${logger.C(
+          'green'
+        )}${subjName}${logger.C()}`
+      )
+    })
+  }
+}
+
+export function updateQuestionsInArray(
+  questionIndexesToRemove: number[][],
+  questions: Question[],
+  newQuestions: Question[]
+): Question[] {
+  const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => {
+    if (x.length > 1) {
+      return [...acc, ...x]
+    }
+    return acc
+  }, [])
+
+  const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => {
+    return questionIndexesToRemove[i].length > 1
+  })
+
+  return [
+    ...questions.filter((_x, i) => {
+      return !indexesToRemove.includes(i)
+    }),
+    ...newQuestionsToAdd.map((x) => {
+      x.data.date = new Date()
+      return x
+    }),
+  ]
+}
+
 export function isQuestionValid(question: Question): boolean {
   if (!question.Q) {
     return false
@@ -435,12 +542,12 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
     const path = './publicDirs/qminingPublic/backs/'
     utils.CreatePath(path)
     try {
-      logger.Log(`Backing up ${data.name}...`)
+      // logger.Log(`Backing up ${data.name}...`)
       writeData(
         data.data,
         `${path}${data.name}_${utils.GetDateString(undefined, true)}.json`
       )
-      logger.Log('Done')
+      // logger.Log('Done')
     } catch (err) {
       logger.Log(
         `Error backing up data file ${data.name}!`,

src/utils/classes.ts

@@ -17,6 +17,7 @@
  along with this program. If not, see <https://www.gnu.org/licenses/>.
 
  ------------------------------------------------------------------------- */
+// FIXME: this should be renamed to worker.ts or something
 
 import { isMainThread, parentPort, workerData } from 'worker_threads'
 
@@ -27,12 +28,13 @@ import {
   QuestionDb,
   Subject,
 } from '../types/basicTypes'
-import { editDb, Edits } from './actions'
+import { editDb, Edits, updateQuestionsInArray } from './actions'
+// import { TaskObject } from './workerPool'
 
 export interface WorkerResult {
   msg: string
   workerIndex: number
-  result?: SearchResultQuestion[]
+  result?: SearchResultQuestion[] | number[][]
   error?: boolean
 }
 
@@ -115,12 +117,15 @@ function removeUnnecesarySpaces(toremove: string): string {
   return normalizeSpaces(toremove).replace(/\s+/g, ' ')
 }
 
-export function compareString(
+function compareString(
   s1: string,
-  s1a: Array<string>,
   s2: string,
-  s2a: Array<string>
+  s1cache?: Array<string>,
+  s2cache?: Array<string>
 ): number {
+  const s1a = s1cache || s1.split(' ')
+  const s2a = s2cache || s2.split(' ')
+
   if (s1 === s2) {
     return 100
   }
@@ -277,16 +282,16 @@ function compareImage(data: QuestionData, data2: QuestionData): number {
   if (data.hashedImages && data2.hashedImages) {
     return compareString(
       data.hashedImages.join(' '),
-      data.hashedImages,
       data2.hashedImages.join(' '),
+      data.hashedImages,
       data2.hashedImages
     )
   } else if (data.images && data2.images) {
     return (
       compareString(
         data.images.join(' '),
-        data.images,
         data2.images.join(' '),
+        data.images,
         data2.images
       ) - 10
     )
@@ -324,7 +329,7 @@ function compareData(q1: Question, q2: Question): number {
 }
 
 function compareQuestion(q1: Question, q2: Question): number {
-  return compareString(q1.Q, q1.cache.Q, q2.Q, q2.cache.Q)
+  return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q)
   // return compareString(
   //   q1.Q,
   //   q1.Q ? q1.Q.split(' ') : [],
@@ -334,7 +339,7 @@ function compareQuestion(q1: Question, q2: Question): number {
 }
 
 function compareAnswer(q1: Question, q2: Question): number {
-  return compareString(q1.A, q1.cache.A, q2.A, q2.cache.A)
+  return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A)
   // return compareString(
   //   q1.A,
   //   q1.A ? q1.A.split(' ') : [],
@@ -648,7 +653,7 @@ if (!isMainThread) {
     `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
   )
 
-  parentPort.on('message', (msg) => {
+  parentPort.on('message', (msg /*: TaskObject */) => {
     if (msg.type === 'work') {
       const {
         subjName,
@@ -814,15 +819,93 @@ if (!isMainThread) {
         workerIndex: workerIndex,
       })
       // console.log(`[THREAD #${workerIndex}]: newdb`)
+    } else if (msg.type === 'dbClean') {
+      const removedIndexes = cleanDb(msg.data, qdbs)
+
+      const workerResult: WorkerResult = {
+        msg: `From thread #${workerIndex}: db clean done`,
+        workerIndex: workerIndex,
+        result: removedIndexes,
+      }
+
+      parentPort.postMessage(workerResult)
+    } else if (msg.type === 'rmQuestions') {
+      const {
+        questionIndexesToRemove,
+        subjIndex,
+        qdbIndex,
+        recievedQuestions,
+      } = msg.data
+
+      qdbs[qdbIndex].data[subjIndex].Questions = updateQuestionsInArray(
+        questionIndexesToRemove,
+        qdbs[qdbIndex].data[subjIndex].Questions,
+        recievedQuestions
+      )
+
+      parentPort.postMessage({
+        msg: `From thread #${workerIndex}: rm question done`,
+        workerIndex: workerIndex,
+      })
     } else {
       logger.Log(`Invalid msg type!`, logger.GetColor('redbg'))
       console.error(msg)
+
+      parentPort.postMessage({
+        msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`,
+        workerIndex: workerIndex,
+      })
     }
   })
 } else {
   // console.log('[THREAD]: Main thread!')
 }
 
+export function cleanDb(
+  {
+    questions: recievedQuestions,
+    subjToClean,
+    overwriteFromDate,
+    qdbIndex,
+  }: {
+    questions: Question[]
+    subjToClean: string
+    overwriteFromDate: number
+    qdbIndex: number
+  },
+  qdbs: QuestionDb[]
+): number[][] {
+  const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
+    return x.Name.toLowerCase().includes(subjToClean.toLowerCase())
+  })
+
+  if (!qdbs[qdbIndex].data[subjIndex]) {
+    return recievedQuestions.map(() => [])
+  }
+
+  const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) =>
+    qdbs[qdbIndex].data[subjIndex].Questions.reduce<number[]>(
+      (acc, question, i) => {
+        const res = compareString(
+          simplifyQuestion(recievedQuestion.Q),
+          simplifyQuestion(question.Q)
+        )
+
+        if (
+          res > minMatchToNotSearchOtherSubjects &&
+          (!question.data.date || question.data.date < overwriteFromDate)
+        ) {
+          return [...acc, i]
+        }
+        return acc
+      },
+      []
+    )
+  )
+
+  return questionIndexesToRemove
+}
+
 // ------------------------------------------------------------------------
 
 export {

src/utils/logger.ts

@@ -41,11 +41,6 @@ const logFileName = 'log'
 const writeInterval = 10
 const debugLevel = parseInt(process.env.NS_LOGLEVEL) || 0
 
-// let vdata = {} // visit data
-// let dvData = {} // visit data, but daily
-// let uData = {} // visit data, but per user
-// let udvData = {} // visit data, but per user and daily
-
 let vData = {} // visit data
 let dvData = {} // visit data, but daily
 let uvData = {} // visit data, but per user
@@ -237,12 +232,19 @@ function LogStat(url: string, hostname: string, userId: number | string): void {
 
   url = hostname + url.split('?')[0]
   Inc(url)
-  AddUserIdStat(userId.toString())
-  IncUserStat(userId.toString())
   AddVisitStat(url)
+  if (shouldAddUserStat(url)) {
+    AddUserIdStat(userId.toString())
+    IncUserStat(userId.toString())
+  }
   Save()
 }
 
+const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
+function shouldAddUserStat(url: string) {
+  return !userStatExcludes.some((x) => url.includes(x))
+}
+
 function IncUserStat(userId: string) {
   try {
     if (uvData[userId] === undefined) {

src/utils/workerPool.ts

@@ -34,8 +34,8 @@ interface WorkerObj {
   free: Boolean
 }
 
-interface TaskObject {
+export interface TaskObject {
-  type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb'
+  type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions'
   data:
     | {
         searchIn: number[]
@@ -50,6 +50,18 @@ interface TaskObject {
     | { dbIndex: number; edits: Edits }
     | QuestionDb
     | Result
+    | {
+        questions: Question[]
+        subjToClean: string
+        overwriteFromDate: number
+        qdbIndex: number
+      }
+    | {
+        questionIndexesToRemove: number[][]
+        subjIndex: number
+        qdbIndex: number
+        recievedQuestions: Question[]
+      }
 }
 
 interface PendingJob {

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit ce63911b68e5604c02f5761dd4b5388272b76c6f
+Subproject commit 9c333008c7049b032e4e3d0dbf5fd5893542ffb7