Possible answers penalty fixes, logging imporvements, added tests

package.json

@@ -20,7 +20,8 @@
     "dev": "npm run build && NS_THREAD_COUNT=2 NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node --inspect ./dist/server.js",
     "build": "tsc && bash -c './scripts/postBuild.sh'",
     "export": "tsc && bash -c './scripts/postBuild.sh'",
-    "test": "jest --detectOpenHandles"
+    "test": "NS_NOLOG=1 NS_THREAD_COUNT=1 jest --detectOpenHandles",
+    "test-debug": "NS_NOLOG=1 NS_THREAD_COUNT=1 node --inspect node_modules/.bin/jest --watch --runInBand src/tests/*.test.ts"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.5.0",

src/tests/possibleAnswerPenalty.test.ts

@@ -0,0 +1,308 @@
+import {
+  setNoPossibleAnswersPenalties,
+  SearchResultQuestion,
+  noPossibleAnswerMatchPenalty,
+} from '../utils/classes'
+import { Question } from '../types/basicTypes'
+
+const matchPercent = 100
+
+const questionWithNormalPossibleAnswers: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'rubber duck' },
+      { type: 'txt', val: 'super laptop' },
+      { type: 'txt', val: 'nothing in particular' },
+      { type: 'txt', val: 'something giberish' },
+    ],
+  },
+}
+
+const questionWithNormalPossibleAnswersWithLabels: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'a) nothing in particular' },
+      { type: 'txt', val: 'b) super laptop' },
+      { type: 'txt', val: 'c) something giberish' },
+      { type: 'txt', val: 'd) rubber duck' },
+    ],
+  },
+}
+
+const questionWithNormalPossibleAnswers2: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'rubber duck' },
+      { type: 'txt', val: 'cat' },
+      { type: 'txt', val: 'nothing in particular' },
+      { type: 'txt', val: 'dog' },
+    ],
+  },
+}
+
+const questionWithNormalPossibleAnswers3: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'rubber duck 2' },
+      { type: 'txt', val: 'whale' },
+      { type: 'txt', val: 'nothing in particular 2' },
+      { type: 'txt', val: 'sea lion' },
+    ],
+  },
+}
+
+const questionWithNormalPossibleAnswers4: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'rubber duck' },
+      { type: 'txt', val: 'super laptop' },
+    ],
+  },
+}
+
+const questionWithSimilarPossibleAnswers: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'asd' },
+      { type: 'txt', val: 'basd' },
+      { type: 'txt', val: 'aaa' },
+      { type: 'txt', val: 'bbb' },
+    ],
+  },
+}
+
+const questionWithTrueFalsePossibleAnser: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+    possibleAnswers: [
+      { type: 'txt', val: 'true' },
+      { type: 'txt', val: 'false' },
+    ],
+  },
+}
+
+const questionWithNoPossibleAnswer: Question = {
+  Q: 'asd',
+  A: 'asd',
+  data: {
+    type: 'simple',
+  },
+}
+
+const resNormal: SearchResultQuestion = {
+  q: questionWithNormalPossibleAnswers,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const resNormal2: SearchResultQuestion = {
+  q: questionWithNormalPossibleAnswers2,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const resNormal3: SearchResultQuestion = {
+  q: questionWithNormalPossibleAnswers3,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const resNormal4: SearchResultQuestion = {
+  q: questionWithNormalPossibleAnswers4,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const resSimilar: SearchResultQuestion = {
+  q: questionWithSimilarPossibleAnswers,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const resTrueFalse: SearchResultQuestion = {
+  q: questionWithTrueFalsePossibleAnser,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const resNoPossibleAnswer: SearchResultQuestion = {
+  q: questionWithNoPossibleAnswer,
+  match: matchPercent,
+  detailedMatch: {
+    qMatch: matchPercent,
+    aMatch: matchPercent,
+    dMatch: matchPercent,
+    matchedSubjName: 'testSubj',
+    avg: matchPercent,
+  },
+}
+
+const testFunction = (
+  question: Question,
+  searchResult: SearchResultQuestion[],
+  index: number
+) => {
+  const updated = setNoPossibleAnswersPenalties(
+    question.data.possibleAnswers,
+    searchResult
+  )
+
+  updated.forEach((x, i) => {
+    if (i !== index) {
+      expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty)
+      expect(x.detailedMatch.qMatch).toBe(
+        matchPercent - noPossibleAnswerMatchPenalty
+      )
+    } else {
+      expect(x.match).toBe(100)
+      expect(x.detailedMatch.qMatch).toBe(100)
+    }
+  })
+
+  return updated
+}
+
+test('Possible answer penalty applies correctly (normal possible answers)', () => {
+  testFunction(
+    questionWithNormalPossibleAnswers,
+    [
+      resNormal,
+      resNormal2,
+      resNormal3,
+      resNormal4,
+      resSimilar,
+      resTrueFalse,
+      resNoPossibleAnswer,
+    ],
+    0
+  )
+})
+
+test('Possible answer penalty applies correctly (normal possible answers, with labels)', () => {
+  testFunction(
+    questionWithNormalPossibleAnswersWithLabels,
+    [
+      resNormal,
+      resNormal2,
+      resNormal3,
+      resNormal4,
+      resSimilar,
+      resTrueFalse,
+      resNoPossibleAnswer,
+    ],
+    0
+  )
+})
+
+test('Possible answer penalty applies correctly (similar possible answers)', () => {
+  testFunction(
+    questionWithSimilarPossibleAnswers,
+    [
+      resNormal,
+      resNormal2,
+      resNormal3,
+      resNormal4,
+      resSimilar,
+      resTrueFalse,
+      resNoPossibleAnswer,
+    ],
+    4
+  )
+})
+
+test('Possible answer penalty applies correctly (true false possible answers)', () => {
+  testFunction(
+    questionWithTrueFalsePossibleAnser,
+    [
+      resNormal,
+      resNormal2,
+      resNormal3,
+      resNormal4,
+      resSimilar,
+      resTrueFalse,
+      resNoPossibleAnswer,
+    ],
+    5
+  )
+})
+
+test('Possible answer penalty applies correctly (no possible answers)', () => {
+  const updated = setNoPossibleAnswersPenalties(
+    questionWithNoPossibleAnswer.data.possibleAnswers,
+    [
+      resNormal,
+      resNormal2,
+      resNormal3,
+      resNormal4,
+      resSimilar,
+      resTrueFalse,
+      resNoPossibleAnswer,
+    ]
+  )
+
+  updated.forEach((x) => {
+    expect(x.match).toBe(100)
+    expect(x.detailedMatch.qMatch).toBe(100)
+  })
+})
+
+test('Possible answer penalty applies correctly (empty searchResult)', () => {
+  const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0)
+  expect(updated.length).toBe(0)
+})

src/types/basicTypes.ts

@@ -31,8 +31,9 @@ export interface QuestionData {
   images?: Array<string>
   hashedImages?: Array<string>
   possibleAnswers?: Array<{
-    text: string
-    selectedByUser: boolean
+    type: string
+    val: string
+    selectedByUser?: boolean
   }>
 }
 
@@ -96,7 +97,11 @@ export interface User {
 export interface Request<T = any> extends express.Request {
   body: T
   cookies: any
-  session: any
+  session: {
+    user?: User
+    sessionID?: string
+    isException?: boolean
+  }
   files: any
   query: { [key: string]: string }
 }

src/utils/actions.ts

@@ -204,7 +204,7 @@ function processIncomingRequestUsingDb(
             const add = result.result.every((res: SearchResultQuestion) => {
               return res.match < minMatchAmmountToAdd
             })
-            if (add) {
+            if (add && !result.error) {
               allQuestions.push(recievedQuestions[i])
             }
           })

src/utils/classes.ts

@@ -33,6 +33,7 @@ export interface WorkerResult {
   msg: string
   workerIndex: number
   result?: SearchResultQuestion[]
+  error?: boolean
 }
 
 interface DetailedMatch {
@@ -63,7 +64,7 @@ const commonUselessAnswerParts = [
 // const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
 /* Percent minus for length difference */
 const lengthDiffMultiplier = 10
-const noPossibleAnswerMatchPenalty = 5
+export const noPossibleAnswerMatchPenalty = 5
 /* Minimum ammount to consider that two questions match during answering */
 const minMatchAmmount = 75
 const magicNumber = 0.7 // same as minMatchAmmount, but /100
@@ -574,39 +575,52 @@ function doSearch(
 }
 
 function setNoPossibleAnswersPenalties(
-  possibleAnswers: QuestionData['possibleAnswers'],
-  result: SearchResultQuestion[]
+  questionPossibleAnswers: QuestionData['possibleAnswers'],
+  results: SearchResultQuestion[]
 ): SearchResultQuestion[] {
-  if (!Array.isArray(possibleAnswers)) {
-    return result
+  if (!Array.isArray(questionPossibleAnswers)) {
+    return results
   }
-  const noneHasPossibleAnswers = result.every((x) => {
+  const noneHasPossibleAnswers = results.every((x) => {
     return !Array.isArray(x.q.data.possibleAnswers)
   })
-  if (noneHasPossibleAnswers) return result
+  if (noneHasPossibleAnswers) return results
 
   let possibleAnswerMatch = false
-  const updated = result.map((result) => {
-    const hasMatch = result.q.data.possibleAnswers.some((possibleAnswer) => {
-      return possibleAnswers.some((questionPossibleAnswer) => {
-        // FIXME: this could be object: questionPossibleAnswer
-        return questionPossibleAnswer.text.includes(possibleAnswer.text)
-      })
-    })
-    if (hasMatch) {
-      possibleAnswerMatch = true
+  const updated = results.map((result) => {
+    const matchCount = Array.isArray(result.q.data.possibleAnswers)
+      ? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => {
+          return questionPossibleAnswers.some((questionPossibleAnswer) => {
+            if (questionPossibleAnswer.val && resultPossibleAnswer.val) {
+              return questionPossibleAnswer.val.includes(
+                resultPossibleAnswer.val
+              )
             } else {
-      result.match = result.match - noPossibleAnswerMatchPenalty
-      result.detailedMatch.qMatch =
-        result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty
+              return false
             }
+          })
+        }).length
+      : 0
+
+    if (matchCount === questionPossibleAnswers.length) {
+      possibleAnswerMatch = true
       return result
+    } else {
+      return {
+        ...result,
+        match: result.match - noPossibleAnswerMatchPenalty,
+        detailedMatch: {
+          ...result.detailedMatch,
+          qMatch: result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty,
+        },
+      }
+    }
   })
 
   if (possibleAnswerMatch) {
     return updated
   } else {
-    return result
+    return results
   }
 }
 
@@ -646,6 +660,7 @@ if (!isMainThread) {
       }: WorkData = msg.data
 
       let searchResult: SearchResultQuestion[] = []
+      let error = false
 
       try {
         qdbs.forEach((qdb) => {
@@ -674,14 +689,21 @@ if (!isMainThread) {
       } catch (err) {
         logger.Log('Error in worker thread!', logger.GetColor('redbg'))
         console.error(err)
-        console.error({
+        console.error(
+          JSON.stringify(
+            {
               subjName: subjName,
               question: question,
               searchTillMatchPercent: searchTillMatchPercent,
               searchInAllIfNoResult: searchInAllIfNoResult,
               searchIn: searchIn,
               index: index,
-        })
+            },
+            null,
+            2
+          )
+        )
+        error = true
       }
 
       // sorting
@@ -703,6 +725,7 @@ if (!isMainThread) {
         }done`,
         workerIndex: workerIndex,
         result: sortedResult,
+        error: error,
       }
 
       // ONDONE:
@@ -810,4 +833,5 @@ export {
   addQuestion,
   dataToString,
   doSearch,
+  setNoPossibleAnswersPenalties,
 }

src/utils/logger.ts

@@ -52,7 +52,7 @@ let uvData = {} // visit data, but per user
 let udvData = {} // visit data, but per user and daily
 let writes = 0
 
-let noLogips: string[] = []
+let noLogIds: string[] = []
 
 function getColoredDateString(): string {
   const date = new Date()
@@ -86,7 +86,9 @@ function Log(msg: string | object, color?: string): void {
     log = getColoredDateString() + delimiter + C(color) + msg + C()
   }
 
+  if (!process.env.NS_NOLOG) {
     console.log(log)
+  }
   utils.AppendToFile(
     typeof log === 'string' ? log : JSON.stringify(log),
     logDir + logFileName
@@ -160,20 +162,20 @@ function LogReq(
       utils.AppendToFile(defLogs, vlogDir + logFileName)
     }
   } catch (err) {
-    console.log(err)
+    console.error(err)
     Log('Error at logging lol', GetColor('redbg'))
   }
 }
 
 function parseNoLogFile(newData: string) {
-  noLogips = newData.split('\n')
-  if (noLogips[noLogips.length - 1] === '') {
-    noLogips.pop()
+  noLogIds = newData.split('\n')
+  if (noLogIds[noLogIds.length - 1] === '') {
+    noLogIds.pop()
   }
-  noLogips = noLogips.filter((noLogip) => {
-    return noLogip !== ''
+  noLogIds = noLogIds.filter((noLogId) => {
+    return noLogId !== ''
   })
-  Log('\tNo Log IP-s changed: ' + noLogips.join(', '))
+  Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
 }
 
 function setNoLogReadInterval() {
@@ -219,16 +221,23 @@ function Load(): void {
   setNoLogReadInterval()
 }
 
-function LogStat(url: string, hostname: string, userId: number): void {
+function LogStat(url: string, hostname: string, userId: number | string): void {
+  const nolog = noLogIds.some((noLogId) => {
+    return noLogId === userId.toString()
+  })
+  if (nolog) {
+    return
+  }
+
   url = hostname + url.split('?')[0]
   Inc(url)
-  AddUserIdStat(userId)
-  IncUserStat(userId)
+  AddUserIdStat(userId.toString())
+  IncUserStat(userId.toString())
   AddVisitStat(url)
   Save()
 }
 
-function IncUserStat(userId: number) {
+function IncUserStat(userId: string) {
   try {
     if (uvData[userId] === undefined) {
       uvData[userId] = 0
@@ -240,7 +249,7 @@ function IncUserStat(userId: number) {
   }
 }
 
-function AddUserIdStat(userId: number) {
+function AddUserIdStat(userId: string) {
   try {
     const date = new Date()
     const now =

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 4f82b9d8c9c32ebbd9d73b951fc1db26ad1ceb22
+Subproject commit 4748c23769d16450731562bb9ffd1553ce2eb704

tsconfig.json

@@ -26,6 +26,7 @@
     "src/**/*"
   ],
   "exclude": [
+    "src/tests/",
     "node_modules",
     "submodules",
     "devel",