Reqlogger exceptions, searchig: search till match percent and search in other subjs

src/middlewares/reqlogger.middleware.ts

@@ -8,10 +8,16 @@ interface Options {
 export default function(options: Options): any {
   const loggableKeywords = options ? options.loggableKeywords : undefined
   const loggableModules = options ? options.loggableModules : undefined
+  const exceptions = options ? options.exceptions : []
 
   return function(req, res, next) {
     res.on('finish', function() {
-      if (req.url.includes('_next/static')) {
+      // TODO: test this
+      const isException = exceptions.some((ex) => {
+        return req.url.includes(ex)
+      })
+
+      if (isException) {
         return
       }
 
@@ -26,7 +32,6 @@ export default function(options: Options): any {
         console.log(req.headers)
       }
 
-      // fixme: regexp includes checking
       const hasLoggableKeyword =
         loggableKeywords &&
         loggableKeywords.some((keyword) => {

src/modules/api/api.ts

@@ -1166,7 +1166,7 @@ function GetApp(): ModuleType {
           question: question,
           subjName: subj,
           questionData: recData,
-          firstMatchOnly: true, // search till the first match only
+          searchInAllIfNoResult: true,
         },
       })
         .then((taskResult) => {

src/server.ts

@@ -148,12 +148,14 @@ app.use(
     // origin: [ /\.frylabs\.net$/ ]
   })
 )
+
 const cookieSecret = uuidv4()
 app.use(cookieParser(cookieSecret))
 app.use(
   reqlogger({
     loggableKeywords: ['news.json'],
     loggableModules: ['dataeditor'],
+    exceptions: ['stable.user.js?up', '_next/static'],
   })
 )
 

src/utils/actions.ts

@@ -32,7 +32,7 @@ import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
 
 // if a recievend question doesnt match at least this % to any other question in the db it gets
 // added to db
-const minMatchToAmmountToAdd = 90
+const minMatchAmmountToAdd = 90
 
 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
 let currWrites = 0
@@ -166,6 +166,7 @@ function processIncomingRequestUsingDb(
         )
         logger.DebugLog(currentQuestion, 'actions', 3)
         recievedQuestions.push(currentQuestion)
+        // This here searches only in relevant subjects, and not all subjects
         questionSearchPromises.push(
           doALongTask({
             type: 'work',
@@ -174,6 +175,7 @@ function processIncomingRequestUsingDb(
               qdb: qdb.data,
               question: currentQuestion,
               subjName: recievedData.subj,
+              searchTillMatchPercent: minMatchAmmountToAdd,
             },
           })
         )
@@ -184,7 +186,7 @@ function processIncomingRequestUsingDb(
           const allQuestions = [] // all new questions here that do not have result
           results.forEach((result, i) => {
             const add = result.result.every((res) => {
-              return res.match < minMatchToAmmountToAdd
+              return res.match < minMatchAmmountToAdd
             })
             if (add) {
               allQuestions.push(recievedQuestions[i])

src/utils/classes.ts

@@ -319,7 +319,7 @@ function searchQuestion(
   subj: Subject,
   question: Question,
   subjName: string,
-  firstMatchOnly?: Boolean
+  searchTillMatchPercent?: number
 ) {
   assert(question)
 
@@ -341,7 +341,7 @@ function searchQuestion(
       })
     }
 
-    if (firstMatchOnly && percent.avg === 100) {
+    if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) {
       return false
     }
 
@@ -388,6 +388,7 @@ function addQuestion(
   assert(typeof question === 'object')
 
   let i = 0
+  // FIXME: this only adds to the first matched subject name. Check if this doesnt cause any bugs
   while (
     i < data.length &&
     !subj
@@ -443,14 +444,13 @@ function dataToString(data: Array<Subject>): string {
   return result.join('\n\n')
 }
 
-// ------------------------------------------------------------------------
-
 function doSearch(
   data: Array<Subject>,
   subjName: string,
   question: Question | string,
   questionData?: QuestionData,
-  firstMatchOnly?: Boolean
+  searchTillMatchPercent?: number,
+  searchInAllIfNoResult?: Boolean
 ): any {
   let result = []
 
@@ -469,53 +469,47 @@ function doSearch(
         subj,
         questionToSearch,
         subjName,
-        firstMatchOnly
+        searchTillMatchPercent
       )
       result = result.concat(subjRes)
-      if (firstMatchOnly) {
+      if (searchTillMatchPercent) {
+        // TODO: this should return true to continue searching?
         return subjRes.every((sr) => {
-          return sr.match < 100
+          return sr.match <= searchTillMatchPercent
         })
       }
       return true
     }
+    // TODO: data.every should always return something! true or false?
   })
 
-  // FIXME: try to remove this? but this is also a good backup plan so idk
-  // its sufficent to check only result[0].match, since its sorted, and the first one should have
-  // the highest match
-  if (
-    result.length === 0 ||
-    result[0].match < minMatchToNotSearchOtherSubjects
-  ) {
-    logger.DebugLog(
-      'Reqults length is zero when comparing names, trying all subjects',
-      'searchworker',
-      1
-    )
-    data.every((subj) => {
-      const subjRes = searchQuestion(
-        subj,
-        questionToSearch,
-        subjName,
-        firstMatchOnly
-      )
-      result = result.concat(subjRes)
-      if (firstMatchOnly) {
-        return subjRes.every((sr) => {
-          return sr.match < 100
-        })
-      }
-      return true
-    })
-
-    if (result.length > 0) {
+  if (searchInAllIfNoResult) {
+    // FIXME: dont research subject searched above
+    if (
+      result.length === 0 ||
+      result[0].match < minMatchToNotSearchOtherSubjects
+    ) {
       logger.DebugLog(
-        `FIXME: '${subjName}' gave no result but '' did!`,
+        'Reqults length is zero when comparing names, trying all subjects',
         'searchworker',
         1
       )
-      console.error(`FIXME: '${subjName}' gave no result but '' did!`)
+      data.every((subj) => {
+        const subjRes = searchQuestion(
+          subj,
+          questionToSearch,
+          subjName,
+          searchTillMatchPercent
+        )
+        result = result.concat(subjRes)
+        if (searchTillMatchPercent) {
+          // TODO: this should return true to continue searching?
+          return subjRes.every((sr) => {
+            return sr.match < searchTillMatchPercent
+          })
+        }
+        return true
+      })
     }
   }
 
@@ -532,6 +526,10 @@ function doSearch(
   return result
 }
 
+// ---------------------------------------------------------------------------------------------------------
+// Multi threaded stuff
+// ---------------------------------------------------------------------------------------------------------
+
 if (!isMainThread) {
   const { workerIndex } = workerData
   let qdbs: Array<any> = workerData.initData
@@ -542,7 +540,14 @@ if (!isMainThread) {
 
   parentPort.on('message', (msg) => {
     if (msg.type === 'work') {
-      const { subjName, question, questionData, firstMatchOnly } = msg.data
+      const {
+        subjName,
+        question,
+        questionData,
+        searchTillMatchPercent,
+        searchInAllIfNoResult,
+      } = msg.data
+
       const index = msg.index
       const searchIn = msg.data.searchIn
 
@@ -562,7 +567,8 @@ if (!isMainThread) {
               subjName,
               question,
               questionData,
-              firstMatchOnly
+              searchTillMatchPercent,
+              searchInAllIfNoResult
             )
             searchResult = [...searchResult, ...res]
           }