ask api changes, so it wont search unnecesary question dbs

src/middlewares/reqlogger.middleware.ts

@@ -3,6 +3,7 @@ import logger from '../utils/logger'
 interface Options {
   loggableKeywords: Array<string>
   loggableModules: Array<string>
+  exceptions: Array<string>
 }
 
 export default function(options: Options): any {

src/modules/api/api.ts

@@ -34,6 +34,7 @@ import {
   logResult,
   backupData,
   shouldSaveDataFile,
+  shouldSearchDataFile,
   loadJSON,
   Result,
 } from '../../utils/actions'
@@ -930,22 +931,7 @@ function GetApp(): ModuleType {
     }
   })
 
-  app.post('/isAdding', function(req: Request, res: any) {
+  function getNewQdb(location, maxIndex) {
-    logger.LogReq(req)
-    const user: User = req.session.user
-    const dryRun = testUsers.includes(user.id)
-
-    try {
-      let maxIndex = -1
-      const suitedQuestionDbs = questionDbs.filter((qdb) => {
-        if (maxIndex < qdb.index) {
-          maxIndex = qdb.index
-        }
-        return shouldSaveDataFile(qdb, req.body)
-      }, [])
-
-      if (suitedQuestionDbs.length === 0) {
-        const location = req.body.location.split('/')[2]
     // TODO: should check if location is a not empty string
     logger.Log(
       `No suitable questiondbs found for ${location}, creating a new one...`
@@ -953,6 +939,11 @@ function GetApp(): ModuleType {
     const newDb: DataFile = {
       path: `${location}.json`,
       name: location,
+      shouldSearch: {
+        location: {
+          val: location,
+        },
+      },
       shouldSave: {
         location: {
           val: location,
@@ -981,13 +972,33 @@ function GetApp(): ModuleType {
     }
     utils.WriteFile('[]', loadedNewDb.path)
 
-        suitedQuestionDbs.push(loadedNewDb)
     questionDbs.push(loadedNewDb)
     // TODO: problem: new dbs wont get to workers before trying to search with them.
     msgAllWorker({
       newdb: loadedNewDb,
       type: 'newdb',
     })
+
+    return loadedNewDb
+  }
+
+  app.post('/isAdding', function(req: Request, res: any) {
+    logger.LogReq(req)
+    const user: User = req.session.user
+    const dryRun = testUsers.includes(user.id)
+    const location = req.body.location.split('/')[2]
+
+    try {
+      let maxIndex = -1
+      const suitedQuestionDbs = questionDbs.filter((qdb) => {
+        if (maxIndex < qdb.index) {
+          maxIndex = qdb.index
+        }
+        return shouldSaveDataFile(qdb, req.body)
+      }, [])
+
+      if (suitedQuestionDbs.length === 0) {
+        suitedQuestionDbs.push(getNewQdb(location, maxIndex))
       }
 
       processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user)
@@ -1048,16 +1059,16 @@ function GetApp(): ModuleType {
     }
   }
 
-  function saveQuestion(questions, subj, location, userid) {
+  function saveQuestion(questions, subj, testUrl, userid) {
     // TODO: clear folder every now and then, check if saved questions exist
     const questionsToSave = {
       questions: questions,
       subj: subj,
       userid: userid,
-      location: location,
+      testUrl: testUrl,
       date: new Date(),
     }
-    const fname = `${utils.GetDateString()}_${userid}_${location}.json`
+    const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json`
     const subject = getSubjNameWithoutYear(subj)
     const subjPath = `${savedQuestionsDir}/${subject}`
     const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}`
@@ -1072,7 +1083,7 @@ function GetApp(): ModuleType {
       fname: fname,
       subj: subj,
       userid: userid,
-      location: location,
+      testUrl: testUrl,
       date: new Date(),
     })
     utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath)
@@ -1094,12 +1105,14 @@ function GetApp(): ModuleType {
       return
     }
     const subj: any = req.body.subj || ''
-    const location = req.body.location.split('/')[2]
+    const testUrl = req.body.testUrl
+      ? req.body.testUrl.split('/')[2]
+      : undefined
 
     writeAskData(req.body)
 
     const resultPromises = req.body.questions.map((question) => {
-      return getResult(question, subj, null, req.query)
+      return getResult(question, subj, null, req.body, testUrl)
     })
 
     Promise.all(resultPromises).then((results) => {
@@ -1120,7 +1133,7 @@ function GetApp(): ModuleType {
         return acc
       }, [])
 
-      saveQuestion(saveableQuestions, subj, location, user.id)
+      saveQuestion(saveableQuestions, subj, testUrl, user.id)
     })
   })
 
@@ -1155,18 +1168,33 @@ function GetApp(): ModuleType {
     }
   })
 
-  function getResult(question, subj, recData, recievedData) {
+  function getResult(question, subj, recData, recievedData, testUrl?) {
     return new Promise((resolve) => {
+      let searchIn = testUrl
+        ? questionDbs.reduce((acc, qdb, i) => {
+            if (shouldSearchDataFile(qdb, testUrl)) {
+              acc.push(i)
+            }
+            return acc
+          }, [])
+        : 'all'
+
+      if (searchIn.length === 0) {
+        searchIn = 'all'
+      }
+
       // searchIn could be [0], [1], ... to search every db in different thread. Put this into a
       // forEach(qdbs) to achieve this
       doALongTask({
         type: 'work',
         data: {
-          searchIn: 'all',
+          searchIn: searchIn,
+          testUrl: testUrl,
           question: question,
           subjName: subj,
           questionData: recData,
           searchInAllIfNoResult: true,
+          searchTillMatchPercent: 30,
         },
       })
         .then((taskResult) => {

src/types/basicTypes.ts

@@ -20,6 +20,13 @@ export interface Subject {
 export interface DataFile {
   path: string
   name: string
+  shouldSearch:
+    | string
+    | {
+        location?: {
+          val: string
+        }
+      }
   shouldSave: {
     location?: {
       val: string

src/utils/actions.ts

@@ -263,6 +263,22 @@ function processIncomingRequestUsingDb(
   })
 }
 
+export function shouldSearchDataFile(df: DataFile, testUrl: string): Boolean {
+  if (typeof df.shouldSearch === 'string' && df.shouldSearch === 'always') {
+    return true
+  }
+
+  if (typeof df.shouldSearch === 'object') {
+    if (df.shouldSearch.location) {
+      const { val } = df.shouldSearch.location
+      return testUrl.includes(val)
+    }
+  }
+
+  // FIXME: always return true if it gets here?
+  return true
+}
+
 export function shouldSaveDataFile(
   df: DataFile,
   recievedData: RecievedData

src/utils/classes.ts

@@ -315,7 +315,7 @@ function questionToString(question: Question) {
 // ---------------------------------------------------------------------------------------------------------
 // Subject
 // ---------------------------------------------------------------------------------------------------------
-function searchQuestion(
+function searchSubject(
   subj: Subject,
   question: Question,
   subjName: string,
@@ -465,7 +465,7 @@ function doSearch(
         .includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
     ) {
       logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2)
-      const subjRes = searchQuestion(
+      const subjRes = searchSubject(
         subj,
         questionToSearch,
         subjName,
@@ -473,14 +473,13 @@ function doSearch(
       )
       result = result.concat(subjRes)
       if (searchTillMatchPercent) {
-        // TODO: this should return true to continue searching?
+        return !subjRes.some((sr) => {
-        return subjRes.every((sr) => {
+          return sr.match >= searchTillMatchPercent
-          return sr.match <= searchTillMatchPercent
         })
       }
       return true
     }
-    // TODO: data.every should always return something! true or false?
+    return true
   })
 
   if (searchInAllIfNoResult) {
@@ -495,18 +494,19 @@ function doSearch(
         1
       )
       data.every((subj) => {
-        const subjRes = searchQuestion(
+        const subjRes = searchSubject(
           subj,
           questionToSearch,
           subjName,
           searchTillMatchPercent
         )
         result = result.concat(subjRes)
+
         if (searchTillMatchPercent) {
-          // TODO: this should return true to continue searching?
+          const continueSearching = !subjRes.some((sr) => {
-          return subjRes.every((sr) => {
+            return sr.match >= searchTillMatchPercent
-            return sr.match < searchTillMatchPercent
           })
+          return continueSearching
         }
         return true
       })
@@ -546,11 +546,10 @@ if (!isMainThread) {
         questionData,
         searchTillMatchPercent,
         searchInAllIfNoResult,
+        searchIn,
+        index,
       } = msg.data
 
-      const index = msg.index
-      const searchIn = msg.data.searchIn
-
       // console.log(
       //   `[THREAD #${workerIndex}]: staring work${
       //     !isNaN(index) ? ` on job index #${index}` : ''