Search speedup by: caching splitted questions/answers, and refactoring string compare algorithym

src/utils/actions.ts

@@ -28,7 +28,13 @@ import utils from '../utils/utils'
 import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
 
 // types
-import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
+import {
+  QuestionDb,
+  Subject,
+  Question,
+  User,
+  DataFile,
+} from '../types/basicTypes'
 
 // if a recievend question doesnt match at least this % to any other question in the db it gets
 // added to db
@@ -219,7 +225,7 @@ function processIncomingRequestUsingDb(
               if (currWrites >= writeAfter && !dryRun) {
                 currWrites = 0
                 logger.DebugLog('Writing data.json', 'isadding', 1)
-                utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
+                writeData(qdb.data, qdb.path)
               }
             }
 
@@ -335,6 +341,20 @@ export function shouldSaveDataFile(
   return false
 }
 
+export function loadData(path: string): Array<Subject> {
+  return JSON.parse(utils.ReadFile(path)).reduce((acc, subj) => {
+    return [
+      ...acc,
+      {
+        Name: subj.Name,
+        Questions: subj.Questions.map((question) => {
+          return createQuestion(question)
+        }),
+      },
+    ]
+  }, [])
+}
+
 export function loadJSON(
   dataFiles: Array<DataFile>,
   dataDir: string
@@ -351,7 +371,7 @@ export function loadJSON(
         ...dataFile,
         path: dataPath,
         index: index,
-        data: JSON.parse(utils.ReadFile(dataPath)),
+        data: loadData(dataPath),
       })
     } catch (err) {
       console.error(err)
@@ -364,14 +384,34 @@ export function loadJSON(
   }, [])
 }
 
+export function writeData(data: Array<Subject>, path: string): void {
+  utils.WriteFile(
+    JSON.stringify(
+      data.map((subj) => {
+        return {
+          Name: subj.Name,
+          Questions: subj.Questions.map((question) => {
+            return {
+              Q: question.Q,
+              A: question.A,
+              data: question.data,
+            }
+          }),
+        }
+      })
+    ),
+    path
+  )
+}
+
 export function backupData(questionDbs: Array<QuestionDb>): void {
   questionDbs.forEach((data) => {
     const path = './publicDirs/qminingPublic/backs/'
     utils.CreatePath(path)
     try {
       logger.Log(`Backing up ${data.name}...`)
-      utils.WriteFile(
-        JSON.stringify(data.data),
+      writeData(
+        data.data,
         `${path}${data.name}_${utils.GetDateString(true)}.json`
       )
       logger.Log('Done')