Handling multiple data files

src/modules/api/api.ts

@@ -29,13 +29,19 @@ import fs from 'fs'
 
 import logger from '../../utils/logger'
 import utils from '../../utils/utils'
-import actions from '../../utils/actions'
+import {
+  processIncomingRequest,
+  logResult,
+  backupData,
+  loadJSON,
+  RecievedData,
+} from '../../utils/actions'
 import dbtools from '../../utils/dbtools'
 import auth from '../../middlewares/auth.middleware'
-import { dataToString, searchData } from '../../utils/classes'
+import { dataToString, searchDatas } from '../../utils/classes'
 
 import { SetupData } from '../../server'
-import { ModuleType, User } from '../../types/basicTypes'
+import { ModuleType, User, DataFile } from '../../types/basicTypes'
 
 // files
 const msgFile = 'stats/msgs'
@@ -62,11 +68,6 @@ let userDB
 let url // eslint-disable-line
 let publicdirs = []
 
-export interface DataFile {
-  path: string
-  name: string
-}
-
 function GetApp(): ModuleType {
   const app = express()
 
@@ -79,10 +80,21 @@ function GetApp(): ModuleType {
   const recivedFiles = publicDir + 'recivedfiles'
   const uloadFiles = publicDir + 'f'
   const dataFiles: Array<DataFile> = [
-    { path: `${publicDir}oldData.json`, name: 'oldData' },
+    {
-    { path: `${publicDir}data.json`, name: 'newData' },
+      path: `${publicDir}oldData.json`,
+      name: 'oldData',
+      shouldSave: (recData: RecievedData): boolean => {
+        return recData.version.includes('2.0.')
+      },
+    },
+    {
+      path: `${publicDir}data.json`,
+      name: 'newData',
+      shouldSave: (recData: RecievedData): boolean => {
+        return recData.version.includes('2.1.')
+      },
+    },
   ]
-  const data: any = {} // TODO: remove
   const motdFile = publicDir + 'motd'
   const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
   const versionFile = publicDir + 'version'
@@ -127,7 +139,7 @@ function GetApp(): ModuleType {
     })
   )
 
-  const questionDbs = actions.LoadJSON(dataFiles)
+  const questionDbs = loadJSON(dataFiles)
   let version = ''
   let motd = ''
   let userSpecificMotd = {}
@@ -196,13 +208,16 @@ function GetApp(): ModuleType {
     }
 
     if (write) {
-      utils.WriteFile(JSON.stringify(userSpecificMotd), userSpecificMotdFile)
+      utils.WriteFile(
+        JSON.stringify(userSpecificMotd, null, 2),
+        userSpecificMotdFile
+      )
     }
     return shouldSee
   }
 
   function Load() {
-    actions.backupData(questionDbs)
+    backupData(questionDbs)
 
     utils.WatchFile(userSpecificMotdFile, () => {
       logger.Log(`User Specific Motd updated`, logger.GetColor('green'))
@@ -939,16 +954,9 @@ function GetApp(): ModuleType {
 
     const dryRun = testUsers.includes(user.id)
 
-    // automatically saves to dataFile every n write
+    processIncomingRequest(req.body, questionDbs, dryRun, user)
-    // FIXME: req.body.datatoadd is for backwards compatibility, remove this sometime in the future
-    actions
-      .ProcessIncomingRequest(
-        req.body.datatoadd || req.body,
-        questionDbs,
-        dryRun,
-        user
-      )
       .then((resultArray) => {
+        logResult(req.body, resultArray)
         res.json({
           success: resultArray.length > 0, // TODO check for -1s
           newQuestions: resultArray,
@@ -986,8 +994,10 @@ function GetApp(): ModuleType {
           )
         }
 
-        searchData(data, question, subj, recData)
+        searchDatas(questionDbs, question, subj, recData)
           .then((result) => {
+            // TODO: test
+            console.log(result)
             res.json({
               result: result,
               success: true,
@@ -1021,22 +1031,39 @@ function GetApp(): ModuleType {
     }
   })
 
+  function getSubjCount(qdbs) {
+    return qdbs.reduce((acc, qdb) => {
+      return acc + qdb.data.length
+    }, 0)
+  }
+
+  function getQuestionCount(qdbs) {
+    return qdbs.reduce((acc, qdb) => {
+      return (
+        acc +
+        qdb.data.reduce((qacc, subject) => {
+          return qacc + subject.Questions.length
+        }, 0)
+      )
+    }, 0)
+  }
+
   function getSimplreRes() {
     return {
-      subjects: data.length,
+      subjects: getSubjCount(questionDbs),
-      questions: data.reduce((acc, subj) => {
+      questions: getQuestionCount(questionDbs),
-        return acc + subj.Questions.length
-      }, 0),
     }
   }
 
   function getDetailedRes() {
-    return data.map((subj) => {
+    return questionDbs.map((qdb) => {
+      return qdb.data.map((subj) => {
         return {
           name: subj.Name,
           count: subj.Questions.length,
         }
       })
+    })
   }
 
   app.get('/datacount', function(req: any, res: any) {
@@ -1091,10 +1118,8 @@ function GetApp(): ModuleType {
     utils.AppendToFile(
       JSON.stringify({
         date: utils.GetDateString(),
-        subjectCount: data.length,
+        subjectCount: getSubjCount(questionDbs),
-        questionCount: data.reduce((acc, subj) => {
+        questionCount: getQuestionCount(questionDbs),
-          return acc + subj.Questions.length
-        }, 0),
         userCount: dbtools.TableInfo(userDB, 'users').dataCount,
       }),
       dailyDataCountFile
@@ -1165,7 +1190,7 @@ function GetApp(): ModuleType {
   }
 
   function DailyAction() {
-    actions.backupData(data)
+    backupData(questionDbs)
     BackupDB()
     ExportDailyDataCount()
     IncrementAvaiblePWs()

src/types/basicTypes.ts

@@ -17,9 +17,13 @@ export interface Subject {
   Questions: Array<Question>
 }
 
-export interface QuestionDb {
+export interface DataFile {
-  name: string
   path: string
+  name: string
+  shouldSave: (recData: any) => boolean
+}
+
+export interface QuestionDb extends DataFile {
   data: Array<Subject>
 }
 

src/utils/actions.ts

@@ -17,12 +17,6 @@ Question Server
 
  ------------------------------------------------------------------------- */
 
-export default {
-  ProcessIncomingRequest: ProcessIncomingRequest,
-  LoadJSON: LoadJSON,
-  backupData: backupData,
-}
-
 const recDataFile = './stats/recdata'
 const dataLockFile = './data/lockData'
 
@@ -33,8 +27,7 @@ import utils from '../utils/utils'
 import { addQuestion, getSubjNameWithoutYear } from './classes'
 
 // types
-import { QuestionDb, Question, User } from '../types/basicTypes'
+import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
-import { DataFile } from '../modules/api/api'
 
 // if a recievend question doesnt match at least this % to any other question in the db it gets
 // added to db
@@ -43,7 +36,7 @@ const minMatchToAmmountToAdd = 90
 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
 let currWrites = 0
 
-interface RecievedData {
+export interface RecievedData {
   quiz: Array<Question>
   subj: string
   id: string
@@ -51,31 +44,49 @@ interface RecievedData {
   scriptVersion: string
 }
 
-function ProcessIncomingRequest(
+interface Result {
+  qdbName: string
+  newQuestions: number
+}
+
+export function logResult(
+  recievedData: RecievedData,
+  result: Array<Result>
+): void {
+  let subjRow = '\t' + recievedData.subj
+  if (recievedData.id) {
+    subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
+  }
+  logger.Log(subjRow)
+
+  result.forEach((res: Result) => {
+    const allQLength = recievedData.quiz.length
+    let msg = `${res.qdbName}: `
+    let color = logger.GetColor('green')
+    if (res.newQuestions > 0) {
+      color = logger.GetColor('blue')
+      msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
+    } else {
+      msg += `No new data ( ${allQLength} )`
+    }
+    if (recievedData.version !== undefined) {
+      msg += '. Version: ' + recievedData.version
+    }
+    logger.Log('\t' + msg, color)
+  })
+}
+
+export function processIncomingRequest(
   recievedData: RecievedData,
   questionDbs: Array<QuestionDb>,
   dryRun: boolean,
   user: User
-): Promise<Array<number>> {
+): Promise<Array<Result>> {
-  return Promise.all(
-    questionDbs.map((qdb) => {
-      return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user)
-    })
-  )
-}
-
-function ProcessIncomingRequestUsingDb(
-  recievedData: RecievedData,
-  qdb: QuestionDb,
-  dryRun: boolean,
-  user: User
-): Promise<number> {
-  return new Promise((resolve, reject) => {
   logger.DebugLog('Processing incoming request', 'actions', 1)
 
   if (recievedData === undefined) {
     logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
-      reject(new Error('Recieved data is undefined!'))
+    throw new Error('Recieved data is undefined!')
   }
 
   try {
@@ -100,12 +111,26 @@ function ProcessIncomingRequestUsingDb(
       'Data lock file exists, skipping recieved data processing',
       logger.GetColor('red')
     )
-      resolve(-1)
+    return new Promise((resolve) => resolve([]))
-      return
   }
 
+  const promises: Array<Promise<Result>> = questionDbs.reduce((acc, qdb) => {
+    if (qdb.shouldSave(recievedData)) {
+      acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user))
+    }
+    return acc
+  }, [])
+  return Promise.all(promises)
+}
+
+function processIncomingRequestUsingDb(
+  recievedData: RecievedData,
+  qdb: QuestionDb,
+  dryRun: boolean,
+  user: User
+): Promise<Result> {
+  return new Promise((resolve, reject) => {
     try {
-      // recievedData: { version: "", id: "", subj: "" quiz: {} }
       const recievedQuestions = []
 
       logger.DebugLog('recievedData JSON  parsed', 'actions', 1)
@@ -145,11 +170,7 @@ function ProcessIncomingRequestUsingDb(
           })
 
           try {
-            let color = logger.GetColor('green')
-            let msg = ''
             if (allQuestions.length > 0) {
-              color = logger.GetColor('blue')
-              msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
               allQuestions.forEach((currentQuestion) => {
                 const sName = getSubjNameWithoutYear(recievedData.subj)
                 logger.DebugLog(
@@ -171,35 +192,26 @@ function ProcessIncomingRequestUsingDb(
                 currWrites = 0
                 logger.DebugLog('Writing data.json', 'actions', 1)
                 utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
-                logger.Log('\tData file written', color)
               } else if (dryRun) {
                 logger.Log('\tDry run')
               }
-            } else {
-              msg += `No new data ( ${allQLength} )`
             }
 
-            let subjRow = '\t' + recievedData.subj
-            if (recievedData.id) {
-              subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
-            }
             idStats.LogId(
               user.id,
               recievedData.subj,
               allQuestions.length,
               allQLength
             )
-            logger.Log(subjRow)
-            if (recievedData.version !== undefined) {
-              msg += '. Version: ' + recievedData.version
-            }
 
-            logger.Log('\t' + msg, color)
             logger.DebugLog('New Questions:', 'actions', 2)
             logger.DebugLog(allQuestions, 'actions', 2)
 
             logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
-            resolve(allQuestions.length)
+            resolve({
+              newQuestions: allQuestions.length,
+              qdbName: qdb.name,
+            })
           } catch (error) {
             console.error(error)
             logger.Log(
@@ -229,7 +241,7 @@ function ProcessIncomingRequestUsingDb(
   })
 }
 
-function LoadJSON(dataFiles: Array<DataFile>) {
+export function loadJSON(dataFiles: Array<DataFile>): Array<QuestionDb> {
   return dataFiles.reduce((acc, dataFile) => {
     if (!utils.FileExists(dataFile.path)) {
       utils.WriteFile(JSON.stringify([]), dataFile.path)
@@ -251,7 +263,7 @@ function LoadJSON(dataFiles: Array<DataFile>) {
   }, [])
 }
 
-function backupData(questionDbs: Array<QuestionDb>) {
+export function backupData(questionDbs: Array<QuestionDb>): void {
   questionDbs.forEach((data) => {
     const path = './publicDirs/qminingPublic/backs/'
     utils.CreatePath(path)

src/utils/classes.ts

@@ -1,6 +1,11 @@
 import { Worker, isMainThread, parentPort, workerData } from 'worker_threads'
 import logger from './logger'
-import { Question, QuestionData, Subject } from '../types/basicTypes'
+import {
+  Question,
+  QuestionDb,
+  QuestionData,
+  Subject,
+} from '../types/basicTypes'
 
 // TODO
 interface SearchResultQuestion extends Question {
@@ -402,6 +407,19 @@ function addQuestion(
   }
 }
 
+function searchDatas(
+  data: Array<QuestionDb>,
+  question: any,
+  subjName: string,
+  questionData?: QuestionData
+): Promise<Array<Array<SearchResultQuestion>>> {
+  return Promise.all(
+    data.map((db) => {
+      return searchData(db.data, question, subjName, questionData)
+    })
+  )
+}
+
 // TODO: remove questionData, make question only Question type
 function searchData(
   data: Array<Subject>,
@@ -568,5 +586,6 @@ export {
   createQuestion,
   addQuestion,
   searchData,
+  searchDatas,
   dataToString,
 }