Handling multiple data files

src/utils/actions.ts

@@ -17,12 +17,6 @@ Question Server
 
  ------------------------------------------------------------------------- */
 
-export default {
-  ProcessIncomingRequest: ProcessIncomingRequest,
-  LoadJSON: LoadJSON,
-  backupData: backupData,
-}
-
 const recDataFile = './stats/recdata'
 const dataLockFile = './data/lockData'
 
@@ -33,8 +27,7 @@ import utils from '../utils/utils'
 import { addQuestion, getSubjNameWithoutYear } from './classes'
 
 // types
-import { QuestionDb, Question, User } from '../types/basicTypes'
-import { DataFile } from '../modules/api/api'
+import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
 
 // if a recievend question doesnt match at least this % to any other question in the db it gets
 // added to db
@@ -43,7 +36,7 @@ const minMatchToAmmountToAdd = 90
 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
 let currWrites = 0
 
-interface RecievedData {
+export interface RecievedData {
   quiz: Array<Question>
   subj: string
   id: string
@@ -51,61 +44,93 @@ interface RecievedData {
   scriptVersion: string
 }
 
-function ProcessIncomingRequest(
+interface Result {
+  qdbName: string
+  newQuestions: number
+}
+
+export function logResult(
+  recievedData: RecievedData,
+  result: Array<Result>
+): void {
+  let subjRow = '\t' + recievedData.subj
+  if (recievedData.id) {
+    subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
+  }
+  logger.Log(subjRow)
+
+  result.forEach((res: Result) => {
+    const allQLength = recievedData.quiz.length
+    let msg = `${res.qdbName}: `
+    let color = logger.GetColor('green')
+    if (res.newQuestions > 0) {
+      color = logger.GetColor('blue')
+      msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
+    } else {
+      msg += `No new data ( ${allQLength} )`
+    }
+    if (recievedData.version !== undefined) {
+      msg += '. Version: ' + recievedData.version
+    }
+    logger.Log('\t' + msg, color)
+  })
+}
+
+export function processIncomingRequest(
   recievedData: RecievedData,
   questionDbs: Array<QuestionDb>,
   dryRun: boolean,
   user: User
-): Promise<Array<number>> {
-  return Promise.all(
-    questionDbs.map((qdb) => {
-      return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user)
-    })
-  )
+): Promise<Array<Result>> {
+  logger.DebugLog('Processing incoming request', 'actions', 1)
+
+  if (recievedData === undefined) {
+    logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
+    throw new Error('Recieved data is undefined!')
+  }
+
+  try {
+    let towrite = logger.GetDateString() + '\n'
+    towrite +=
+      '------------------------------------------------------------------------------\n'
+    if (typeof recievedData === 'object') {
+      towrite += JSON.stringify(recievedData)
+    } else {
+      towrite += recievedData
+    }
+    towrite +=
+      '\n------------------------------------------------------------------------------\n'
+    utils.AppendToFile(towrite, recDataFile)
+    logger.DebugLog('recDataFile written', 'actions', 1)
+  } catch (err) {
+    logger.Log('Error writing recieved data.')
+  }
+
+  if (utils.FileExists(dataLockFile)) {
+    logger.Log(
+      'Data lock file exists, skipping recieved data processing',
+      logger.GetColor('red')
+    )
+    return new Promise((resolve) => resolve([]))
+  }
+
+  const promises: Array<Promise<Result>> = questionDbs.reduce((acc, qdb) => {
+    if (qdb.shouldSave(recievedData)) {
+      acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user))
+    }
+    return acc
+  }, [])
+  return Promise.all(promises)
 }
 
-function ProcessIncomingRequestUsingDb(
+function processIncomingRequestUsingDb(
   recievedData: RecievedData,
   qdb: QuestionDb,
   dryRun: boolean,
   user: User
-): Promise<number> {
+): Promise<Result> {
   return new Promise((resolve, reject) => {
-    logger.DebugLog('Processing incoming request', 'actions', 1)
-
-    if (recievedData === undefined) {
-      logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
-      reject(new Error('Recieved data is undefined!'))
-    }
-
     try {
-      let towrite = logger.GetDateString() + '\n'
-      towrite +=
-        '------------------------------------------------------------------------------\n'
-      if (typeof recievedData === 'object') {
-        towrite += JSON.stringify(recievedData)
-      } else {
-        towrite += recievedData
-      }
-      towrite +=
-        '\n------------------------------------------------------------------------------\n'
-      utils.AppendToFile(towrite, recDataFile)
-      logger.DebugLog('recDataFile written', 'actions', 1)
-    } catch (err) {
-      logger.Log('Error writing recieved data.')
-    }
-
-    if (utils.FileExists(dataLockFile)) {
-      logger.Log(
-        'Data lock file exists, skipping recieved data processing',
-        logger.GetColor('red')
-      )
-      resolve(-1)
-      return
-    }
-
-    try {
-      // recievedData: { version: "", id: "", subj: "" quiz: {} }
       const recievedQuestions = []
 
       logger.DebugLog('recievedData JSON  parsed', 'actions', 1)
@@ -145,11 +170,7 @@ function ProcessIncomingRequestUsingDb(
           })
 
           try {
-            let color = logger.GetColor('green')
-            let msg = ''
             if (allQuestions.length > 0) {
-              color = logger.GetColor('blue')
-              msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
               allQuestions.forEach((currentQuestion) => {
                 const sName = getSubjNameWithoutYear(recievedData.subj)
                 logger.DebugLog(
@@ -171,35 +192,26 @@ function ProcessIncomingRequestUsingDb(
                 currWrites = 0
                 logger.DebugLog('Writing data.json', 'actions', 1)
                 utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
-                logger.Log('\tData file written', color)
               } else if (dryRun) {
                 logger.Log('\tDry run')
               }
-            } else {
-              msg += `No new data ( ${allQLength} )`
             }
 
-            let subjRow = '\t' + recievedData.subj
-            if (recievedData.id) {
-              subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
-            }
             idStats.LogId(
               user.id,
               recievedData.subj,
               allQuestions.length,
               allQLength
             )
-            logger.Log(subjRow)
-            if (recievedData.version !== undefined) {
-              msg += '. Version: ' + recievedData.version
-            }
 
-            logger.Log('\t' + msg, color)
             logger.DebugLog('New Questions:', 'actions', 2)
             logger.DebugLog(allQuestions, 'actions', 2)
 
             logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
-            resolve(allQuestions.length)
+            resolve({
+              newQuestions: allQuestions.length,
+              qdbName: qdb.name,
+            })
           } catch (error) {
             console.error(error)
             logger.Log(
@@ -229,7 +241,7 @@ function ProcessIncomingRequestUsingDb(
   })
 }
 
-function LoadJSON(dataFiles: Array<DataFile>) {
+export function loadJSON(dataFiles: Array<DataFile>): Array<QuestionDb> {
   return dataFiles.reduce((acc, dataFile) => {
     if (!utils.FileExists(dataFile.path)) {
       utils.WriteFile(JSON.stringify([]), dataFile.path)
@@ -251,7 +263,7 @@ function LoadJSON(dataFiles: Array<DataFile>) {
   }, [])
 }
 
-function backupData(questionDbs: Array<QuestionDb>) {
+export function backupData(questionDbs: Array<QuestionDb>): void {
   questionDbs.forEach((data) => {
     const path = './publicDirs/qminingPublic/backs/'
     utils.CreatePath(path)