Added api to qmining

utils/actions.js

@@ -29,14 +29,17 @@ const versionFile = './public/version'
 const motdFile = './public/motd'
 const qaFile = './public/qa'
 
-var logger = require('../utils/logger.js')
+const logger = require('../utils/logger.js')
 const idStats = require('../utils/ids.js')
 idStats.Load() // TODO: dont always load when actions.js is used
-var utils = require('../utils/utils.js')
+const utils = require('../utils/utils.js')
 const classes = require('./question-classes/classes.js')
 
-function ProcessIncomingRequest (data) {
-  if (data === undefined) {
+const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
+var currWrites = 0
+
+function ProcessIncomingRequest (recievedData, qdb) {
+  if (recievedData === undefined) {
     logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
     return
   }
@@ -44,7 +47,7 @@ function ProcessIncomingRequest (data) {
   try {
     let towrite = logger.GetDateString() + '\n'
     towrite += '------------------------------------------------------------------------------\n'
-    towrite += data
+    towrite += recievedData
     towrite += '\n------------------------------------------------------------------------------\n'
     utils.AppendToFile(towrite, recDataFile)
   } catch (e) {
@@ -52,44 +55,49 @@ function ProcessIncomingRequest (data) {
   }
 
   try {
-    var d = JSON.parse(data)
-    var allQuestions = []
+    // recievedData: { version: "", id: "", subj: "" quiz: {} }
+    let d = JSON.parse(recievedData)
+    let allQLength = d.quiz.length
+    let allQuestions = []
 
-    d = ConvertToNewFormat(d)
-
-    for (let i = 0; i < d.allData.length; i++) {
-      allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data))
-    }
+    d.quiz.forEach((question) => {
+      let q = new classes.Question(question.Q, question.A, question.data)
+      let sames = qdb.Search(q, '') // TODO d.subj
+      // if it didnt find any question, or every found questions match is lower thatn 80
+      let isNew = sames.length === 0 || sames.every(searchResItem => {
+        return searchResItem.match < classes.minMatchAmmount
+      })
+      if (isNew) {
+        allQuestions.push(q)
+      }
+    })
 
     let color = logger.GetColor('green')
-    let questions = []
     let msg = ''
-    if (d.data.length > 0) {
-      msg += `New questions: ${d.data.length} ( All: ${allQuestions.length} )`
-    } else {
-      msg += `No new data ( ${allQuestions.length} )`
-    }
-    if (d.data.length > 0) {
-      let qdb = LoadJSON(utils.ReadFile(dataFile))
-      d.data.forEach((x) => {
-        let q = new classes.Question(x.Q, x.A, x.data)
-        questions.push(q)
+    if (allQuestions.length > 0) {
+      color = logger.GetColor('blue')
+      msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
+
+      allQuestions.forEach((q) => {
         qdb.AddQuestion(d.subj, q)
       })
 
-      try {
-        qdb.version = utils.ReadFile(versionFile)
-        qdb.motd = utils.ReadFile(motdFile)
-      } catch (e) {
-        logger.Log('MOTD/Version writing/reading error!')
-      }
-
-      if (qdb !== undefined && d.data.length > 0) {
-        utils.WriteBackup()
+      currWrites++
+      console.log(currWrites)
+      if (currWrites >= writeAfter) {
+        currWrites = 0
+        try {
+          // TODO: with utils.WatchFile
+          qdb.version = utils.ReadFile(versionFile)
+          qdb.motd = utils.ReadFile(motdFile)
+        } catch (e) {
+          logger.Log('MOTD/Version writing/reading error!')
+        }
         utils.WriteFile(JSON.stringify(qdb), dataFile)
-        msg += ' - Data file written!'
-        color = logger.GetColor('blue')
+        logger.Log('\tData file written', color)
       }
+    } else {
+      msg += `No new data ( ${allQLength} )`
     }
 
     let subjRow = '\t' + d.subj
@@ -101,16 +109,18 @@ function ProcessIncomingRequest (data) {
     if (d.version !== undefined) { msg += '. Version: ' + d.version }
 
     logger.Log('\t' + msg, color)
+    return allQuestions.length
   } catch (e) {
     console.log(e)
     logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))
+    return -1
   }
 }
 
 // loading stuff
-function LoadJSON (resource) {
+function LoadJSON (dataFile) {
   try {
-    var d = JSON.parse(resource)
+    var d = JSON.parse(utils.ReadFile(dataFile))
     var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y))
     var rt = []
 
@@ -155,40 +165,3 @@ function ProcessQA () {
 
   return r
 }
-
-function ConvertToNewFormat (d) {
-  if (d.allData.length < 0) {
-    return d
-  }
-
-  try {
-    if (!d.allData[0].data) {
-      logger.Log('\tConverting old data to new...')
-      d.allData = d.allData.map((x) => {
-        if (x.I) {
-          x.data = {
-            type: 'image',
-            images: JSON.parse(x.I)
-          }
-          delete x.I
-        }
-        return x
-      })
-      d.data = d.data.map((x) => {
-        if (x.I) {
-          x.data = {
-            type: 'image',
-            images: JSON.parse(x.I)
-          }
-          delete x.I
-        }
-        return x
-      })
-    }
-  } catch (e) {
-    console.log(e)
-    logger.Log('Couldnt convert old data to new!', logger.GetColor('redbg'))
-  }
-
-  return d
-}