Added api to qmining

modules/qmining/qmining.js

@@ -37,6 +37,7 @@ const uloadFiles = 'public/f'
 const dataFile = 'public/data.json'
 const msgFile = 'stats/msgs'
 const motdFile = 'public/motd'
+const versionFile = 'public/version'
 let donateURL = ''
 try {
   donateURL = utils.ReadFile('./data/donateURL')
@@ -64,6 +65,8 @@ app.use(bodyParser.json({
   limit: '5mb'
 }))
 
+var data = actions.LoadJSON(dataFile)
+
 // --------------------------------------------------------------
 
 app.get('/', function (req, res) {
@@ -90,14 +93,14 @@ app.get('/manual', function (req, res) {
 })
 
 app.get('/legacy', function (req, res) {
-  var f = utils.ReadFile(dataFile)
+  let qcount = data.Subjects.reduce((acc, currItem) => {
-  var d = actions.LoadJSON(f)
+    acc += currItem.length
-  let qcount = 0
+    return acc
-  for (let i = 0; i < d.length; i++) { qcount += d.Subjects[i].length }
+  }, 0)
-  let scount = d.length
+  let scount = data.length
 
   res.render('alldata', {
-    data: d,
+    data: data,
     scount: scount,
     qcount: qcount,
     siteurl: url
@@ -116,13 +119,6 @@ app.get('/postfeedback', function (req, res) {
   res.redirect('/')
 })
 
-app.post('/isAdding', function (req, res) {
-  res.end('OK')
-  logger.LogReq(req)
-  actions.ProcessIncomingRequest(req.body.datatoadd)
-  utils.WriteBackup()
-})
-
 app.get('/lred', function (req, res) {
   res.redirect('/legacy')
   res.end()
@@ -226,6 +222,90 @@ app.route('/badtestsender').post(function (req, res, next) {
   logger.LogReq(req)
 })
 
+// -------------------------------------------------------------------------------------------
+// API
+
+app.post('/isAdding', function (req, res) {
+  logger.LogReq(req)
+  let result = actions.ProcessIncomingRequest(req.body.datatoadd, data)
+  res.json({
+    success: result !== -1,
+    newQuestions: result
+  })
+  utils.WriteBackup() // TODO: write every n times / after n minutes
+})
+
+app.get('/q', function (req, res) {
+  logger.LogReq(req)
+  if (Object.keys(req.query).length === 0) {
+    res.send('ask something!')
+  } else {
+    if (req.query.q) {
+      let subj = req.query.subj || ''
+      let recData = {}
+      try {
+        recData = JSON.parse(req.query.data)
+      } catch (e) {
+        console.log(e) // TODO: normal error logging
+      }
+      let r = data.Search(decodeURIComponent(req.query.q), subj, recData)
+
+      res.json(r)
+    } else {
+      res.send('no key "q" in query' + JSON.stringify(req.query))
+    }
+  }
+})
+
+function getSimplreRes () {
+  return {
+    subjects: data.length,
+    questions: data.Subjects.reduce((acc, subj) => {
+      return acc + subj.length
+    }, 0)
+  }
+}
+function getDetailedRes () {
+  return data.Subjects.map((subj) => {
+    return {
+      name: subj.Name,
+      count: subj.length
+    }
+  })
+}
+
+app.get('/datacount', function (req, res) {
+  logger.LogReq(req)
+  if (req.query.detailed === 'all') {
+    res.json({
+      detailed: getDetailedRes(),
+      simple: getSimplreRes()
+    })
+  } else if (req.query.detailed) {
+    res.json(getDetailedRes())
+  } else {
+    res.json(getSimplreRes())
+  }
+})
+
+app.get('/infos', function (req, res) {
+  let result = {}
+  if (req.query.subjinfo) {
+    result.subjinfo = getSimplreRes()
+  }
+  // TODO: watch file, and global variable for motd version
+  if (req.query.version) {
+    result.version = utils.ReadFile(versionFile)
+  }
+  if (req.query.motd) {
+    result.motd = utils.ReadFile(motdFile)
+  }
+  logger.LogReq(req)
+  res.json(result)
+})
+
+// -------------------------------------------------------------------------------------------
+
 app.get('*', function (req, res) {
   res.status(404).render('404')
 })

utils/actions.js

@@ -29,14 +29,17 @@ const versionFile = './public/version'
 const motdFile = './public/motd'
 const qaFile = './public/qa'
 
-var logger = require('../utils/logger.js')
+const logger = require('../utils/logger.js')
 const idStats = require('../utils/ids.js')
 idStats.Load() // TODO: dont always load when actions.js is used
-var utils = require('../utils/utils.js')
+const utils = require('../utils/utils.js')
 const classes = require('./question-classes/classes.js')
 
-function ProcessIncomingRequest (data) {
+const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
-  if (data === undefined) {
+var currWrites = 0
+
+function ProcessIncomingRequest (recievedData, qdb) {
+  if (recievedData === undefined) {
     logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
     return
   }
@@ -44,7 +47,7 @@ function ProcessIncomingRequest (data) {
   try {
     let towrite = logger.GetDateString() + '\n'
     towrite += '------------------------------------------------------------------------------\n'
-    towrite += data
+    towrite += recievedData
     towrite += '\n------------------------------------------------------------------------------\n'
     utils.AppendToFile(towrite, recDataFile)
   } catch (e) {
@@ -52,44 +55,49 @@ function ProcessIncomingRequest (data) {
   }
 
   try {
-    var d = JSON.parse(data)
+    // recievedData: { version: "", id: "", subj: "" quiz: {} }
-    var allQuestions = []
+    let d = JSON.parse(recievedData)
+    let allQLength = d.quiz.length
+    let allQuestions = []
 
-    d = ConvertToNewFormat(d)
+    d.quiz.forEach((question) => {
-
+      let q = new classes.Question(question.Q, question.A, question.data)
-    for (let i = 0; i < d.allData.length; i++) {
+      let sames = qdb.Search(q, '') // TODO d.subj
-      allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data))
+      // if it didnt find any question, or every found questions match is lower thatn 80
+      let isNew = sames.length === 0 || sames.every(searchResItem => {
+        return searchResItem.match < classes.minMatchAmmount
+      })
+      if (isNew) {
+        allQuestions.push(q)
       }
+    })
 
     let color = logger.GetColor('green')
-    let questions = []
     let msg = ''
-    if (d.data.length > 0) {
+    if (allQuestions.length > 0) {
-      msg += `New questions: ${d.data.length} ( All: ${allQuestions.length} )`
+      color = logger.GetColor('blue')
-    } else {
+      msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
-      msg += `No new data ( ${allQuestions.length} )`
+
-    }
+      allQuestions.forEach((q) => {
-    if (d.data.length > 0) {
-      let qdb = LoadJSON(utils.ReadFile(dataFile))
-      d.data.forEach((x) => {
-        let q = new classes.Question(x.Q, x.A, x.data)
-        questions.push(q)
         qdb.AddQuestion(d.subj, q)
       })
 
+      currWrites++
+      console.log(currWrites)
+      if (currWrites >= writeAfter) {
+        currWrites = 0
         try {
+          // TODO: with utils.WatchFile
           qdb.version = utils.ReadFile(versionFile)
           qdb.motd = utils.ReadFile(motdFile)
         } catch (e) {
           logger.Log('MOTD/Version writing/reading error!')
         }
-
-      if (qdb !== undefined && d.data.length > 0) {
-        utils.WriteBackup()
         utils.WriteFile(JSON.stringify(qdb), dataFile)
-        msg += ' - Data file written!'
+        logger.Log('\tData file written', color)
-        color = logger.GetColor('blue')
       }
+    } else {
+      msg += `No new data ( ${allQLength} )`
     }
 
     let subjRow = '\t' + d.subj
@@ -101,16 +109,18 @@ function ProcessIncomingRequest (data) {
     if (d.version !== undefined) { msg += '. Version: ' + d.version }
 
     logger.Log('\t' + msg, color)
+    return allQuestions.length
   } catch (e) {
     console.log(e)
     logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))
+    return -1
   }
 }
 
 // loading stuff
-function LoadJSON (resource) {
+function LoadJSON (dataFile) {
   try {
-    var d = JSON.parse(resource)
+    var d = JSON.parse(utils.ReadFile(dataFile))
     var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y))
     var rt = []
 
@@ -155,40 +165,3 @@ function ProcessQA () {
 
   return r
 }
-
-function ConvertToNewFormat (d) {
-  if (d.allData.length < 0) {
-    return d
-  }
-
-  try {
-    if (!d.allData[0].data) {
-      logger.Log('\tConverting old data to new...')
-      d.allData = d.allData.map((x) => {
-        if (x.I) {
-          x.data = {
-            type: 'image',
-            images: JSON.parse(x.I)
-          }
-          delete x.I
-        }
-        return x
-      })
-      d.data = d.data.map((x) => {
-        if (x.I) {
-          x.data = {
-            type: 'image',
-            images: JSON.parse(x.I)
-          }
-          delete x.I
-        }
-        return x
-      })
-    }
-  } catch (e) {
-    console.log(e)
-    logger.Log('Couldnt convert old data to new!', logger.GetColor('redbg'))
-  }
-
-  return d
-}