mrfrys-node-server/src/utils/actions.js

/* ----------------------------------------------------------------------------
Question Server
 GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.

 ------------------------------------------------------------------------- */

module.exports = {
  ProcessIncomingRequest: ProcessIncomingRequest,
  LoadJSON: LoadJSON,
}

const dataFile = './publicDirs/qminingPublic/data.json'
const recDataFile = './stats/recdata'
const processDataWorkerFile = './src/utils/processData.js'

const { Worker } = require('worker_threads')
const logger = require('../utils/logger.js')
const idStats = require('../utils/ids.js')
const utils = require('../utils/utils.js')
const { addQuestion, getSubjNameWithoutYear } = require('./classes.js')
// if a recievend question doesnt match at least this % to any other question in the db it gets
// added to db

const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
var currWrites = 0

function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
  return new Promise((resolve, reject) => {
    logger.DebugLog('Processing incoming request', 'actions', 1)
    if (recievedData === undefined) {
      logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
      reject(new Error('Recieved data is undefined!'))
    }

    try {
      let towrite = logger.GetDateString() + '\n'
      towrite +=
        '------------------------------------------------------------------------------\n'
      if (typeof recievedData === 'object') {
        towrite += JSON.stringify(recievedData)
      } else {
        towrite += recievedData
      }
      towrite +=
        '\n------------------------------------------------------------------------------\n'
      utils.AppendToFile(towrite, recDataFile)
      logger.DebugLog('recDataFile written', 'actions', 1)
    } catch (err) {
      logger.log('Error writing recieved data.')
    }

    try {
      // recievedData: { version: "", id: "", subj: "" quiz: {} }
      let data = recievedData
      // FIXME: if is for backwards compatibility, remove this sometime in the future
      if (typeof data !== 'object') {
        data = JSON.parse(recievedData)
      }

      logger.DebugLog('recievedData JSON  parsed', 'actions', 1)
      logger.DebugLog(data, 'actions', 3)
      let allQLength = data.quiz.length

      const worker = new Worker(processDataWorkerFile, {
        workerData: {
          data: data,
          qdb: qdb,
        },
      })

      worker.on('error', (err) => {
        logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
        console.error(err)
        reject(err)
      })

      worker.on('exit', (code) => {
        logger.DebugLog('Process Data exit, code: ' + code, 'actions', 1)
        if (code !== 0) {
          logger.Log(
            'Process Data Worker error! Exit code is not 0',
            logger.GetColor('redbg')
          )
          reject(new Error('Process Data Worker exit code is not 0!'))
        }
      })

      worker.on('message', (workerMsg) => {
        logger.DebugLog('Message from processData', 'actions', 1)
        logger.DebugLog(workerMsg, 'actions', 3)

        const allQuestions = workerMsg

        try {
          let color = logger.GetColor('green')
          let msg = ''
          if (allQuestions.length > 0) {
            color = logger.GetColor('blue')
            msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
            allQuestions.forEach((currentQuestion) => {
              const sName = getSubjNameWithoutYear(data.subj)
              logger.DebugLog(
                'Adding question with subjName: ' + sName + ' :',
                'actions',
                3
              )
              logger.DebugLog(currentQuestion, 'actions', 3)
              qdb = addQuestion(qdb, sName, currentQuestion)
            })

            currWrites++
            logger.DebugLog(
              'currWrites for data.json: ' + currWrites,
              'actions',
              1
            )
            if (currWrites >= writeAfter && !dryRun) {
              currWrites = 0
              try {
                qdb.version = infos.version
                qdb.motd = infos.motd
                logger.DebugLog(
                  'version and motd set for data.json',
                  'actions',
                  3
                )
              } catch (err) {
                logger.Log('MOTD/Version writing/reading error!')
              }
              logger.DebugLog('Writing data.json', 'actions', 1)
              utils.WriteFile(JSON.stringify(qdb), dataFile)
              logger.Log('\tData file written', color)
            } else if (dryRun) {
              logger.Log('\tDry run')
            }
          } else {
            msg += `No new data ( ${allQLength} )`
          }

          let subjRow = '\t' + data.subj
          if (data.id) {
            subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')'
            idStats.LogId(user.id, data.subj)
          }
          logger.Log(subjRow)
          if (data.version !== undefined) {
            msg += '. Version: ' + data.version
          }

          logger.Log('\t' + msg, color)
          logger.DebugLog('New Questions:', 'actions', 2)
          logger.DebugLog(allQuestions, 'actions', 2)

          logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
          resolve({
            newQuestions: allQLength.length,
            newDb: qdb,
          })
        } catch (error) {
          console.log(error)
          logger.Log(
            'Error while processing processData worker result!',
            logger.GetColor('redbg')
          )
          reject(new Error('Error while processing processData worker result!'))
        }
      })
    } catch (err) {
      console.log(err)
      logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))
      reject(new Error('Couldnt parse JSON data'))
    }
  })
}

// loading stuff
function LoadJSON(dataFile) {
  var data = JSON.parse(utils.ReadFile(dataFile))
  if (!data) {
    logger.Log(
      "data is undefined! Couldn't load data!",
      logger.GetColor('redbg')
    )
  }
  return data
}