mrfrys-node-server/src/utils/actions.ts

/* ----------------------------------------------------------------------------
Question Server
 GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.

 ------------------------------------------------------------------------- */

const recDataFile = './stats/recdata'
const dataLockFile = './data/lockData'

import logger from '../utils/logger'
import { createQuestion } from '../utils/classes'
import { doALongTask } from './workerPool'
import idStats from '../utils/ids'
import utils from '../utils/utils'
import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'

// types
import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'

// if a recievend question doesnt match at least this % to any other question in the db it gets
// added to db
const minMatchAmmountToAdd = 90

const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
let currWrites = 0

export interface RecievedData {
  quiz: Array<Question>
  subj: string
  id: string
  version: string
  scriptVersion: string
  location: string
}

export interface Result {
  qdbName: string
  newQuestions: number
}

export function logResult(
  recievedData: RecievedData,
  result: Array<Result>,
  userId: number,
  dryRun?: boolean
): void {
  logger.Log('\t' + recievedData.subj)

  if (dryRun) {
    logger.Log('\tDry run')
  }

  let idRow = '\t'
  if (recievedData.version) {
    idRow += 'Version: ' + logger.logHashed(recievedData.version)
  }
  if (recievedData.id) {
    idRow += ', CID: ' + logger.logHashed(recievedData.id)
  }
  idRow += `, User #${logger.logHashed(userId.toString())}`
  logger.Log(idRow)

  if (result.length > 0) {
    result.forEach((res: Result) => {
      const allQLength = recievedData.quiz.length
      let msg = `${res.qdbName}: `
      let color = logger.GetColor('green')
      if (res.newQuestions > 0) {
        color = logger.GetColor('blue')
        msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
      } else {
        msg += `No new data ( ${allQLength} )`
      }
      logger.Log('\t' + msg, color)
    })
  } else {
    logger.Log('\tResults length is zero!', logger.GetColor('red'))
  }
}

export function processIncomingRequest(
  recievedData: RecievedData,
  questionDbs: Array<QuestionDb>,
  dryRun: boolean,
  user: User
): Promise<Array<Result>> {
  logger.DebugLog('Processing incoming request', 'isadding', 1)

  if (recievedData === undefined) {
    logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
    throw new Error('Recieved data is undefined!')
  }

  try {
    let towrite = utils.GetDateString() + '\n'
    towrite +=
      '------------------------------------------------------------------------------\n'
    if (typeof recievedData === 'object') {
      towrite += JSON.stringify(recievedData)
    } else {
      towrite += recievedData
    }
    towrite +=
      '\n------------------------------------------------------------------------------\n'
    utils.AppendToFile(towrite, recDataFile)
    logger.DebugLog('recDataFile written', 'isadding', 1)
  } catch (err) {
    logger.Log('Error writing recieved data.')
  }

  if (utils.FileExists(dataLockFile)) {
    logger.Log(
      'Data lock file exists, skipping recieved data processing',
      logger.GetColor('red')
    )
    return new Promise((resolve) => resolve([]))
  }

  // FIXME: this many promises and stuff might be unnecesarry
  const promises: Array<Promise<Result>> = questionDbs.reduce((acc, qdb) => {
    acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user))
    return acc
  }, [])
  return Promise.all(promises)
}

function processIncomingRequestUsingDb(
  recievedData: RecievedData,
  qdb: QuestionDb,
  dryRun: boolean,
  user: User
): Promise<Result> {
  return new Promise((resolve, reject) => {
    try {
      const recievedQuestions = []

      logger.DebugLog('recievedData JSON  parsed', 'isadding', 1)
      logger.DebugLog(recievedData, 'isadding', 3)
      const allQLength = recievedData.quiz.length
      const questionSearchPromises = []
      recievedData.quiz.forEach((question) => {
        logger.DebugLog('Question:', 'isadding', 2)
        logger.DebugLog(question, 'isadding', 2)
        const currentQuestion = createQuestion(
          question.Q,
          question.A,
          question.data
        )
        logger.DebugLog(
          'Searching for question in subj ' + recievedData.subj,
          'isadding',
          3
        )
        logger.DebugLog(currentQuestion, 'isadding', 3)
        if (isQuestionValid(currentQuestion)) {
          recievedQuestions.push(currentQuestion)
          // This here searches only in relevant subjects, and not all subjects
          questionSearchPromises.push(
            doALongTask({
              type: 'work',
              data: {
                searchIn: [qdb.index],
                qdb: qdb.data,
                question: currentQuestion,
                subjName: recievedData.subj,
                searchTillMatchPercent: minMatchAmmountToAdd,
              },
            })
          )
        } else {
          logger.DebugLog('Question isnt valid', 'isadding', 1)
          logger.DebugLog(currentQuestion, 'isadding', 1)
        }
      })

      Promise.all(questionSearchPromises)
        .then((results: Array<SearchResult>) => {
          const allQuestions = [] // all new questions here that do not have result
          results.forEach((result, i) => {
            const add = result.result.every((res) => {
              return res.match < minMatchAmmountToAdd
            })
            if (add) {
              allQuestions.push(recievedQuestions[i])
            }
          })

          try {
            if (allQuestions.length > 0) {
              allQuestions.forEach((currentQuestion) => {
                const sName = getSubjNameWithoutYear(recievedData.subj)
                logger.DebugLog(
                  'Adding question with subjName: ' + sName + ' :',
                  'isadding',
                  3
                )
                logger.DebugLog(currentQuestion, 'isadding', 3)
                addQuestion(qdb.data, sName, currentQuestion)
              })

              currWrites++
              logger.DebugLog(
                'currWrites for data.json: ' + currWrites,
                'isadding',
                1
              )
              if (currWrites >= writeAfter && !dryRun) {
                currWrites = 0
                logger.DebugLog('Writing data.json', 'isadding', 1)
                utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
              }
            }

            idStats.LogId(
              user.id,
              recievedData.subj,
              allQuestions.length,
              allQLength
            )

            logger.DebugLog('New Questions:', 'isadding', 2)
            logger.DebugLog(allQuestions, 'isadding', 2)

            logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1)
            resolve({
              newQuestions: allQuestions.length,
              qdbName: qdb.name,
            })
          } catch (error) {
            console.error(error)
            logger.Log(
              'Error while processing processData worker result!',
              logger.GetColor('redbg')
            )
            reject(
              new Error('Error while processing processData worker result!')
            )
          }
        })
        .catch((err) => {
          logger.Log(
            'Error while searching for questions in ProcessIncomingRequest!',
            logger.GetColor('redbg')
          )
          console.error(err)
        })
    } catch (err) {
      console.error(err)
      logger.Log(
        'There was en error handling incoming quiz data, see stderr',
        logger.GetColor('redbg')
      )
      reject(new Error('Couldnt parse JSON data'))
    }
  })
}

function isQuestionValid(question: Question) {
  if (!question.Q) {
    return false
  }
  if (!question.A && question.data.type === 'simple') {
    return false
  }
  return true
}

export function shouldSearchDataFile(
  df: DataFile,
  testUrl: string,
  trueIfAlways?: Boolean
): Boolean {
  if (
    typeof df.shouldSearch === 'string' &&
    df.shouldSearch === 'always' &&
    trueIfAlways
  ) {
    return true
  }

  if (typeof df.shouldSearch === 'object') {
    if (df.shouldSearch.location) {
      const { val } = df.shouldSearch.location
      return testUrl.includes(val)
    }
  }

  logger.DebugLog(`no suitable dbs for ${testUrl}`, 'shouldSearchDataFile', 1)
  return false
}

export function shouldSaveDataFile(
  df: DataFile,
  recievedData: RecievedData
): Boolean {
  if (df.shouldSave.version) {
    const { compare, val } = df.shouldSave.version
    if (compare === 'equals') {
      return recievedData.version === val
    } else if (compare === 'startsWith') {
      return recievedData.version.startsWith(val)
    }
  }

  if (df.shouldSave.version) {
    const { compare, val } = df.shouldSave.version
    if (compare === 'equals') {
      return recievedData.version === val
    } else if (compare === 'startsWith') {
      return recievedData.version.startsWith(val)
    }
  }

  if (df.shouldSave.location) {
    const { val } = df.shouldSave.location
    return recievedData.location.includes(val)
  }

  return false
}

export function loadJSON(
  dataFiles: Array<DataFile>,
  dataDir: string
): Array<QuestionDb> {
  return dataFiles.reduce((acc, dataFile, index) => {
    const dataPath = dataDir + dataFile.path

    if (!utils.FileExists(dataPath)) {
      utils.WriteFile(JSON.stringify([]), dataPath)
    }

    try {
      acc.push({
        ...dataFile,
        path: dataPath,
        index: index,
        data: JSON.parse(utils.ReadFile(dataPath)),
      })
    } catch (err) {
      console.error(err)
      logger.Log(
        "data is undefined! Couldn't load data!",
        logger.GetColor('redbg')
      )
    }
    return acc
  }, [])
}

export function backupData(questionDbs: Array<QuestionDb>): void {
  questionDbs.forEach((data) => {
    const path = './publicDirs/qminingPublic/backs/'
    utils.CreatePath(path)
    try {
      logger.Log(`Backing up ${data.name}...`)
      utils.WriteFile(
        JSON.stringify(data.data),
        `${path}${data.name}_${utils.GetDateString(true)}.json`
      )
      logger.Log('Done')
    } catch (err) {
      logger.Log(
        `Error backing up data file ${data.name}!`,
        logger.GetColor('redbg')
      )
      console.error(err)
    }
  })
}