Moved submodules and every stuff into seperate folders neatly #4

src/utils/actions.js

@@ -0,0 +1,178 @@
+/* ----------------------------------------------------------------------------
+Question Server
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+module.exports = {
+  ProcessIncomingRequest: ProcessIncomingRequest,
+  LoadJSON: LoadJSON
+}
+
+const dataFile = './qminingPublic/data.json'
+const recDataFile = './stats/recdata'
+
+const logger = require('../utils/logger.js')
+const idStats = require('../utils/ids.js')
+idStats.Load() // FIXME: dont always load when actions.js is used
+const utils = require('../utils/utils.js')
+const classes = require('./classes.js')
+classes.initLogger(logger.DebugLog)
+// if a recievend question doesnt match at least this % to any other question in the db it gets
+// added to db
+const minMatchAmmountToAdd = 90 // FIXME: test this value
+
+const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
+var currWrites = 0
+
+function ProcessIncomingRequest (recievedData, qdb, infos, dryRun) {
+  return new Promise((resolve, reject) => {
+    logger.DebugLog('Processing incoming request', 'actions', 1)
+    if (recievedData === undefined) {
+      logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
+      reject(new Error('Recieved data is undefined!'))
+    }
+
+    try {
+      let towrite = logger.GetDateString() + '\n'
+      towrite += '------------------------------------------------------------------------------\n'
+      if (typeof recievedData === 'object') {
+        towrite += JSON.stringify(recievedData)
+      } else {
+        towrite += recievedData
+      }
+      towrite += '\n------------------------------------------------------------------------------\n'
+      utils.AppendToFile(towrite, recDataFile)
+      logger.DebugLog('recDataFile written', 'actions', 1)
+    } catch (e) {
+      logger.log('Error writing recieved data.')
+    }
+
+    try {
+    // recievedData: { version: "", id: "", subj: "" quiz: {} }
+      let d = recievedData
+      // FIXME: if is for backwards compatibility, remove this sometime in the future
+      if (typeof d !== 'object') {
+        d = JSON.parse(recievedData)
+      }
+
+      logger.DebugLog('recievedData JSON  parsed', 'actions', 1)
+      logger.DebugLog(d, 'actions', 3)
+      let allQLength = d.quiz.length
+      let allQuestions = []
+
+      d.quiz.forEach((question) => {
+        logger.DebugLog('Question:', 'actions', 2)
+        logger.DebugLog(question, 'actions', 2)
+        let q = new classes.Question(question.Q, question.A, question.data)
+        logger.DebugLog('Searching for question in subj ' + d.subj, 'actions', 3)
+        logger.DebugLog(q, 'actions', 3)
+
+        let sames = qdb.Search(q, d.subj)
+        logger.DebugLog('Same questions:', 'actions', 2)
+        logger.DebugLog('Length: ' + sames.length, 'actions', 2)
+        logger.DebugLog(sames, 'actions', 3)
+        // if it didnt find any question, or every found questions match is lower thatn 80
+        let isNew = sames.length === 0 || sames.every(searchResItem => {
+          return searchResItem.match < minMatchAmmountToAdd
+        })
+        logger.DebugLog('isNew: ' + isNew, 'actions', 2)
+        if (isNew) {
+          allQuestions.push(q)
+        }
+      })
+
+      let color = logger.GetColor('green')
+      let msg = ''
+      if (allQuestions.length > 0) {
+        color = logger.GetColor('blue')
+        msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
+        allQuestions.forEach((q) => {
+          const sName = classes.SUtils.GetSubjNameWithoutYear(d.subj)
+          logger.DebugLog('Adding question with subjName: ' + sName + ' :', 'actions', 3)
+          logger.DebugLog(q, 'actions', 3)
+          qdb.AddQuestion(sName, q)
+        })
+
+        currWrites++
+        logger.DebugLog('currWrites for data.json: ' + currWrites, 'actions', 1)
+        if (currWrites >= writeAfter && !dryRun) {
+          currWrites = 0
+          try {
+            qdb.version = infos.version
+            qdb.motd = infos.motd
+            logger.DebugLog('version and motd set for data.json', 'actions', 3)
+          } catch (e) {
+            logger.Log('MOTD/Version writing/reading error!')
+          }
+          logger.DebugLog('Writing data.json', 'actions', 1)
+          utils.WriteFile(JSON.stringify(qdb), dataFile)
+          logger.Log('\tData file written', color)
+        } else if (dryRun) {
+          logger.Log('\tDry run')
+        }
+      } else {
+        msg += `No new data ( ${allQLength} )`
+      }
+
+      let subjRow = '\t' + d.subj
+      if (d.id) {
+        subjRow += ' ( CID: ' + logger.logHashed(d.id) + ')'
+        idStats.LogId(d.id, d.subj)
+      }
+      logger.Log(subjRow)
+      if (d.version !== undefined) { msg += '. Version: ' + d.version }
+
+      logger.Log('\t' + msg, color)
+      logger.DebugLog('New Questions:', 'actions', 2)
+      logger.DebugLog(allQuestions, 'actions', 2)
+
+      logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
+      resolve(allQLength.length)
+    } catch (e) {
+      console.log(e)
+      logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))
+      reject(new Error('Couldnt parse JSON data'))
+    }
+  })
+}
+
+// loading stuff
+function LoadJSON (dataFile) {
+  try {
+    var d = JSON.parse(utils.ReadFile(dataFile))
+    var r = new classes.QuestionDB()
+    var rt = []
+
+    for (var i = 0; i < d.Subjects.length; i++) {
+      let s = new classes.Subject(d.Subjects[i].Name)
+      var j = 0
+      for (j = 0; j < d.Subjects[i].Questions.length; j++) {
+        var currQ = d.Subjects[i].Questions[j]
+        s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data))
+      }
+      rt.push({
+        name: d.Subjects[i].Name,
+        count: j
+      })
+      r.AddSubject(s)
+    }
+    return r
+  } catch (e) {
+    logger.Log('Error loading sutff', logger.GetColor('redbg'), true)
+    console.log(e)
+  }
+}

src/utils/changedataversion.js

@@ -0,0 +1,31 @@
+const utils = require('../utils/utils.js')
+const dataFile = '../public/data.json'
+const versionFile = '../public/version'
+
+var p = GetParams()
+if (p.length <= 0) {
+  console.log('no params!')
+  process.exit(0)
+}
+
+var param = p.join(' ')
+
+console.log('param: ' + param)
+
+var d = utils.ReadFile(dataFile)
+var parsed = JSON.parse(d)
+
+console.log('Old version:')
+console.log(parsed.version)
+
+parsed.version = param
+
+console.log('New version:')
+console.log(parsed.version)
+
+utils.WriteFile(JSON.stringify(parsed), dataFile)
+utils.WriteFile(parsed.version, versionFile)
+
+function GetParams () {
+  return process.argv.splice(2)
+}

src/utils/classes.js

@@ -0,0 +1,508 @@
+var debugLogger = null
+
+function initLogger (logger) {
+  debugLogger = logger
+}
+
+function debugLog (msg, name, lvl) {
+  if (debugLogger) {
+    debugLogger(msg, name, lvl)
+  }
+}
+
+const commonUselessAnswerParts = [
+  'A helyes válasz az ',
+  'A helyes válasz a ',
+  'A helyes válaszok: ',
+  'A helyes válaszok:',
+  'A helyes válasz: ',
+  'A helyes válasz:',
+  'The correct answer is:',
+  '\''
+]
+const commonUselessStringParts = [
+  ',',
+  '\\.',
+  ':',
+  '!',
+  '\\+',
+  '\\s*\\.'
+]
+const specialChars = [ '&', '\\+' ]
+const lengthDiffMultiplier = 10 /* Percent minus for length difference */
+const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */
+
+const assert = (val) => {
+  if (!val) { throw new Error('Assertion failed') }
+}
+
+class StringUtils {
+  GetSubjNameWithoutYear (subjName) {
+    let t = subjName.split(' - ')
+    if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
+      return t[1] || subjName
+    } else {
+      return subjName
+    }
+  }
+
+  RemoveStuff (value, removableStrings, toReplace) {
+    removableStrings.forEach((x) => {
+      var regex = new RegExp(x, 'g')
+      value = value.replace(regex, toReplace || '')
+    })
+    return value
+  }
+
+  SimplifyQuery (q) {
+    assert(q)
+
+    var result = q.replace(/\n/g, ' ').replace(/\s/g, ' ')
+    return this.RemoveUnnecesarySpaces(result)
+  }
+
+  ShortenString (toShorten, ammount) {
+    assert(toShorten)
+
+    var result = ''
+    var i = 0
+    while (i < toShorten.length && i < ammount) {
+      result += toShorten[i]
+      i++
+    }
+    return result
+  }
+
+  ReplaceCharsWithSpace (val, char) {
+    assert(val)
+    assert(char)
+
+    var toremove = this.NormalizeSpaces(val)
+
+    var regex = new RegExp(char, 'g')
+    toremove = toremove.replace(regex, ' ')
+
+    return this.RemoveUnnecesarySpaces(toremove)
+  }
+
+  // removes whitespace from begining and and, and replaces multiple spaces with one space
+  RemoveUnnecesarySpaces (toremove) {
+    assert(toremove)
+
+    toremove = this.NormalizeSpaces(toremove)
+    while (toremove.includes('  ')) {
+      toremove = toremove.replace(/ {2}/g, ' ')
+    }
+    return toremove.trim()
+  }
+
+  // simplifies a string for easier comparison
+  SimplifyStringForComparison (value) {
+    assert(value)
+
+    value = this.RemoveUnnecesarySpaces(value).toLowerCase()
+    return this.RemoveStuff(value, commonUselessStringParts)
+  }
+
+  RemoveSpecialChars (value) {
+    assert(value)
+
+    return this.RemoveStuff(value, specialChars, ' ')
+  }
+
+  // if the value is empty, or whitespace
+  EmptyOrWhiteSpace (value) {
+    // replaces /n-s with "". then replaces spaces with "". if it equals "", then its empty, or only consists of white space
+    if (value === undefined) { return true }
+    return (value.replace(/\n/g, '').replace(/ /g, '').replace(/\s/g, ' ') === '')
+  }
+
+  // damn nonbreaking space
+  NormalizeSpaces (input) {
+    assert(input)
+
+    return input.replace(/\s/g, ' ')
+  }
+
+  CompareString (s1, s2) {
+    if (!s1 || !s2) {
+      if (!s1 && !s2) {
+        return 100
+      } else {
+        return 0
+      }
+    }
+
+    s1 = this.SimplifyStringForComparison(s1).split(' ')
+    s2 = this.SimplifyStringForComparison(s2).split(' ')
+    var match = 0
+    for (var i = 0; i < s1.length; i++) {
+      if (s2.includes(s1[i])) { match++ }
+    }
+    var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent
+    var lengthDifference = Math.abs(s2.length - s1.length)
+    percent -= lengthDifference * lengthDiffMultiplier
+    if (percent < 0) { percent = 0 }
+    return percent
+  }
+
+  AnswerPreProcessor (value) {
+    assert(value)
+
+    return this.RemoveStuff(
+      value, commonUselessAnswerParts)
+  }
+
+  // 'a. pécsi sör' -> 'pécsi sör'
+  RemoveAnswerLetters (value) {
+    assert(value)
+
+    let s = value.split('. ')
+    if (s[0].length < 2 && s.length > 1) {
+      s.shift()
+      return s.join(' ')
+    } else {
+      return value
+    }
+  }
+
+  SimplifyQA (value, mods) {
+    if (!value) { return }
+
+    const reducer = (res, fn) => {
+      return fn(res)
+    }
+
+    return mods.reduce(reducer, value)
+  }
+
+  SimplifyAnswer (value) {
+    return this.SimplifyQA(
+      value,
+      [
+        this.RemoveSpecialChars.bind(this),
+        this.RemoveUnnecesarySpaces.bind(this),
+        this.AnswerPreProcessor.bind(this),
+        this.RemoveAnswerLetters.bind(this)
+      ])
+  }
+
+  SimplifyQuestion (value) {
+    return this.SimplifyQA(
+      value,
+      [
+        this.RemoveSpecialChars.bind(this),
+        this.RemoveUnnecesarySpaces.bind(this)
+      ])
+  }
+
+  SimplifyStack (stack) {
+    return this.SimplifyQuery(stack)
+  }
+}
+
+const SUtils = new StringUtils()
+
+class Question {
+  constructor (q, a, data) {
+    this.Q = SUtils.SimplifyQuestion(q)
+    this.A = SUtils.SimplifyAnswer(a)
+    this.data = { ...data }
+  }
+
+  toString () {
+    if (this.data.type !== 'simple') {
+      return '?' + this.Q + '\n!' + this.A + '\n>' + JSON.stringify(this.data)
+    } else {
+      return '?' + this.Q + '\n!' + this.A
+    }
+  }
+
+  HasQuestion () {
+    return this.Q !== undefined
+  }
+
+  HasAnswer () {
+    return this.A !== undefined
+  }
+
+  HasImage () {
+    return this.data.type === 'image'
+  }
+
+  IsComplete () {
+    return this.HasQuestion() && this.HasAnswer()
+  }
+
+  CompareImage (data2) {
+    return SUtils.CompareString(this.data.images.join(' '), data2.images.join(' '))
+  }
+
+  // returns -1 if botth is simple
+  CompareData (qObj) {
+    try {
+      if (qObj.data.type === this.data.type) {
+        let dataType = qObj.data.type
+        if (dataType === 'simple') {
+          return -1
+        } else if (dataType === 'image') {
+          return this.CompareImage(qObj.data)
+        } else {
+          debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1)
+          debugLog(qObj, 'Compare question data', 2)
+        }
+      } else {
+        return 0
+      }
+    } catch (e) {
+      debugLog('Error comparing data', 'Compare question data', 1)
+      debugLog(e.message, 'Compare question data', 1)
+      debugLog(e, 'Compare question data', 2)
+    }
+    return 0
+  }
+
+  CompareQuestion (qObj) {
+    return SUtils.CompareString(this.Q, qObj.Q)
+  }
+
+  CompareAnswer (qObj) {
+    return SUtils.CompareString(this.A, qObj.A)
+  }
+
+  Compare (q2, data) {
+    assert(q2)
+    let qObj
+
+    if (typeof q2 === 'string') {
+      qObj = {
+        Q: q2,
+        data: data
+      }
+    } else {
+      qObj = q2
+    }
+
+    const qMatch = this.CompareQuestion(qObj)
+    const aMatch = this.CompareAnswer(qObj)
+    // -1 if botth questions are simple
+    const dMatch = this.CompareData(qObj)
+
+    let avg = -1
+    if (qObj.A) {
+      if (dMatch === -1) {
+        avg = (qMatch + aMatch) / 2
+      } else {
+        avg = (qMatch + aMatch + dMatch) / 3
+      }
+    } else {
+      if (dMatch === -1) {
+        avg = qMatch
+      } else {
+        avg = (qMatch + dMatch) / 2
+      }
+    }
+
+    return {
+      qMatch: qMatch,
+      aMatch: aMatch,
+      dMatch: dMatch,
+      avg: avg
+    }
+  }
+}
+
+class Subject {
+  constructor (n) {
+    assert(n)
+
+    this.Name = n
+    this.Questions = []
+  }
+
+  setIndex (i) {
+    this.index = i
+  }
+
+  getIndex () {
+    return this.index
+  }
+
+  get length () {
+    return this.Questions.length
+  }
+
+  AddQuestion (q) {
+    assert(q)
+
+    this.Questions.push(q)
+  }
+
+  getSubjNameWithoutYear () {
+    return SUtils.GetSubjNameWithoutYear(this.Name)
+  }
+
+  getYear () {
+    let t = this.Name.split(' - ')[0]
+    if (t.match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
+      return t
+    } else {
+      return ''
+    }
+  }
+
+  Search (q, data) {
+    assert(q)
+
+    var r = []
+    for (let i = 0; i < this.length; i++) {
+      let percent = this.Questions[i].Compare(q, data)
+      if (percent.avg > minMatchAmmount) {
+        r.push({
+          q: this.Questions[i],
+          match: percent.avg,
+          detailedMatch: percent
+        })
+      }
+    }
+
+    for (let i = 0; i < r.length; i++) {
+      for (var j = i; j < r.length; j++) {
+        if (r[i].match < r[j].match) {
+          var tmp = r[i]
+          r[i] = r[j]
+          r[j] = tmp
+        }
+      }
+    }
+
+    return r
+  }
+
+  toString () {
+    var r = []
+    for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) }
+    return '+' + this.Name + '\n' + r.join('\n')
+  }
+}
+
+class QuestionDB {
+  constructor () {
+    this.Subjects = []
+  }
+
+  get length () {
+    return this.Subjects.length
+  }
+
+  AddQuestion (subj, q) {
+    debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
+    debugLog(q, 'qdb add', 3)
+    assert(subj)
+
+    var i = 0
+    while (i < this.Subjects.length &&
+      !subj.toLowerCase().includes(this.Subjects[i].getSubjNameWithoutYear().toLowerCase())) {
+      i++
+    }
+
+    if (i < this.Subjects.length) {
+      debugLog('Adding new question to existing subject', 'qdb add', 1)
+      this.Subjects[i].AddQuestion(q)
+    } else {
+      debugLog('Creating new subject for question', 'qdb add', 1)
+      const n = new Subject(subj)
+      n.AddQuestion(q)
+      this.Subjects.push(n)
+    }
+  }
+
+  SimplifyQuestion (q) {
+    if (typeof q === 'string') {
+      return SUtils.SimplifyQuestion(q)
+    } else {
+      q.Q = SUtils.SimplifyQuestion(q.Q)
+      q.A = SUtils.SimplifyQuestion(q.A)
+      return q
+    }
+  }
+
+  Search (q, subjName, data) {
+    assert(q)
+    debugLog('Searching for question', 'qdb search', 1)
+    debugLog('Question:', 'qdb search', 2)
+    debugLog(q, 'qdb search', 2)
+    debugLog(`Subject name: ${subjName}`, 'qdb search', 2)
+    debugLog('Data:', 'qdb search', 2)
+    debugLog(data || q.data, 'qdb search', 2)
+
+    if (!data) {
+      data = q.data || { type: 'simple' }
+    }
+    if (!subjName) {
+      subjName = ''
+      debugLog('No subject name as param!', 'qdb search', 1)
+    }
+    q = this.SimplifyQuestion(q)
+
+    var r = []
+    this.Subjects.forEach((subj) => {
+      if (subjName.toLowerCase().includes(subj.getSubjNameWithoutYear().toLowerCase())) {
+        debugLog(`Searching in ${subj.Name} `, 2)
+        r = r.concat(subj.Search(q, data))
+      }
+    })
+
+    // FIXME: try to remove this? but this is also a good backup plan so idk
+    if (r.length === 0) {
+      debugLog('Reqults length is zero when comparing names, trying all subjects', 'qdb search', 1)
+      this.Subjects.forEach((subj) => {
+        r = r.concat(subj.Search(q, data))
+      })
+      if (r.length > 0) {
+        debugLog(`FIXME: '${subjName}' gave no result but '' did!`, 'qdb search', 1)
+        console.error(`FIXME: '${subjName}' gave no result but '' did!`)
+      }
+    }
+
+    for (let i = 0; i < r.length; i++) {
+      for (var j = i; j < r.length; j++) {
+        if (r[i].match < r[j].match) {
+          var tmp = r[i]
+          r[i] = r[j]
+          r[j] = tmp
+        }
+      }
+    }
+
+    debugLog(`QDB search result length: ${r.length}`, 'qdb search', 1)
+    return r
+  }
+
+  AddSubject (subj) {
+    assert(subj)
+
+    var i = 0
+    while (i < this.length && subj.Name !== this.Subjects[i].Name) { i++ }
+
+    if (i < this.length) {
+      this.Subjects.concat(subj.Questions)
+    } else {
+      this.Subjects.push(subj)
+    }
+  }
+
+  toString () {
+    var r = []
+    for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) }
+    return r.join('\n\n')
+  }
+}
+
+module.exports.StringUtils = StringUtils // TODO: export singleton string utils, remove nea StringUtils from other files
+module.exports.SUtils = SUtils
+module.exports.Question = Question
+module.exports.Subject = Subject
+module.exports.QuestionDB = QuestionDB
+module.exports.minMatchAmmount = minMatchAmmount
+module.exports.initLogger = initLogger

src/utils/dbSetup.js

@@ -0,0 +1,62 @@
+const utils = require('../utils/utils.js')
+const logger = require('../utils/logger.js')
+const dbtools = require('../utils/dbtools.js')
+const dbStructPath = '../modules/api/apiDBStruct.json'
+const usersDBPath = '../data/dbs/users.db'
+const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported
+
+let authDB
+
+console.clear()
+CreateDB()
+
+authDB.close()
+
+function CreateDB () {
+  const dbStruct = utils.ReadJSON(dbStructPath)
+  // authDB = dbtools.GetDB(':memory:')
+  authDB = dbtools.GetDB(usersDBPath)
+  authDB.pragma('synchronous = OFF')
+
+  Object.keys(dbStruct).forEach((tableName) => {
+    const tableData = dbStruct[tableName]
+    dbtools.CreateTable(authDB, tableName, tableData.tableStruct, tableData.foreignKey)
+  })
+
+  try {
+    if (utils.FileExists('./ids')) {
+      const uids = utils.ReadFile('./ids').split('\n')
+
+      uids.forEach((cid, i) => {
+        if (!cid) { return }
+        logger.Log(`[ ${i} / ${uids.length} ]`)
+        try {
+          dbtools.Insert(authDB, 'users', {
+            pw: uuidv4(),
+            oldCID: cid,
+            avaiblePWRequests: 4,
+            created: utils.GetDateString()
+          })
+        } catch (e) {
+          logger.Log('Error during inserting', logger.GetColor('redbg'))
+          console.error(e)
+        }
+      })
+    }
+  } catch (e) {
+    console.error(e)
+  }
+
+  const dir = `./dbSetupResult/${utils.GetDateString().replace(/ /g, '_')}`
+  utils.CreatePath(dir)
+  Object.keys(dbStruct).forEach((key) => {
+    const path = `${dir}/${key}.json`
+    logger.Log(`Writing ${path}...`)
+    utils.WriteFile(JSON.stringify({
+      tableInfo: dbtools.TableInfo(authDB, key),
+      tableRows: dbtools.SelectAll(authDB, key)
+    }), path)
+  })
+
+  logger.Log('Done')
+}

src/utils/dbtools.js

@@ -0,0 +1,224 @@
+// https://www.sqlitetutorial.net/sqlite-nodejs/
+// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
+
+module.exports = {
+  GetDB,
+  AddColumn,
+  TableInfo,
+  Update,
+  Delete,
+  CreateTable,
+  SelectAll,
+  Select,
+  Insert,
+  CloseDB
+}
+
+const Sqlite = require('better-sqlite3')
+const logger = require('../utils/logger.js')
+const utils = require('../utils/utils.js')
+
+const debugLog = process.env.NS_SQL_DEBUG_LOG
+
+// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
+function GetSqlQuerry (conditions, type) {
+  const res = Object.keys(conditions).reduce((acc, key) => {
+    const item = conditions[key]
+    if (typeof item === 'string') {
+      acc.push(`${key} = '${conditions[key]}'`)
+    } else {
+      acc.push(`${key} = ${conditions[key]}`)
+    }
+    return acc
+  }, [])
+  if (type === 'where') {
+    return res.join(' AND ')
+  } else {
+    return res.join(', ')
+  }
+}
+
+// -------------------------------------------------------------------------
+
+function GetDB (path) {
+  utils.CreatePath(path)
+  const res = new Sqlite(path)
+  res.pragma('synchronous = OFF')
+  return res
+}
+
+function DebugLog (msg) {
+  if (debugLog) {
+    logger.DebugLog(msg, 'sql', 0)
+  }
+}
+
+function AddColumn (db, table, col) {
+  try {
+    const colName = Object.keys(col)[0]
+    const colType = col.type
+
+    const s = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
+    const stmt = PrepareStatement(db, s)
+
+    return stmt.run()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function TableInfo (db, table) {
+  try {
+
+  } catch (e) {
+    console.error(e)
+  }
+  const s = `PRAGMA table_info(${table})`
+  const stmt = PrepareStatement(db, s)
+
+  const infoRes = stmt.all()
+
+  const s2 = `SELECT COUNT(*) FROM ${table}`
+  const stmt2 = PrepareStatement(db, s2)
+
+  const countRes = stmt2.get()
+
+  return {
+    columns: infoRes,
+    dataCount: countRes[Object.keys(countRes)[0]]
+  }
+}
+
+function Update (db, table, newData, conditions) {
+  try {
+    const s = `UPDATE ${table} SET ${GetSqlQuerry(newData, 'set')} WHERE ${GetSqlQuerry(conditions, 'where')}`
+    const stmt = PrepareStatement(db, s)
+
+    return stmt.run()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function Delete (db, table, conditions) {
+  try {
+    const s = `DELETE FROM ${table} WHERE ${GetSqlQuerry(conditions, 'where')}`
+    const stmt = PrepareStatement(db, s)
+
+    return stmt.run()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function CreateTable (db, name, columns, foreignKeys) {
+  // CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
+  // number, lastLogin text, lastAccess text
+  //
+  // FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
+
+  try {
+    const cols = Object.keys(columns).reduce((acc, key) => {
+      const item = columns[key]
+      // FIXME: array, and push stuff, then join()
+      const flags = []
+      const toCheck = {
+        primary: 'PRIMARY KEY',
+        notNull: 'NOT NULL',
+        unique: 'UNIQUE',
+        autoIncrement: 'AUTOINCREMENT',
+        defaultZero: 'DEFAULT 0'
+      }
+      Object.keys(toCheck).forEach((key) => {
+        if (item[key]) {
+          flags.push(toCheck[key])
+        }
+      })
+
+      acc.push(`${key} ${item.type} ${flags.join(' ')}`)
+      return acc
+    }, []).join(', ')
+
+    let fKeys = []
+    if (foreignKeys) {
+      foreignKeys.forEach((f) => {
+        const { keysFrom, table, keysTo } = f
+        fKeys.push(`, FOREIGN KEY(${keysFrom.join(', ')}) REFERENCES ${table}(${keysTo.join(', ')})`)
+      })
+    }
+
+    // IF NOT EXISTS
+    const s = `CREATE TABLE ${name}(${cols}${fKeys.join(', ')})`
+    const stmt = PrepareStatement(db, s)
+    return stmt.run()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function SelectAll (db, from) {
+  try {
+    const s = `SELECT * from ${from}`
+
+    const stmt = PrepareStatement(db, s)
+    return stmt.all()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function Select (db, from, conditions) {
+  try {
+    const s = `SELECT * from ${from} WHERE ${GetSqlQuerry(conditions, 'where')}`
+
+    const stmt = PrepareStatement(db, s)
+    return stmt.all()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function Insert (db, table, data) {
+  try {
+    const cols = Object.keys(data).reduce((acc, key) => {
+      acc.push(`${key}`)
+      return acc
+    }, []).join(', ')
+
+    const values = Object.keys(data).reduce((acc, key) => {
+      const item = data[key]
+      if (typeof item === 'string') {
+        acc.push(`'${item}'`)
+      } else {
+        acc.push(`${item}`)
+      }
+      return acc
+    }, []).join(', ')
+
+    const s = `INSERT INTO ${table} (${cols}) VALUES (${values})`
+    const stmt = PrepareStatement(db, s)
+
+    return stmt.run()
+  } catch (e) {
+    console.error(e)
+  }
+}
+
+function CloseDB (db) {
+  db.close((err) => {
+    if (err) {
+      return console.error(err.message)
+    }
+    DebugLog('Close the database connection.')
+  })
+}
+
+// -------------------------------------------------------------------------
+
+function PrepareStatement (db, s) {
+  if (!db) {
+    throw new Error('DB is undefined in prepare statement! DB action called with undefined db')
+  }
+  DebugLog(s)
+  return db.prepare(s)
+}

src/utils/ids.js

@@ -0,0 +1,112 @@
+/* ----------------------------------------------------------------------------
+
+ Question Server
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+module.exports = {
+  LogId: LogId,
+  Load: Load
+}
+
+const utils = require('../utils/utils.js')
+const logger = require('../utils/logger.js')
+const idStatFile = 'stats/idstats'
+const idVStatFile = 'stats/idvstats'
+
+const writeInterval = 1
+
+let data = {}
+let vData = {}
+let writes = 0
+
+function Load () {
+  try {
+    var prevData = utils.ReadFile(idStatFile)
+    data = JSON.parse(prevData)
+  } catch (e) {
+    logger.Log('Error at loading id logs! (@ first run its normal)', logger.GetColor('redbg'))
+    console.log(e)
+  }
+
+  try {
+    var prevVData = utils.ReadFile(idVStatFile)
+    vData = JSON.parse(prevVData)
+  } catch (e) {
+    logger.Log('Error at loading id logs! (@ first run its normal)', logger.GetColor('redbg'))
+    console.log(e)
+  }
+}
+
+function LogId (id, subj) {
+  Inc(id, subj)
+  AddVisitStat(id, subj)
+  Save()
+}
+
+function AddSubjToList (list, subj) {
+  if (!list[subj]) {
+    list[subj] = 0
+  }
+  list[subj]++
+}
+
+function Inc (value, subj) {
+  if (data[value] === undefined) {
+    data[value] = {
+      count: 0,
+      subjs: {}
+    }
+  }
+  data[value].count++
+  AddSubjToList(data[value].subjs, subj)
+}
+
+function AddVisitStat (name, subj) {
+  var m = new Date()
+  const now = m.getFullYear() + '/' + ('0' + (m.getMonth() + 1)).slice(-2) + '/' + ('0' + m.getDate()).slice(-2)
+  if (vData[now] === undefined) { vData[now] = {} }
+  if (vData[now][name] === undefined) {
+    vData[now][name] = {
+      count: 0,
+      subjs: {}
+    }
+  }
+  vData[now][name].count++
+  AddSubjToList(vData[now][name].subjs, subj)
+}
+
+function Save () {
+  writes++
+  if (writes === writeInterval) {
+    try {
+      utils.WriteFile(JSON.stringify(data), idStatFile)
+      // Log("Stats wrote.");
+    } catch (e) {
+      logger.Log('Error at writing logs!', logger.GetColor('redbg'))
+      console.log(e)
+    }
+    try {
+      utils.WriteFile(JSON.stringify(vData), idVStatFile)
+      // Log("Stats wrote.");
+    } catch (e) {
+      logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
+      console.log(e)
+    }
+    writes = 0
+  }
+}

src/utils/logger.js

@@ -0,0 +1,329 @@
+/* ----------------------------------------------------------------------------
+
+ Question Server
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+const hr = '---------------------------------------------------------------------------------'
+
+module.exports = {
+  GetDateString: GetDateString,
+  Log: Log,
+  DebugLog: DebugLog,
+  GetColor: GetColor,
+  LogReq: LogReq,
+  LogStat: LogStat,
+  Load: Load,
+  logHashed: logHashed,
+  hr: hr,
+  C: C
+}
+
+const DELIM = C('green') + '|' + C()
+
+const utils = require('../utils/utils.js')
+const locLogFile = './stats/logs'
+const logFile = '/nlogs/nlogs'
+const allLogFile = '/nlogs/log'
+const statFile = 'stats/stats'
+const vStatFile = 'stats/vstats'
+const uStatsFile = 'stats/ustats'
+const uvStatsFile = 'stats/uvstats'
+const nologFile = './nolog'
+
+const colors = [
+  'green',
+  'red',
+  'yellow',
+  'blue',
+  'magenta',
+  'cyan'
+]
+
+const writeInterval = 10
+const debugLevel = parseInt(process.env.NS_LOGLEVEL) || 0
+Log('Loglevel is: ' + debugLevel)
+
+let data = {} // visit data
+let vData = {} // visit data, but daily
+let uData = {} // visit data, but per user
+let uvData = {} // visit data, but per user and daily
+let writes = 0
+
+let noLogips = []
+
+function GetDateString () {
+  const m = new Date()
+  const d = utils.GetDateString()
+  return GetRandomColor(m.getHours().toString()) + d + C()
+}
+
+function DebugLog (msg, name, lvl) {
+  if (lvl <= debugLevel) {
+    if (msg === 'hr') {
+      msg = hr
+    }
+    let s = msg
+    let header = `${C('red')}#DEBUG${lvl}#${C('yellow')}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
+    if (typeof msg !== 'object') {
+      s = header + msg
+    } else {
+      Log(header + 'OBJECT:', 'yellow')
+      s = msg
+    }
+    Log(s, 'yellow')
+  }
+}
+
+function Log (s, c) {
+  let log = s
+  if (typeof s !== 'object') {
+    let dl = DELIM + C(c)
+    log = C(c) + GetDateString() + dl + s + C()
+  }
+
+  console.log(log)
+  utils.AppendToFile(log, logFile)
+}
+
+function LogReq (req, toFile, sc) {
+  try {
+    let ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
+
+    let nolog = noLogips.some((x) => {
+      return ip.includes(x)
+    })
+    if (nolog) {
+      return
+    }
+
+    let logEntry = GetRandomColor(ip) + ip + C()
+    let dl = DELIM
+    if (req.url.includes('lred')) {
+      dl += C('red')
+    }
+
+    let hostname
+    if (req.hostname) {
+      hostname = req.hostname.replace('www.', '').split('.')[0]
+    } else {
+      hostname = 'NOHOST'
+      Log('req.hostname is undefined! req.hostname: ' + req.hostname, GetColor('redbg'))
+    }
+    logEntry += dl +
+      hostname + dl +
+      req.headers['user-agent'] + dl +
+      req.method + dl
+
+    if (req.session && req.session.user) {
+      logEntry += C('cyan') + req.session.user.id + C() + dl
+    } else if (req.session && req.session.isException === true) {
+      logEntry += C('cyan') + 'EX' + C() + dl
+    } else {
+      logEntry += C('red') + 'NOUSER' + C() + dl
+    }
+
+    logEntry += GetRandomColor(req.url.split('?')[0]) + req.url
+
+    if (sc !== undefined) { logEntry += dl + sc }
+
+    logEntry += C()
+    if (!toFile) {
+      Log(logEntry)
+    } else {
+      let defLogs = GetDateString() + dl + logEntry
+
+      utils.AppendToFile(defLogs, locLogFile)
+      utils.AppendToFile(defLogs, allLogFile)
+    }
+  } catch (e) {
+    console.log(e)
+    Log('Error at logging lol', GetColor('redbg'), true)
+  }
+}
+
+function parseNoLogFile (newData) {
+  noLogips = newData.split('\n')
+  if (noLogips[noLogips.length - 1] === '') {
+    noLogips.pop()
+  }
+  noLogips = noLogips.filter((x) => {
+    return x !== ''
+  })
+  Log('\tNo Log IP-s changed: ' + noLogips.join(', '))
+}
+
+function setNoLogReadInterval () {
+  utils.WatchFile(nologFile, (newData) => {
+    parseNoLogFile(newData)
+  })
+
+  parseNoLogFile(utils.ReadFile(nologFile))
+}
+
+function Load () {
+  Log('Loading logger...')
+  try {
+    uData = JSON.parse(utils.ReadFile(uStatsFile))
+  } catch (e) {
+    Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
+    console.log(e)
+  }
+
+  try {
+    uvData = JSON.parse(utils.ReadFile(uvStatsFile))
+  } catch (e) {
+    Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
+    console.log(e)
+  }
+
+  try {
+    var prevData = utils.ReadFile(statFile)
+    data = JSON.parse(prevData)
+  } catch (e) {
+    Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
+    console.log(e)
+  }
+
+  try {
+    var prevVData = utils.ReadFile(vStatFile)
+    vData = JSON.parse(prevVData)
+  } catch (e) {
+    Log('Error at loading visit logs! (@ first run its normal)', GetColor('redbg'))
+    console.log(e)
+  }
+  setNoLogReadInterval()
+}
+
+function LogStat (url, ip, hostname, userId) {
+  let nolog = noLogips.some((x) => {
+    return x.includes(ip)
+  })
+  if (nolog) {
+    return
+  }
+
+  url = hostname + url.split('?')[0]
+  Inc(url)
+  AddUserIdStat(userId)
+  IncUserStat(userId)
+  AddVisitStat(url)
+  Save()
+}
+
+function IncUserStat (userId) {
+  try {
+    if (uData[userId] === undefined) { uData[userId] = 0 }
+    uData[userId]++
+  } catch (e) {
+    Log('Error at making user ID stats!', GetColor('redbg'))
+    console.error(e)
+  }
+}
+
+function AddUserIdStat (userId) {
+  try {
+    var m = new Date()
+    const now = m.getFullYear() + '/' + ('0' + (m.getMonth() + 1)).slice(-2) + '/' + ('0' + m.getDate()).slice(-2)
+    if (uvData[now] === undefined) { uvData[now] = {} }
+    if (uvData[now][userId] === undefined) { uvData[now][userId] = 0 }
+    uvData[now][userId]++
+  } catch (e) {
+    Log('Error at making user ID stats!', GetColor('redbg'))
+    console.error(e)
+  }
+}
+
+function Inc (value) {
+  if (value.startsWith('/?')) { value = '/' }
+  if (data[value] === undefined) { data[value] = 0 }
+  data[value]++
+}
+
+function AddVisitStat (name) {
+  var m = new Date()
+  const now = m.getFullYear() + '/' + ('0' + (m.getMonth() + 1)).slice(-2) + '/' + ('0' + m.getDate()).slice(-2)
+  if (vData[now] === undefined) { vData[now] = {} }
+  if (vData[now][name] === undefined) { vData[now][name] = 0 }
+  vData[now][name]++
+}
+
+function Save () {
+  writes++
+  if (writes === writeInterval) {
+    try {
+      utils.WriteFile(JSON.stringify(uData), uStatsFile)
+    } catch (e) {
+      Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
+      console.error(e)
+    }
+    try {
+      utils.WriteFile(JSON.stringify(uvData), uvStatsFile)
+    } catch (e) {
+      Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
+      console.error(e)
+    }
+    try {
+      utils.WriteFile(JSON.stringify(data), statFile)
+      // Log("Stats wrote.");
+    } catch (e) {
+      Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
+      console.error(e)
+    }
+    try {
+      utils.WriteFile(JSON.stringify(vData), vStatFile)
+      // Log("Stats wrote.");
+    } catch (e) {
+      Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
+      console.error(e)
+    }
+    writes = 0
+  }
+}
+
+function logHashed (x) {
+  return GetRandomColor(x.toString()) + x + C()
+}
+
+function GetRandomColor (ip) {
+  if (!ip) {
+    return 'red'
+  }
+
+  let res = ip.split('').reduce((res, x) => {
+    return res + x.charCodeAt(0)
+  }, 0)
+  return C(colors[res % colors.length])
+}
+
+function GetColor (c) {
+  return c
+}
+
+function C (c) {
+  if (c !== undefined) { c = c.toLowerCase() }
+
+  if (c === 'redbg') { return '\x1b[41m' }
+  if (c === 'bluebg') { return '\x1b[44m' }
+  if (c === 'green') { return '\x1b[32m' }
+  if (c === 'red') { return '\x1b[31m' }
+  if (c === 'yellow') { return '\x1b[33m' }
+  if (c === 'blue') { return '\x1b[34m' }
+  if (c === 'magenta') { return '\x1b[35m' }
+  if (c === 'cyan') { return '\x1b[36m' }
+  return '\x1b[0m'
+}

src/utils/merge.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+p=$(echo $PWD | rev | cut -d'/' -f2- | rev)
+
+cp -v $p/public/data.json /tmp/data.json
+node $p/utils/rmDuplicates.js /tmp/data.json 2> /dev/null
+
+mkdir -p "$p/public/backs/"
+mv -v $p/public/data.json "$p/public/backs/data.json $(date)"
+mv -v $p/utils/res.json $p/public/data.json
+
+echo Done

src/utils/motd.js

@@ -0,0 +1,22 @@
+const utils = require('../utils/utils.js')
+const dataFile = '../public/data.json'
+const motdFile = '../public/motd'
+
+var p = GetParams()
+if (p.length <= 0) {
+  console.log('no params!')
+  process.exit(0)
+}
+
+var param = p.join(' ')
+console.log('param: ' + param)
+var d = utils.ReadFile(dataFile)
+var parsed = JSON.parse(d)
+parsed.motd = param
+utils.WriteFile(JSON.stringify(parsed), dataFile)
+
+utils.WriteFile(parsed.motd, motdFile)
+
+function GetParams () {
+  return process.argv.splice(2)
+}

src/utils/readme.md

@@ -0,0 +1,86 @@
+# Contents
+
+Utils mappa tartalom
+
+## merger.js
+
+Törölve lesz
+
+## merge.sh
+
+Törölve lesz
+
+## rmDuplicates.js
+
+Tervezve
+
+Kitörli a két vagy többször szereplő kérdéseket
+
+Params:
+
+1. Elérési út a JSON adatbázishoz
+
+## actions.js
+
+Át lesz nevezve: `questionProcessor.js`
+
+JSON adatbázist betölti, és az új bejövő kérdéseket hozzáadja
+
+Exportált funkciók:
+
+1. ProcessIncomingRequest
+2. LoadJSON
+
+## utils.js
+
+Általános eszközök
+
+Exportált funkciók:
+
+1. ReadFile
+2. WriteFile
+3. writeFileAsync
+4. AppendToFile
+5. Beep
+6. WriteBackup
+7. FileExists
+8. CreatePath
+9. WatchFile
+10. ReadDir
+
+## logger.js
+
+Logolást kezeli
+
+Exportált funkciók:
+
+1. GetDateString
+2. Log
+3. DebugLog
+4. GetColor
+5. LogReq
+6. LogStat
+7. Load
+8. logHashed
+9. hr
+
+## motd.js
+
+Az adatbázis MOTD-ét változtatja meg. Elvileg már nem kell
+
+`node motd.js "new motd"`
+
+## changedataversion.js
+
+Az adatbázis jelenlegi legfrissebb kliens verzióját változtatja meg. Elvileg már nem kell
+
+`node motd.js "new version"`
+
+## ids.js
+
+`./stats/idstats` és `./stats/idvstats` ba írja a egyedi kliens ID statisztikákat
+
+Exportált funkciók:
+
+1. LogId
+2. Load

src/utils/rmDuplicates.js

@@ -0,0 +1,260 @@
+/* ----------------------------------------------------------------------------
+
+ Question Server question file merger
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+const utils = require('./utils.js')
+const classes = require('./classes.js')
+const actions = require('./actions.js')
+const logger = require('./logger.js')
+
+const resultFileName = 'res.json'
+const minMatchAmmount = 100
+
+const logPath = './mergeLogs/mergelog_' + GetDateString().replace(/ /g, '_')
+
+Main()
+
+function Main () {
+  const params = GetParams()
+  console.log(params)
+  if (params.length === 0) {
+    console.error('No params! Need a path to a question database!')
+    process.exit()
+  }
+  const data = actions.LoadJSON(params[0])
+
+  PrintDB(data)
+  console.log(hr('='))
+
+  const { res, stats } = RemoveDuplicates(data)
+  console.log(hr('='))
+
+  LogStats(stats, data, res)
+  console.log(hr('='))
+
+  console.log('Result database:')
+  PrintDB(res)
+  console.log(hr('='))
+
+  utils.WriteFile(JSON.stringify(res), resultFileName)
+  console.log(C('green') + resultFileName + ' written!' + C())
+  console.log(hr('='))
+
+  console.log(C('green') + 'Done' + C())
+}
+
+function LogStats (stats, oldData, newData) {
+  const maxSubjNameLength = MaxLengthOf(stats, 'name')
+  const maxPrevLength = MaxLengthOf(stats, 'prevQuestions')
+  const maxAddedLength = MaxLengthOf(stats, 'addedQuestions')
+  const maxRemovedLength = MaxLengthOf(stats, 'removedQuestions')
+
+  stats.forEach((currStat) => {
+    const { name, prevQuestions, addedQuestions, removedQuestions } = currStat
+    let toLog = ''
+
+    toLog += C('green')
+    toLog += GetExactLength(name, maxSubjNameLength)
+    toLog += C()
+    toLog += ' '
+    toLog += C('magenta')
+    toLog += GetExactLength(prevQuestions, maxPrevLength)
+    toLog += C()
+    toLog += C('cyan')
+    toLog += ' -> '
+    toLog += C()
+    toLog += C('green')
+    toLog += GetExactLength(addedQuestions, maxAddedLength)
+    toLog += C()
+    toLog += ' [ '
+    toLog += C('red')
+    toLog += GetExactLength(removedQuestions, maxRemovedLength)
+    toLog += C()
+    toLog += ' ]'
+
+    console.log(toLog)
+  })
+  console.log(hr())
+  console.log('Old data:')
+  LogDataCount(oldData)
+  console.log('New data:')
+  LogDataCount(newData)
+}
+
+function LogDataCount (data) {
+  const subjLength = data.Subjects.length
+  const qLength = data.Subjects.reduce((acc, subj) => {
+    return acc + subj.Questions.length
+  }, 0)
+
+  console.log('Subjects: ' + C('green') + subjLength + C() + ', Questions: ' + C('green') + qLength + C())
+}
+
+function PrintDB (data) {
+  const maxSubjNameLength = MaxLengthOf(data.Subjects, 'Name')
+
+  data.Subjects.forEach((subj) => {
+    let toLog = ''
+    toLog += C('green')
+    toLog += GetExactLength(subj.Name, maxSubjNameLength)
+    toLog += C()
+    toLog += ' [ '
+    toLog += C('cyan')
+    toLog += subj.Questions.length
+    toLog += C()
+    toLog += ' ]'
+
+    console.log(toLog)
+  })
+  console.log(hr())
+  LogDataCount(data)
+  console.log(hr())
+}
+
+function GetExactLength (s, length) {
+  let toLog = s.toString()
+  const lengthDiff = length - toLog.length
+  for (let i = 0; i < lengthDiff; i++) {
+    toLog += ' '
+  }
+
+  return toLog
+}
+
+function MaxLengthOf (prop, key) {
+  return prop.reduce((acc, currStat) => {
+    if (acc < currStat[key].toString().length) {
+      acc = currStat[key].toString().length
+    }
+    return acc
+  }, 0)
+}
+
+function RemoveDuplicates (data) {
+  console.log(C('yellow') + 'Removing duplicates' + C())
+  const res = new classes.QuestionDB()
+  const stats = []
+
+  data.Subjects.forEach((subj, i) => {
+    const logFile = logPath + '/' + subj.Name.replace(/ /g, '_').replace(/\//g, '-')
+    LogSubjProgress(i, subj, data.Subjects.length)
+    let addedQuestions = 0
+    let removedQuestions = 0
+    subj.Questions.forEach((question) => {
+      // Searching for same question in result database
+      let r = res.Search(question).reduce((acc, r) => {
+        if (r.match >= minMatchAmmount) {
+          acc.push(r)
+        }
+        return acc
+      }, [])
+
+      // if htere are more that one same questions in the new database
+      if (r.length > 0) {
+        utils.AppendToFile(hr('#'), logFile)
+        utils.AppendToFile('QUESTION', logFile)
+        utils.AppendToFile(JSON.stringify(question, null, 2), logFile)
+        utils.AppendToFile(hr(), logFile)
+        utils.AppendToFile('SAMES', logFile)
+        utils.AppendToFile(JSON.stringify(r, null, 2), logFile)
+        removedQuestions++
+      } else {
+        // if no same questions are fount then adding it to then new db
+        res.AddQuestion(subj.getSubjNameWithoutYear(), question)
+        addedQuestions++
+      }
+    })
+    LogResultProgress(subj, addedQuestions, removedQuestions)
+    stats.push({
+      name: subj.Name,
+      prevQuestions: subj.Questions.length,
+      addedQuestions: addedQuestions,
+      removedQuestions: removedQuestions
+    })
+  })
+  return { res, stats }
+}
+
+function LogSubjProgress (i, subj, subjCount) {
+  log(
+    '[ ' +
+    C('cyan') +
+    (i + 1) +
+    C() +
+    ' / ' +
+    C('green') +
+    subjCount +
+    C() +
+    ' ] ' +
+    C('yellow') +
+    subj.Name +
+    C() +
+    ': ' +
+    C('green') +
+    subj.Questions.length
+  )
+}
+
+function LogResultProgress (subj, addedQuestions, removedQuestions) {
+  log(
+    ' ' +
+    C('cyan') +
+    '->  ' +
+    C('green') +
+    addedQuestions +
+    C() +
+    ', removed: ' +
+    C('red') +
+    removedQuestions +
+    C() +
+    '\n'
+  )
+}
+
+function log (msg) {
+  process.stdout.write(msg)
+}
+
+function hr (char) {
+  let h = ''
+  const cols = process.stdout.columns || 20
+  for (let i = 0; i < cols; i++) {
+    h += char || '-'
+  }
+  return h
+}
+
+function C (color) {
+  return logger.C(color)
+}
+
+function GetParams () {
+  return process.argv.splice(2)
+}
+
+function GetDateString () {
+  const m = new Date()
+  const d = m.getFullYear() + '-' +
+    ('0' + (m.getMonth() + 1)).slice(-2) + '-' +
+    ('0' + m.getDate()).slice(-2) + ' ' +
+    ('0' + m.getHours()).slice(-2) + ':' +
+    ('0' + m.getMinutes()).slice(-2) + ':' +
+    ('0' + m.getSeconds()).slice(-2)
+  return d
+}

src/utils/runSqliteCmds.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ "$#" -lt "2" ]; then
+  echo "No params! 2 file required: db, commands file"
+  echo "usage: ./runSqliteCmds db.db commands"
+  exit 1
+fi
+
+echo "Executing:"
+cat $2
+echo
+
+cmd=''
+
+while read p; do
+  cmd="$cmd -cmd \"${p}\" -cmd \".shell echo\""
+done <"$2"
+
+echo "sqlite3 -bail $1 $cmd"
+eval "sqlite3 -bail $1 $cmd" > cmdRes 2> /dev/null
+
+echo "Done, result written to cmdRes file!"

src/utils/sqliteBatchCommands/showAll

@@ -0,0 +1,9 @@
+.mode column
+.headers ON
+select * from users 
+select * from sessions
+select * from veteranPWRequests
+select * from accesses
+.tables
+.bail
+select * from EXIT

src/utils/utils.js

@@ -0,0 +1,138 @@
+module.exports = {
+  ReadFile: ReadFile,
+  ReadJSON: ReadJSON,
+  WriteFile: WriteFile,
+  writeFileAsync: WriteFileAsync,
+  AppendToFile: AppendToFile,
+  Beep: Beep,
+  WriteBackup: WriteBackup,
+  FileExists: FileExists,
+  CreatePath: CreatePath,
+  WatchFile: WatchFile,
+  ReadDir: ReadDir,
+  CopyFile: CopyFile,
+  GetDateString: GetDateString
+}
+
+var fs = require('fs')
+
+var logger = require('../utils/logger.js')
+
+const dataFile = './qminingPublic/data.json'
+
+function GetDateString () {
+  const m = new Date()
+  return m.getFullYear() + '-' +
+    ('0' + (m.getMonth() + 1)).slice(-2) + '-' +
+    ('0' + m.getDate()).slice(-2) + ' ' +
+    ('0' + m.getHours()).slice(-2) + ':' +
+    ('0' + m.getMinutes()).slice(-2) + ':' +
+    ('0' + m.getSeconds()).slice(-2)
+}
+
+function CopyFile (from, to) {
+  CreatePath(to)
+  fs.copyFileSync(from, to)
+}
+
+function ReadDir (path) {
+  return fs.readdirSync(path)
+}
+
+function ReadJSON (name) {
+  try {
+    return JSON.parse(ReadFile(name))
+  } catch (e) {
+    console.log(e)
+    throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
+  }
+}
+
+function ReadFile (name) {
+  if (!FileExists(name)) { throw new Error('No such file: ' + name) }
+  return fs.readFileSync(name, 'utf8')
+}
+
+function FileExists (path) {
+  return fs.existsSync(path)
+}
+
+function WatchFile (file, callback) {
+  if (FileExists(file)) {
+    fs.watchFile(file, (curr, prev) => {
+      fs.readFile(file, 'utf8', (err, data) => {
+        if (err) {
+          // console.log(err)
+        } else {
+          callback(data)
+        }
+      })
+    })
+  } else {
+    console.log(file + ' does not eadjsalék')
+    setTimeout(() => {
+      WatchFile(file)
+    }, 1000)
+  }
+}
+
+function CreatePath (path, onlyPath) {
+  if (FileExists(path)) { return }
+
+  var p = path.split('/')
+  var currDir = p[0]
+  for (var i = 1; i < p.length; i++) {
+    if (currDir !== '' && !fs.existsSync(currDir)) {
+      try {
+        fs.mkdirSync(currDir)
+      } catch (e) { console.log('Failed to make ' + currDir + ' directory... ') }
+    }
+    currDir += '/' + p[i]
+  }
+  if (onlyPath === undefined || onlyPath === false) {
+  } else {
+    fs.mkdirSync(path)
+  }
+}
+
+function WriteFile (content, path) {
+  CreatePath(path)
+  fs.writeFileSync(path, content)
+}
+
+function WriteFileAsync (content, path) {
+  CreatePath(path)
+  fs.writeFile(path, content, function (err) {
+    if (err) {
+      logger.Log('Error writing file: ' + path + ' (sync)', logger.GetColor('redbg'))
+    }
+  })
+}
+
+function AppendToFile (data, file) {
+  CreatePath(file)
+  try {
+    fs.appendFileSync(file, '\n' + data)
+  } catch (e) {
+    logger.Log('Error appendig to file log file: ' + file + ' (sync)', logger.GetColor('redbg'))
+    logger.Log(data)
+    console.log(e)
+  }
+}
+
+function Beep () {
+  try {
+    process.stdout.write('\x07')
+  } catch (e) {
+    console.log('error beepin')
+  }
+}
+
+function WriteBackup () {
+  try {
+    WriteFileAsync(ReadFile(dataFile), 'public/backs/data_' + new Date().toString())
+  } catch (e) {
+    logger.Log('Error backing up data json file!', logger.GetColor('redbg'))
+    console.log(e)
+  }
+}