Merge branch 'devel'

utils/actions.js

@@ -54,8 +54,11 @@ function ProcessIncomingRequest (data) {
   try {
     var d = JSON.parse(data)
     var allQuestions = []
+
+    d = ConvertToNewFormat(d)
+
     for (let i = 0; i < d.allData.length; i++) {
-      allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].I))
+      allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data))
     }
 
     let color = logger.GetColor('green')
@@ -69,7 +72,7 @@ function ProcessIncomingRequest (data) {
     if (d.data.length > 0) {
       let qdb = LoadJSON(utils.ReadFile(dataFile))
       d.data.forEach((x) => {
-        let q = new classes.Question(x.Q, x.A, x.I)
+        let q = new classes.Question(x.Q, x.A, x.data)
         questions.push(q)
         qdb.AddQuestion(d.subj, q)
       })
@@ -116,7 +119,7 @@ function LoadJSON (resource) {
       var j = 0
       for (j = 0; j < d.Subjects[i].Questions.length; j++) {
         var currQ = d.Subjects[i].Questions[j]
-        s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I))
+        s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data))
       }
       rt.push({
         name: d.Subjects[i].Name,
@@ -152,3 +155,40 @@ function ProcessQA () {
 
   return r
 }
+
+function ConvertToNewFormat (d) {
+  if (d.allData.length < 0) {
+    return d
+  }
+
+  try {
+    if (!d.allData[0].data) {
+      logger.Log('\tConverting old data to new...')
+      d.allData = d.allData.map((x) => {
+        if (x.I) {
+          x.data = {
+            type: 'image',
+            images: JSON.parse(x.I)
+          }
+          delete x.I
+        }
+        return x
+      })
+      d.data = d.data.map((x) => {
+        if (x.I) {
+          x.data = {
+            type: 'image',
+            images: JSON.parse(x.I)
+          }
+          delete x.I
+        }
+        return x
+      })
+    }
+  } catch (e) {
+    console.log(e)
+    logger.Log('Couldnt convert old data to new!', logger.GetColor('redbg'))
+  }
+
+  return d
+}

utils/dataUpdater.js

@@ -0,0 +1,637 @@
+/* ----------------------------------------------------------------------------
+
+ Question Server question file merger
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+const commonUselessAnswerParts = [
+  'A helyes válasz az ',
+  'A helyes válasz a ',
+  'A helyes válaszok: ',
+  'A helyes válaszok:',
+  'A helyes válasz: ',
+  'A helyes válasz:',
+  'The correct answer is:',
+  '\''
+]
+const commonUselessStringParts = [',', '\\.', ':', '!', '\\+']
+const specialChars = [ '&', '\\+' ]
+const lengthDiffMultiplier = 10 /* Percent minus for length difference */
+const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */
+
+const assert = (val) => {
+  if (!val) { throw new Error('Assertion failed') }
+}
+
+class StringUtils {
+  RemoveStuff (value, removableStrings, toReplace) {
+    removableStrings.forEach((x) => {
+      var regex = new RegExp(x, 'g')
+      value = value.replace(regex, toReplace || '')
+    })
+    return value
+  }
+
+  SimplifyQuery (q) {
+    assert(q)
+
+    var result = q.replace(/\n/g, ' ').replace(/\s/g, ' ')
+    return this.RemoveUnnecesarySpaces(result)
+  }
+
+  ShortenString (toShorten, ammount) {
+    assert(toShorten)
+
+    var result = ''
+    var i = 0
+    while (i < toShorten.length && i < ammount) {
+      result += toShorten[i]
+      i++
+    }
+    return result
+  }
+
+  ReplaceCharsWithSpace (val, char) {
+    assert(val)
+    assert(char)
+
+    var toremove = this.NormalizeSpaces(val)
+
+    var regex = new RegExp(char, 'g')
+    toremove = toremove.replace(regex, ' ')
+
+    return this.RemoveUnnecesarySpaces(toremove)
+  }
+
+  // removes whitespace from begining and and, and replaces multiple spaces with one space
+  RemoveUnnecesarySpaces (toremove) {
+    assert(toremove)
+
+    toremove = this.NormalizeSpaces(toremove)
+    while (toremove.includes('  ')) {
+      toremove = toremove.replace(/ {2}/g, ' ')
+    }
+    return toremove.trim()
+  }
+
+  // simplifies a string for easier comparison
+  SimplifyStringForComparison (value) {
+    assert(value)
+
+    value = this.RemoveUnnecesarySpaces(value).toLowerCase()
+    return this.RemoveStuff(value, commonUselessStringParts)
+  }
+
+  RemoveSpecialChars (value) {
+    assert(value)
+
+    return this.RemoveStuff(value, specialChars, ' ')
+  }
+
+  // if the value is empty, or whitespace
+  EmptyOrWhiteSpace (value) {
+    // replaces /n-s with "". then replaces spaces with "". if it equals "", then its empty, or only consists of white space
+    if (value === undefined) { return true }
+    return (value.replace(/\n/g, '').replace(/ /g, '').replace(/\s/g, ' ') === '')
+  }
+
+  // damn nonbreaking space
+  NormalizeSpaces (input) {
+    assert(input)
+
+    return input.replace(/\s/g, ' ')
+  }
+
+  CompareString (s1, s2) {
+    if (!s1 || !s2) {
+      return 0
+    }
+
+    s1 = this.SimplifyStringForComparison(s1).split(' ')
+    s2 = this.SimplifyStringForComparison(s2).split(' ')
+    var match = 0
+    for (var i = 0; i < s1.length; i++) {
+      if (s2.includes(s1[i])) { match++ }
+    }
+    var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent
+    var lengthDifference = Math.abs(s2.length - s1.length)
+    percent -= lengthDifference * lengthDiffMultiplier
+    if (percent < 0) { percent = 0 }
+    return percent
+  }
+
+  AnswerPreProcessor (value) {
+    assert(value)
+
+    return this.RemoveStuff(
+      value, commonUselessAnswerParts)
+  }
+
+  // 'a. pécsi sör' -> 'pécsi sör'
+  RemoveAnswerLetters (value) {
+    assert(value)
+
+    let s = value.split('. ')
+    if (s[0].length < 2 && s.length > 1) {
+      s.shift()
+      return s.join(' ')
+    } else {
+      return value
+    }
+  }
+
+  SimplifyQA (value, mods) {
+    if (!value) { return }
+
+    const reducer = (res, fn) => {
+      return fn(res)
+    }
+
+    return mods.reduce(reducer, value)
+  }
+
+  SimplifyAnswer (value) {
+    return this.SimplifyQA(
+      value,
+      [
+        this.RemoveSpecialChars.bind(this),
+        this.RemoveUnnecesarySpaces.bind(this),
+        this.AnswerPreProcessor.bind(this),
+        this.RemoveAnswerLetters.bind(this)
+      ])
+  }
+
+  SimplifyQuestion (value) {
+    return this.SimplifyQA(
+      value,
+      [
+        this.RemoveSpecialChars.bind(this),
+        this.RemoveUnnecesarySpaces.bind(this)
+      ])
+  }
+
+  SimplifyStack (stack) {
+    return this.SimplifyQuery(stack)
+  }
+}
+
+const SUtils = new StringUtils()
+
+class Question {
+  constructor (q, a, i) {
+    this.Q = SUtils.SimplifyQuestion(q)
+    this.A = SUtils.SimplifyAnswer(a)
+    this.I = i
+  }
+
+  toString () {
+    var r = '?' + this.Q + '\n!' + this.A
+    if (this.I) { r += '\n>' + this.I }
+    return r
+  }
+
+  HasQuestion () {
+    return this.Q !== undefined
+  }
+
+  HasAnswer () {
+    return this.A !== undefined
+  }
+
+  HasImage () {
+    return this.I !== undefined && (typeof this.I === 'string' || Array.isArray(this.I))
+  }
+
+  IsComplete () {
+    return this.HasQuestion() && this.HasAnswer()
+  }
+
+  Compare (q2, i) {
+    assert(q2)
+
+    if (typeof q2 === 'string') {
+      var qmatchpercent = SUtils.CompareString(this.Q, q2)
+
+      if (i === undefined || i.length === 0) { return qmatchpercent } else {
+        if (this.HasImage()) {
+          const imatchpercent = this.HasImage() ? SUtils.CompareString(this.I.join(' '), i.join(' '))
+            : 0
+          return (qmatchpercent + imatchpercent) / 2
+        } else {
+          qmatchpercent -= 30
+          if (qmatchpercent < 0) { return 0 } else { return qmatchpercent }
+        }
+      }
+    } else {
+      const qmatchpercent = SUtils.CompareString(this.Q, q2.Q)
+      const amatchpercent = SUtils.CompareString(this.A, q2.A)
+      if (this.I !== undefined) {
+        const imatchpercent = this.I === undefined ? SUtils.CompareString(this.I.join(' '), q2.I.join(
+          ' ')) : 0
+        return (qmatchpercent + amatchpercent + imatchpercent) / 3
+      } else {
+        return (qmatchpercent + amatchpercent) / 2
+      }
+    }
+  }
+}
+
+class Subject {
+  constructor (n) {
+    assert(n)
+
+    this.Name = n
+    this.Questions = []
+    this.active = false
+  }
+
+  setIndex (i) {
+    this.index = i
+  }
+
+  getIndex () {
+    return this.index || -1
+  }
+
+  get length () {
+    return this.Questions.length
+  }
+
+  markActive () {
+    this.active = true
+  }
+
+  getIfActive () {
+    return this.active
+  }
+
+  AddQuestion (q) {
+    assert(q)
+
+    this.Questions.push(q)
+  }
+
+  getSubjNameWithoutYear () {
+    let t = this.Name.split(' - ')
+    if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
+      return t[1] || ''
+    } else {
+      return ''
+    }
+  }
+
+  getYear () {
+    let t = this.Name.split(' - ')[0]
+    if (t.match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
+      return t
+    } else {
+      return ''
+    }
+  }
+
+  Search (q, img) {
+    assert(q)
+
+    var r = []
+    for (let i = 0; i < this.length; i++) {
+      let percent = this.Questions[i].Compare(q, img)
+      if (percent > minMatchAmmount) {
+        r.push({
+          q: this.Questions[i],
+          match: percent
+        })
+      }
+    }
+
+    for (let i = 0; i < r.length; i++) {
+      for (var j = i; j < r.length; j++) {
+        if (r[i].match < r[j].match) {
+          var tmp = r[i]
+          r[i] = r[j]
+          r[j] = tmp
+        }
+      }
+    }
+
+    return r
+  }
+
+  toString () {
+    var r = []
+    for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) }
+    return '+' + this.Name + '\n' + r.join('\n')
+  }
+}
+
+class QuestionDB {
+  constructor (getVal, setVal) {
+    this.Subjects = []
+    this.getVal = getVal
+    this.setVal = setVal
+  }
+
+  get length () {
+    return this.Subjects.length
+  }
+
+  get activeIndexes () {
+    var r = []
+    for (var i = 0; i < this.length; i++) {
+      if (this.getVal('Is' + i + 'Active')) {
+        r.push(i)
+      }
+    }
+    return r
+  }
+
+  GetIfActive (ind) {
+    return this.getVal('Is' + ind + 'Active')
+  }
+
+  ChangeActive (i, value) {
+    this.setVal('Is' + i + 'Active', !!value)
+  }
+
+  AddQuestion (subj, q) {
+    assert(subj)
+
+    var i = 0
+    while (i < this.Subjects.length && this.Subjects[i].Name !== subj) { i++ }
+    if (i < this.Subjects.length) { this.Subjects[i].AddQuestion(q) } else {
+      const n = new Subject(subj)
+      n.AddQuestion(q)
+      this.Subjects.push(n)
+    }
+  }
+
+  Search (q, img) {
+    assert(q)
+
+    var r = []
+    for (let i = 0; i < this.length; i++) {
+      if (this.GetIfActive(i)) { r = r.concat(this.Subjects[i].Search(q, img)) }
+    }
+
+    for (let i = 0; i < r.length; i++) {
+      for (var j = i; j < r.length; j++) {
+        if (r[i].match < r[j].match) {
+          var tmp = r[i]
+          r[i] = r[j]
+          r[j] = tmp
+        }
+      }
+    }
+
+    return r
+  }
+
+  AddSubject (subj) {
+    assert(subj)
+
+    var i = 0
+    while (i < this.length && subj.Name !== this.Subjects[i].Name) { i++ }
+
+    if (i < this.length) {
+      this.Subjects.concat(subj.Questions)
+    } else {
+      this.Subjects.push(subj)
+    }
+  }
+
+  toString () {
+    var r = []
+    for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) }
+    return r.join('\n\n')
+  }
+}
+
+module.exports.StringUtils = StringUtils
+module.exports.Question = Question
+module.exports.Subject = Subject
+module.exports.QuestionDB = QuestionDB
+const utils = require('./utils.js')
+
+Main()
+
+function Main () {
+  console.clear()
+  const params = GetParams()
+  console.log(params)
+  var db = []
+
+  PrintLN()
+  console.log(params[0] + ': ')
+  try {
+    db = ParseJSONData(utils.ReadFile(params[0]))
+    console.log('JSON data added')
+  } catch (e) {
+    console.log(e)
+    console.log('Trying with old format...')
+    db = ReadData(utils.ReadFile(params[0])).result
+  }
+  PrintLN()
+
+  PrintDB(db)
+
+  PrintLN()
+
+  RefactorDb(db)
+
+  utils.WriteFile(JSON.stringify(db), 'newData')
+}
+
+function RefactorDb (db) {
+  db.Subjects.forEach((subj) => {
+    subj.Questions.forEach((question) => {
+      if (question.I) {
+        question.data = {
+          type: 'image',
+          images: typeof question.I === 'string' ? JSON.parse(question.I) : question.I
+        }
+        delete question.I
+      } else {
+        question.data = {
+          type: 'simple'
+        }
+      }
+    })
+  })
+}
+
+// -----------------------------------------------------------------------------------------
+
+function PrintLN () {
+  console.log('------------------------------------------------------')
+}
+
+function PrintDB (r, olds) {
+  console.log('Data subject count: ' + r.length)
+  var maxLength = 0
+  for (let i = 0; i < r.length; i++) {
+    if (maxLength < r.Subjects[i].Name.length) { maxLength = r.Subjects[i].Name.length }
+  }
+
+  let qcount = 0
+
+  for (let i = 0; i < r.length; i++) {
+    let line = i
+    if (line < 10) { line += ' ' }
+
+    line += ': '
+    var currLength = line.length + maxLength + 4
+    line += r.Subjects[i].Name
+
+    while (line.length < currLength) {
+      if (i % 4 === 0) { line += '.' } else { line += ' ' }
+    }
+
+    if (olds && olds.length > 0) {
+      // TODO: check if correct row! should be now, but well...
+      if (olds[i] < 10) { line += ' ' }
+      if (olds[i] < 100) { line += ' ' }
+
+      line += olds[i]
+      line += ' -> '
+    }
+
+    if (r.Subjects[i].length < 10) { line += ' ' }
+    if (r.Subjects[i].length < 100) { line += ' ' }
+
+    line += r.Subjects[i].length
+    qcount += r.Subjects[i].length
+
+    line += ' db'
+
+    console.log(line)
+  }
+
+  console.log('Total questions: ' + qcount)
+
+  PrintLN()
+}
+
+function GetParams () {
+  return process.argv.splice(2)
+}
+
+function ParseJSONData (data) {
+  var d = JSON.parse(data)
+  var r = new QuestionDB((x) => true, (x, y) => console.log(x, y))
+  var rt = []
+
+  for (var i = 0; i < d.Subjects.length; i++) {
+    let s = new Subject(d.Subjects[i].Name)
+    var j = 0
+    for (j = 0; j < d.Subjects[i].Questions.length; j++) {
+      var currQ = d.Subjects[i].Questions[j]
+      s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I))
+    }
+    rt.push({
+      name: d.Subjects[i].Name,
+      count: j
+    })
+    r.AddSubject(s)
+  }
+  return r
+}
+
+function ReadData (data) {
+  const d = data.split('\n')
+  const r = new QuestionDB((x) => true, (x, y) => console.log(x, y))
+  var logs = []
+  var currSubj = '' // the current subjects name
+  var ExpectedIdentifier = ['+', '?']
+  let currQuestion = new Question()
+
+  var i = -1
+  while (i < d.length) {
+    let currIdentifier
+    let skipped = 0
+    do {
+      if (skipped >= 1) { logs.push(i + ': ' + d[i]) }
+      i++
+      if (i >= d.length) {
+        if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) }
+        return {
+          result: r,
+          logs: logs
+        }
+      }
+      currIdentifier = d[i][0]
+      skipped++
+    } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length)
+
+    let currData = d[i].substring(1).trim()
+
+    if (currIdentifier === '+') {
+      if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) }
+      currQuestion = new Question()
+      currSubj = currData
+      ExpectedIdentifier = ['?']
+      continue
+    }
+
+    if (currIdentifier === '?') {
+      if (currQuestion.IsComplete()) {
+        r.AddQuestion(currSubj, currQuestion)
+        currQuestion = new Question()
+      }
+      // overwriting is allowed here, bcus:
+      // ?????!>
+      currQuestion.Q = currData
+      ExpectedIdentifier = ['!', '?']
+      continue
+    }
+
+    if (currIdentifier === '!') {
+      // if dont have question continue
+      if (!currQuestion.HasQuestion()) { throw new Error('No question! (A)') }
+      // dont allow overwriting
+      // ?!!!!
+      if (!currQuestion.HasAnswer()) {
+        currData = currData.replace('A helyes válaszok: ', '')
+        currData = currData.replace('A helyes válasz: ', '')
+
+        currQuestion.A = currData
+      }
+      ExpectedIdentifier = ['?', '>', '+']
+      continue
+    }
+
+    if (currIdentifier === '>') {
+      // if dont have question or answer continue
+      if (!currQuestion.HasQuestion()) { throw new Error('No question! (I)') }
+      if (!currQuestion.HasAnswer()) { throw new Error('No asnwer! (I)') }
+      // dont allow overwriting
+      // ?!>>>
+      if (!currQuestion.HasImage()) {
+        try {
+          currQuestion.I = JSON.parse(currData)
+        } catch (e) {
+          currQuestion.I = currData.split(',')
+        }
+      }
+      ExpectedIdentifier = ['?', '+']
+      continue
+    }
+  }
+
+  return {
+    result: r,
+    logs: logs
+  }
+}

utils/merger.js

@@ -41,8 +41,7 @@ function Main () {
       console.log('JSON data added')
     } catch (e) {
       console.log(e)
-      console.log('Trying with old format...')
-      dbs.push(ReadData(utils.ReadFile(params[i])).result)
+      return
     }
   }
   PrintLN()
@@ -155,98 +154,6 @@ function MergeDatabases (dbs) {
   return db
 }
 
-/*
- * Returns a question database from the given data.
- * Parameter should be raw read file in string with "\n"-s
- * TODO: ??? -s are not listed as errors, tho works correctly
- * */
-function ReadData (data) {
-  const d = data.split('\n')
-  const r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y))
-  var logs = []
-  var currSubj = '' // the current subjects name
-  var ExpectedIdentifier = ['+', '?']
-  let currQuestion = new classes.Question()
-
-  var i = -1
-  while (i < d.length) {
-    let currIdentifier
-    let skipped = 0
-    do {
-      if (skipped >= 1) { logs.push(i + ': ' + d[i]) }
-      i++
-      if (i >= d.length) {
-        if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) }
-        return {
-          result: r,
-          logs: logs
-        }
-      }
-      currIdentifier = d[i][0]
-      skipped++
-    } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length)
-
-    let currData = d[i].substring(1).trim()
-
-    if (currIdentifier === '+') {
-      if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) }
-      currQuestion = new classes.Question()
-      currSubj = currData
-      ExpectedIdentifier = ['?']
-      continue
-    }
-
-    if (currIdentifier === '?') {
-      if (currQuestion.IsComplete()) {
-        r.AddQuestion(currSubj, currQuestion)
-        currQuestion = new classes.Question()
-      }
-      // overwriting is allowed here, bcus:
-      // ?????!>
-      currQuestion.Q = currData
-      ExpectedIdentifier = ['!', '?']
-      continue
-    }
-
-    if (currIdentifier === '!') {
-      // if dont have question continue
-      if (!currQuestion.HasQuestion()) { throw new Error('No question! (A)') }
-      // dont allow overwriting
-      // ?!!!!
-      if (!currQuestion.HasAnswer()) {
-        currData = currData.replace('A helyes válaszok: ', '')
-        currData = currData.replace('A helyes válasz: ', '')
-
-        currQuestion.A = currData
-      }
-      ExpectedIdentifier = ['?', '>', '+']
-      continue
-    }
-
-    if (currIdentifier === '>') {
-      // if dont have question or answer continue
-      if (!currQuestion.HasQuestion()) { throw new Error('No question! (I)') }
-      if (!currQuestion.HasAnswer()) { throw new Error('No asnwer! (I)') }
-      // dont allow overwriting
-      // ?!>>>
-      if (!currQuestion.HasImage()) {
-        try {
-          currQuestion.I = JSON.parse(currData)
-        } catch (e) {
-          currQuestion.I = currData.split(',')
-        }
-      }
-      ExpectedIdentifier = ['?', '+']
-      continue
-    }
-  }
-
-  return {
-    result: r,
-    logs: logs
-  }
-}
-
 function RemoveDuplicates (dataObj) {
   for (var i = 0; i < dataObj.length; i++) { RemoveDuplFromSubject(dataObj.Subjects[i]) }
   return dataObj