diff --git a/utils/actions.js b/utils/actions.js index 9bdb2e1..6673476 100644 --- a/utils/actions.js +++ b/utils/actions.js @@ -55,7 +55,7 @@ function ProcessIncomingRequest (data) { var d = JSON.parse(data) var allQuestions = [] for (let i = 0; i < d.allData.length; i++) { - allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].I)) + allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data)) } let color = logger.GetColor('green') @@ -69,7 +69,7 @@ function ProcessIncomingRequest (data) { if (d.data.length > 0) { let qdb = LoadJSON(utils.ReadFile(dataFile)) d.data.forEach((x) => { - let q = new classes.Question(x.Q, x.A, x.I) + let q = new classes.Question(x.Q, x.A, x.data) questions.push(q) qdb.AddQuestion(d.subj, q) }) @@ -116,7 +116,7 @@ function LoadJSON (resource) { var j = 0 for (j = 0; j < d.Subjects[i].Questions.length; j++) { var currQ = d.Subjects[i].Questions[j] - s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I)) + s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data)) } rt.push({ name: d.Subjects[i].Name, diff --git a/utils/dataUpdater.js b/utils/dataUpdater.js index 333555e..be22220 100644 --- a/utils/dataUpdater.js +++ b/utils/dataUpdater.js @@ -18,7 +18,411 @@ ------------------------------------------------------------------------- */ -const classes = require('./question-classes/classes.js') +const commonUselessAnswerParts = [ + 'A helyes válasz az ', + 'A helyes válasz a ', + 'A helyes válaszok: ', + 'A helyes válaszok:', + 'A helyes válasz: ', + 'A helyes válasz:', + 'The correct answer is:', + '\'' +] +const commonUselessStringParts = [',', '\\.', ':', '!', '\\+'] +const specialChars = [ '&', '\\+' ] +const lengthDiffMultiplier = 10 /* Percent minus for length difference */ +const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */ + +const assert = (val) => { + if (!val) { throw new Error('Assertion failed') } +} + +class StringUtils { + RemoveStuff (value, removableStrings, toReplace) { + removableStrings.forEach((x) => { + var regex = new RegExp(x, 'g') + value = value.replace(regex, toReplace || '') + }) + return value + } + + SimplifyQuery (q) { + assert(q) + + var result = q.replace(/\n/g, ' ').replace(/\s/g, ' ') + return this.RemoveUnnecesarySpaces(result) + } + + ShortenString (toShorten, ammount) { + assert(toShorten) + + var result = '' + var i = 0 + while (i < toShorten.length && i < ammount) { + result += toShorten[i] + i++ + } + return result + } + + ReplaceCharsWithSpace (val, char) { + assert(val) + assert(char) + + var toremove = this.NormalizeSpaces(val) + + var regex = new RegExp(char, 'g') + toremove = toremove.replace(regex, ' ') + + return this.RemoveUnnecesarySpaces(toremove) + } + + // removes whitespace from begining and and, and replaces multiple spaces with one space + RemoveUnnecesarySpaces (toremove) { + assert(toremove) + + toremove = this.NormalizeSpaces(toremove) + while (toremove.includes(' ')) { + toremove = toremove.replace(/ {2}/g, ' ') + } + return toremove.trim() + } + + // simplifies a string for easier comparison + SimplifyStringForComparison (value) { + assert(value) + + value = this.RemoveUnnecesarySpaces(value).toLowerCase() + return this.RemoveStuff(value, commonUselessStringParts) + } + + RemoveSpecialChars (value) { + assert(value) + + return this.RemoveStuff(value, specialChars, ' ') + } + + // if the value is empty, or whitespace + EmptyOrWhiteSpace (value) { + // replaces /n-s with "". then replaces spaces with "". if it equals "", then its empty, or only consists of white space + if (value === undefined) { return true } + return (value.replace(/\n/g, '').replace(/ /g, '').replace(/\s/g, ' ') === '') + } + + // damn nonbreaking space + NormalizeSpaces (input) { + assert(input) + + return input.replace(/\s/g, ' ') + } + + CompareString (s1, s2) { + if (!s1 || !s2) { + return 0 + } + + s1 = this.SimplifyStringForComparison(s1).split(' ') + s2 = this.SimplifyStringForComparison(s2).split(' ') + var match = 0 + for (var i = 0; i < s1.length; i++) { + if (s2.includes(s1[i])) { match++ } + } + var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length) + percent -= lengthDifference * lengthDiffMultiplier + if (percent < 0) { percent = 0 } + return percent + } + + AnswerPreProcessor (value) { + assert(value) + + return this.RemoveStuff( + value, commonUselessAnswerParts) + } + + // 'a. pécsi sör' -> 'pécsi sör' + RemoveAnswerLetters (value) { + assert(value) + + let s = value.split('. ') + if (s[0].length < 2 && s.length > 1) { + s.shift() + return s.join(' ') + } else { + return value + } + } + + SimplifyQA (value, mods) { + if (!value) { return } + + const reducer = (res, fn) => { + return fn(res) + } + + return mods.reduce(reducer, value) + } + + SimplifyAnswer (value) { + return this.SimplifyQA( + value, + [ + this.RemoveSpecialChars.bind(this), + this.RemoveUnnecesarySpaces.bind(this), + this.AnswerPreProcessor.bind(this), + this.RemoveAnswerLetters.bind(this) + ]) + } + + SimplifyQuestion (value) { + return this.SimplifyQA( + value, + [ + this.RemoveSpecialChars.bind(this), + this.RemoveUnnecesarySpaces.bind(this) + ]) + } + + SimplifyStack (stack) { + return this.SimplifyQuery(stack) + } +} + +const SUtils = new StringUtils() + +class Question { + constructor (q, a, i) { + this.Q = SUtils.SimplifyQuestion(q) + this.A = SUtils.SimplifyAnswer(a) + this.I = i + } + + toString () { + var r = '?' + this.Q + '\n!' + this.A + if (this.I) { r += '\n>' + this.I } + return r + } + + HasQuestion () { + return this.Q !== undefined + } + + HasAnswer () { + return this.A !== undefined + } + + HasImage () { + return this.I !== undefined && (typeof this.I === 'string' || Array.isArray(this.I)) + } + + IsComplete () { + return this.HasQuestion() && this.HasAnswer() + } + + Compare (q2, i) { + assert(q2) + + if (typeof q2 === 'string') { + var qmatchpercent = SUtils.CompareString(this.Q, q2) + + if (i === undefined || i.length === 0) { return qmatchpercent } else { + if (this.HasImage()) { + const imatchpercent = this.HasImage() ? SUtils.CompareString(this.I.join(' '), i.join(' ')) + : 0 + return (qmatchpercent + imatchpercent) / 2 + } else { + qmatchpercent -= 30 + if (qmatchpercent < 0) { return 0 } else { return qmatchpercent } + } + } + } else { + const qmatchpercent = SUtils.CompareString(this.Q, q2.Q) + const amatchpercent = SUtils.CompareString(this.A, q2.A) + if (this.I !== undefined) { + const imatchpercent = this.I === undefined ? SUtils.CompareString(this.I.join(' '), q2.I.join( + ' ')) : 0 + return (qmatchpercent + amatchpercent + imatchpercent) / 3 + } else { + return (qmatchpercent + amatchpercent) / 2 + } + } + } +} + +class Subject { + constructor (n) { + assert(n) + + this.Name = n + this.Questions = [] + this.active = false + } + + setIndex (i) { + this.index = i + } + + getIndex () { + return this.index || -1 + } + + get length () { + return this.Questions.length + } + + markActive () { + this.active = true + } + + getIfActive () { + return this.active + } + + AddQuestion (q) { + assert(q) + + this.Questions.push(q) + } + + getSubjNameWithoutYear () { + let t = this.Name.split(' - ') + if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t[1] || '' + } else { + return '' + } + } + + getYear () { + let t = this.Name.split(' - ')[0] + if (t.match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t + } else { + return '' + } + } + + Search (q, img) { + assert(q) + + var r = [] + for (let i = 0; i < this.length; i++) { + let percent = this.Questions[i].Compare(q, img) + if (percent > minMatchAmmount) { + r.push({ + q: this.Questions[i], + match: percent + }) + } + } + + for (let i = 0; i < r.length; i++) { + for (var j = i; j < r.length; j++) { + if (r[i].match < r[j].match) { + var tmp = r[i] + r[i] = r[j] + r[j] = tmp + } + } + } + + return r + } + + toString () { + var r = [] + for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) } + return '+' + this.Name + '\n' + r.join('\n') + } +} + +class QuestionDB { + constructor (getVal, setVal) { + this.Subjects = [] + this.getVal = getVal + this.setVal = setVal + } + + get length () { + return this.Subjects.length + } + + get activeIndexes () { + var r = [] + for (var i = 0; i < this.length; i++) { + if (this.getVal('Is' + i + 'Active')) { + r.push(i) + } + } + return r + } + + GetIfActive (ind) { + return this.getVal('Is' + ind + 'Active') + } + + ChangeActive (i, value) { + this.setVal('Is' + i + 'Active', !!value) + } + + AddQuestion (subj, q) { + assert(subj) + + var i = 0 + while (i < this.Subjects.length && this.Subjects[i].Name !== subj) { i++ } + if (i < this.Subjects.length) { this.Subjects[i].AddQuestion(q) } else { + const n = new Subject(subj) + n.AddQuestion(q) + this.Subjects.push(n) + } + } + + Search (q, img) { + assert(q) + + var r = [] + for (let i = 0; i < this.length; i++) { + if (this.GetIfActive(i)) { r = r.concat(this.Subjects[i].Search(q, img)) } + } + + for (let i = 0; i < r.length; i++) { + for (var j = i; j < r.length; j++) { + if (r[i].match < r[j].match) { + var tmp = r[i] + r[i] = r[j] + r[j] = tmp + } + } + } + + return r + } + + AddSubject (subj) { + assert(subj) + + var i = 0 + while (i < this.length && subj.Name !== this.Subjects[i].Name) { i++ } + + if (i < this.length) { + this.Subjects.concat(subj.Questions) + } else { + this.Subjects.push(subj) + } + } + + toString () { + var r = [] + for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) } + return r.join('\n\n') + } +} + +module.exports.StringUtils = StringUtils +module.exports.Question = Question +module.exports.Subject = Subject +module.exports.QuestionDB = QuestionDB const utils = require('./utils.js') Main() @@ -56,7 +460,7 @@ function RefactorDb (db) { if (question.I) { question.data = { type: 'image', - images: question.I + images: typeof question.I === 'string' ? JSON.parse(question.I) : question.I } delete question.I } else { @@ -126,15 +530,15 @@ function GetParams () { function ParseJSONData (data) { var d = JSON.parse(data) - var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) + var r = new QuestionDB((x) => true, (x, y) => console.log(x, y)) var rt = [] for (var i = 0; i < d.Subjects.length; i++) { - let s = new classes.Subject(d.Subjects[i].Name) + let s = new Subject(d.Subjects[i].Name) var j = 0 for (j = 0; j < d.Subjects[i].Questions.length; j++) { var currQ = d.Subjects[i].Questions[j] - s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I)) + s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I)) } rt.push({ name: d.Subjects[i].Name, @@ -147,11 +551,11 @@ function ParseJSONData (data) { function ReadData (data) { const d = data.split('\n') - const r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) + const r = new QuestionDB((x) => true, (x, y) => console.log(x, y)) var logs = [] var currSubj = '' // the current subjects name var ExpectedIdentifier = ['+', '?'] - let currQuestion = new classes.Question() + let currQuestion = new Question() var i = -1 while (i < d.length) { @@ -175,7 +579,7 @@ function ReadData (data) { if (currIdentifier === '+') { if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } - currQuestion = new classes.Question() + currQuestion = new Question() currSubj = currData ExpectedIdentifier = ['?'] continue @@ -184,7 +588,7 @@ function ReadData (data) { if (currIdentifier === '?') { if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) - currQuestion = new classes.Question() + currQuestion = new Question() } // overwriting is allowed here, bcus: // ?????!> diff --git a/utils/question-classes b/utils/question-classes index 0cfd9cf..5dac788 160000 --- a/utils/question-classes +++ b/utils/question-classes @@ -1 +1 @@ -Subproject commit 0cfd9cfda377ff8917df50e68f105ec43f5e9311 +Subproject commit 5dac788b18b7240a4194a2a37ecbee857292800d