From 50a1dd91eb1de0c34bd18cf384db17a3adc811be Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer Date: Wed, 14 Aug 2019 10:34:55 +0200 Subject: [PATCH] merger, changedataversion, motd refactor --- utils/changedataversion.js | 39 +-- utils/merger.js | 690 ++++++++++++++++++------------------- utils/motd.js | 33 +- 3 files changed, 364 insertions(+), 398 deletions(-) diff --git a/utils/changedataversion.js b/utils/changedataversion.js index 2b5052a..94582e2 100644 --- a/utils/changedataversion.js +++ b/utils/changedataversion.js @@ -1,32 +1,31 @@ -const utils = require('./utils.js'); -const dataFile = "public/data.json"; -const versionFile = "public/version"; +const utils = require('../utils/utils.js') +const dataFile = '../public/data.json' +const versionFile = '../public/version' -var p = GetParams(); +var p = GetParams() if (p.length <= 0) { - console.log("no params!"); - process.exit(0); + console.log('no params!') + process.exit(0) } +var param = p.join(' ') -var param = p.join(" "); +console.log('param: ' + param) -console.log("param: " + param); +var d = utils.ReadFile(dataFile) +var parsed = JSON.parse(d) -var d = utils.ReadFile(dataFile); -var parsed = JSON.parse(d); +console.log('Old version:') +console.log(parsed.version) -console.log("Old version:"); -console.log(parsed.version); +parsed.version = param -parsed.version = param; +console.log('New version:') +console.log(parsed.version) -console.log("New version:"); -console.log(parsed.version); +utils.WriteFile(JSON.stringify(parsed), dataFile) +utils.WriteFile(parsed.version, versionFile) -utils.WriteFile(JSON.stringify(parsed), dataFile); -utils.WriteFile(parsed.version, versionFile); - -function GetParams() { - return process.argv.splice(2); +function GetParams () { + return process.argv.splice(2) } diff --git a/utils/merger.js b/utils/merger.js index 2833705..b3b0e07 100644 --- a/utils/merger.js +++ b/utils/merger.js @@ -1,4 +1,4 @@ -/* ---------------------------------------------------------------------------- +/* ---------------------------------------------------------------------------- Question Server question file merger GitLab: @@ -22,280 +22,256 @@ // join json datas, or raw datas // or something else -const minMatchAmmount = 55; -const minResultMatchPercent = 99; -const lengthDiffMultiplier = 10; +const minMatchAmmount = 55 +const minResultMatchPercent = 99 +const lengthDiffMultiplier = 10 class Question { - constructor(q, a, i) { - this.Q = q; - this.A = a; - this.I = i; - } - toString() { - var r = "?" + this.Q + "\n!" + this.A; - if (this.I) - r += "\n>" + this.I; - return r; - } - HasQuestion() { - return this.Q != undefined; - } - HasAnswer() { - return this.A != undefined; - } - HasImage() { - return this.I != undefined; - } - IsComplete() { - return this.HasQuestion() && this.HasAnswer(); - } - // TODO: TEST DIS - Compare(q2, i) { - if (typeof q2 == 'string') { - var qmatchpercent = Question.CompareString(this.Q, q2); + constructor (q, a, i) { + this.Q = q + this.A = a + this.I = i + } + toString () { + var r = '?' + this.Q + '\n!' + this.A + if (this.I) { r += '\n>' + this.I } + return r + } + HasQuestion () { + return this.Q != undefined + } + HasAnswer () { + return this.A != undefined + } + HasImage () { + return this.I != undefined + } + IsComplete () { + return this.HasQuestion() && this.HasAnswer() + } + // TODO: TEST DIS + Compare (q2, i) { + if (typeof q2 === 'string') { + var qmatchpercent = Question.CompareString(this.Q, q2) - if (i == undefined || i.length == 0) - return qmatchpercent; - else { - if (this.HasImage()) { - const imatchpercent = this.HasImage() ? Question.CompareString(this.I.join(" "), i.join(" ")) : - 0; - return (qmatchpercent + imatchpercent) / 2; - } else { - qmatchpercent -= 30; - if (qmatchpercent < 0) - return 0; - else - return qmatchpercent; - } - } - } else { - const qmatchpercent = Question.CompareString(this.Q, q2.Q); - const amatchpercent = Question.CompareString(this.A, q2.A); - if (this.I != undefined) { - const imatchpercent = this.I == undefined ? Question.CompareString(this.I.join(" "), q2.I.join( - " ")) : 0; - return (qmatchpercent + amatchpercent + imatchpercent) / 3; - } else { - return (qmatchpercent + amatchpercent) / 2; - } - } - } - static CompareString(s1, s2) { - s1 = SimplifyStringForComparison(s1).split(" "); - s2 = SimplifyStringForComparison(s2).split(" "); - var match = 0; - for (var i = 0; i < s1.length; i++) - if (s2.includes(s1[i])) - match++; - var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent - var lengthDifference = Math.abs(s2.length - s1.length); - percent -= lengthDifference * lengthDiffMultiplier; - if (percent < 0) - percent = 0; - return percent; - } + if (i == undefined || i.length == 0) { return qmatchpercent } else { + if (this.HasImage()) { + const imatchpercent = this.HasImage() ? Question.CompareString(this.I.join(' '), i.join(' ')) + : 0 + return (qmatchpercent + imatchpercent) / 2 + } else { + qmatchpercent -= 30 + if (qmatchpercent < 0) { return 0 } else { return qmatchpercent } + } + } + } else { + const qmatchpercent = Question.CompareString(this.Q, q2.Q) + const amatchpercent = Question.CompareString(this.A, q2.A) + if (this.I != undefined) { + const imatchpercent = this.I == undefined ? Question.CompareString(this.I.join(' '), q2.I.join( + ' ')) : 0 + return (qmatchpercent + amatchpercent + imatchpercent) / 3 + } else { + return (qmatchpercent + amatchpercent) / 2 + } + } + } + static CompareString (s1, s2) { + s1 = SimplifyStringForComparison(s1).split(' ') + s2 = SimplifyStringForComparison(s2).split(' ') + var match = 0 + for (var i = 0; i < s1.length; i++) { + if (s2.includes(s1[i])) { match++ } + } + var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length) + percent -= lengthDifference * lengthDiffMultiplier + if (percent < 0) { percent = 0 } + return percent + } } class Subject { - constructor(n) { - this.Name = n; - this.Questions = []; - } - get length() { - return this.Questions.length; - } - AddQuestion(q) { - this.Questions.push(q); - } - toString() { - var r = []; - for (var i = 0; i < this.Questions.length; i++) - r.push(this.Questions[i].toString()); - return "+" + this.Name + "\n" + r.join("\n"); - } + constructor (n) { + this.Name = n + this.Questions = [] + } + get length () { + return this.Questions.length + } + AddQuestion (q) { + this.Questions.push(q) + } + toString () { + var r = [] + for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) } + return '+' + this.Name + '\n' + r.join('\n') + } } class QuestionDB { - constructor() { - this.Subjects = []; - } - get length() { - return this.Subjects.length; - } - AddQuestion(subj, q) { - var i = 0; - while (i < this.Subjects.length && this.Subjects[i].Name != subj) - i++; - if (i < this.Subjects.length) - this.Subjects[i].AddQuestion(q); - else { - const n = new Subject(subj); - n.AddQuestion(q); - this.Subjects.push(n); - } - } - AddSubject(subj) { - var i = 0; - while (i < this.length && subj.Name != this.Subjects[i].Name) - i++; + constructor () { + this.Subjects = [] + } + get length () { + return this.Subjects.length + } + AddQuestion (subj, q) { + var i = 0 + while (i < this.Subjects.length && this.Subjects[i].Name != subj) { i++ } + if (i < this.Subjects.length) { this.Subjects[i].AddQuestion(q) } else { + const n = new Subject(subj) + n.AddQuestion(q) + this.Subjects.push(n) + } + } + AddSubject (subj) { + var i = 0 + while (i < this.length && subj.Name != this.Subjects[i].Name) { i++ } - if (i < this.length) { - this.Subjects.concat(subj.Questions); - } else { - this.Subjects.push(subj); - } - } - toString() { - var r = []; - for (var i = 0; i < this.Subjects.length; i++) - r.push(this.Subjects[i].toString()); - return r.join("\n\n"); - } + if (i < this.length) { + this.Subjects.concat(subj.Questions) + } else { + this.Subjects.push(subj) + } + } + toString () { + var r = [] + for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) } + return r.join('\n\n') + } } -var utils = require('./utils.js'); -var actions = require('./actions.js'); +var utils = require('./utils.js') +var actions = require('./actions.js') -Main(); +Main() -function Main() { - console.clear(); - const params = GetParams(); - console.log(params); - var dbs = []; +function Main () { + console.clear() + const params = GetParams() + console.log(params) + var dbs = [] + for (var i = 0; i < params.length; i++) { + PrintLN() + console.log(params[i] + ': ') + try { + dbs.push(ParseJSONData(utils.ReadFile(params[i]))) + console.log('JSON data added') + } catch (e) { + console.log(e) + console.log('Trying with old format...') + dbs.push(ReadData(utils.ReadFile(params[i])).result) + } + } + PrintLN() - for (var i = 0; i < params.length; i++) { - PrintLN(); - console.log(params[i] + ": "); - try { - dbs.push(ParseJSONData(utils.ReadFile(params[i]))); - console.log("JSON data added"); - } catch (e) { - console.log(e); - console.log("Trying with old format..."); - dbs.push(ReadData(utils.ReadFile(params[i])).result); - } - } - PrintLN(); + dbs.forEach((item) => { + PrintDB(item) + }) - dbs.forEach((item) => { - PrintDB(item); - }); + var olds = [] + if (dbs.length == 1) { + for (let i = 0; i < dbs[0].length; i++) { olds.push(dbs[0].Subjects[i].length) } + } - var olds = []; - if (dbs.length == 1) { - for ( let i = 0; i < dbs[0].length; i++) - olds.push(dbs[0].Subjects[i].length); - } + console.log('Parsed data count: ' + dbs.length) + PrintLN() - console.log("Parsed data count: " + dbs.length); - PrintLN(); + console.log('Merging databases...') + var db = MergeDatabases(dbs) - console.log("Merging databases..."); - var db = MergeDatabases(dbs); + console.log('Removing duplicates...') + var r = RemoveDuplicates(db) - console.log("Removing duplicates..."); - var r = RemoveDuplicates(db); + console.log('RESULT:') + PrintDB(r, olds) - console.log("RESULT:"); - PrintDB(r, olds); - - utils.WriteFile(JSON.stringify(r), "newData"); - console.log("File written!"); + utils.WriteFile(JSON.stringify(r), 'newData') + console.log('File written!') } -function PrintLN() { - console.log("------------------------------------------------------"); +function PrintLN () { + console.log('------------------------------------------------------') } -function PrintDB(r, olds) { - console.log("Data subject count: " + r.length); - var maxLength = 0; - for (var i = 0; i < r.length; i++) { - if (maxLength < r.Subjects[i].Name.length) - maxLength = r.Subjects[i].Name.length; - } +function PrintDB (r, olds) { + console.log('Data subject count: ' + r.length) + var maxLength = 0 + for (var i = 0; i < r.length; i++) { + if (maxLength < r.Subjects[i].Name.length) { maxLength = r.Subjects[i].Name.length } + } - let qcount = 0; + let qcount = 0 - for (var i = 0; i < r.length; i++) { - let line = i; - if (line < 10) - line += ' '; + for (var i = 0; i < r.length; i++) { + let line = i + if (line < 10) { line += ' ' } - line += ": "; - var currLength = line.length + maxLength + 4; - line += r.Subjects[i].Name; + line += ': ' + var currLength = line.length + maxLength + 4 + line += r.Subjects[i].Name - while (line.length < currLength) { - if (i % 4 == 0) - line += "."; - else - line += " "; - } + while (line.length < currLength) { + if (i % 4 == 0) { line += '.' } else { line += ' ' } + } - if (olds && olds.length > 0) { - // TODO: check if correct row! should be now, but well... - if (olds[i] < 10) - line += " "; - if (olds[i] < 100) - line += " "; + if (olds && olds.length > 0) { + // TODO: check if correct row! should be now, but well... + if (olds[i] < 10) { line += ' ' } + if (olds[i] < 100) { line += ' ' } - line += olds[i]; - line += " -> "; - } + line += olds[i] + line += ' -> ' + } - if (r.Subjects[i].length < 10) - line += " "; - if (r.Subjects[i].length < 100) - line += " "; + if (r.Subjects[i].length < 10) { line += ' ' } + if (r.Subjects[i].length < 100) { line += ' ' } - line += r.Subjects[i].length; - qcount += r.Subjects[i].length; - - line += " db"; - - console.log(line); - } + line += r.Subjects[i].length + qcount += r.Subjects[i].length - console.log("Total questions: " + qcount); + line += ' db' - PrintLN(); + console.log(line) + } + + console.log('Total questions: ' + qcount) + + PrintLN() } -function GetParams() { - return process.argv.splice(2); +function GetParams () { + return process.argv.splice(2) } -function ParseJSONData(data) { - var d = JSON.parse(data); - var r = new QuestionDB(); - var rt = []; +function ParseJSONData (data) { + var d = JSON.parse(data) + var r = new QuestionDB() + var rt = [] - for (var i = 0; i < d.Subjects.length; i++) { - let s = new Subject(d.Subjects[i].Name); - var j = 0; - for (j = 0; j < d.Subjects[i].Questions.length; j++) { - var currQ = d.Subjects[i].Questions[j]; - s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I)); - } - rt.push({ - name: d.Subjects[i].Name, - count: j - }); - r.AddSubject(s); - } - return r; + for (var i = 0; i < d.Subjects.length; i++) { + let s = new Subject(d.Subjects[i].Name) + var j = 0 + for (j = 0; j < d.Subjects[i].Questions.length; j++) { + var currQ = d.Subjects[i].Questions[j] + s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I)) + } + rt.push({ + name: d.Subjects[i].Name, + count: j + }) + r.AddSubject(s) + } + return r } -function MergeDatabases(dbs) { - var db = new QuestionDB(); - for (var i = 0; i < dbs.length; i++) - for (var j = 0; j < dbs[i].length; j++) - db.AddSubject(dbs[i].Subjects[j]); - return db; +function MergeDatabases (dbs) { + var db = new QuestionDB() + for (var i = 0; i < dbs.length; i++) { + for (var j = 0; j < dbs[i].length; j++) { db.AddSubject(dbs[i].Subjects[j]) } + } + return db } /* @@ -303,150 +279,142 @@ function MergeDatabases(dbs) { * Parameter should be raw read file in string with "\n"-s * TODO: ??? -s are not listed as errors, tho works correctly * */ -function ReadData(data) { +function ReadData (data) { + const d = data.split('\n') + const r = new QuestionDB() + var logs = [] + var currSubj = '' // the current subjects name + var ExpectedIdentifier = ['+', '?'] + let currQuestion = new Question() - const d = data.split("\n"); - const r = new QuestionDB(); - var logs = []; - var currSubj = ""; // the current subjects name - var ExpectedIdentifier = ['+', '?']; - let currQuestion = new Question(); + var i = -1 + while (i < d.length) { + let currIdentifier + let skipped = 0 + do { + if (skipped >= 1) { logs.push(i + ': ' + d[i]) } + i++ + if (i >= d.length) { + if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } + return { + result: r, + logs: logs + } + } + currIdentifier = d[i][0] + skipped++ + } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length) - var i = -1; - while (i < d.length) { - let currIdentifier; - let skipped = 0; - do { - if (skipped >= 1) - logs.push(i + ": " + d[i]); - i++; - if (i >= d.length) { - if (currQuestion.IsComplete()) - r.AddQuestion(currSubj, currQuestion); - return { - result: r, - logs: logs - }; - } - currIdentifier = d[i][0]; - skipped++; - } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length); + let currData = d[i].substring(1).trim() - let currData = d[i].substring(1).trim(); + if (currIdentifier == '+') { + if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } + currQuestion = new Question() + currSubj = currData + ExpectedIdentifier = ['?'] + continue + } - if (currIdentifier == '+') { - if (currQuestion.IsComplete()) - r.AddQuestion(currSubj, currQuestion); - currQuestion = new Question(); - currSubj = currData; - ExpectedIdentifier = ['?']; - continue; - } + if (currIdentifier == '?') { + if (currQuestion.IsComplete()) { + r.AddQuestion(currSubj, currQuestion) + currQuestion = new Question() + } + // overwriting is allowed here, bcus: + // ?????!> + currQuestion.Q = currData + ExpectedIdentifier = ['!', '?'] + continue + } - if (currIdentifier == '?') { - if (currQuestion.IsComplete()) { - r.AddQuestion(currSubj, currQuestion); - currQuestion = new Question(); - } - // overwriting is allowed here, bcus: - // ?????!> - currQuestion.Q = currData; - ExpectedIdentifier = ['!', '?']; - continue; - } + if (currIdentifier == '!') { + // if dont have question continue + if (!currQuestion.HasQuestion()) { throw 'No question! (A)' } + // dont allow overwriting + // ?!!!! + if (!currQuestion.HasAnswer()) { + currData = currData.replace('A helyes válaszok: ', '') + currData = currData.replace('A helyes válasz: ', '') - if (currIdentifier == '!') { - // if dont have question continue - if (!currQuestion.HasQuestion()) - throw "No question! (A)"; - // dont allow overwriting - // ?!!!! - if (!currQuestion.HasAnswer()) { - currData = currData.replace("A helyes válaszok: ", ""); - currData = currData.replace("A helyes válasz: ", ""); + currQuestion.A = currData + } + ExpectedIdentifier = ['?', '>', '+'] + continue + } - currQuestion.A = currData; - } - ExpectedIdentifier = ['?', '>', '+']; - continue; - } + if (currIdentifier == '>') { + // if dont have question or answer continue + if (!currQuestion.HasQuestion()) { throw 'No question! (I)' } + if (!currQuestion.HasAnswer()) { throw 'No asnwer! (I)' } + // dont allow overwriting + // ?!>>> + if (!currQuestion.HasImage()) { + try { + currQuestion.I = JSON.parse(currData) + } catch (e) { + currQuestion.I = currData.split(',') + } + } + ExpectedIdentifier = ['?', '+'] + continue + } + } - if (currIdentifier == '>') { - // if dont have question or answer continue - if (!currQuestion.HasQuestion()) - throw "No question! (I)"; - if (!currQuestion.HasAnswer()) - throw "No asnwer! (I)"; - // dont allow overwriting - // ?!>>> - if (!currQuestion.HasImage()) { - try { - currQuestion.I = JSON.parse(currData); - } catch (e) { - currQuestion.I = currData.split(','); - } - } - ExpectedIdentifier = ['?', '+']; - continue; - } - } - - return { - result: r, - logs: logs - }; + return { + result: r, + logs: logs + } } -function RemoveDuplicates(dataObj) { - for (var i = 0; i < dataObj.length; i++) - RemoveDuplFromSubject(dataObj.Subjects[i]); - return dataObj; +function RemoveDuplicates (dataObj) { + for (var i = 0; i < dataObj.length; i++) { RemoveDuplFromSubject(dataObj.Subjects[i]) } + return dataObj } -function RemoveDuplFromSubject(subj) { - var cp = subj.Questions; - subj.Questions = []; - for (var i = 0; i < cp.length; i++) { - var j = 0; - // Only removes 100% match! - while (j < subj.length && cp[i].Compare(subj.Questions[j]) != 100) { - j++; - } - if (j < subj.length) { - //console.log("----------------------------------------------------------"); - //console.log(cp[i].toString()); - //console.log(" VS "); - //console.log(subj.Questions[j].toString()); - //console.log(cp[i].Compare(subj.Questions[j])); - //console.log(j); - //console.log("removed:"); - //console.log(subj.Questions.splice(j, 1).toString()); - //console.log("----------------------------------------------------------"); - } else { - subj.AddQuestion(cp[i]); - } - } +function RemoveDuplFromSubject (subj) { + var cp = subj.Questions + subj.Questions = [] + for (var i = 0; i < cp.length; i++) { + var j = 0 + // Only removes 100% match! + while (j < subj.length && cp[i].Compare(subj.Questions[j]) != 100) { + j++ + } + if (j < subj.length) { + // console.log("----------------------------------------------------------"); + // console.log(cp[i].toString()); + // console.log(" VS "); + // console.log(subj.Questions[j].toString()); + // console.log(cp[i].Compare(subj.Questions[j])); + // console.log(j); + // console.log("removed:"); + // console.log(subj.Questions.splice(j, 1).toString()); + // console.log("----------------------------------------------------------"); + } else { + subj.AddQuestion(cp[i]) + } + } } -function SimplifyStringForComparison(value) { - value = RemoveUnnecesarySpaces(value).toLowerCase(); - var removableChars = [",", ".", ":", "!"]; - for (var i = 0; i < removableChars.length; i++) { - var regex = new RegExp(removableChars[i], "g"); - value.replace(regex, ""); - } - return value; +function SimplifyStringForComparison (value) { + value = RemoveUnnecesarySpaces(value).toLowerCase() + var removableChars = [',', '.', ':', '!'] + for (var i = 0; i < removableChars.length; i++) { + var regex = new RegExp(removableChars[i], 'g') + value.replace(regex, '') + } + return value } -function RemoveUnnecesarySpaces(toremove) { - toremove = NormalizeSpaces(toremove); - while (toremove.includes(" ")) // while the text includes double spaces replaces all of them with a single one - { - toremove = toremove.replace(/ /g, " "); - } - return toremove.trim(); +function RemoveUnnecesarySpaces (toremove) { + toremove = NormalizeSpaces(toremove) + while (toremove.includes(' ')) // while the text includes double spaces replaces all of them with a single one + { + toremove = toremove.replace(/ {2}/g, ' ') + } + return toremove.trim() } -function NormalizeSpaces(input) { - return input.replace(/\s/g, ' '); +function NormalizeSpaces (input) { + return input.replace(/\s/g, ' ') } diff --git a/utils/motd.js b/utils/motd.js index 6a8bcc9..6f50a6a 100644 --- a/utils/motd.js +++ b/utils/motd.js @@ -1,24 +1,23 @@ -const utils = require('./utils.js'); -const dataFile = "public/data.json"; -const versionFile = "public/version"; -const motdFile = "public/motd"; +const utils = require('../utils/utils.js') +const dataFile = '../public/data.json' +const versionFile = '../public/version' +const motdFile = '../public/motd' -var p = GetParams(); +var p = GetParams() if (p.length <= 0) { - console.log("no params!"); - process.exit(0); + console.log('no params!') + process.exit(0) } +var param = p.join(' ') +console.log('param: ' + param) +var d = utils.ReadFile(dataFile) +var parsed = JSON.parse(d) +parsed.motd = param +utils.WriteFile(JSON.stringify(parsed), dataFile) -var param = p.join(" "); -console.log("param: " + param); -var d = utils.ReadFile(dataFile); -var parsed = JSON.parse(d); -parsed.motd = param; -utils.WriteFile(JSON.stringify(parsed), dataFile); +utils.WriteFile(parsed.motd, motdFile) -utils.WriteFile(parsed.motd, motdFile); - -function GetParams() { - return process.argv.splice(2); +function GetParams () { + return process.argv.splice(2) }