From f9361895f1cf365ca696351bccc5a2dd041ce6bb Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Tue, 11 Dec 2018 19:56:00 +0100 Subject: [PATCH 1/6] Started merger.js, which simplifies, merges and removes duplicates from question data --- actions.js | 3 +- merger.js | 180 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 merger.js diff --git a/actions.js b/actions.js index 2420384..555e487 100755 --- a/actions.js +++ b/actions.js @@ -20,7 +20,8 @@ module.exports = { ProcessIncomingRequest: ProcessIncomingRequest, - CheckData: CheckData + CheckData: CheckData, + NLoad: NLoad }; var recievedFile = "stats/recieved"; diff --git a/merger.js b/merger.js new file mode 100644 index 0000000..0e3c6fe --- /dev/null +++ b/merger.js @@ -0,0 +1,180 @@ +/* ---------------------------------------------------------------------------- + + Question Server question file merger + GitLab: + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + + ------------------------------------------------------------------------- */ + +var utils = require('./utils.js'); +var actions = require('./actions.js'); + +Main(); + +function Main() { + const params = GetParams(); + console.log(params); + var oldData = ""; + var sum = 0; + for (var i = 0; i < params.length; i++) { + const data = utils.ReadFile(params[i]); + var r = actions.NLoad(data); + if (r.count > 0) { + sum += r.count; + console.log(params[i] + ": " + r.count); + + oldData += "\n" + data; + } else + console.log("error loading: " + params[i]); + } + console.log("-------------"); + console.log("Sum: " + sum); + console.log("Simplifying..."); + var newData = Simplify(oldData); + var nr = actions.NLoad(newData); + console.log("New result: " + nr.count); + utils.WriteFile(newData, "newData"); + console.log("File written!"); +} + +function GetParams() { + return process.argv.splice(2); +} + +/*Question object + * q: question + * a: answer + * i: image + * */ +function Question(q, a, i) { + this.q = q; + this.a = a; + this.i = i; + this.toString = function() { + var r = "?" + this.q + "\n!" + a; + if (i) + r += "\n>" + i; + return r; + }; + this.Compare = function(q2) { + const qmatchpercent = CompareString(this.q, q2.q); + const amatchpercent = CompareString(this.a, q2.a); + const imatchpercent = i ? CompareString(this.i, q2.i) : 0; + + return (qmatchpercent + amatchpercent + imatchpercent) / 3; + }; + const CompareString = function(s1, s2) { + s1 = SimplifyStringForComparison(s1).split(" "); + s2 = SimplifyStringForComparison(s2).split(" "); + var match = 0; + for (var i = 0; i < s1.length; i++) { + if (s2.includes(s1[i])) { + match++; + } + } + var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length); + percent -= lengthDifference * 3; + if (percent < 0) + percent = 0; + return percent; + }; +} + +function Subject(n) { + this.Name = n; + this.Questions = []; + this.AddQuestion = function(q) { + this.Questions.push(q); + }; +} + +function QuestionDB () { + this.Subjects = []; + this.AddQuestion = function(subj, q) { + + + }; +} + +/* + * Returns a question database from the given data. + * Parameter should be raw read file in string + * */ +function ReadData(data) { + + function AddQuestion(q) { + + } + var d = data.split("\n"); + var result = + + + /* + * sliding window + * check if its a ? + * then searches next ! + * then checks if next is ? or > + * adds to different subj on + + * + * ERROR HANDLING: + * skips unexpected start characters + * resets curr question if there are multiple ?-s after each other + * + * DOES NOT COMPARES + * */ + +} + +function SortQuestions(dataObj) { + /* + * Sort questions by something + * */ +} + +function RemoveDuplicates(dataObj) { + /* + * removes duplicates from a read data by comparing every question after sorting + * */ +} + +function QuestionDbToString(questionObj) { + /* + * Converst a question db object to string for writing out + * */ +} + +function SimplifyStringForComparison(value) { + value = RemoveUnnecesarySpaces(value).toLowerCase(); + var removableChars = [",", ".", ":", "!"]; + for (var i = 0; i < removableChars.length; i++) { + var regex = new RegExp(removableChars[i], "g"); + value.replace(regex, ""); + } + return value; +} + +function RemoveUnnecesarySpaces(toremove) { + toremove = NormalizeSpaces(toremove); + while (toremove.includes(" ")) // while the text includes double spaces replaces all of them with a single one + { + toremove = toremove.replace(/ /g, " "); + } + return toremove.trim(); +} + +function NormalizeSpaces(input) { + return input.replace(/\s/g, ' '); +} From 848a9011ce600e9a472f0859f0a966e80d6774a7 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Tue, 11 Dec 2018 21:27:51 +0100 Subject: [PATCH 2/6] Merger reading --- merger.js | 181 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 132 insertions(+), 49 deletions(-) diff --git a/merger.js b/merger.js index 0e3c6fe..a02b3b4 100644 --- a/merger.js +++ b/merger.js @@ -28,16 +28,15 @@ function Main() { console.log(params); var oldData = ""; var sum = 0; - for (var i = 0; i < params.length; i++) { - const data = utils.ReadFile(params[i]); - var r = actions.NLoad(data); - if (r.count > 0) { - sum += r.count; - console.log(params[i] + ": " + r.count); - oldData += "\n" + data; - } else - console.log("error loading: " + params[i]); + var r = ReadData(utils.ReadFile(params[0])); + console.log(r.result.toString()); + console.log("\n\n"); + console.log("errors:"); + console.log(r.logs.join("\n")); + + /*for (var i = 0; i < params.length; i++) { + const data = utils.ReadFile(params[i]); } console.log("-------------"); console.log("Sum: " + sum); @@ -46,7 +45,7 @@ function Main() { var nr = actions.NLoad(newData); console.log("New result: " + nr.count); utils.WriteFile(newData, "newData"); - console.log("File written!"); + console.log("File written!");*/ } function GetParams() { @@ -59,19 +58,31 @@ function GetParams() { * i: image * */ function Question(q, a, i) { - this.q = q; - this.a = a; - this.i = i; + this.Q = q; + this.A = a; + this.I = i; this.toString = function() { - var r = "?" + this.q + "\n!" + a; - if (i) - r += "\n>" + i; + var r = "?" + this.Q + "\n!" + this.A; + if (this.I) + r += "\n>" + this.I; return r; }; + this.HasQuestion = function() { + return this.Q != undefined; + }; + this.HasAnswer = function() { + return this.A != undefined; + }; + this.HasImage = function() { + return this.I != undefined; + }; + this.IsComplete = function() { + return this.HasQuestion() && this.HasAnswer(); + }; this.Compare = function(q2) { - const qmatchpercent = CompareString(this.q, q2.q); - const amatchpercent = CompareString(this.a, q2.a); - const imatchpercent = i ? CompareString(this.i, q2.i) : 0; + const qmatchpercent = CompareString(this.Q, q2.Q); + const amatchpercent = CompareString(this.A, q2.A); + const imatchpercent = I ? CompareString(this.I, q2.I) : 0; return (qmatchpercent + amatchpercent + imatchpercent) / 3; }; @@ -99,43 +110,120 @@ function Subject(n) { this.AddQuestion = function(q) { this.Questions.push(q); }; + this.toString = function() { + var r = []; + for (var i = 0; i < this.Questions.length; i++) + r.push(this.Questions[i].toString()); + return "+" + this.Name + "\n" + r.join("\n"); + }; } -function QuestionDB () { +function QuestionDB() { this.Subjects = []; this.AddQuestion = function(subj, q) { - - + var i = 0; + while (i < this.Subjects.length && this.Subjects[i].Name != subj) + i++; + if (i < this.Subjects.length) + this.Subjects[i].AddQuestion(q); + else { + const n = new Subject(subj); + n.AddQuestion(q); + this.Subjects.push(n); + } + }; + this.toString = function() { + var r = []; + for (var i = 0; i < this.Subjects.length; i++) + r.push(this.Subjects[i].toString()); + return r.join("\n\n"); }; } /* - * Returns a question database from the given data. - * Parameter should be raw read file in string - * */ + * Returns a question database from the given data. + * Parameter should be raw read file in string with "\n"-s + * TODO: ??? -s are not listed as errors, tho works correctly + * */ function ReadData(data) { - function AddQuestion(q) { + const d = data.split("\n"); + const r = new QuestionDB(); + var logs = []; + var currSubj = ""; // the current subjects name + var ExpectedIdentifier = ['+', '?']; + let currQuestion = new Question(); + var i = -1; + while (i < d.length) { + let currIdentifier; + let skipped = 0; + do { + if (skipped >= 1) + logs.push(i + ": " + d[i]); + i++; + if (i >= d.length) + return { + result: r, + logs: logs + }; + currIdentifier = d[i][0]; + skipped++; + } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length); + + let currData = d[i].substring(1); + + if (currIdentifier == '+') { + currSubj = currData; + ExpectedIdentifier = ['?']; + continue; + } + + if (currIdentifier == '?') { + if (currQuestion.IsComplete()) { + r.AddQuestion(currSubj, currQuestion); + currQuestion = new Question(); + } + // overwriting is allowed here, bcus: + // ?????!> + currQuestion.Q = currData; + ExpectedIdentifier = ['!', '?']; + continue; + } + + if (currIdentifier == '!') { + // if dont have question continue + if (!currQuestion.HasQuestion()) + throw "No question! (A)"; + // dont allow overwriting + // ?!!!! + if (!currQuestion.HasAnswer()) { + currQuestion.A = currData; + } + ExpectedIdentifier = ['?', '>', '+']; + continue; + } + + if (currIdentifier == '>') { + // if dont have question or answer continue + if (!currQuestion.HasQuestion()) + throw "No question! (I)"; + if (!currQuestion.HasAnswer()) + throw "No asnwer! (I)"; + // dont allow overwriting + // ?!>>> + if (!currQuestion.HasImage()) { + currQuestion.I = currData; + } + ExpectedIdentifier = ['?', '+']; + continue; + } } - var d = data.split("\n"); - var result = - - - /* - * sliding window - * check if its a ? - * then searches next ! - * then checks if next is ? or > - * adds to different subj on + - * - * ERROR HANDLING: - * skips unexpected start characters - * resets curr question if there are multiple ?-s after each other - * - * DOES NOT COMPARES - * */ + return { + result: r, + logs: logs + }; } function SortQuestions(dataObj) { @@ -145,17 +233,12 @@ function SortQuestions(dataObj) { } function RemoveDuplicates(dataObj) { + // TODO: compare and delete /* * removes duplicates from a read data by comparing every question after sorting * */ } -function QuestionDbToString(questionObj) { - /* - * Converst a question db object to string for writing out - * */ -} - function SimplifyStringForComparison(value) { value = RemoveUnnecesarySpaces(value).toLowerCase(); var removableChars = [",", ".", ":", "!"]; From 147f66031997a8867ad714608d3d8da102096e49 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Wed, 12 Dec 2018 18:10:58 +0100 Subject: [PATCH 3/6] Merger finnish --- merger.js | 221 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 136 insertions(+), 85 deletions(-) diff --git a/merger.js b/merger.js index a02b3b4..0dfedc6 100644 --- a/merger.js +++ b/merger.js @@ -18,109 +18,90 @@ ------------------------------------------------------------------------- */ -var utils = require('./utils.js'); -var actions = require('./actions.js'); - -Main(); - -function Main() { - const params = GetParams(); - console.log(params); - var oldData = ""; - var sum = 0; - - var r = ReadData(utils.ReadFile(params[0])); - console.log(r.result.toString()); - console.log("\n\n"); - console.log("errors:"); - console.log(r.logs.join("\n")); - - /*for (var i = 0; i < params.length; i++) { - const data = utils.ReadFile(params[i]); - } - console.log("-------------"); - console.log("Sum: " + sum); - console.log("Simplifying..."); - var newData = Simplify(oldData); - var nr = actions.NLoad(newData); - console.log("New result: " + nr.count); - utils.WriteFile(newData, "newData"); - console.log("File written!");*/ -} - -function GetParams() { - return process.argv.splice(2); -} - /*Question object * q: question * a: answer * i: image * */ -function Question(q, a, i) { - this.Q = q; - this.A = a; - this.I = i; - this.toString = function() { +class Question { + constructor(q, a, i) { + this.Q = q; + this.A = a; + this.I = i; + } + toString() { var r = "?" + this.Q + "\n!" + this.A; if (this.I) r += "\n>" + this.I; return r; - }; - this.HasQuestion = function() { + } + HasQuestion() { return this.Q != undefined; - }; - this.HasAnswer = function() { + } + HasAnswer() { return this.A != undefined; - }; - this.HasImage = function() { + } + HasImage() { return this.I != undefined; - }; - this.IsComplete = function() { + } + IsComplete() { return this.HasQuestion() && this.HasAnswer(); - }; - this.Compare = function(q2) { - const qmatchpercent = CompareString(this.Q, q2.Q); - const amatchpercent = CompareString(this.A, q2.A); - const imatchpercent = I ? CompareString(this.I, q2.I) : 0; - - return (qmatchpercent + amatchpercent + imatchpercent) / 3; - }; - const CompareString = function(s1, s2) { + } + Compare(q2) { + const qmatchpercent = Question.CompareString(this.Q, q2.Q); + const amatchpercent = Question.CompareString(this.A, q2.A); + if (this.I != undefined) { + const imatchpercent = this.I == undefined ? Question.CompareString(this.I, q2.I) : 0; + return (qmatchpercent + amatchpercent + imatchpercent) / 3; + } else { + return (qmatchpercent + amatchpercent) / 2; + } + } + static CompareString(s1, s2) { + //if (s1 == undefined || s2 == undefined) + // return 0; s1 = SimplifyStringForComparison(s1).split(" "); s2 = SimplifyStringForComparison(s2).split(" "); var match = 0; - for (var i = 0; i < s1.length; i++) { - if (s2.includes(s1[i])) { + for (var i = 0; i < s1.length; i++) + if (s2.includes(s1[i])) match++; - } - } var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent var lengthDifference = Math.abs(s2.length - s1.length); percent -= lengthDifference * 3; if (percent < 0) percent = 0; return percent; - }; + } } -function Subject(n) { - this.Name = n; - this.Questions = []; - this.AddQuestion = function(q) { +class Subject { + constructor(n) { + this.Name = n; + this.Questions = []; + } + get length() { + return this.Questions.length; + } + AddQuestion(q) { this.Questions.push(q); - }; - this.toString = function() { + } + toString() { var r = []; for (var i = 0; i < this.Questions.length; i++) r.push(this.Questions[i].toString()); return "+" + this.Name + "\n" + r.join("\n"); - }; + } } -function QuestionDB() { - this.Subjects = []; - this.AddQuestion = function(subj, q) { +class QuestionDB { + constructor() { + this.Subjects = []; + } + get length() { + return this.Subjects.length; + } + AddQuestion(subj, q) { var i = 0; while (i < this.Subjects.length && this.Subjects[i].Name != subj) i++; @@ -131,13 +112,60 @@ function QuestionDB() { n.AddQuestion(q); this.Subjects.push(n); } - }; - this.toString = function() { + } + AddSubject(subj) { + var i = 0; + while (i < this.length && subj.Name != this.Subjects[i].Name) + i++; + + if (i < this.length) { + this.Subjects.concat(subj.Questions); + } else { + this.Subjects.push(subj); + } + } + toString() { var r = []; for (var i = 0; i < this.Subjects.length; i++) r.push(this.Subjects[i].toString()); return r.join("\n\n"); - }; + } +} +var utils = require('./utils.js'); +var actions = require('./actions.js'); + +Main(); + +function Main() { + const params = GetParams(); + console.log(params); + var dbs = []; + + for (var i = 0; i < params.length; i++) + dbs.push(ReadData(utils.ReadFile(params[i])).result); + + var db = MergeDatabases(dbs); + + var r = RemoveDuplicates(db); + + for (var i = 0; i < r.length; i++) { + console.log(r.Subjects[i].Name + ": " + r.Subjects[i].length + " darab kérdés."); + } + + utils.WriteFile(JSON.stringify(r), "newData"); + console.log("File written!"); +} + +function GetParams() { + return process.argv.splice(2); +} + +function MergeDatabases(dbs) { + var db = new QuestionDB(); + for (var i = 0; i < dbs.length; i++) + for (var j = 0; j < dbs[i].length; j++) + db.AddSubject(dbs[i].Subjects[j]); + return db; } /* @@ -162,18 +190,24 @@ function ReadData(data) { if (skipped >= 1) logs.push(i + ": " + d[i]); i++; - if (i >= d.length) + if (i >= d.length) { + if (currQuestion.IsComplete()) + r.AddQuestion(currSubj, currQuestion); return { result: r, logs: logs }; + } currIdentifier = d[i][0]; skipped++; } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length); - let currData = d[i].substring(1); + let currData = d[i].substring(1).trim(); if (currIdentifier == '+') { + if (currQuestion.IsComplete()) + r.AddQuestion(currSubj, currQuestion); + currQuestion = new Question(); currSubj = currData; ExpectedIdentifier = ['?']; continue; @@ -226,17 +260,34 @@ function ReadData(data) { }; } -function SortQuestions(dataObj) { - /* - * Sort questions by something - * */ +function RemoveDuplicates(dataObj) { + for (var i = 0; i < dataObj.length; i++) + RemoveDuplFromSubject(dataObj.Subjects[i]); + return dataObj; } -function RemoveDuplicates(dataObj) { - // TODO: compare and delete - /* - * removes duplicates from a read data by comparing every question after sorting - * */ +function RemoveDuplFromSubject(subj) { + var cp = subj.Questions; + subj.Questions = []; + for (var i = 0; i < cp.length; i++) { + var j = 0; + while (j < subj.length && cp[i].Compare(subj.Questions[j]) != 100) { + j++; + } + if (j < subj.length) { + //console.log("----------------------------------------------------------"); + //console.log(cp[i].toString()); + //console.log(" VS "); + //console.log(subj.Questions[j].toString()); + //console.log(cp[i].Compare(subj.Questions[j])); + //console.log(j); + //console.log("removed:"); + //console.log(subj.Questions.splice(j, 1).toString()); + //console.log("----------------------------------------------------------"); + } else { + subj.AddQuestion(cp[i]); + } + } } function SimplifyStringForComparison(value) { From 1c5d24a3266caad5a88b99d9231a5b47e1746c1c Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Thu, 13 Dec 2018 09:04:32 +0100 Subject: [PATCH 4/6] Merger removing unnecesarry parts of answers --- merger.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/merger.js b/merger.js index 0dfedc6..1a4cec1 100644 --- a/merger.js +++ b/merger.js @@ -18,11 +18,10 @@ ------------------------------------------------------------------------- */ -/*Question object - * q: question - * a: answer - * i: image - * */ +// TODO: handle flags +// join json datas, or raw datas +// or something else + class Question { constructor(q, a, i) { this.Q = q; @@ -232,6 +231,9 @@ function ReadData(data) { // dont allow overwriting // ?!!!! if (!currQuestion.HasAnswer()) { + currData = currData.replace("A helyes válaszok: ", ""); + currData = currData.replace("A helyes válasz: ", ""); + currQuestion.A = currData; } ExpectedIdentifier = ['?', '>', '+']; From 98857436bd3ac9d680b2f6e6887dc205a0587248 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Fri, 14 Dec 2018 09:34:33 +0100 Subject: [PATCH 5/6] Now correctly using image data for new data format --- merger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merger.js b/merger.js index 1a4cec1..4360724 100644 --- a/merger.js +++ b/merger.js @@ -249,7 +249,7 @@ function ReadData(data) { // dont allow overwriting // ?!>>> if (!currQuestion.HasImage()) { - currQuestion.I = currData; + currQuestion.I = JSON.parse(currData); } ExpectedIdentifier = ['?', '+']; continue; From 989b761b0dfcbbe914b704a5af8a9a5586bc3f81 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Fri, 14 Dec 2018 17:57:51 +0100 Subject: [PATCH 6/6] Fixed invalid image data handling --- merger.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/merger.js b/merger.js index 4360724..cbf1984 100644 --- a/merger.js +++ b/merger.js @@ -249,7 +249,11 @@ function ReadData(data) { // dont allow overwriting // ?!>>> if (!currQuestion.HasImage()) { - currQuestion.I = JSON.parse(currData); + try { + currQuestion.I = JSON.parse(currData); + } catch (e) { + currQuestion.I = [currData]; + } } ExpectedIdentifier = ['?', '+']; continue;