diff --git a/actions.js b/actions.js index 86531d6..999d202 100755 --- a/actions.js +++ b/actions.js @@ -20,7 +20,8 @@ module.exports = { ProcessIncomingRequest: ProcessIncomingRequest, - CheckData: CheckData + CheckData: CheckData, + NLoad: NLoad }; var recievedFile = "stats/recieved"; diff --git a/merger.js b/merger.js new file mode 100644 index 0000000..cbf1984 --- /dev/null +++ b/merger.js @@ -0,0 +1,320 @@ +/* ---------------------------------------------------------------------------- + + Question Server question file merger + GitLab: + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + + ------------------------------------------------------------------------- */ + +// TODO: handle flags +// join json datas, or raw datas +// or something else + +class Question { + constructor(q, a, i) { + this.Q = q; + this.A = a; + this.I = i; + } + toString() { + var r = "?" + this.Q + "\n!" + this.A; + if (this.I) + r += "\n>" + this.I; + return r; + } + HasQuestion() { + return this.Q != undefined; + } + HasAnswer() { + return this.A != undefined; + } + HasImage() { + return this.I != undefined; + } + IsComplete() { + return this.HasQuestion() && this.HasAnswer(); + } + Compare(q2) { + const qmatchpercent = Question.CompareString(this.Q, q2.Q); + const amatchpercent = Question.CompareString(this.A, q2.A); + if (this.I != undefined) { + const imatchpercent = this.I == undefined ? Question.CompareString(this.I, q2.I) : 0; + return (qmatchpercent + amatchpercent + imatchpercent) / 3; + } else { + return (qmatchpercent + amatchpercent) / 2; + } + } + static CompareString(s1, s2) { + //if (s1 == undefined || s2 == undefined) + // return 0; + s1 = SimplifyStringForComparison(s1).split(" "); + s2 = SimplifyStringForComparison(s2).split(" "); + var match = 0; + for (var i = 0; i < s1.length; i++) + if (s2.includes(s1[i])) + match++; + var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length); + percent -= lengthDifference * 3; + if (percent < 0) + percent = 0; + return percent; + } +} + +class Subject { + constructor(n) { + this.Name = n; + this.Questions = []; + } + get length() { + return this.Questions.length; + } + AddQuestion(q) { + this.Questions.push(q); + } + toString() { + var r = []; + for (var i = 0; i < this.Questions.length; i++) + r.push(this.Questions[i].toString()); + return "+" + this.Name + "\n" + r.join("\n"); + } +} + +class QuestionDB { + constructor() { + this.Subjects = []; + } + get length() { + return this.Subjects.length; + } + AddQuestion(subj, q) { + var i = 0; + while (i < this.Subjects.length && this.Subjects[i].Name != subj) + i++; + if (i < this.Subjects.length) + this.Subjects[i].AddQuestion(q); + else { + const n = new Subject(subj); + n.AddQuestion(q); + this.Subjects.push(n); + } + } + AddSubject(subj) { + var i = 0; + while (i < this.length && subj.Name != this.Subjects[i].Name) + i++; + + if (i < this.length) { + this.Subjects.concat(subj.Questions); + } else { + this.Subjects.push(subj); + } + } + toString() { + var r = []; + for (var i = 0; i < this.Subjects.length; i++) + r.push(this.Subjects[i].toString()); + return r.join("\n\n"); + } +} +var utils = require('./utils.js'); +var actions = require('./actions.js'); + +Main(); + +function Main() { + const params = GetParams(); + console.log(params); + var dbs = []; + + for (var i = 0; i < params.length; i++) + dbs.push(ReadData(utils.ReadFile(params[i])).result); + + var db = MergeDatabases(dbs); + + var r = RemoveDuplicates(db); + + for (var i = 0; i < r.length; i++) { + console.log(r.Subjects[i].Name + ": " + r.Subjects[i].length + " darab kérdés."); + } + + utils.WriteFile(JSON.stringify(r), "newData"); + console.log("File written!"); +} + +function GetParams() { + return process.argv.splice(2); +} + +function MergeDatabases(dbs) { + var db = new QuestionDB(); + for (var i = 0; i < dbs.length; i++) + for (var j = 0; j < dbs[i].length; j++) + db.AddSubject(dbs[i].Subjects[j]); + return db; +} + +/* + * Returns a question database from the given data. + * Parameter should be raw read file in string with "\n"-s + * TODO: ??? -s are not listed as errors, tho works correctly + * */ +function ReadData(data) { + + const d = data.split("\n"); + const r = new QuestionDB(); + var logs = []; + var currSubj = ""; // the current subjects name + var ExpectedIdentifier = ['+', '?']; + let currQuestion = new Question(); + + var i = -1; + while (i < d.length) { + let currIdentifier; + let skipped = 0; + do { + if (skipped >= 1) + logs.push(i + ": " + d[i]); + i++; + if (i >= d.length) { + if (currQuestion.IsComplete()) + r.AddQuestion(currSubj, currQuestion); + return { + result: r, + logs: logs + }; + } + currIdentifier = d[i][0]; + skipped++; + } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length); + + let currData = d[i].substring(1).trim(); + + if (currIdentifier == '+') { + if (currQuestion.IsComplete()) + r.AddQuestion(currSubj, currQuestion); + currQuestion = new Question(); + currSubj = currData; + ExpectedIdentifier = ['?']; + continue; + } + + if (currIdentifier == '?') { + if (currQuestion.IsComplete()) { + r.AddQuestion(currSubj, currQuestion); + currQuestion = new Question(); + } + // overwriting is allowed here, bcus: + // ?????!> + currQuestion.Q = currData; + ExpectedIdentifier = ['!', '?']; + continue; + } + + if (currIdentifier == '!') { + // if dont have question continue + if (!currQuestion.HasQuestion()) + throw "No question! (A)"; + // dont allow overwriting + // ?!!!! + if (!currQuestion.HasAnswer()) { + currData = currData.replace("A helyes válaszok: ", ""); + currData = currData.replace("A helyes válasz: ", ""); + + currQuestion.A = currData; + } + ExpectedIdentifier = ['?', '>', '+']; + continue; + } + + if (currIdentifier == '>') { + // if dont have question or answer continue + if (!currQuestion.HasQuestion()) + throw "No question! (I)"; + if (!currQuestion.HasAnswer()) + throw "No asnwer! (I)"; + // dont allow overwriting + // ?!>>> + if (!currQuestion.HasImage()) { + try { + currQuestion.I = JSON.parse(currData); + } catch (e) { + currQuestion.I = [currData]; + } + } + ExpectedIdentifier = ['?', '+']; + continue; + } + } + + return { + result: r, + logs: logs + }; +} + +function RemoveDuplicates(dataObj) { + for (var i = 0; i < dataObj.length; i++) + RemoveDuplFromSubject(dataObj.Subjects[i]); + return dataObj; +} + +function RemoveDuplFromSubject(subj) { + var cp = subj.Questions; + subj.Questions = []; + for (var i = 0; i < cp.length; i++) { + var j = 0; + while (j < subj.length && cp[i].Compare(subj.Questions[j]) != 100) { + j++; + } + if (j < subj.length) { + //console.log("----------------------------------------------------------"); + //console.log(cp[i].toString()); + //console.log(" VS "); + //console.log(subj.Questions[j].toString()); + //console.log(cp[i].Compare(subj.Questions[j])); + //console.log(j); + //console.log("removed:"); + //console.log(subj.Questions.splice(j, 1).toString()); + //console.log("----------------------------------------------------------"); + } else { + subj.AddQuestion(cp[i]); + } + } +} + +function SimplifyStringForComparison(value) { + value = RemoveUnnecesarySpaces(value).toLowerCase(); + var removableChars = [",", ".", ":", "!"]; + for (var i = 0; i < removableChars.length; i++) { + var regex = new RegExp(removableChars[i], "g"); + value.replace(regex, ""); + } + return value; +} + +function RemoveUnnecesarySpaces(toremove) { + toremove = NormalizeSpaces(toremove); + while (toremove.includes(" ")) // while the text includes double spaces replaces all of them with a single one + { + toremove = toremove.replace(/ /g, " "); + } + return toremove.trim(); +} + +function NormalizeSpaces(input) { + return input.replace(/\s/g, ' '); +}