/* ---------------------------------------------------------------------------- Question Server question file merger GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ // TODO: handle flags // join json datas, or raw datas // or something else class Question { constructor(q, a, i) { this.Q = q; this.A = a; this.I = i; } toString() { var r = "?" + this.Q + "\n!" + this.A; if (this.I) r += "\n>" + this.I; return r; } HasQuestion() { return this.Q != undefined; } HasAnswer() { return this.A != undefined; } HasImage() { return this.I != undefined; } IsComplete() { return this.HasQuestion() && this.HasAnswer(); } Compare(q2) { const qmatchpercent = Question.CompareString(this.Q, q2.Q); const amatchpercent = Question.CompareString(this.A, q2.A); if (this.I != undefined) { const imatchpercent = this.I == undefined ? Question.CompareString(this.I, q2.I) : 0; return (qmatchpercent + amatchpercent + imatchpercent) / 3; } else { return (qmatchpercent + amatchpercent) / 2; } } static CompareString(s1, s2) { //if (s1 == undefined || s2 == undefined) // return 0; s1 = SimplifyStringForComparison(s1).split(" "); s2 = SimplifyStringForComparison(s2).split(" "); var match = 0; for (var i = 0; i < s1.length; i++) if (s2.includes(s1[i])) match++; var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent var lengthDifference = Math.abs(s2.length - s1.length); percent -= lengthDifference * 3; if (percent < 0) percent = 0; return percent; } } class Subject { constructor(n) { this.Name = n; this.Questions = []; } get length() { return this.Questions.length; } AddQuestion(q) { this.Questions.push(q); } toString() { var r = []; for (var i = 0; i < this.Questions.length; i++) r.push(this.Questions[i].toString()); return "+" + this.Name + "\n" + r.join("\n"); } } class QuestionDB { constructor() { this.Subjects = []; } get length() { return this.Subjects.length; } AddQuestion(subj, q) { var i = 0; while (i < this.Subjects.length && this.Subjects[i].Name != subj) i++; if (i < this.Subjects.length) this.Subjects[i].AddQuestion(q); else { const n = new Subject(subj); n.AddQuestion(q); this.Subjects.push(n); } } AddSubject(subj) { var i = 0; while (i < this.length && subj.Name != this.Subjects[i].Name) i++; if (i < this.length) { this.Subjects.concat(subj.Questions); } else { this.Subjects.push(subj); } } toString() { var r = []; for (var i = 0; i < this.Subjects.length; i++) r.push(this.Subjects[i].toString()); return r.join("\n\n"); } } var utils = require('./utils.js'); var actions = require('./actions.js'); Main(); function Main() { console.clear(); const params = GetParams(); console.log(params); var dbs = []; for (var i = 0; i < params.length; i++) { PrintLN(); console.log(params[i] + ": "); try { dbs.push(ParseJSONData(utils.ReadFile(params[i]))); console.log("JSON data added"); } catch (e) { console.log(e); console.log("Trying with old format..."); dbs.push(ReadData(utils.ReadFile(params[i])).result); } } PrintLN(); dbs.forEach((item) => { PrintDB(item); }); var olds = []; if (dbs.length == 1) { for ( let i = 0; i < dbs[0].length; i++) olds.push(dbs[0].Subjects[i].length); } console.log("Parsed data count: " + dbs.length); PrintLN(); console.log("Merging databases..."); var db = MergeDatabases(dbs); console.log("Removing duplicates..."); var r = RemoveDuplicates(db); console.log("RESULT:"); PrintDB(r, olds); utils.WriteFile(JSON.stringify(r), "newData"); console.log("File written!"); } function PrintLN() { console.log("------------------------------------------------------"); } function PrintDB(r, olds) { console.log("Data subject count: " + r.length); var maxLength = 0; for (var i = 0; i < r.length; i++) { if (maxLength < r.Subjects[i].Name.length) maxLength = r.Subjects[i].Name.length; } for (var i = 0; i < r.length; i++) { let line = i; if (line < 10) line += ' '; line += ": "; var currLength = line.length + maxLength + 4; line += r.Subjects[i].Name; while (line.length < currLength) { if (i % 4 == 0) line += "."; else line += " "; } if (olds && olds.length > 0) { // TODO: check if correct row! should be now, but well... if (olds[i] < 10) line += " "; if (olds[i] < 100) line += " "; line += olds[i]; line += " -> "; } if (r.Subjects[i].length < 10) line += " "; if (r.Subjects[i].length < 100) line += " "; line += r.Subjects[i].length; line += " db"; console.log(line); } PrintLN(); } function GetParams() { return process.argv.splice(2); } function ParseJSONData(data) { var d = JSON.parse(data); var r = new QuestionDB(); var rt = []; for (var i = 0; i < d.Subjects.length; i++) { let s = new Subject(d.Subjects[i].Name); var j = 0; for (j = 0; j < d.Subjects[i].Questions.length; j++) { var currQ = d.Subjects[i].Questions[j]; s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I)); } rt.push({ name: d.Subjects[i].Name, count: j }); r.AddSubject(s); } return r; } function MergeDatabases(dbs) { var db = new QuestionDB(); for (var i = 0; i < dbs.length; i++) for (var j = 0; j < dbs[i].length; j++) db.AddSubject(dbs[i].Subjects[j]); return db; } /* * Returns a question database from the given data. * Parameter should be raw read file in string with "\n"-s * TODO: ??? -s are not listed as errors, tho works correctly * */ function ReadData(data) { const d = data.split("\n"); const r = new QuestionDB(); var logs = []; var currSubj = ""; // the current subjects name var ExpectedIdentifier = ['+', '?']; let currQuestion = new Question(); var i = -1; while (i < d.length) { let currIdentifier; let skipped = 0; do { if (skipped >= 1) logs.push(i + ": " + d[i]); i++; if (i >= d.length) { if (currQuestion.IsComplete()) r.AddQuestion(currSubj, currQuestion); return { result: r, logs: logs }; } currIdentifier = d[i][0]; skipped++; } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length); let currData = d[i].substring(1).trim(); if (currIdentifier == '+') { if (currQuestion.IsComplete()) r.AddQuestion(currSubj, currQuestion); currQuestion = new Question(); currSubj = currData; ExpectedIdentifier = ['?']; continue; } if (currIdentifier == '?') { if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion); currQuestion = new Question(); } // overwriting is allowed here, bcus: // ?????!> currQuestion.Q = currData; ExpectedIdentifier = ['!', '?']; continue; } if (currIdentifier == '!') { // if dont have question continue if (!currQuestion.HasQuestion()) throw "No question! (A)"; // dont allow overwriting // ?!!!! if (!currQuestion.HasAnswer()) { currData = currData.replace("A helyes válaszok: ", ""); currData = currData.replace("A helyes válasz: ", ""); currQuestion.A = currData; } ExpectedIdentifier = ['?', '>', '+']; continue; } if (currIdentifier == '>') { // if dont have question or answer continue if (!currQuestion.HasQuestion()) throw "No question! (I)"; if (!currQuestion.HasAnswer()) throw "No asnwer! (I)"; // dont allow overwriting // ?!>>> if (!currQuestion.HasImage()) { try { currQuestion.I = JSON.parse(currData); } catch (e) { currQuestion.I = [currData]; } } ExpectedIdentifier = ['?', '+']; continue; } } return { result: r, logs: logs }; } function RemoveDuplicates(dataObj) { for (var i = 0; i < dataObj.length; i++) RemoveDuplFromSubject(dataObj.Subjects[i]); return dataObj; } function RemoveDuplFromSubject(subj) { var cp = subj.Questions; subj.Questions = []; for (var i = 0; i < cp.length; i++) { var j = 0; // Only removes 100% match! while (j < subj.length && cp[i].Compare(subj.Questions[j]) != 100) { j++; } if (j < subj.length) { //console.log("----------------------------------------------------------"); //console.log(cp[i].toString()); //console.log(" VS "); //console.log(subj.Questions[j].toString()); //console.log(cp[i].Compare(subj.Questions[j])); //console.log(j); //console.log("removed:"); //console.log(subj.Questions.splice(j, 1).toString()); //console.log("----------------------------------------------------------"); } else { subj.AddQuestion(cp[i]); } } } function SimplifyStringForComparison(value) { value = RemoveUnnecesarySpaces(value).toLowerCase(); var removableChars = [",", ".", ":", "!"]; for (var i = 0; i < removableChars.length; i++) { var regex = new RegExp(removableChars[i], "g"); value.replace(regex, ""); } return value; } function RemoveUnnecesarySpaces(toremove) { toremove = NormalizeSpaces(toremove); while (toremove.includes(" ")) // while the text includes double spaces replaces all of them with a single one { toremove = toremove.replace(/ /g, " "); } return toremove.trim(); } function NormalizeSpaces(input) { return input.replace(/\s/g, ' '); }