Moving stuff around

utils/actions.js

@@ -0,0 +1,533 @@
+/* ----------------------------------------------------------------------------    
+Question Server
+ GitLab: <https://gitlab.com/YourFriendlyNeighborhoodDealer/question-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+module.exports = {
+	ProcessIncomingRequest: ProcessIncomingRequest,
+	CheckData: CheckData,
+	NLoad: NLoad,
+	LoadJSON: LoadJSON,
+	ProcessQA: ProcessQA
+};
+
+var staticFile = "public/data/static";
+var manFile = "public/man.html";
+var dataFile = "public/data.json";
+const recDataFile = "stats/recdata";
+const versionFile = "public/version";
+const motdFile = "public/motd";
+const qaFile = "public/qa";
+
+var logger = require('./logger.js');
+var utils = require('./utils.js');
+
+class Question {
+	constructor(q, a, i) {
+		this.Q = q;
+		this.A = a;
+		this.I = i;
+	}
+	toString() {
+		var r = "?" + this.Q + "\n!" + this.A;
+		if (this.I)
+			r += "\n>" + this.I;
+		return r;
+	}
+	HasQuestion() {
+		return this.Q != undefined;
+	}
+	HasAnswer() {
+		return this.A != undefined;
+	}
+	HasImage() {
+		return this.I != undefined;
+	}
+	IsComplete() {
+		return this.HasQuestion() && this.HasAnswer();
+	}
+	Compare(q2, i) {
+		if (typeof q2 == 'string') {
+			var qmatchpercent = Question.CompareString(this.Q, q2);
+
+			if (i == undefined || i.length == 0)
+				return qmatchpercent;
+			else {
+				if (this.HasImage()) {
+					const imatchpercent = this.HasImage() ? Question.CompareString(this.I.join(" "), i.join(" ")) :
+						0;
+					return (qmatchpercent + imatchpercent) / 2;
+				} else {
+					qmatchpercent -= 30;
+					if (qmatchpercent < 0)
+						return 0;
+					else
+						return qmatchpercent;
+				}
+			}
+		} else {
+			const qmatchpercent = Question.CompareString(this.Q, q2.Q);
+			const amatchpercent = Question.CompareString(this.A, q2.A);
+			if (this.I != undefined) {
+				const imatchpercent = this.I == undefined ? Question.CompareString(this.I.join(" "), q2.I.join(
+					" ")) : 0;
+				return (qmatchpercent + amatchpercent + imatchpercent) / 3;
+			} else {
+				return (qmatchpercent + amatchpercent) / 2;
+			}
+		}
+	}
+	static CompareString(s1, s2) {
+		//if (s1 == undefined || s2 == undefined)
+		//	return 0;
+		s1 = SimplifyStringForComparison(s1).split(" ");
+		s2 = SimplifyStringForComparison(s2).split(" ");
+		var match = 0;
+		for (var i = 0; i < s1.length; i++)
+			if (s2.includes(s1[i]))
+				match++;
+		var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent
+		var lengthDifference = Math.abs(s2.length - s1.length);
+		percent -= lengthDifference * 3;
+		if (percent < 0)
+			percent = 0;
+		return percent;
+	}
+}
+
+class Subject {
+	constructor(n) {
+		this.Name = n;
+		this.Questions = [];
+	}
+	get length() {
+		return this.Questions.length;
+	}
+	AddQuestion(q) {
+		this.Questions.push(q);
+	}
+	Search(q, img) {
+		var r = [];
+		for (var i = 0; i < this.length; i++) {
+			let percent = this.Questions[i].Compare(q, img);
+			if (percent > minMatchAmmount)
+				r.push({
+					q: this.Questions[i],
+					match: percent
+				});
+		}
+
+		for (var i = 0; i < r.length; i++)
+			for (var j = i; j < r.length; j++)
+				if (r[i].match < r[j].match) {
+					var tmp = r[i];
+					r[i] = r[j];
+					r[j] = tmp;
+				}
+
+		return r;
+	}
+	toString() {
+		var r = [];
+		for (var i = 0; i < this.Questions.length; i++)
+			r.push(this.Questions[i].toString());
+		return "+" + this.Name + "\n" + r.join("\n");
+	}
+}
+
+class QuestionDB {
+	constructor() {
+		this.Subjects = [];
+	}
+	get length() {
+		return this.Subjects.length;
+	}
+	get activeIndexes() {
+		var r = [];
+		for (var i = 0; i < this.length; i++) {
+			if (GM_getValue("Is" + i + "Active")) {
+				r.push(i);
+			}
+		}
+		return r;
+	}
+	GetIfActive(ind) {
+		return GM_getValue("Is" + ind + "Active");
+	}
+	ChangeActive(i, value) {
+		GM_setValue("Is" + i + "Active", !!value);
+	}
+	AddQuestion(subj, q) {
+		var i = 0;
+		while (i < this.Subjects.length && this.Subjects[i].Name != subj)
+			i++;
+		if (i < this.Subjects.length)
+			this.Subjects[i].AddQuestion(q);
+		else {
+			const n = new Subject(subj);
+			n.AddQuestion(q);
+			this.Subjects.push(n);
+		}
+	}
+	Search(q, img) {
+		var r = [];
+		for (var i = 0; i < this.length; i++)
+			if (this.GetIfActive(i))
+				r = r.concat(this.Subjects[i].Search(q, img));
+
+		for (var i = 0; i < r.length; i++)
+			for (var j = i; j < r.length; j++)
+				if (r[i].match < r[j].match) {
+					var tmp = r[i];
+					r[i] = r[j];
+					r[j] = tmp;
+				}
+
+		return r;
+	}
+	AddSubject(subj) {
+		var i = 0;
+		while (i < this.length && subj.Name != this.Subjects[i].Name)
+			i++;
+
+		if (i < this.length) {
+			this.Subjects.concat(subj.Questions);
+		} else {
+			this.Subjects.push(subj);
+		}
+	}
+	toString() {
+		var r = [];
+		for (var i = 0; i < this.Subjects.length; i++)
+			r.push(this.Subjects[i].toString());
+		return r.join("\n\n");
+	}
+}
+
+
+function Process(d, file) {
+	try {
+		logger.Log("File: " + file);
+		if (d.data.split("\n").length > 1) {
+			var oldFile = utils.ReadFile(file);
+			var newFile = oldFile + "\n";
+			if (d.data[0] == '+')
+				newFile += d.data;
+			else
+				newFile += "+" + d.data;
+
+			var newRes = CheckData(newFile);
+			var oldRes = CheckData(oldFile);
+
+			if (oldRes.count > 0)
+				logger.Log("\t\told public result: " + oldRes.count, logger.GetColor("blue"));
+			else
+				logger.Log("\t\told public NLOD error, " + oldRes.log, logger.GetColor("redbg"), true);
+
+			if (newRes.count > 0)
+				logger.Log("\t\tnew file result: " + newRes.count, logger.GetColor("blue"));
+			else
+				logger.Log("\t\tnew file NLOD error, " + newRes.log, logger.GetColor("redbg"), true);
+
+			utils.WriteFile(newFile, file);
+			logger.Log("\t\tNew data written to: " + file);
+
+			return newRes.count - oldRes.count;
+		} else
+			logger.Log("\t\tNo new data");
+
+
+	} catch (e) {
+		Beep();
+		logger.Log("\tError at processing data! File: " + file, logger.GetColor("redbg"));
+		logger.Log(e.toString(), logger.GetColor("redbg"));
+	}
+	return -1;
+}
+
+function ProcessIncomingRequest(data) {
+
+	if (data == undefined) {
+		logger.Log("\tRecieved data is undefined!", logger.GetColor("redbg"));
+		return;
+	}
+
+	try {
+		let towrite = logger.GetDateString() + "\n";
+		towrite += "------------------------------------------------------------------------------\n";
+		towrite += data
+		towrite += "\n------------------------------------------------------------------------------\n";
+		utils.AppendToFile(towrite, recDataFile);
+	} catch (e) {
+		logger.log("Error writing recieved data.");
+	}
+
+	try {
+		var d = JSON.parse(data);
+		var dfile = utils.ReadFile(dataFile);
+		var data = LoadJSON(dfile);
+		var allQuestions = [];
+		for ( var i = 0; i < d.allData.length; i++){
+			allQuestions.push(new Question(d.allData[i].Q, d.allData[i].A, d.allData[i].I));
+		}
+		var questions = [];
+		for ( var i = 0; i < d.data.length; i++){
+			let q = new Question(d.data[i].Q, d.data[i].A, d.data[i].I);
+			questions.push(q);
+			data.AddQuestion(d.subj, q);
+		}
+		
+		logger.Log("\t" + d.subj);
+		var msg = "All / new count: " + allQuestions.length + " / " + questions.length;
+		if (d.version != undefined)
+			msg += ". Version: " + d.version;
+		var color = logger.GetColor("green");
+
+		try {
+			data.version = utils.ReadFile(versionFile);
+			data.motd = utils.ReadFile(motdFile);
+		} catch (e) {
+			Log("MOTD/Version writing/reading error!");
+		}
+
+		if (data != undefined && d.data.length > 0){
+			utils.WriteBackup();
+			utils.WriteFile(JSON.stringify(data), dataFile);
+			msg += " - Data file written!";
+			var color = logger.GetColor("blue");
+		}
+		logger.Log("\t" + msg, color);
+
+	} catch (e) {
+		logger.Log("Couldnt parse JSON data, trying old format...", logger.GetColor("redbg"));
+		var d = SetupData(data);
+		var qcount = -1;
+		try {
+			var splitted = d.alldata.split("\n");
+			var count = 0;
+			for (var i = 0; i < splitted.length; i++)
+				if (splitted[i][0] == '?')
+					count ++;
+			qcount = count;
+		} catch (e) {console.log("Error :c"); console.log(e);}
+
+		logger.Log("\tProcessing data: " + d.subj + " (" + d.type + "), count: " + qcount, logger.GetColor("green"));
+		if (d.subj == undefined){
+			logger.Log(JSON.stringify(d), logger.GetColor("red"));
+			return;
+		}
+
+		var newItems = 0;
+
+		var newStatItems = Process(d, staticFile);
+
+		PrintNewCount(d, newStatItems, staticFile);
+	}
+}
+
+function PrintNewCount(d, newItems, file) {
+	if (newItems > 0) {
+		var count = 0;
+		var splitted = d.alldata.split("\n");
+		for (var i = 0; i < splitted.length; i++)
+			if (splitted[i].startsWith("?"))
+				count++;
+		logger.Log("\t" + file + " All / New: " + count + " / " + newItems, logger.GetColor("cyan"));
+	}
+}
+
+function SetupData(data) {
+	var pdata = data.split("<#>");
+	if (pdata.length <= 0){
+		logger.Log("Data length is zero !", logger.GetColor("redbg"));
+		throw "No data recieved!";
+	}
+
+	var d = {}; // parsed data
+	for (var i = 0; i < pdata.length; i++) {
+		var td = pdata[i].split("<=>");
+		if (td.length == 2)
+			d[td[0]] = td[1];
+		else {
+			logger.Log("Invalid parameter!", logger.GetColor("redbg"));
+			throw "Invalid parameter recieved!";
+		}
+	}
+	return d;
+}
+
+function CheckData(data) {
+	try {
+		var presult = NLoad(data);
+		return presult;
+	} catch (e) {
+		logger.Log("Load error, " + e.toString(), logger.GetColor("redbg"), true);
+		return {
+			count: -1,
+			log: [e.toString()]
+		};
+	}
+}
+
+function NLoad(resource) {
+	var resultLog = [];
+	var count = -1;
+	var allCount = 0;
+	if (resource == undefined)
+		throw "A megadott adat undefined!";
+	resource = resource.split("\n"); // splitting by enters
+	if (resource.length == 1)
+		throw "A megadott adat nem sorokból áll!";
+	var data = []; // initializing data declared at the begining
+
+	function AddNewSubj(name) // adds a new subject to the data
+	{
+		data.push({
+			"questions": []
+		}); // ads aa new object, with an empty array in it
+		GetCurrSubj().name = name; // sets the name for it
+		GetCurrSubj().count = 0; // setting count to default
+		// setting if its active only if its undefined, otherwise previous user setting shouldt be overwritten
+	}
+
+	function AddItem() // adds an item to the last subjects questions
+	{
+		GetCurrSubj().count++;
+		allCount++; // incrementing all count
+		GetCurrSubj().questions.push({}); // adding a new empty object to the last item in the data
+	}
+
+	function GetLastItem() // returns the last item of the last item of data
+	{
+		var q = GetCurrSubj().questions.length; // questions length
+		return GetCurrSubj().questions[q - 1];
+	}
+
+	function GetCurrSubj() {
+		return data[data.length - 1];
+	}
+
+	// ? : question
+	// ! : answer
+	// > : image JSON data
+	// + : subject name
+
+	// checking for name
+	for (var j = 0; j < resource.length; j++) // goes through resources
+	{
+		if (resource[j][0] == '+') // if there is a name identifier
+		{
+			break; // breaks, couse there will be a name
+		}
+		if (resource[j][0] == '?' || resource[j][0] == '!' || resource[j][0] == '>') // if it begins with another identifier:
+		{
+			AddNewSubj("NONAME"); // there is no name (for the first question at least), so setting it noname.
+			break;
+		}
+		// else it does nothing, continues to check
+	}
+	var jumped = 0; // the amount of lines it processed
+	for (var i = 0; i < resource.length; i += jumped) // gouing through the resource
+	{
+		jumped = 0; // resetting it to 0
+		var currRawDataQ = resource[i]; // current question
+		var currRawDataA = resource[i + 1]; // current answer
+		var currRawDataI = resource[i + 2]; // current image
+		if (currRawDataQ != undefined && currRawDataQ[0] == '?' && currRawDataA != undefined &&
+			currRawDataA[0] == '!') // if the current line is ? and the next is ! its a data
+		{
+			AddItem();
+			GetLastItem().q = currRawDataQ.substr(1);
+			GetLastItem().a = currRawDataA.substr(1);
+			jumped += 2;
+			if (currRawDataI != undefined && currRawDataI[0] == '>') {
+				GetLastItem().i = currRawDataI.substr(1);
+				jumped++;
+			}
+		} else if (currRawDataQ[0] == '+') // if its a new subject
+		{
+			AddNewSubj(currRawDataQ.substr(1));
+			jumped++;
+		} else {
+			// this should be invalid question order
+			resultLog.push("Warning @ line " + i + ":" + currRawDataQ + " " + currRawDataA + " " +
+				currRawDataI);
+			jumped++;
+		}
+	} // end of parsing all data
+	return {
+		count: allCount,
+		log: resultLog
+	};
+}
+
+// loading stuff
+function LoadJSON(resource) {
+	var count = -1;
+	try {
+		var	d = JSON.parse(resource);
+		var r = new QuestionDB();
+		var rt = [];
+		var allCount = -1;
+
+		for (var i = 0; i < d.Subjects.length; i++) {
+			let s = new Subject(d.Subjects[i].Name);
+			var j = 0;
+			for (j = 0; j < d.Subjects[i].Questions.length; j++) {
+				var currQ = d.Subjects[i].Questions[j];
+				s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I));
+			}
+			rt.push({
+				name: d.Subjects[i].Name,
+				count: j
+			});
+			allCount += j;
+			r.AddSubject(s);
+		}
+		return r;
+	} catch (e) {
+			logger.Log("Error loading sutff", logger.GetColor("redbg"), true);
+	}
+}
+
+function ProcessQA() {
+	if (!utils.FileExists(qaFile))
+		utils.WriteFile("", qaFile);
+
+	let a = utils.ReadFile(qaFile).split("\n");
+	let r = [];
+	let ind = 0;
+	for (let i = 0; i < a.length; i++) {
+		if ( a[i] == "#")
+			ind ++;
+		else {
+			if (r[ind] == undefined)
+				r[ind] = {};
+
+			if (r[ind].q == undefined) {
+				r[ind].q = a[i];
+			} else {
+				if (r[ind].a == undefined)
+					r[ind].a = [];
+
+				r[ind].a.push(a[i]);
+			}
+		}
+	}
+	
+	return r;
+}

utils/changedataversion.js

@@ -0,0 +1,32 @@
+const utils = require('./utils.js');
+const dataFile = "public/data.json";
+const versionFile = "public/version";
+
+var p = GetParams();
+if (p.length <= 0) {
+	console.log("no params!");
+	process.exit(0);
+}
+
+
+var param = p.join(" ");
+
+console.log("param: " + param);
+
+var d = utils.ReadFile(dataFile);
+var parsed = JSON.parse(d);
+
+console.log("Old version:");
+console.log(parsed.version);
+
+parsed.version = param;
+
+console.log("New version:");
+console.log(parsed.version);
+
+utils.WriteFile(JSON.stringify(parsed), dataFile);
+utils.WriteFile(parsed.version, versionFile);
+
+function GetParams() {
+	return process.argv.splice(2);
+}

utils/logger.js

@@ -0,0 +1,104 @@
+/* ----------------------------------------------------------------------------    
+
+ Question Server
+ GitLab: <https://gitlab.com/YourFriendlyNeighborhoodDealer/question-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+module.exports = {
+	GetDateString: GetDateString,
+	Log: Log,
+	GetColor: GetColor,
+	LogReq: LogReq
+};
+
+const DELIM = "|";
+
+var utils = require('./utils.js');
+const nlogFile = "stats/nlogs";
+const logFile = "/nlogs/nlogs";
+const locLogFile = "stats/logs";
+const allLogFile = "/nlogs/log";
+
+function GetDateString() {
+	var m = new Date();
+	return m.getFullYear() + "/" +
+		("0" + (m.getMonth() + 1)).slice(-2) + "/" +
+		("0" + m.getDate()).slice(-2) + " " +
+		("0" + m.getHours()).slice(-2) + ":" +
+		("0" + m.getMinutes()).slice(-2) + ":" +
+		("0" + m.getSeconds()).slice(-2);
+}
+
+function Log(s, c, b) {
+	if (c != undefined)
+		console.log(c, GetDateString() + DELIM + s);
+	else
+		console.log(GetDateString() + DELIM + s);
+
+	if (b)
+		utils.Beep();
+
+	utils.AppendToFile(GetDateString() + DELIM + s, nlogFile);
+	utils.AppendToFile(GetDateString() + DELIM + s, logFile);
+}
+
+function LogReq(req, toFile, sc) {
+	try {
+		var ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress;
+		var logEntry = ip + DELIM + req.hostname + DELIM + req.headers['user-agent'] +
+			DELIM + req.method + DELIM;
+
+		logEntry += req.url;
+
+		if (sc != undefined && sc == 404)
+			logEntry += DELIM + sc;
+		var color = GetColor("green");
+
+		if (req.url.toLowerCase().includes("isadding"))
+			color = GetColor("yellow");
+		if (!toFile) {
+			Log(logEntry, color);
+		} else {
+			var defLogs = GetDateString() + DELIM + logEntry;
+			var extraLogs = "\n\t" + JSON.stringify(req.headers) + "\n\t" + JSON.stringify(req.body) + "\n";
+
+			utils.AppendToFile(defLogs, locLogFile);
+			utils.AppendToFile(defLogs, allLogFile);
+		}
+
+	} catch (e) {
+		console.log(e);
+		Log("Error at logging lol", GetColor("redbg"), true);
+	}
+}
+
+function GetColor(c) {
+	if (c == "redbg")
+		return '\x1b[41m%s\x1b[0m';
+	if (c == "bluebg")
+		return '\x1b[44m%s\x1b[0m';
+	if (c == "red")
+		return '\x1b[31m%s\x1b[0m';
+	if (c == "green")
+		return '\x1b[32m%s\x1b[0m';
+	if (c == "yellow")
+		return '\x1b[33m%s\x1b[0m';
+	if (c == "blue")
+		return '\x1b[34m%s\x1b[0m';
+	if (c == "cyan")
+		return '\x1b[36m%s\x1b[0m';
+}

utils/merger.js

@@ -0,0 +1,452 @@
+/* ----------------------------------------------------------------------------    
+
+ Question Server question file merger
+ GitLab: <https://gitlab.com/YourFriendlyNeighborhoodDealer/question-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+// TODO: handle flags
+// join json datas, or raw datas
+// or something else
+
+const minMatchAmmount = 55;
+const minResultMatchPercent = 99;
+const lengthDiffMultiplier = 10;
+
+class Question {
+	constructor(q, a, i) {
+		this.Q = q;
+		this.A = a;
+		this.I = i;
+	}
+	toString() {
+		var r = "?" + this.Q + "\n!" + this.A;
+		if (this.I)
+			r += "\n>" + this.I;
+		return r;
+	}
+	HasQuestion() {
+		return this.Q != undefined;
+	}
+	HasAnswer() {
+		return this.A != undefined;
+	}
+	HasImage() {
+		return this.I != undefined;
+	}
+	IsComplete() {
+		return this.HasQuestion() && this.HasAnswer();
+	}
+	// TODO: TEST DIS
+	Compare(q2, i) {
+		if (typeof q2 == 'string') {
+			var qmatchpercent = Question.CompareString(this.Q, q2);
+
+			if (i == undefined || i.length == 0)
+				return qmatchpercent;
+			else {
+				if (this.HasImage()) {
+					const imatchpercent = this.HasImage() ? Question.CompareString(this.I.join(" "), i.join(" ")) :
+						0;
+					return (qmatchpercent + imatchpercent) / 2;
+				} else {
+					qmatchpercent -= 30;
+					if (qmatchpercent < 0)
+						return 0;
+					else
+						return qmatchpercent;
+				}
+			}
+		} else {
+			const qmatchpercent = Question.CompareString(this.Q, q2.Q);
+			const amatchpercent = Question.CompareString(this.A, q2.A);
+			if (this.I != undefined) {
+				const imatchpercent = this.I == undefined ? Question.CompareString(this.I.join(" "), q2.I.join(
+					" ")) : 0;
+				return (qmatchpercent + amatchpercent + imatchpercent) / 3;
+			} else {
+				return (qmatchpercent + amatchpercent) / 2;
+			}
+		}
+	}
+	static CompareString(s1, s2) {
+		s1 = SimplifyStringForComparison(s1).split(" ");
+		s2 = SimplifyStringForComparison(s2).split(" ");
+		var match = 0;
+		for (var i = 0; i < s1.length; i++)
+			if (s2.includes(s1[i]))
+				match++;
+		var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent
+		var lengthDifference = Math.abs(s2.length - s1.length);
+		percent -= lengthDifference * lengthDiffMultiplier;
+		if (percent < 0)
+			percent = 0;
+		return percent;
+	}
+}
+
+class Subject {
+	constructor(n) {
+		this.Name = n;
+		this.Questions = [];
+	}
+	get length() {
+		return this.Questions.length;
+	}
+	AddQuestion(q) {
+		this.Questions.push(q);
+	}
+	toString() {
+		var r = [];
+		for (var i = 0; i < this.Questions.length; i++)
+			r.push(this.Questions[i].toString());
+		return "+" + this.Name + "\n" + r.join("\n");
+	}
+}
+
+class QuestionDB {
+	constructor() {
+		this.Subjects = [];
+	}
+	get length() {
+		return this.Subjects.length;
+	}
+	AddQuestion(subj, q) {
+		var i = 0;
+		while (i < this.Subjects.length && this.Subjects[i].Name != subj)
+			i++;
+		if (i < this.Subjects.length)
+			this.Subjects[i].AddQuestion(q);
+		else {
+			const n = new Subject(subj);
+			n.AddQuestion(q);
+			this.Subjects.push(n);
+		}
+	}
+	AddSubject(subj) {
+		var i = 0;
+		while (i < this.length && subj.Name != this.Subjects[i].Name)
+			i++;
+
+		if (i < this.length) {
+			this.Subjects.concat(subj.Questions);
+		} else {
+			this.Subjects.push(subj);
+		}
+	}
+	toString() {
+		var r = [];
+		for (var i = 0; i < this.Subjects.length; i++)
+			r.push(this.Subjects[i].toString());
+		return r.join("\n\n");
+	}
+}
+var utils = require('./utils.js');
+var actions = require('./actions.js');
+
+Main();
+
+function Main() {
+	console.clear();
+	const params = GetParams();
+	console.log(params);
+	var dbs = [];
+
+
+	for (var i = 0; i < params.length; i++) {
+		PrintLN();
+		console.log(params[i] + ": ");
+		try {
+			dbs.push(ParseJSONData(utils.ReadFile(params[i])));
+			console.log("JSON data added");
+		} catch (e) {
+			console.log(e);
+			console.log("Trying with old format...");
+			dbs.push(ReadData(utils.ReadFile(params[i])).result);
+		}
+	}
+	PrintLN();
+
+	dbs.forEach((item) => {
+		PrintDB(item);
+	});
+
+	var olds = [];
+	if (dbs.length == 1) {
+		for ( let i = 0; i < dbs[0].length; i++)
+			olds.push(dbs[0].Subjects[i].length);
+	}
+
+	console.log("Parsed data count: " + dbs.length);
+	PrintLN();
+
+	console.log("Merging databases...");
+	var db = MergeDatabases(dbs);
+
+	console.log("Removing duplicates...");
+	var r = RemoveDuplicates(db);
+
+	console.log("RESULT:");
+	PrintDB(r, olds);
+
+	utils.WriteFile(JSON.stringify(r), "newData");
+	console.log("File written!");
+}
+
+function PrintLN() {
+	console.log("------------------------------------------------------");
+}
+
+function PrintDB(r, olds) {
+	console.log("Data subject count: " + r.length);
+	var maxLength = 0;
+	for (var i = 0; i < r.length; i++) {
+		if (maxLength < r.Subjects[i].Name.length)
+			maxLength = r.Subjects[i].Name.length;
+	}
+
+	let qcount = 0;
+
+	for (var i = 0; i < r.length; i++) {
+		let line = i;
+		if (line < 10)
+			line += ' ';
+
+		line += ": ";
+		var currLength = line.length + maxLength + 4;
+		line += r.Subjects[i].Name;
+
+		while (line.length < currLength) {
+			if (i % 4 == 0)
+				line += ".";
+			else
+				line += " ";
+		}
+
+		if (olds && olds.length > 0) {
+			// TODO: check if correct row! should be now, but well...
+			if (olds[i] < 10)
+				line += " ";
+			if (olds[i] < 100)
+				line += " ";
+
+			line += olds[i];
+			line += " -> ";
+		}
+
+		if (r.Subjects[i].length < 10)
+			line += " ";
+		if (r.Subjects[i].length < 100)
+			line += " ";
+
+		line += r.Subjects[i].length;
+		qcount += r.Subjects[i].length;
+			
+		line += " db";
+		
+		console.log(line);
+	}
+
+	console.log("Total questions: " + qcount);
+
+	PrintLN();
+}
+
+function GetParams() {
+	return process.argv.splice(2);
+}
+
+function ParseJSONData(data) {
+	var d = JSON.parse(data);
+	var r = new QuestionDB();
+	var rt = [];
+
+	for (var i = 0; i < d.Subjects.length; i++) {
+		let s = new Subject(d.Subjects[i].Name);
+		var j = 0;
+		for (j = 0; j < d.Subjects[i].Questions.length; j++) {
+			var currQ = d.Subjects[i].Questions[j];
+			s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I));
+		}
+		rt.push({
+			name: d.Subjects[i].Name,
+			count: j
+		});
+		r.AddSubject(s);
+	}
+	return r;
+}
+
+function MergeDatabases(dbs) {
+	var db = new QuestionDB();
+	for (var i = 0; i < dbs.length; i++)
+		for (var j = 0; j < dbs[i].length; j++)
+			db.AddSubject(dbs[i].Subjects[j]);
+	return db;
+}
+
+/*
+ * Returns a question database from the given data.
+ * Parameter should be raw read file in string with "\n"-s
+ * TODO: ??? -s are not listed as errors, tho works correctly
+ * */
+function ReadData(data) {
+
+	const d = data.split("\n");
+	const r = new QuestionDB();
+	var logs = [];
+	var currSubj = ""; // the current subjects name
+	var ExpectedIdentifier = ['+', '?'];
+	let currQuestion = new Question();
+
+	var i = -1;
+	while (i < d.length) {
+		let currIdentifier;
+		let skipped = 0;
+		do {
+			if (skipped >= 1)
+				logs.push(i + ": " + d[i]);
+			i++;
+			if (i >= d.length) {
+				if (currQuestion.IsComplete())
+					r.AddQuestion(currSubj, currQuestion);
+				return {
+					result: r,
+					logs: logs
+				};
+			}
+			currIdentifier = d[i][0];
+			skipped++;
+		} while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length);
+
+		let currData = d[i].substring(1).trim();
+
+		if (currIdentifier == '+') {
+			if (currQuestion.IsComplete())
+				r.AddQuestion(currSubj, currQuestion);
+			currQuestion = new Question();
+			currSubj = currData;
+			ExpectedIdentifier = ['?'];
+			continue;
+		}
+
+		if (currIdentifier == '?') {
+			if (currQuestion.IsComplete()) {
+				r.AddQuestion(currSubj, currQuestion);
+				currQuestion = new Question();
+			}
+			// overwriting is allowed here, bcus:
+			// ?????!>
+			currQuestion.Q = currData;
+			ExpectedIdentifier = ['!', '?'];
+			continue;
+		}
+
+		if (currIdentifier == '!') {
+			// if dont have question continue
+			if (!currQuestion.HasQuestion())
+				throw "No question! (A)";
+			// dont allow overwriting
+			// ?!!!! 
+			if (!currQuestion.HasAnswer()) {
+				currData = currData.replace("A helyes válaszok: ", "");
+				currData = currData.replace("A helyes válasz: ", "");
+
+				currQuestion.A = currData;
+			}
+			ExpectedIdentifier = ['?', '>', '+'];
+			continue;
+		}
+
+		if (currIdentifier == '>') {
+			// if dont have question or answer continue
+			if (!currQuestion.HasQuestion())
+				throw "No question! (I)";
+			if (!currQuestion.HasAnswer())
+				throw "No asnwer! (I)";
+			// dont allow overwriting
+			// ?!>>>
+			if (!currQuestion.HasImage()) {
+				try {
+					currQuestion.I = JSON.parse(currData);
+				} catch (e) {
+					currQuestion.I = currData.split(',');
+				}
+			}
+			ExpectedIdentifier = ['?', '+'];
+			continue;
+		}
+	}
+
+	return {
+		result: r,
+		logs: logs
+	};
+}
+
+function RemoveDuplicates(dataObj) {
+	for (var i = 0; i < dataObj.length; i++)
+		RemoveDuplFromSubject(dataObj.Subjects[i]);
+	return dataObj;
+}
+
+function RemoveDuplFromSubject(subj) {
+	var cp = subj.Questions;
+	subj.Questions = [];
+	for (var i = 0; i < cp.length; i++) {
+		var j = 0;
+		// Only removes 100% match!
+		while (j < subj.length && cp[i].Compare(subj.Questions[j]) != 100) {
+			j++;
+		}
+		if (j < subj.length) {
+			//console.log("----------------------------------------------------------");
+			//console.log(cp[i].toString());
+			//console.log("  VS  ");
+			//console.log(subj.Questions[j].toString());
+			//console.log(cp[i].Compare(subj.Questions[j]));
+			//console.log(j);
+			//console.log("removed:");
+			//console.log(subj.Questions.splice(j, 1).toString());
+			//console.log("----------------------------------------------------------");
+		} else {
+			subj.AddQuestion(cp[i]);
+		}
+	}
+}
+
+function SimplifyStringForComparison(value) {
+	value = RemoveUnnecesarySpaces(value).toLowerCase();
+	var removableChars = [",", ".", ":", "!"];
+	for (var i = 0; i < removableChars.length; i++) {
+		var regex = new RegExp(removableChars[i], "g");
+		value.replace(regex, "");
+	}
+	return value;
+}
+
+function RemoveUnnecesarySpaces(toremove) {
+	toremove = NormalizeSpaces(toremove);
+	while (toremove.includes("  ")) // while the text includes double spaces replaces all of them with a single one
+	{
+		toremove = toremove.replace(/  /g, " ");
+	}
+	return toremove.trim();
+}
+
+function NormalizeSpaces(input) {
+	return input.replace(/\s/g, ' ');
+}

utils/motd.js

@@ -0,0 +1,24 @@
+const utils = require('./utils.js');
+const dataFile = "public/data.json";
+const versionFile = "public/version";
+const motdFile = "public/motd";
+
+var p = GetParams();
+if (p.length <= 0) {
+	console.log("no params!");
+	process.exit(0);
+}
+
+
+var param = p.join(" ");
+console.log("param: " + param);
+var d = utils.ReadFile(dataFile);
+var parsed = JSON.parse(d);
+parsed.motd = param;
+utils.WriteFile(JSON.stringify(parsed), dataFile);
+
+utils.WriteFile(parsed.motd, motdFile);
+
+function GetParams() {
+	return process.argv.splice(2);
+}

utils/stat.js

@@ -0,0 +1,112 @@
+/* ----------------------------------------------------------------------------    
+
+ Question Server
+ GitLab: <https://gitlab.com/YourFriendlyNeighborhoodDealer/question-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+module.exports = {
+	LogStat: LogStat,
+	Load: Load
+};
+
+var utils = require('./utils.js');
+var logger = require('./logger.js');
+
+const statFile = "stats/stats";
+const vStatFile = "stats/vstats";
+const writeInterval = 10;
+
+var data = {};
+var vData = {};
+var writes = 0;
+
+function Load() {
+	try {
+		var prevData = utils.ReadFile(statFile);
+		data = JSON.parse(prevData);
+	} catch (e) {
+		logger.Log("Error at loading logs! (@ first run its normal)", logger.GetColor("redbg"));
+		console.log(e);
+	}
+
+	try {
+		var prevVData = utils.ReadFile(vStatFile);
+		vData = JSON.parse(prevVData);
+	} catch (e) {
+		logger.Log("Error at loading visit logs! (@ first run its normal)", logger.GetColor("redbg"));
+		console.log(e);
+	}
+}
+
+function LogStat(url) {
+	Inc(url);
+	AddVisitStat(url);
+	Save();
+}
+
+function Inc(value) {
+	if (value.startsWith("/?"))
+		value = "/";
+	if (data[value] == undefined)
+		data[value] = 0;
+	data[value]++;
+}
+
+function AddVisitStat(name) {
+	var m = new Date();
+	const now = m.getFullYear() + "/" +
+		("0" + (m.getMonth() + 1)).slice(-2) + "/" +
+		("0" + m.getDate()).slice(-2);
+	if (vData[now] == undefined)
+		vData[now] = {};
+	if (vData[now][name] == undefined)
+		vData[now][name] = 0;
+	vData[now][name]++;
+}
+
+function AddVisitStat(name) {
+	var m = new Date();
+	const now = m.getFullYear() + "/" +
+		("0" + (m.getMonth() + 1)).slice(-2) + "/" +
+		("0" + m.getDate()).slice(-2);
+	if (vData[now] == undefined)
+		vData[now] = {};
+	if (vData[now][name] == undefined)
+		vData[now][name] = 0;
+	vData[now][name]++;
+}
+
+function Save() {
+	writes++;
+	if (writes == writeInterval) {
+		try {
+			utils.WriteFile(JSON.stringify(data), statFile);
+			// logger.Log("Stats wrote.");
+		} catch (e) {
+			logger.Log("Error at writing logs!", logger.GetColor("redbg"));
+			console.log(e);
+		}
+		try {
+			utils.WriteFile(JSON.stringify(vData), vStatFile);
+			// logger.Log("Stats wrote.");
+		} catch (e) {
+			logger.Log("Error at writing visit logs!", logger.GetColor("redbg"));
+			console.log(e);
+		}
+		writes = 0;
+	}
+}

utils/utils.js

@@ -0,0 +1,109 @@
+module.exports = {
+	ReadFile: ReadFile,
+	WriteFile: WriteFile,
+	writeFileAsync: WriteFileAsync,
+	AppendToFile: AppendToFile,
+	Beep: Beep,
+	WriteBackup: WriteBackup,
+	FileExists: FileExists,
+	CreatePath: CreatePath,
+	GetAllFilesFromFolder: GetAllFilesFromFolder
+};
+
+var fs = require('fs');
+
+var logger = require('./logger.js');
+
+const recievedFile = "stats/recieved";
+const manFile = "public/man.html";
+const logFile = "stats/logs";
+const dataFile = "public/data.json";
+
+function ReadFile(name) {
+	if (!FileExists(name))
+		throw "No such file: " + name;
+	return fs.readFileSync(name, "utf8");
+}
+
+function FileExists(path) {
+	return fs.existsSync(path);
+}
+
+function CreatePath(path, onlyPath) {
+	if (FileExists(path))
+		return;
+
+	var p = path.split("/");
+	var currDir = p[0];
+	for (var i = 1; i < p.length; i++) {
+		if (currDir != "" && !fs.existsSync(currDir)) {
+			try {
+				fs.mkdirSync(currDir);
+			} catch (e) { console.log("Failed to make " + currDir + " directory... "); }
+		}
+		currDir += "/" + p[i];
+	}
+	if (onlyPath == undefined || onlyPath == false)
+		fs.writeFileSync(path, "");
+	else
+		fs.mkdirSync(path);
+
+}
+
+function WriteFile(content, path) {
+	CreatePath(path);
+	fs.writeFileSync(path, content);
+}
+
+function WriteFileAsync(content, path) {
+	CreatePath(path);
+	fs.writeFile(path, content, function(err) {
+		if (err) {
+			logger.Log("Error writing file: " + path + " (sync)", logger.GetColor("redbg"));
+		}
+	});
+}
+
+function AppendToFile(data, file) {
+	CreatePath(file);
+	fs.appendFile(file, "\n" + data, function(err) {
+		if (err)
+			logger.Log("Error writing log file: " + file + " (sync)", logger.GetColor("redbg"));
+	});
+}
+
+function Beep() {
+	try {
+		process.stdout.write('\x07');
+	} catch (e) {
+		console.log("error beepin");
+	}
+}
+
+function WriteBackup() {
+	try {
+		WriteFileAsync(ReadFile(dataFile), 'public/backs/data_' + new Date().toString());
+	} catch (e) {
+		logger.Log("Error backing up data json file!", logger.GetColor("redbg"));
+		console.log(e);
+	}
+}
+
+function GetAllFilesFromFolder(dir) {
+
+	var results = [];
+
+	fs.readdirSync(dir).forEach(function(file) {
+
+		file = dir + '/' + file;
+		var stat = fs.statSync(file);
+
+		if (stat && stat.isDirectory()) {
+			results = results.concat(_getAllFilesFromFolder(file));
+		} else results.push(file);
+
+	});
+
+	return results;
+}
+