Refactored server to handle new JSON post requests

actions.js

@@ -26,10 +26,195 @@ module.exports = {
 var recievedFile = "stats/recieved";
 var staticFile = "public/data/static";
 var manFile = "public/man.html";
+var dataFile = "public/data.json";
+var motdFile = "public/motd";
 
 var logger = require('./logger.js');
 var utils = require('./utils.js');
 
+class Question {
+	constructor(q, a, i) {
+		this.Q = q;
+		this.A = a;
+		this.I = i;
+	}
+	toString() {
+		var r = "?" + this.Q + "\n!" + this.A;
+		if (this.I)
+			r += "\n>" + this.I;
+		return r;
+	}
+	HasQuestion() {
+		return this.Q != undefined;
+	}
+	HasAnswer() {
+		return this.A != undefined;
+	}
+	HasImage() {
+		return this.I != undefined;
+	}
+	IsComplete() {
+		return this.HasQuestion() && this.HasAnswer();
+	}
+	Compare(q2, i) {
+		if (typeof q2 == 'string') {
+			var qmatchpercent = Question.CompareString(this.Q, q2);
+
+			if (i == undefined || i.length == 0)
+				return qmatchpercent;
+			else {
+				if (this.HasImage()) {
+					const imatchpercent = this.HasImage() ? Question.CompareString(this.I.join(" "), i.join(" ")) :
+						0;
+					return (qmatchpercent + imatchpercent) / 2;
+				} else {
+					qmatchpercent -= 30;
+					if (qmatchpercent < 0)
+						return 0;
+					else
+						return qmatchpercent;
+				}
+			}
+		} else {
+			const qmatchpercent = Question.CompareString(this.Q, q2.Q);
+			const amatchpercent = Question.CompareString(this.A, q2.A);
+			if (this.I != undefined) {
+				const imatchpercent = this.I == undefined ? Question.CompareString(this.I.join(" "), q2.I.join(
+					" ")) : 0;
+				return (qmatchpercent + amatchpercent + imatchpercent) / 3;
+			} else {
+				return (qmatchpercent + amatchpercent) / 2;
+			}
+		}
+	}
+	static CompareString(s1, s2) {
+		//if (s1 == undefined || s2 == undefined)
+		//	return 0;
+		s1 = SimplifyStringForComparison(s1).split(" ");
+		s2 = SimplifyStringForComparison(s2).split(" ");
+		var match = 0;
+		for (var i = 0; i < s1.length; i++)
+			if (s2.includes(s1[i]))
+				match++;
+		var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent
+		var lengthDifference = Math.abs(s2.length - s1.length);
+		percent -= lengthDifference * 3;
+		if (percent < 0)
+			percent = 0;
+		return percent;
+	}
+}
+
+class Subject {
+	constructor(n) {
+		this.Name = n;
+		this.Questions = [];
+	}
+	get length() {
+		return this.Questions.length;
+	}
+	AddQuestion(q) {
+		this.Questions.push(q);
+	}
+	Search(q, img) {
+		var r = [];
+		for (var i = 0; i < this.length; i++) {
+			let percent = this.Questions[i].Compare(q, img);
+			if (percent > minMatchAmmount)
+				r.push({
+					q: this.Questions[i],
+					match: percent
+				});
+		}
+
+		for (var i = 0; i < r.length; i++)
+			for (var j = i; j < r.length; j++)
+				if (r[i].match < r[j].match) {
+					var tmp = r[i];
+					r[i] = r[j];
+					r[j] = tmp;
+				}
+
+		return r;
+	}
+	toString() {
+		var r = [];
+		for (var i = 0; i < this.Questions.length; i++)
+			r.push(this.Questions[i].toString());
+		return "+" + this.Name + "\n" + r.join("\n");
+	}
+}
+
+class QuestionDB {
+	constructor() {
+		this.Subjects = [];
+	}
+	get length() {
+		return this.Subjects.length;
+	}
+	get activeIndexes() {
+		var r = [];
+		for (var i = 0; i < this.length; i++) {
+			if (GM_getValue("Is" + i + "Active")) {
+				r.push(i);
+			}
+		}
+		return r;
+	}
+	GetIfActive(ind) {
+		return GM_getValue("Is" + ind + "Active");
+	}
+	ChangeActive(i, value) {
+		GM_setValue("Is" + i + "Active", !!value);
+	}
+	AddQuestion(subj, q) {
+		var i = 0;
+		while (i < this.Subjects.length && this.Subjects[i].Name != subj)
+			i++;
+		if (i < this.Subjects.length)
+			this.Subjects[i].AddQuestion(q);
+		else {
+			const n = new Subject(subj);
+			n.AddQuestion(q);
+			this.Subjects.push(n);
+		}
+	}
+	Search(q, img) {
+		var r = [];
+		for (var i = 0; i < this.length; i++)
+			if (this.GetIfActive(i))
+				r = r.concat(this.Subjects[i].Search(q, img));
+
+		for (var i = 0; i < r.length; i++)
+			for (var j = i; j < r.length; j++)
+				if (r[i].match < r[j].match) {
+					var tmp = r[i];
+					r[i] = r[j];
+					r[j] = tmp;
+				}
+
+		return r;
+	}
+	AddSubject(subj) {
+		var i = 0;
+		while (i < this.length && subj.Name != this.Subjects[i].Name)
+			i++;
+
+		if (i < this.length) {
+			this.Subjects.concat(subj.Questions);
+		} else {
+			this.Subjects.push(subj);
+		}
+	}
+	toString() {
+		var r = [];
+		for (var i = 0; i < this.Subjects.length; i++)
+			r.push(this.Subjects[i].toString());
+		return r.join("\n\n");
+	}
+}
+
+
 function Process(d, file) {
 	try {
 		logger.Log("[PCES]:\tFile: " + file);
@@ -59,7 +244,7 @@ function Process(d, file) {
 
 			return newRes.count - oldRes.count;
 		} else
-			logger.Log("[PCES]:\t\tNo new data");
+			logger.Log("[PCES]:\t\tNo new data, and one line of stuff only");
 
 
 	} catch (e) {
@@ -76,30 +261,61 @@ function ProcessIncomingRequest(data) {
 		logger.Log("[PCES]:\tRecieved data is undefined!", logger.GetColor("redbg"));
 		return;
 	}
-
-	var d = SetupData(data);
-	var qcount = -1;
 	try {
-		var splitted = d.alldata.split("\n");
+		var d = JSON.parse(data);
-		var count = 0;
+		var dfile = utils.ReadFile(dataFile);
-		for (var i = 0; i < splitted.length; i++)
+		var data = LoadJSON(dfile);
-			if (splitted[i][0] == '?')
+		var allQuestions = [];
-				count ++;
+		for ( var i = 0; i < d.allData.length; i++){
-		qcount = count;
+			allQuestions.push(new Question(d.allData[i].Q, d.allData[i].A, d.allData[i].I));
-	} catch (e) {console.log("Error :c"); console.log(e);}
+		}
+		var questions = [];
+		for ( var i = 0; i < d.data.length; i++){
+			let q = new Question(d.data[i].Q, d.data[i].A, d.data[i].I);
+			questions.push(q);
+			data.AddQuestion(d.subj, q);
+		}
 		
-	logger.Log("[PCES]:\tProcessing data: " + d.subj + " (" + d.type + "), count: " + qcount, logger.GetColor("green"));
+		var motd = utils.ReadFile(motdFile);
-	if (d.subj == undefined){
+		data.motd = motd;
-		logger.Log(JSON.stringify(d), logger.GetColor("red"));
+		logger.Log("[PCES]:\t" + d.subj);
-		return;
+		var msg = "All / new count: " + allQuestions.length + " / " + questions.length;
+		var color = logger.GetColor("green");
+
+		if (data != undefined && d.data.length > 0){
+			utils.WriteBackup();
+			utils.WriteFile(JSON.stringify(data), dataFile);
+			msg += " - Data file written!";
+			var color = logger.GetColor("blue");
+		}
+		logger.Log("[PCES]:\t" + msg, color);
+
+	} catch (e) {
+		logger.Log("[PCES]: Couldnt parse JSON data, trying old format...");
+		var d = SetupData(data);
+		var qcount = -1;
+		try {
+			var splitted = d.alldata.split("\n");
+			var count = 0;
+			for (var i = 0; i < splitted.length; i++)
+				if (splitted[i][0] == '?')
+					count ++;
+			qcount = count;
+		} catch (e) {console.log("Error :c"); console.log(e);}
+
+		logger.Log("[PCES]:\tProcessing data: " + d.subj + " (" + d.type + "), count: " + qcount, logger.GetColor("green"));
+		if (d.subj == undefined){
+			logger.Log(JSON.stringify(d), logger.GetColor("red"));
+			return;
+		}
+
+		var newItems = 0;
+
+		var newStatItems = Process(d, staticFile);
+
+		utils.AppendToFile(logger.GetDateString() + "\n" + d.data, recievedFile);
+		PrintNewCount(d, newStatItems, staticFile);
 	}
-
-	var newItems = 0;
-
-	var newStatItems = Process(d, staticFile);
-
-	utils.AppendToFile(logger.GetDateString() + "\n" + d.data, recievedFile);
-	PrintNewCount(d, newStatItems, staticFile);
 }
 
 function PrintNewCount(d, newItems, file) {
@@ -115,15 +331,20 @@ function PrintNewCount(d, newItems, file) {
 
 function SetupData(data) {
 	var pdata = data.split("<#>");
-	if (pdata.length <= 0)
+	if (pdata.length <= 0){
+		logger.Log("[SUPD]: Data length is zero !", logger.GetColor("redbg"));
 		throw "No data recieved!";
+	}
+
 	var d = {}; // parsed data
 	for (var i = 0; i < pdata.length; i++) {
 		var td = pdata[i].split("<=>");
 		if (td.length == 2)
 			d[td[0]] = td[1];
-		else
+		else {
+			logger.Log("[SUPD]: Invalid parameter!", logger.GetColor("redbg"));
 			throw "Invalid parameter recieved!";
+		}
 	}
 	return d;
 }
@@ -232,3 +453,32 @@ function NLoad(resource) {
 		log: resultLog
 	};
 }
+
+// loading stuff
+function LoadJSON(resource) {
+	var count = -1;
+	try {
+		var	d = JSON.parse(resource);
+		var r = new QuestionDB();
+		var rt = [];
+		var allCount = -1;
+
+		for (var i = 0; i < d.Subjects.length; i++) {
+			let s = new Subject(d.Subjects[i].Name);
+			var j = 0;
+			for (j = 0; j < d.Subjects[i].Questions.length; j++) {
+				var currQ = d.Subjects[i].Questions[j];
+				s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I));
+			}
+			rt.push({
+				name: d.Subjects[i].Name,
+				count: j
+			});
+			allCount += j;
+			r.AddSubject(s);
+		}
+		return r;
+	} catch (e) {
+			logger.Log("[LOAD]:Error loading sutff", logger.GetColor("redbg"), true);
+	}
+}

server.js

@@ -34,6 +34,7 @@ var bodyParser = require('body-parser');
 const recivedFiles = "public/recivedfiles";
 const motdFile = "public/motd";
 const staticFile = "public/data/static";
+const dataFile = "public/data/data";
 const countFile = "public/data/count";
 const manFile = "public/man.html";
 const simpOutFile = "public/simplified";

utils.js

@@ -15,6 +15,7 @@ const recievedFile = "stats/recieved";
 const staticFile = "public/data/static";
 const manFile = "public/man.html";
 const logFile = "stats/logs";
+const dataFile = "public/data.json";
 
 function ReadFile(name) {
 	return fs.readFileSync(name, "utf8");
@@ -56,4 +57,12 @@ function WriteBackup() {
 		logger.Log("[ERR ]: Error backing up recieved file!", logger.GetColor("redbg"));
 		console.log(e);
 	}
+
+	try {
+		WriteFileAsync(ReadFile(dataFile), 'public/backs/data_' + new Date().toString());
+
+	} catch (e) {
+		logger.Log("[ERR ]: Error backing up data json file!", logger.GetColor("redbg"));
+		console.log(e);
+	}
 }