From 0d96738c4d58f0d6f294d7ecbc131f8d1c26778d Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Mon, 10 Dec 2018 13:38:24 +0100 Subject: [PATCH 01/13] Fixed [] to {} --- stat.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stat.js b/stat.js index 43882e9..a676758 100755 --- a/stat.js +++ b/stat.js @@ -84,7 +84,7 @@ function AddVisitStat(name) { ("0" + (m.getMonth() + 1)).slice(-2) + "/" + ("0" + m.getDate()).slice(-2); if (vData[now] == undefined) - vData[now] = []; + vData[now] = {}; if (vData[now][name] == undefined) vData[now][name] = 0; vData[now][name]++; From 42c46caf7c494315fb7fe78f2c38e0fde3a69ff3 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Tue, 11 Dec 2018 19:44:11 +0100 Subject: [PATCH 02/13] Removed public data backup --- utils.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/utils.js b/utils.js index b61d506..22c6b5d 100644 --- a/utils.js +++ b/utils.js @@ -12,7 +12,6 @@ var fs = require('fs'); var logger = require('./logger.js'); const recievedFile = "stats/recieved"; -const publicFile = "public/data/public"; const staticFile = "public/data/static"; const manFile = "public/man.html"; const logFile = "stats/logs"; @@ -57,11 +56,4 @@ function WriteBackup() { logger.Log("[ERR ]: Error backing up recieved file!", logger.GetColor("redbg")); console.log(e); } - try { - WriteFileAsync(ReadFile(publicFile), 'public/backs/public_' + new Date().toString()); - //logger.Log('[SAVE]: Public questions backup wrote'); - } catch (e) { - logger.Log("[ERR ]: Error backing up public file!", logger.GetColor("redbg"), true); - console.log(e); - } } From 0f71d6705b80f0aa3fa2b01836be2cd4d20a1f00 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Tue, 11 Dec 2018 19:45:53 +0100 Subject: [PATCH 03/13] Removed some commented out, outdated code --- actions.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/actions.js b/actions.js index 2420384..3482a68 100755 --- a/actions.js +++ b/actions.js @@ -96,15 +96,6 @@ function ProcessIncomingRequest(data) { var newItems = 0; - /*if (d.type == 0) // -> safe, static - newItems = Process(d, staticFile); - else if (d.type == 1) // -> unsafe, public - newItems = Process(d, publicFile); - else { // -> error, or old client - logger.Log("[PCES]: Old client..."); - Process(d, publicFile); - Process(d, staticFile); - }*/ var newStatItems = Process(d, staticFile); utils.AppendToFile(logger.GetDateString() + "\n" + d.data, recievedFile); From 3c0e593ed428a9a31584d29d43abaf808402ef69 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Fri, 14 Dec 2018 18:32:04 +0100 Subject: [PATCH 04/13] Refactored server to handle new JSON post requests --- actions.js | 298 ++++++++++++++++++++++++++++++++++++++++++++++++----- server.js | 1 + utils.js | 9 ++ 3 files changed, 284 insertions(+), 24 deletions(-) diff --git a/actions.js b/actions.js index 3482a68..86531d6 100755 --- a/actions.js +++ b/actions.js @@ -26,10 +26,195 @@ module.exports = { var recievedFile = "stats/recieved"; var staticFile = "public/data/static"; var manFile = "public/man.html"; +var dataFile = "public/data.json"; +var motdFile = "public/motd"; var logger = require('./logger.js'); var utils = require('./utils.js'); +class Question { + constructor(q, a, i) { + this.Q = q; + this.A = a; + this.I = i; + } + toString() { + var r = "?" + this.Q + "\n!" + this.A; + if (this.I) + r += "\n>" + this.I; + return r; + } + HasQuestion() { + return this.Q != undefined; + } + HasAnswer() { + return this.A != undefined; + } + HasImage() { + return this.I != undefined; + } + IsComplete() { + return this.HasQuestion() && this.HasAnswer(); + } + Compare(q2, i) { + if (typeof q2 == 'string') { + var qmatchpercent = Question.CompareString(this.Q, q2); + + if (i == undefined || i.length == 0) + return qmatchpercent; + else { + if (this.HasImage()) { + const imatchpercent = this.HasImage() ? Question.CompareString(this.I.join(" "), i.join(" ")) : + 0; + return (qmatchpercent + imatchpercent) / 2; + } else { + qmatchpercent -= 30; + if (qmatchpercent < 0) + return 0; + else + return qmatchpercent; + } + } + } else { + const qmatchpercent = Question.CompareString(this.Q, q2.Q); + const amatchpercent = Question.CompareString(this.A, q2.A); + if (this.I != undefined) { + const imatchpercent = this.I == undefined ? Question.CompareString(this.I.join(" "), q2.I.join( + " ")) : 0; + return (qmatchpercent + amatchpercent + imatchpercent) / 3; + } else { + return (qmatchpercent + amatchpercent) / 2; + } + } + } + static CompareString(s1, s2) { + //if (s1 == undefined || s2 == undefined) + // return 0; + s1 = SimplifyStringForComparison(s1).split(" "); + s2 = SimplifyStringForComparison(s2).split(" "); + var match = 0; + for (var i = 0; i < s1.length; i++) + if (s2.includes(s1[i])) + match++; + var percent = Math.round(((match / s1.length) * 100).toFixed(2)); // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length); + percent -= lengthDifference * 3; + if (percent < 0) + percent = 0; + return percent; + } +} + +class Subject { + constructor(n) { + this.Name = n; + this.Questions = []; + } + get length() { + return this.Questions.length; + } + AddQuestion(q) { + this.Questions.push(q); + } + Search(q, img) { + var r = []; + for (var i = 0; i < this.length; i++) { + let percent = this.Questions[i].Compare(q, img); + if (percent > minMatchAmmount) + r.push({ + q: this.Questions[i], + match: percent + }); + } + + for (var i = 0; i < r.length; i++) + for (var j = i; j < r.length; j++) + if (r[i].match < r[j].match) { + var tmp = r[i]; + r[i] = r[j]; + r[j] = tmp; + } + + return r; + } + toString() { + var r = []; + for (var i = 0; i < this.Questions.length; i++) + r.push(this.Questions[i].toString()); + return "+" + this.Name + "\n" + r.join("\n"); + } +} + +class QuestionDB { + constructor() { + this.Subjects = []; + } + get length() { + return this.Subjects.length; + } + get activeIndexes() { + var r = []; + for (var i = 0; i < this.length; i++) { + if (GM_getValue("Is" + i + "Active")) { + r.push(i); + } + } + return r; + } + GetIfActive(ind) { + return GM_getValue("Is" + ind + "Active"); + } + ChangeActive(i, value) { + GM_setValue("Is" + i + "Active", !!value); + } + AddQuestion(subj, q) { + var i = 0; + while (i < this.Subjects.length && this.Subjects[i].Name != subj) + i++; + if (i < this.Subjects.length) + this.Subjects[i].AddQuestion(q); + else { + const n = new Subject(subj); + n.AddQuestion(q); + this.Subjects.push(n); + } + } + Search(q, img) { + var r = []; + for (var i = 0; i < this.length; i++) + if (this.GetIfActive(i)) + r = r.concat(this.Subjects[i].Search(q, img)); + + for (var i = 0; i < r.length; i++) + for (var j = i; j < r.length; j++) + if (r[i].match < r[j].match) { + var tmp = r[i]; + r[i] = r[j]; + r[j] = tmp; + } + + return r; + } + AddSubject(subj) { + var i = 0; + while (i < this.length && subj.Name != this.Subjects[i].Name) + i++; + + if (i < this.length) { + this.Subjects.concat(subj.Questions); + } else { + this.Subjects.push(subj); + } + } + toString() { + var r = []; + for (var i = 0; i < this.Subjects.length; i++) + r.push(this.Subjects[i].toString()); + return r.join("\n\n"); + } +} + + function Process(d, file) { try { logger.Log("[PCES]:\tFile: " + file); @@ -59,7 +244,7 @@ function Process(d, file) { return newRes.count - oldRes.count; } else - logger.Log("[PCES]:\t\tNo new data"); + logger.Log("[PCES]:\t\tNo new data, and one line of stuff only"); } catch (e) { @@ -76,30 +261,61 @@ function ProcessIncomingRequest(data) { logger.Log("[PCES]:\tRecieved data is undefined!", logger.GetColor("redbg")); return; } - - var d = SetupData(data); - var qcount = -1; try { - var splitted = d.alldata.split("\n"); - var count = 0; - for (var i = 0; i < splitted.length; i++) - if (splitted[i][0] == '?') - count ++; - qcount = count; - } catch (e) {console.log("Error :c"); console.log(e);} + var d = JSON.parse(data); + var dfile = utils.ReadFile(dataFile); + var data = LoadJSON(dfile); + var allQuestions = []; + for ( var i = 0; i < d.allData.length; i++){ + allQuestions.push(new Question(d.allData[i].Q, d.allData[i].A, d.allData[i].I)); + } + var questions = []; + for ( var i = 0; i < d.data.length; i++){ + let q = new Question(d.data[i].Q, d.data[i].A, d.data[i].I); + questions.push(q); + data.AddQuestion(d.subj, q); + } + + var motd = utils.ReadFile(motdFile); + data.motd = motd; + logger.Log("[PCES]:\t" + d.subj); + var msg = "All / new count: " + allQuestions.length + " / " + questions.length; + var color = logger.GetColor("green"); - logger.Log("[PCES]:\tProcessing data: " + d.subj + " (" + d.type + "), count: " + qcount, logger.GetColor("green")); - if (d.subj == undefined){ - logger.Log(JSON.stringify(d), logger.GetColor("red")); - return; + if (data != undefined && d.data.length > 0){ + utils.WriteBackup(); + utils.WriteFile(JSON.stringify(data), dataFile); + msg += " - Data file written!"; + var color = logger.GetColor("blue"); + } + logger.Log("[PCES]:\t" + msg, color); + + } catch (e) { + logger.Log("[PCES]: Couldnt parse JSON data, trying old format..."); + var d = SetupData(data); + var qcount = -1; + try { + var splitted = d.alldata.split("\n"); + var count = 0; + for (var i = 0; i < splitted.length; i++) + if (splitted[i][0] == '?') + count ++; + qcount = count; + } catch (e) {console.log("Error :c"); console.log(e);} + + logger.Log("[PCES]:\tProcessing data: " + d.subj + " (" + d.type + "), count: " + qcount, logger.GetColor("green")); + if (d.subj == undefined){ + logger.Log(JSON.stringify(d), logger.GetColor("red")); + return; + } + + var newItems = 0; + + var newStatItems = Process(d, staticFile); + + utils.AppendToFile(logger.GetDateString() + "\n" + d.data, recievedFile); + PrintNewCount(d, newStatItems, staticFile); } - - var newItems = 0; - - var newStatItems = Process(d, staticFile); - - utils.AppendToFile(logger.GetDateString() + "\n" + d.data, recievedFile); - PrintNewCount(d, newStatItems, staticFile); } function PrintNewCount(d, newItems, file) { @@ -115,15 +331,20 @@ function PrintNewCount(d, newItems, file) { function SetupData(data) { var pdata = data.split("<#>"); - if (pdata.length <= 0) + if (pdata.length <= 0){ + logger.Log("[SUPD]: Data length is zero !", logger.GetColor("redbg")); throw "No data recieved!"; + } + var d = {}; // parsed data for (var i = 0; i < pdata.length; i++) { var td = pdata[i].split("<=>"); if (td.length == 2) d[td[0]] = td[1]; - else + else { + logger.Log("[SUPD]: Invalid parameter!", logger.GetColor("redbg")); throw "Invalid parameter recieved!"; + } } return d; } @@ -232,3 +453,32 @@ function NLoad(resource) { log: resultLog }; } + +// loading stuff +function LoadJSON(resource) { + var count = -1; + try { + var d = JSON.parse(resource); + var r = new QuestionDB(); + var rt = []; + var allCount = -1; + + for (var i = 0; i < d.Subjects.length; i++) { + let s = new Subject(d.Subjects[i].Name); + var j = 0; + for (j = 0; j < d.Subjects[i].Questions.length; j++) { + var currQ = d.Subjects[i].Questions[j]; + s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I)); + } + rt.push({ + name: d.Subjects[i].Name, + count: j + }); + allCount += j; + r.AddSubject(s); + } + return r; + } catch (e) { + logger.Log("[LOAD]:Error loading sutff", logger.GetColor("redbg"), true); + } +} diff --git a/server.js b/server.js index 8d81ae8..b38bdbe 100755 --- a/server.js +++ b/server.js @@ -34,6 +34,7 @@ var bodyParser = require('body-parser'); const recivedFiles = "public/recivedfiles"; const motdFile = "public/motd"; const staticFile = "public/data/static"; +const dataFile = "public/data/data"; const countFile = "public/data/count"; const manFile = "public/man.html"; const simpOutFile = "public/simplified"; diff --git a/utils.js b/utils.js index 22c6b5d..d84d0e6 100644 --- a/utils.js +++ b/utils.js @@ -15,6 +15,7 @@ const recievedFile = "stats/recieved"; const staticFile = "public/data/static"; const manFile = "public/man.html"; const logFile = "stats/logs"; +const dataFile = "public/data.json"; function ReadFile(name) { return fs.readFileSync(name, "utf8"); @@ -56,4 +57,12 @@ function WriteBackup() { logger.Log("[ERR ]: Error backing up recieved file!", logger.GetColor("redbg")); console.log(e); } + + try { + WriteFileAsync(ReadFile(dataFile), 'public/backs/data_' + new Date().toString()); + + } catch (e) { + logger.Log("[ERR ]: Error backing up data json file!", logger.GetColor("redbg")); + console.log(e); + } } From 774ff64b362a6b95ac1868f7e341fff3124e5162 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Fri, 14 Dec 2018 20:35:44 +0100 Subject: [PATCH 05/13] Small website redesign --- server.js | 15 ++++++++++++++- views/main.ejs | 15 +++++---------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/server.js b/server.js index b38bdbe..5552901 100755 --- a/server.js +++ b/server.js @@ -34,7 +34,7 @@ var bodyParser = require('body-parser'); const recivedFiles = "public/recivedfiles"; const motdFile = "public/motd"; const staticFile = "public/data/static"; -const dataFile = "public/data/data"; +const dataFile = "public/data.json"; const countFile = "public/data/count"; const manFile = "public/man.html"; const simpOutFile = "public/simplified"; @@ -114,6 +114,19 @@ app.get('/static', function(req, res) { res.end(); }); +app.get('/legacy', function(req, res) { + var f = utils.ReadFile(dataFile); + var r = ''; + r += ''; + var d = actions.LoadJSON(f).toString().split("\n"); + for (var i = 0; i < d.length; i++) + r += d[i] + "
"; + r += ''; + + res.write(r); + res.end(); +}); + app.post('/postfeedback', function(req, res) { res.redirect('back'); newMessages += "\n" + req.body.message_field; diff --git a/views/main.ejs b/views/main.ejs index 76b9226..1ada181 100755 --- a/views/main.ejs +++ b/views/main.ejs @@ -2,6 +2,7 @@ + '; From 05b034974c4ad5dfec14090056b34194f9315d67 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Sun, 16 Dec 2018 11:01:01 +0100 Subject: [PATCH 07/13] Added https support --- server.js | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/server.js b/server.js index f6a9745..a587858 100755 --- a/server.js +++ b/server.js @@ -23,6 +23,8 @@ var bodyParser = require('body-parser'); var busboy = require('connect-busboy'); var fs = require('fs'); var app = express(); +var http = require('http'); +var https = require('https'); var logger = require('./logger.js'); var utils = require('./utils.js'); @@ -41,7 +43,14 @@ const simpOutFile = "public/simplified"; const inputFile = "stats/inputs"; const msgFile = "stats/msgs"; const logFile = "stats/logs"; + +// https://certbot.eff.org/ +const key = fs.readFileSync("/etc/letsencrypt/live/questionmining.tk/privkey.pem", "utf8"); +const cert = fs.readFileSync("/etc/letsencrypt/live/questionmining.tk/fullchain.pem", "utf8"); +const ca = fs.readFileSync("/etc/letsencrypt/live/questionmining.tk/chain.pem", "utf8"); +var certs = {key:key, cert:cert, ca:ca}; const port = 8080; +const httpsPort = 8443; var highlights = ["public", "static", "manual", "isgetting", "postfeedback", "postquestions" @@ -66,6 +75,13 @@ try { var newMessages = ""; app.set('view engine', 'ejs'); +//app.all('*', function(req, res, next) { +// if(req.secure) { +// next(); +// } else { +// res.redirect('https://' + req.hostname + req.url); +// } +//}); app.use(function(req, res, next) { res.on('finish', function() { Log(req, true); @@ -203,14 +219,18 @@ app.post('*', function(req, res) { }); stat.Load(); -app.listen(port); +const httpServer = http.createServer(app); +const httpsServer = https.createServer(certs, app); +httpServer.listen(port); +httpsServer.listen(httpsPort); logger.Log("[STRT]: Server listening on port " + port + "...", logger.GetColor("yellow")); function Log(req, toFile) { try { var ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress; var logEntry = "[RSND]: " + ip + ", " + req.headers['user-agent'] + - " " + req.method + " " + req.url; + " " + req.method + " "; + logEntry += req.url; var color = logger.GetColor("green"); for (var i = 0; i < highlights.length; i++) if (req.url.toLowerCase().includes(highlights[i])) { From 9f8d8329cd2cffb97617245c3eb82c3a06ed5778 Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Mon, 17 Dec 2018 09:59:58 +0100 Subject: [PATCH 08/13] Logging improvements, wont add 404 requests to daily stats --- server.js | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/server.js b/server.js index a587858..58a9da1 100755 --- a/server.js +++ b/server.js @@ -56,22 +56,6 @@ var highlights = ["public", "static", "manual", "isgetting", "postfeedback", "postquestions" ]; -try { - var stdin = process.openStdin(); - stdin.addListener("data", function(d) { - var input = d.toString().trim(); - - if (input == "hello") - console.log("hello, messages: \n" + newMessages); - if (input == "clear" || input == "c") - console.clear(); - - }); -} catch (e) { - console.log("Failed to open STDIN"); - console.log(e); -} - var newMessages = ""; app.set('view engine', 'ejs'); @@ -84,9 +68,11 @@ app.set('view engine', 'ejs'); //}); app.use(function(req, res, next) { res.on('finish', function() { + console.log(res.statusCode); Log(req, true); + if (res.statusCode != 404) + stat.LogStat(req.url); }); - stat.LogStat(req.url); next(); }); app.use(express.static('public')); @@ -223,7 +209,7 @@ const httpServer = http.createServer(app); const httpsServer = https.createServer(certs, app); httpServer.listen(port); httpsServer.listen(httpsPort); -logger.Log("[STRT]: Server listening on port " + port + "...", logger.GetColor("yellow")); +logger.Log("[STRT]: Server listening on port " + port + " (http), and " + httpsPort + " (https)...", logger.GetColor("yellow")); function Log(req, toFile) { try { From 7cb0a3b3787b758850d707dc89e89940e85d21ff Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Mon, 17 Dec 2018 10:03:10 +0100 Subject: [PATCH 09/13] More minor logging changes --- server.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server.js b/server.js index 58a9da1..d9a7ba7 100755 --- a/server.js +++ b/server.js @@ -68,8 +68,7 @@ app.set('view engine', 'ejs'); //}); app.use(function(req, res, next) { res.on('finish', function() { - console.log(res.statusCode); - Log(req, true); + Log(req, true, res.statusCode); if (res.statusCode != 404) stat.LogStat(req.url); }); @@ -195,13 +194,12 @@ app.get('/servergit', function(req, res) { app.get('*', function(req, res) { res.render('404'); res.status(404); - utils.AppendToFile(logger.GetDateString() + ": " + "404 GET", logFile); + // utils.AppendToFile(logger.GetDateString() + ": " + "404 GET", logFile); }); app.post('*', function(req, res) { res.status(404); - utils.AppendToFile(logger.GetDateString() + ": " + "404 POST", logFile); - Log(req, true); + // utils.AppendToFile(logger.GetDateString() + ": " + "404 POST", logFile); }); stat.Load(); @@ -211,12 +209,14 @@ httpServer.listen(port); httpsServer.listen(httpsPort); logger.Log("[STRT]: Server listening on port " + port + " (http), and " + httpsPort + " (https)...", logger.GetColor("yellow")); -function Log(req, toFile) { +function Log(req, toFile, sc) { try { var ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress; var logEntry = "[RSND]: " + ip + ", " + req.headers['user-agent'] + " " + req.method + " "; logEntry += req.url; + if (sc == 404) + logEntry += " " + sc; var color = logger.GetColor("green"); for (var i = 0; i < highlights.length; i++) if (req.url.toLowerCase().includes(highlights[i])) { From 3028a6f0138e9b2d83997b09963fe559cfe390de Mon Sep 17 00:00:00 2001 From: YourFriendlyNeighborhoodDealer <3167982-YourFriendlyNeighborhoodDealer@users.noreply.gitlab.com> Date: Mon, 17 Dec 2018 12:42:08 +0100 Subject: [PATCH 10/13] Small logging, and manual changes --- actions.js | 1 + public/man.html | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/actions.js b/actions.js index 9432e1d..574576a 100755 --- a/actions.js +++ b/actions.js @@ -293,6 +293,7 @@ function ProcessIncomingRequest(data) { logger.Log("[PCES]:\t" + msg, color); } catch (e) { + console.log(e); logger.Log("[PCES]: Couldnt parse JSON data, trying old format..."); var d = SetupData(data); var qcount = -1; diff --git a/public/man.html b/public/man.html index 3ae54b7..4d5fe7e 100644 --- a/public/man.html +++ b/public/man.html @@ -132,6 +132,8 @@ változtatni akarsz, akkor illik tudni

A script radiobuttonos, tickboxos, képes és szöveges kérdéseknél is jól működik. Egyéb típusú kérdésekre nincsen (nagyon) felkészítve. +

Érdemes az éppen nem használt tárgyakat nem-bepipálni, mert akkor baromi sok kérdés + közül kell keresni, és az előfordulhat, hogy lassú.

Egyéb funkciók: