From 75ba9905b48f0551b84920c5778e45d05e1d1830 Mon Sep 17 00:00:00 2001 From: MrFry Date: Sat, 7 Mar 2020 14:19:03 +0100 Subject: [PATCH 1/7] Allqr downloadable txt --- modules/qmining/qmining.js | 8 ++++---- modules/qmining/views/alldata.ejs | 2 +- modules/qmining/views/allqr.ejs | 23 ----------------------- public/moodle-test-userscript | 2 +- utils/question-classes | 2 +- 5 files changed, 7 insertions(+), 30 deletions(-) delete mode 100755 modules/qmining/views/allqr.ejs diff --git a/modules/qmining/qmining.js b/modules/qmining/qmining.js index 7c70ca0..a4eb481 100644 --- a/modules/qmining/qmining.js +++ b/modules/qmining/qmining.js @@ -179,10 +179,10 @@ app.get('/menuClick', function (req, res) { }) // all questions readable -app.get('/allqr', function (req, res) { - res.render('allqr', { - d: data.toString().split('\n') - }) +app.get('/allqr.txt', function (req, res) { + res.set('Content-Type', 'text/plain') + res.send(data.toString()) + res.end() logger.LogReq(req) }) diff --git a/modules/qmining/views/alldata.ejs b/modules/qmining/views/alldata.ejs index b19de9e..62371ef 100755 --- a/modules/qmining/views/alldata.ejs +++ b/modules/qmining/views/alldata.ejs @@ -78,7 +78,7 @@ <%=scount%> tárgy és <%=qcount%> kérdés

- Letöltés + Letöltés (ctrl-s ha nem tölti le file-ba) diff --git a/modules/qmining/views/allqr.ejs b/modules/qmining/views/allqr.ejs deleted file mode 100755 index 6e6b48d..0000000 --- a/modules/qmining/views/allqr.ejs +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - All questions - - - - - - <% for (var i = 0; i < d.length; i++) { %> - <%= d[i] %> -
- <% } %> - - - diff --git a/public/moodle-test-userscript b/public/moodle-test-userscript index 1e81548..fe74436 160000 --- a/public/moodle-test-userscript +++ b/public/moodle-test-userscript @@ -1 +1 @@ -Subproject commit 1e8154802cf4c17a6c8e71a577c6fa9f4957c2b1 +Subproject commit fe74436ab119a31fdb35f7a3bbae50a572b0fb21 diff --git a/utils/question-classes b/utils/question-classes index 40f1878..e84ff44 160000 --- a/utils/question-classes +++ b/utils/question-classes @@ -1 +1 @@ -Subproject commit 40f187881337f2c3f3fe50a3799d7f46d4c6361c +Subproject commit e84ff4451251f3e8403ebb8b31ebd13d7830e44a From e610a7e722ed8208ba58e519ef9a47bf073ff241 Mon Sep 17 00:00:00 2001 From: MrFry Date: Sat, 7 Mar 2020 14:25:22 +0100 Subject: [PATCH 2/7] Removed logging from /ask api --- modules/qmining/qmining.js | 2 +- public/moodle-test-userscript | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/qmining/qmining.js b/modules/qmining/qmining.js index 7c70ca0..ac29e4a 100644 --- a/modules/qmining/qmining.js +++ b/modules/qmining/qmining.js @@ -286,7 +286,7 @@ app.post('/isAdding', function (req, res) { }) app.get('/ask', function (req, res) { - logger.LogReq(req) + // logger.LogReq(req) if (Object.keys(req.query).length === 0) { res.json({ message: `ask something! ?q=[question]&subj=[subject]&data=[question data]. 'subj' is optimal for faster result`, diff --git a/public/moodle-test-userscript b/public/moodle-test-userscript index 1e81548..3d1399c 160000 --- a/public/moodle-test-userscript +++ b/public/moodle-test-userscript @@ -1 +1 @@ -Subproject commit 1e8154802cf4c17a6c8e71a577c6fa9f4957c2b1 +Subproject commit 3d1399c13b54b4c881c3c7af95003b0397d36290 From 72aefca297f24a7428bad4f2809163c40fea7d55 Mon Sep 17 00:00:00 2001 From: MrFry Date: Sat, 7 Mar 2020 16:18:01 +0100 Subject: [PATCH 3/7] logging decode uri errors --- modules/qmining/qmining.js | 8 +++++++- utils/question-classes | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/qmining/qmining.js b/modules/qmining/qmining.js index a4eb481..4eb7acd 100644 --- a/modules/qmining/qmining.js +++ b/modules/qmining/qmining.js @@ -297,7 +297,13 @@ app.get('/ask', function (req, res) { } else { if (req.query.q && req.query.data) { let subj = req.query.subj || '' - let question = decodeURIComponent(req.query.q) + let question + try { + question = decodeURIComponent(req.query.q) + } catch (e) { + console.err(req.query) + throw e + } let recData = {} try { recData = JSON.parse(req.query.data) diff --git a/utils/question-classes b/utils/question-classes index e84ff44..a117239 160000 --- a/utils/question-classes +++ b/utils/question-classes @@ -1 +1 @@ -Subproject commit e84ff4451251f3e8403ebb8b31ebd13d7830e44a +Subproject commit a117239f35610520706e402382c3911d422d743e From a7a75bd9d43010919aa4dc834d70db156d46e756 Mon Sep 17 00:00:00 2001 From: MrFry Date: Thu, 12 Mar 2020 22:25:26 +0100 Subject: [PATCH 4/7] Added debug logging --- modules/qmining/qmining.js | 8 ++++++++ server.js | 12 +++++++++++- utils/actions.js | 16 +++++++++++++++- utils/logger.js | 32 ++++++++++++++++++++++++++++---- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/modules/qmining/qmining.js b/modules/qmining/qmining.js index 4eb7acd..8038918 100644 --- a/modules/qmining/qmining.js +++ b/modules/qmining/qmining.js @@ -273,6 +273,8 @@ app.post('/isAdding', function (req, res) { logger.LogReq(req) // automatically saves to dataFile every n write + logger.DebugLog('request.body:', 4) + logger.DebugLog(req.body, 4) let result = actions.ProcessIncomingRequest( req.body.datatoadd, data, @@ -299,6 +301,7 @@ app.get('/ask', function (req, res) { let subj = req.query.subj || '' let question try { + logger.DebugLog('Decoding URI component', 4) question = decodeURIComponent(req.query.q) } catch (e) { console.err(req.query) @@ -306,11 +309,16 @@ app.get('/ask', function (req, res) { } let recData = {} try { + logger.DebugLog('Parsing req.query.data', 4) recData = JSON.parse(req.query.data) } catch (e) { logger.Log(`Unable to parse recieved question data! '${req.query.data}'`, logger.GetColor('redbg')) } + logger.DebugLog('Searching in DB', 4) let r = data.Search(question, subj, recData) + logger.DebugLog('Result length: ' + r.length, 1) + logger.DebugLog('Result:', 3) + logger.DebugLog(r, 3) res.json({ result: r, diff --git a/server.js b/server.js index 9b50137..7dd86a2 100755 --- a/server.js +++ b/server.js @@ -25,7 +25,6 @@ const httpsport = 8443 const express = require('express') const vhost = require('vhost') const logger = require('./utils/logger.js') -logger.Load() const utils = require('./utils/utils.js') const http = require('http') const https = require('https') @@ -38,6 +37,17 @@ const loggableKeywords = [ ] let modules = JSON.parse(utils.ReadFile(modulesFile)) +let logLevel = parseInt(GetParams()[0]) +if (isNaN(logLevel)) { + logLevel = 0 +} +logger.Log('Loglevel is: ' + logLevel) +logger.Load(logLevel) // TODO: debug level in enviromet variable / param + +function GetParams () { + return process.argv.splice(2) +} + try { if (utils.FileExists(extraModulesFile)) { const extraModules = JSON.parse(utils.ReadFile(extraModulesFile)) diff --git a/utils/actions.js b/utils/actions.js index ff14935..a957b56 100755 --- a/utils/actions.js +++ b/utils/actions.js @@ -37,6 +37,7 @@ const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate var currWrites = 0 function ProcessIncomingRequest (recievedData, qdb, infos) { + logger.DebugLog('Processing incoming request', 1) if (recievedData === undefined) { logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) return @@ -48,6 +49,7 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { towrite += recievedData towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) + logger.DebugLog('recDataFile written', 1) } catch (e) { logger.log('Error writing recieved data.') } @@ -55,20 +57,29 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { try { // recievedData: { version: "", id: "", subj: "" quiz: {} } let d = JSON.parse(recievedData) + logger.DebugLog('recievedData JSON parsed', 1) let allQLength = d.quiz.length let allQuestions = [] + logger.DebugLog(logger.hr, 2) d.quiz.forEach((question) => { + logger.DebugLog('Question:', 2) + logger.DebugLog(question, 2) let q = new classes.Question(question.Q, question.A, question.data) let sames = qdb.Search(q, d.subj) + logger.DebugLog('Same questions:', 2) + logger.DebugLog('Length: ' + sames.length, 2) + logger.DebugLog(sames, 3) // if it didnt find any question, or every found questions match is lower thatn 80 let isNew = sames.length === 0 || sames.every(searchResItem => { return searchResItem.match < classes.minMatchAmmount }) + logger.DebugLog('isNew: ' + isNew, 2) if (isNew) { allQuestions.push(q) } }) + logger.DebugLog(logger.hr, 2) let color = logger.GetColor('green') let msg = '' @@ -81,15 +92,17 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { }) currWrites++ - console.log(currWrites) + logger.DebugLog('currWrites for data.json: ' + currWrites, 1) if (currWrites >= writeAfter) { currWrites = 0 try { qdb.version = infos.version qdb.motd = infos.motd + logger.DebugLog('version and motd set for data.json', 3) } catch (e) { logger.Log('MOTD/Version writing/reading error!') } + logger.DebugLog('Writing data.json', 1) utils.WriteFile(JSON.stringify(qdb), dataFile) logger.Log('\tData file written', color) } @@ -106,6 +119,7 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { if (d.version !== undefined) { msg += '. Version: ' + d.version } logger.Log('\t' + msg, color) + logger.DebugLog('ProcessIncomingRequest done', 1) return allQuestions.length } catch (e) { console.log(e) diff --git a/utils/logger.js b/utils/logger.js index 5ce78d0..8993bc3 100755 --- a/utils/logger.js +++ b/utils/logger.js @@ -18,14 +18,18 @@ ------------------------------------------------------------------------- */ +const hr = '---------------------------------------------------------------------------------' + module.exports = { GetDateString: GetDateString, Log: Log, + DebugLog: DebugLog, GetColor: GetColor, LogReq: LogReq, LogStat: LogStat, Load: Load, - logHashed: logHashed + logHashed: logHashed, + hr: hr } const DELIM = C('green') + '|' + C() @@ -43,6 +47,7 @@ const writeInterval = 10 let data = {} let vData = {} let writes = 0 +let debugLevel = 0 const colors = [ 'green', @@ -66,9 +71,25 @@ function GetDateString () { return GetRandomColor(m.getHours().toString()) + d + C() } +function DebugLog (msg, lvl) { + if (lvl <= debugLevel) { + let s = msg + let header = C('red') + '#DEBUG ' + lvl + '#' + C() + DELIM + C() + if (typeof msg !== 'object') { + s = header + msg + } else { + Log(header + 'OBJECT:', 'yellow') + } + Log(s, 'yellow') + } +} + function Log (s, c) { - let dl = DELIM + C(c) - let log = C(c) + GetDateString() + dl + s + let log = s + if (typeof s !== 'object') { + let dl = DELIM + C(c) + log = C(c) + GetDateString() + dl + s + } console.log(log) utils.AppendToFile(log, logFile) @@ -132,7 +153,10 @@ function setNoLogReadInterval () { parseNoLogFile(utils.ReadFile(nologFile)) } -function Load () { +function Load (debugLvl) { + if (debugLvl) { + debugLevel = debugLvl + } Log('Loading logger...') try { var prevData = utils.ReadFile(statFile) From d1e03f4c4116d75faeec0ad472d90d6d99c9032c Mon Sep 17 00:00:00 2001 From: MrFry Date: Sun, 15 Mar 2020 09:58:20 +0100 Subject: [PATCH 5/7] Added debug more debug logging, saving every question that doesnt has a match with at leas 90% --- utils/actions.js | 34 ++++++++++++++++++---------------- utils/logger.js | 7 +++++-- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/utils/actions.js b/utils/actions.js index a957b56..90afc49 100755 --- a/utils/actions.js +++ b/utils/actions.js @@ -32,12 +32,16 @@ const idStats = require('../utils/ids.js') idStats.Load() // FIXME: dont always load when actions.js is used const utils = require('../utils/utils.js') const classes = require('./question-classes/classes.js') +classes.initLogger(logger.DebugLog) +// if a recievend question doesnt match at least this % to any other question in the db it gets +// added to db +const minMatchAmmountToAdd = 90 // FIXME: test this value const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate var currWrites = 0 function ProcessIncomingRequest (recievedData, qdb, infos) { - logger.DebugLog('Processing incoming request', 1) + logger.DebugLog('Processing incoming request', 'actions', 1) if (recievedData === undefined) { logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) return @@ -49,7 +53,7 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { towrite += recievedData towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) - logger.DebugLog('recDataFile written', 1) + logger.DebugLog('recDataFile written', 'actions', 1) } catch (e) { logger.log('Error writing recieved data.') } @@ -57,29 +61,27 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { try { // recievedData: { version: "", id: "", subj: "" quiz: {} } let d = JSON.parse(recievedData) - logger.DebugLog('recievedData JSON parsed', 1) + logger.DebugLog('recievedData JSON parsed', 'actions', 1) let allQLength = d.quiz.length let allQuestions = [] - logger.DebugLog(logger.hr, 2) d.quiz.forEach((question) => { - logger.DebugLog('Question:', 2) - logger.DebugLog(question, 2) + logger.DebugLog('Question:', 'actions', 2) + logger.DebugLog(question, 'actions', 2) let q = new classes.Question(question.Q, question.A, question.data) let sames = qdb.Search(q, d.subj) - logger.DebugLog('Same questions:', 2) - logger.DebugLog('Length: ' + sames.length, 2) - logger.DebugLog(sames, 3) + logger.DebugLog('Same questions:', 'actions', 2) + logger.DebugLog('Length: ' + sames.length, 'actions', 2) + logger.DebugLog(sames, 'actions', 3) // if it didnt find any question, or every found questions match is lower thatn 80 let isNew = sames.length === 0 || sames.every(searchResItem => { - return searchResItem.match < classes.minMatchAmmount + return searchResItem.match < minMatchAmmountToAdd }) - logger.DebugLog('isNew: ' + isNew, 2) + logger.DebugLog('isNew: ' + isNew, 'actions', 2) if (isNew) { allQuestions.push(q) } }) - logger.DebugLog(logger.hr, 2) let color = logger.GetColor('green') let msg = '' @@ -92,17 +94,17 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { }) currWrites++ - logger.DebugLog('currWrites for data.json: ' + currWrites, 1) + logger.DebugLog('currWrites for data.json: ' + currWrites, 'actions', 1) if (currWrites >= writeAfter) { currWrites = 0 try { qdb.version = infos.version qdb.motd = infos.motd - logger.DebugLog('version and motd set for data.json', 3) + logger.DebugLog('version and motd set for data.json', 'actions', 3) } catch (e) { logger.Log('MOTD/Version writing/reading error!') } - logger.DebugLog('Writing data.json', 1) + logger.DebugLog('Writing data.json', 'actions', 1) utils.WriteFile(JSON.stringify(qdb), dataFile) logger.Log('\tData file written', color) } @@ -119,7 +121,7 @@ function ProcessIncomingRequest (recievedData, qdb, infos) { if (d.version !== undefined) { msg += '. Version: ' + d.version } logger.Log('\t' + msg, color) - logger.DebugLog('ProcessIncomingRequest done', 1) + logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) return allQuestions.length } catch (e) { console.log(e) diff --git a/utils/logger.js b/utils/logger.js index 8993bc3..35a6f09 100755 --- a/utils/logger.js +++ b/utils/logger.js @@ -71,10 +71,13 @@ function GetDateString () { return GetRandomColor(m.getHours().toString()) + d + C() } -function DebugLog (msg, lvl) { +function DebugLog (msg, name, lvl) { if (lvl <= debugLevel) { + if (msg === 'hr') { + msg = hr + } let s = msg - let header = C('red') + '#DEBUG ' + lvl + '#' + C() + DELIM + C() + let header = `${C('red')}#DEBUG${lvl}#${C('yellow')}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}` if (typeof msg !== 'object') { s = header + msg } else { From 6a711818611a53b1f2a1cba38311bb4840aa22f4 Mon Sep 17 00:00:00 2001 From: MrFry Date: Sun, 15 Mar 2020 10:00:20 +0100 Subject: [PATCH 6/7] Updated submodules --- utils/question-classes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/question-classes b/utils/question-classes index a117239..7f9ea08 160000 --- a/utils/question-classes +++ b/utils/question-classes @@ -1 +1 @@ -Subproject commit a117239f35610520706e402382c3911d422d743e +Subproject commit 7f9ea08717b902a902ff924442d5bba230d74b19 From b6c24952443d778cb7c84a05006fb603f30a792f Mon Sep 17 00:00:00 2001 From: MrFry Date: Sun, 15 Mar 2020 10:01:59 +0100 Subject: [PATCH 7/7] decodeURIComponent in try --- modules/qmining/qmining.js | 8 +++++++- public/moodle-test-userscript | 2 +- utils/question-classes | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/qmining/qmining.js b/modules/qmining/qmining.js index c7ea4ad..e7a4db4 100644 --- a/modules/qmining/qmining.js +++ b/modules/qmining/qmining.js @@ -297,7 +297,13 @@ app.get('/ask', function (req, res) { } else { if (req.query.q && req.query.data) { let subj = req.query.subj || '' - let question = decodeURIComponent(req.query.q) + let question + try { + question = decodeURIComponent(req.query.q) + } catch (e) { + console.error(req.query) + throw e + } let recData = {} try { recData = JSON.parse(req.query.data) diff --git a/public/moodle-test-userscript b/public/moodle-test-userscript index 194fc22..f69734e 160000 --- a/public/moodle-test-userscript +++ b/public/moodle-test-userscript @@ -1 +1 @@ -Subproject commit 194fc22d06adc664aa1a8a3a02a81edae613f888 +Subproject commit f69734e571ec02d6c7de6ecabe8ca83c8bc62623 diff --git a/utils/question-classes b/utils/question-classes index 40f1878..a117239 160000 --- a/utils/question-classes +++ b/utils/question-classes @@ -1 +1 @@ -Subproject commit 40f187881337f2c3f3fe50a3799d7f46d4c6361c +Subproject commit a117239f35610520706e402382c3911d422d743e