From f8a4ccef8360132a2d9c701e661525a718088ccc Mon Sep 17 00:00:00 2001 From: mrfry Date: Wed, 24 Feb 2021 17:25:11 +0100 Subject: [PATCH 1/3] Moved extra modules, qmining redirect minor fixes --- .gitignore | 9 +++ devel/tests/testData/withLocation.json | 78 ++++++++++++++++++++++++++ src/modules/qmining/qmining.ts | 14 +++-- src/server.ts | 2 +- submodules/qmining-page | 2 +- 5 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 .gitignore create mode 100644 devel/tests/testData/withLocation.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ee333a8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +data/ +stats/ +node_modules/ +dist/ +nextStatic/ +publicDirs/ +extraModules/ +nolog + diff --git a/devel/tests/testData/withLocation.json b/devel/tests/testData/withLocation.json new file mode 100644 index 0000000..6c8ae4c --- /dev/null +++ b/devel/tests/testData/withLocation.json @@ -0,0 +1,78 @@ +{ + "subj": "Válgazd II. 2020S BSc Nap EA", + "version": "2.1.0.0", + "id": "1582138883656", + "location": "https://bacon.com", + "quiz": [ + { + "Q": "Ha a forgóeszközök értéke 42000, a készletek 10000 és a rövid lejáratú kötelezettségek 33600, mekkora a likviditási ráta (L2) (két tizedesig)?", + "A": "A helyes válasz: 1,25.", + "data": { + "type": "simple" + } + }, + { + "Q": "Folyó források lehetnek", + "A": "c. Rövid lejáratú hitel, szállítói kötelezettségek, ÁFA tartozás", + "data": { + "type": "simple" + } + }, + { + "Q": "A pénzügyi tervezés és gazdálkodás módszerei a következők:", + "A": "b. mobilitás vizsgálat,likviditási terv, diszponibilitás", + "data": { + "type": "simple" + } + }, + { + "Q": "A mérlegben az eszközök pénzzétételi lehetősége és a források esedékessége között egyensúly kell hogy fenn álljon.", + "A": "Igaz", + "data": { + "type": "simple" + } + }, + { + "Q": "Ha a nettó forgótőke pozitív, akkor a vállalkozás rövid lejáratú forrásokból fedezi befektetett eszközei egy részét.", + "A": "Hamis", + "data": { + "type": "simple" + } + }, + { + "Q": "Az adósságszolgálati mutató a vállalati tiszta jövedelem és az azt terhelő kötelezettségek hányadosa.", + "A": "Igaz", + "data": { + "type": "simple" + } + }, + { + "Q": "Az éves beszámoló részei:", + "A": "b. Mérleg, eredménykimutatás, kiegészítő melléklet", + "data": { + "type": "simple" + } + }, + { + "Q": "Ha a vállalati saját tőkéje 25M, és az összes kötelezettsége 100M, mekkora a vállalat tőkeellátottsági mutatója (%)?", + "A": "A helyes válasz: 20.", + "data": { + "type": "simple" + } + }, + { + "Q": "Egy vállalat tavalyi 2019-es eredményét a mérlegen a saját tőke részeként a mérleg szerinti eredmény soron láthatjuk.\n", + "A": "Hamis", + "data": { + "type": "simple" + } + }, + { + "Q": "A készletezési periódus a szállítói számla kiegyenlítésétől a vevővel szembeni követelés beérkezéséig tart.", + "A": "Hamis", + "data": { + "type": "simple" + } + } + ] +} diff --git a/src/modules/qmining/qmining.ts b/src/modules/qmining/qmining.ts index 6919e8b..7f86234 100644 --- a/src/modules/qmining/qmining.ts +++ b/src/modules/qmining/qmining.ts @@ -98,10 +98,14 @@ function GetApp(): ModuleType { loadDonateURL() - utils.WatchFile(linksFile, (newData) => { - logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`) - loadDonateURL() - }) + if (utils.FileExists(linksFile)) { + utils.WatchFile(linksFile, (newData) => { + logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`) + loadDonateURL() + }) + } else { + logger.Log('Couldnt read donate URL file!', logger.GetColor('red')) + } // -------------------------------------------------------------- // REDIRECTS @@ -187,8 +191,6 @@ function GetApp(): ModuleType { }, ] - console.log(links, simpleRedirects) - simpleRedirects.forEach((redirect) => { app.get(redirect.from, function(req: Request, res) { if (!redirect.nolog) { diff --git a/src/server.ts b/src/server.ts index 6663bf3..2e6d758 100755 --- a/src/server.ts +++ b/src/server.ts @@ -40,7 +40,7 @@ import utils from './utils/utils' import dbtools from './utils/dbtools' import reqlogger from './middlewares/reqlogger.middleware' import idStats from './utils/ids' -const extraModulesFile = './data/extraModules.json' +const extraModulesFile = './extraModules/extraModules.json' const statExcludeFile = './data/statExclude.json' const modulesFile = './src/modules.json' const usersDBPath = './data/dbs/users.db' diff --git a/submodules/qmining-page b/submodules/qmining-page index 6b6b9a4..599ef00 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 6b6b9a43d4d9c2e429adaf76c461c0304dad1133 +Subproject commit 599ef00635c15d500dfcc1a9c226212000e74707 From 6424a01758ac04f0c2ba4cd090149c234f8dee42 Mon Sep 17 00:00:00 2001 From: mrfry Date: Wed, 24 Feb 2021 17:41:13 +0100 Subject: [PATCH 2/3] git fixes --- submodules/moodle-test-userscript | 2 +- submodules/qmining-data-editor | 2 +- submodules/qmining-page | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/submodules/moodle-test-userscript b/submodules/moodle-test-userscript index dc07c90..969c251 160000 --- a/submodules/moodle-test-userscript +++ b/submodules/moodle-test-userscript @@ -1 +1 @@ -Subproject commit dc07c903d39003a5cc13302a9ee6e6e3e0b6ee17 +Subproject commit 969c2516c3bc8d454ebf6bffb621c79c968da334 diff --git a/submodules/qmining-data-editor b/submodules/qmining-data-editor index 83ed7f3..33e8b3a 160000 --- a/submodules/qmining-data-editor +++ b/submodules/qmining-data-editor @@ -1 +1 @@ -Subproject commit 83ed7f3b312e0de67505601d13774bdcf2971420 +Subproject commit 33e8b3a49e7ddbf5c52721c51e655dc28b6ff877 diff --git a/submodules/qmining-page b/submodules/qmining-page index 599ef00..8e0c03b 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 599ef00635c15d500dfcc1a9c226212000e74707 +Subproject commit 8e0c03ba360f79c6dbbab30b7f0166fce3174529 From 0336860a9d62ef42a813dd669f0e3ad3605d45a5 Mon Sep 17 00:00:00 2001 From: mrfry Date: Thu, 25 Feb 2021 10:19:35 +0100 Subject: [PATCH 3/3] server stats in javascript --- scripts/serverStats.js | 286 ++++++++++++++++++++++++++++++++++------- 1 file changed, 237 insertions(+), 49 deletions(-) diff --git a/scripts/serverStats.js b/scripts/serverStats.js index f1b1e1f..afeeed9 100644 --- a/scripts/serverStats.js +++ b/scripts/serverStats.js @@ -2,15 +2,61 @@ // CONFIG // ----------------------------------------------------------------- -const maxStatLength = 15 +const cols = process.stdout.columns +// const rows = process.stdout.rows +const maxStatLength = Math.floor(cols / 4) const statNameSpacing = 5 +const beforeRowSpace = 13 const coloredWords = { - red: ['lred'], + red: ['lred', 'thanks'], + cyan: [ + 'getveteranpw', + 'pwrequest', + 'getpw', + 'availablepws', + 'login', + 'logout', + ], + green: [ + 'manual', + 'todos', + 'allquestions', + 'subjectbrowser', + 'contribute', + 'feedback', + 'ranklist', + 'allqr', + 'possibleAnswers', + ], + blue: ['isadding', 'ask'], + magenta: [ + 'donate', + 'tiszai', + 'install', + 'irc', + 'discord', + 'postfeedback', + 'votetodo', + 'registerscript', + 'quickvote', + ], } +const filterFromDailyStats = [ + 'savedQuestions', + 'sio/f', + 'sound/', + '/img/', + '.php', + 'favicon', + 'robots.txt', + 'ads.txt', + '/f/', + '.git', +] // ----------------------------------------------------------------- -const fs = require('fs') +const fs = require('fs') // eslint-disable-line const dir = process.argv[2] if (!dir) { @@ -23,19 +69,29 @@ function getDayIndex(offset) { offset = 0 } - let date = new Date() + const date = new Date() if (offset) { date.setDate(date.getDate() + offset) } return ( date.getFullYear() + - '/' + + '-' + ('0' + (date.getMonth() + 1)).slice(-2) + - '/' + + '-' + ('0' + date.getDate()).slice(-2) ) } +function hr(char) { + console.log(C('blue') + getLetterNTimes(char || '-', cols) + C()) +} + +function printHeader(text) { + hr() + console.log(C('green') + text + C()) + hr() +} + function C(color) { if (color !== undefined) { color = color.toLowerCase() @@ -73,7 +129,7 @@ function readJSON(name) { } function tail(text, number) { - let splitedText = text.split('\n') + const splitedText = text.split('\n') return splitedText.slice(Math.max(splitedText.length - number, 1)).join('\n') } @@ -84,6 +140,16 @@ function head(text, number) { .join('\n') } +function countLinesMatching(text, toMatch) { + let count = 0 + text.split('\n').forEach((line) => { + if (line.includes(toMatch.toLowerCase())) { + count++ + } + }) + return count +} + function readFile(name) { return fs.readFileSync(name, 'utf8') } @@ -96,99 +162,196 @@ function getLetterNTimes(letter, number) { return res } -function pCols() { - // console.log(arguments) +function pCols(cols, rowTitles, colorNames, firstRowColor, showDiff) { + // console.log(cols) let maxLength = 0 - Object.keys(arguments).forEach((key) => { - if (arguments[key].length > maxLength) { - maxLength = arguments[key].length + Object.keys(cols).forEach((key) => { + if (cols[key].length > maxLength) { + maxLength = cols[key].length } }) for (let i = 0; i < maxLength; i++) { - let row = [] - Object.keys(arguments) + const row = [] + const lastItems = [] + Object.keys(cols) .reverse() .forEach((key) => { - const val = arguments[key] + const val = cols[key] if (!val[i]) { return } - const keyName = val[i].name || '' + lastItems.push(val[i]) + const keyName = val[i].name || val[i] let slicedName = keyName.slice(0, maxStatLength) - let toColor = Object.keys(coloredWords).reduce((acc, key) => { - const colorArray = coloredWords[key] + const toColor = colorNames + ? Object.keys(coloredWords).reduce((acc, key) => { + const colorArray = coloredWords[key] - const includes = colorArray.some((colorableIdName) => { - return keyName.includes(colorableIdName) - }) + const includes = colorArray.some((colorableIdName) => { + return keyName + .toLowerCase() + .includes(colorableIdName.toLowerCase()) + }) - if (includes) { - return key - } + if (includes) { + return key + } - return acc - }, '') + return acc + }, '') + : false + + const sep = (i + 1) % 5 === 0 ? '.' : ' ' while (slicedName.length < maxStatLength) { - slicedName = ' ' + slicedName + slicedName = slicedName + sep } - let ammount = val[i].val ? val[i].val.toString() : 0 + let ammount = val[i].val ? val[i].val.toString() : '' while (ammount.length < 4) { ammount = ammount + ' ' } if (toColor) { - row.push(C(toColor) + slicedName + ': ' + ammount + C()) + row.push(C(toColor) + slicedName + ' ' + ammount + C()) } else { row.push(slicedName + ' ' + ammount) } }) - console.log(row.join(getLetterNTimes(' ', statNameSpacing))) + + // ROW TITLE --------------------------------------------------- + let currRowTitle = + rowTitles && rowTitles[i] + ? rowTitles[i] + : getLetterNTimes(' ', beforeRowSpace) + + while (currRowTitle.length < beforeRowSpace) { + currRowTitle = currRowTitle + ' ' + } + currRowTitle = C('blue') + currRowTitle + C() + // COLORING ---------------------------------------------------- + let res = '' + if (firstRowColor && i === 0) { + res = + currRowTitle + + C('green') + + row.join(getLetterNTimes(' ', statNameSpacing)) + + C() + } else { + res = currRowTitle + row.join(getLetterNTimes(' ', statNameSpacing)) + } + // SHOW DIFF --------------------------------------------------- + if (showDiff && i !== 0) { + let diff = + lastItems[lastItems.length - 1] - lastItems[lastItems.length - 2] + if (diff > 0) { + diff = '+' + diff.toString() + } + res += C('blue') + diff + C() + } + + console.log(res) } } // ------------------------------------------------------------------------------ +printHeader('Daily stats') + const dailyStats = readJSON(`${dir}stats/vstats`) function preProcessDailyStats(obj) { - return Object.keys(obj).map((key) => { - return { name: key, val: obj[key] } + const formatted = Object.keys(obj).reduce((acc, key) => { + const includes = filterFromDailyStats.some((keyword) => { + return key.toLowerCase().includes(keyword.toLowerCase()) + }) + if (!includes) { + acc.push({ name: key, val: obj[key] }) + } + return acc + }, []) + return formatted.sort((a, b) => { + if (a.name > b.name) { + return 1 + } else if (a.name < b.name) { + return -1 + } else { + return 0 + } }) } pCols( - preProcessDailyStats(dailyStats[getDayIndex()]), - preProcessDailyStats(dailyStats[getDayIndex(-1)]), - preProcessDailyStats(dailyStats[getDayIndex(-2)]) + [ + preProcessDailyStats(dailyStats[getDayIndex()]), + preProcessDailyStats(dailyStats[getDayIndex(-1)]), + preProcessDailyStats(dailyStats[getDayIndex(-2)]), + ], + null, + true ) // ------------------------------------------------------------------------------ +printHeader('User id test solving') const userIdTestSolving = readJSON(`${dir}stats/idvstats`) -function preProcessUIdTestSolving(obj) { +function preProcessUIdTestSolving(obj, minLength) { if (!obj) { - return [{ val: 0 }] + return '0' + } + if (minLength) { + return Object.keys(obj) + .filter((key) => { + return obj[key] > minLength + }) + .length.toString() + } else { + return Object.keys(obj).length.toString() } - return [{ val: Object.keys(obj).length }] } pCols( - preProcessUIdTestSolving(userIdTestSolving[getDayIndex()]), - preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-1)]), - preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-2)]) + [ + ['Today', preProcessUIdTestSolving(userIdTestSolving[getDayIndex()])], + ['Yesterday', preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-1)])], + [ + 'Before yesterday', + preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-2)]), + ], + ], + null, + false, + 'green' ) // ------------------------------------------------------------------------------ +printHeader('User id requests') const clientIdTestSolving = readJSON(`${dir}stats/uvstats`) pCols( - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()]), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)]), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)]) + [ + [ + 'Today', + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()]), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()], 2), + ], + [ + 'Yesterday', + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)]), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)], 2), + ], + [ + 'Before Yesterday', + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)]), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)], 2), + ], + ], + ['', 'All', 'More than 2'], + false, + 'green' ) // ------------------------------------------------------------------------------ +printHeader('Daily data count') const dailyDataCount = readFile(`${dir}stats/dailyDataCount`) printLastDataCount([ @@ -198,12 +361,37 @@ printLastDataCount([ ]) function printLastDataCount(data) { - const res = [[], [], []] + const res = [['Today'], ['Yesterday'], ['Before yesterday']] data.forEach((dataCount, i) => { - res[i].push({ val: dataCount.userCount }) - res[i].push({ val: dataCount.subjectCount }) - res[i].push({ val: dataCount.questionCount }) + res[i].push(dataCount.userCount.toString()) + res[i].push(dataCount.subjectCount.toString()) + res[i].push(dataCount.questionCount.toString()) }) - pCols(...res) + pCols(res, ['', 'Users', 'Subjects', 'Questions'], false, 'green', true) } +// ------------------------------------------------------------------------------ +printHeader('Daily script install / update check count') +const todaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex()}`) +const yesterdaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex(-1)}`) +const beforeYesterdaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex(-2)}`) + +const installs = [ + [ + 'Today', + countLinesMatching(todaysLogs, '?install').toString(), + countLinesMatching(todaysLogs, '?up').toString(), + ], + [ + 'Yesterday', + countLinesMatching(yesterdaysLogs, '?install').toString(), + countLinesMatching(yesterdaysLogs, '?up').toString(), + ], + [ + 'Before yesterday', + countLinesMatching(beforeYesterdaysLogs, '?install').toString(), + countLinesMatching(beforeYesterdaysLogs, '?up').toString(), + ], +] + +pCols(installs, ['', 'Installs', 'Updates'], false, 'green')