diff --git a/package-lock.json b/package-lock.json index 0349465..f219bec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,6 @@ "dependencies": { "@types/express": "^4.17.9", "@types/node": "^15.0.1", - "@types/socket.io": "^3.0.2", "better-sqlite3": "^7.1.5", "connect-busboy": "0.0.2", "cookie-parser": "^1.4.5", @@ -1144,15 +1143,6 @@ "@types/node": "*" } }, - "node_modules/@types/socket.io": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-3.0.2.tgz", - "integrity": "sha512-pu0sN9m5VjCxBZVK8hW37ZcMe8rjn4HHggBN5CbaRTvFwv5jOmuIRZEuddsBPa9Th0ts0SIo3Niukq+95cMBbQ==", - "deprecated": "This is a stub types definition. socket.io provides its own type definitions, so you do not need this installed.", - "dependencies": { - "socket.io": "*" - } - }, "node_modules/@types/stack-utils": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.0.tgz", @@ -9461,14 +9451,6 @@ "@types/node": "*" } }, - "@types/socket.io": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-3.0.2.tgz", - "integrity": "sha512-pu0sN9m5VjCxBZVK8hW37ZcMe8rjn4HHggBN5CbaRTvFwv5jOmuIRZEuddsBPa9Th0ts0SIo3Niukq+95cMBbQ==", - "requires": { - "socket.io": "*" - } - }, "@types/stack-utils": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.0.tgz", diff --git a/package.json b/package.json index b81d889..2ef4b75 100755 --- a/package.json +++ b/package.json @@ -4,7 +4,6 @@ "dependencies": { "@types/express": "^4.17.9", "@types/node": "^15.0.1", - "@types/socket.io": "^3.0.2", "better-sqlite3": "^7.1.5", "connect-busboy": "0.0.2", "cookie-parser": "^1.4.5", @@ -23,7 +22,7 @@ "dev": "npm run build && NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node ./dist/server.js", "build": "tsc && bash -c './scripts/postBuild.sh'", "export": "tsc && bash -c './scripts/postBuild.sh'", - "test": "jest" + "test": "jest --detectOpenHandles" }, "devDependencies": { "@types/jest": "^26.0.23", diff --git a/scripts/make.sh b/scripts/make.sh old mode 100755 new mode 100644 index 721127b..d3366d2 --- a/scripts/make.sh +++ b/scripts/make.sh @@ -1,79 +1,85 @@ #!/bin/bash +hr() { + printf '\033[0;32m%*s\033[0m\n' "$(tput cols)" '' | tr ' ' '=' +} + +log() { + hr + echo -e "\033[0;32m${@}\033[0m" + hr +} + checkFile() { if [ ! -f "$@" ]; then - echo "$@ does not exists, exiting" + log "$@ does not exists, exiting" exit fi } -checkFile "$PWD/src/server.js" +makeNextSubmodule() { + log "Making ${1}" + pushd "submodules/${1}/" + npm install + npm audit fix + npm run export + popd + ln -sf "$PWD/submodules/${1}/out" "$PWD/nextStatic/${2}" +} + +checkFile "$PWD/src/server.ts" checkFile "$PWD/package.json" git pull git submodule update --init --recursive -echo "Installing node modules for server" +log "Installing node modules for server" npm install +npm audit fix +npm run export -echo "Seting up next.js static stuff..." +log "Seting up next.js static stuff..." mkdir "$PWD/nextStatic" -echo "Making qmining page" -pushd submodules/qmining-page/ -npm install -npm run export -popd -checkFile "$PWD/submodules/qmining-page/out" -ln -sf "$PWD/submodules/qmining-page/out" "$PWD/nextStatic/qminingPagePublic" +# ------------------------------------------------------------------------------------ +makeNextSubmodule "qmining-page" "qminingPagePublic" +makeNextSubmodule "qmining-data-editor" "dataEditorPublic" +# ------------------------------------------------------------------------------------ -echo "Making data editor page" -pushd submodules/qmining-data-editor/ -npm install -npm run export -popd -checkFile "$PWD/submodules/qmining-data-editor/out" -ln -sf "$PWD/submodules/qmining-data-editor/out" "$PWD/nextStatic/dataEditorPublic" - -echo "Making moodle test userscript" -mkdir -p "$PWD/publicDirs/qminingPublic/moodle-test-userscript" +log "Making moodle test userscript" +mkdir -pv "$PWD/publicDirs/qminingPublic/moodle-test-userscript" checkFile "$PWD/submodules/moodle-test-userscript/stable.user.js" -ln -sf "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/" -ln -sf "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic" +ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/" +ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic" -echo "mkdir-ing/touching :3" +log "mkdir-ing/touching :3" # TODO: make server create these itself mkdir -p stats mkdir -p stats/logs mkdir -p stats/vlogs -touch nolog +mkdir -p data +mkdir -p data/dbs +touch data/nolog #JSONS echo '{}' >stats/stats echo '{}' >stats/vstats echo '{}' >stats/idstats echo '{}' >stats/idvstats -echo '{}' >qminingPublic/userSpecificMotd.json -echo '{}' >qminingPublic/news.json +echo '{}' >publicDirs/qminingPublic/userSpecificMotd.json +echo '{}' >publicDirs/qminingPublic/news.json -touch qminingPublic/version -touch qminingPublic/motd +touch publicDirs/qminingPublic/version +touch publicDirs/qminingPublic/motd -echo "wgetting data.json from frylabs..." -wget "http://qmining.frylabs.net/data.json" -O ./qminingPublic/data.json - -if [ "$?" -ne "0" ]; then - echo "Failed to wget data.json, please create it yourself!" - echo "Now starting with empty data!" - echo '{"Subjects":[],"version":"TESET","motd":"hai"}' >./qminingPublic/data.json -fi - -pushd standaloneUtils -rm -v ../data/dbs/users.db -NS_SQL_DEBUG_LOG=true NS_LOGLEVEL=2 node dbSetup.js +pushd src/standaloneUtils +NS_LOGLEVEL=2 node dbSetup.js +mv *db ../../data/dbs popd +hr echo "Done!" echo "npm start {loglevel}" echo "To start server" +hr diff --git a/scripts/makeQminingPage.sh b/scripts/makeQminingPage.sh deleted file mode 100755 index 6d919d5..0000000 --- a/scripts/makeQminingPage.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -pushd modules/qmining/qmining-page/ -./make.sh -popd diff --git a/scripts/serverStats.js b/scripts/serverStats.js index 3ee1903..0eb907d 100644 --- a/scripts/serverStats.js +++ b/scripts/serverStats.js @@ -8,12 +8,10 @@ const colWidth = 40 const maxStatLength = colWidth // Math.floor(cols / 4) const statNameSpacing = 4 const beforeRowSpace = 13 -const rowsToPrint = Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || 1 +const colsToPrint = + Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || 1 const coloredWords = { - red: [ - 'lred', - 'thanks' - ], + red: ['lred', 'thanks'], cyan: [ 'getveteranpw', 'pwrequest', @@ -53,12 +51,7 @@ const coloredWords = { 'registerscript', 'install', ], - magenta: [ - 'addPost', - 'comment', - 'postfeedback', - 'quickvote', - ], + magenta: ['addPost', 'comment', 'postfeedback', 'quickvote'], } const filterFromDailyStats = [ 'savedQuestions', @@ -164,10 +157,7 @@ function tail(text, number) { } function head(text, number) { - return text - .split('\n') - .slice(0, number) - .join('\n') + return text.split('\n').slice(0, number).join('\n') } function countLinesMatching(text, toMatch) { @@ -180,6 +170,20 @@ function countLinesMatching(text, toMatch) { return count } +function getDayName(day) { + switch (day) { + case 0: + case undefined: + return 'Today' + case -1: + return 'Yesterday' + case -2: + return 'Before yesterday' + default: + return `Day ${day.toString()}` + } +} + function readFile(name) { if (fs.existsSync(name)) { return fs.readFileSync(name, 'utf8') @@ -198,7 +202,7 @@ function pCols(cols, rowTitles, colorNames, firstRowColor) { // console.log(cols) let maxLength = 0 cols.reverse().forEach((col, i) => { - if (i >= rowsToPrint) { + if (i >= colsToPrint) { return } if (col.length > maxLength) { @@ -211,54 +215,53 @@ function pCols(cols, rowTitles, colorNames, firstRowColor) { for (let i = 0; i < maxLength; i++) { const row = [] - cols - .forEach((val, colIndex) => { - if (colIndex >= rowsToPrint) { - return - } - if (!val[i]) { - row.push(getLetterNTimes(' ', maxStatLength + statNameSpacing + 2)) - return - } + cols.forEach((val, colIndex) => { + if (colIndex >= colsToPrint) { + return + } + if (!val[i]) { + row.push(getLetterNTimes(' ', maxStatLength + statNameSpacing + 2)) + return + } - const keyName = val[i].name || val[i] + const keyName = val[i].name || val[i] - let slicedName = keyName.slice(0, maxStatLength) - const toColor = colorNames - ? Object.keys(coloredWords).reduce((acc, key) => { - const colorArray = coloredWords[key] + let slicedName = keyName.slice(0, maxStatLength) + const toColor = colorNames + ? Object.keys(coloredWords).reduce((acc, key) => { + const colorArray = coloredWords[key] - const includes = colorArray.some((colorableIdName) => { - return keyName - .toLowerCase() - .includes(colorableIdName.toLowerCase()) - }) + const includes = colorArray.some((colorableIdName) => { + return keyName + .toLowerCase() + .includes(colorableIdName.toLowerCase()) + }) - if (includes) { - return key - } + if (includes) { + return key + } - return acc - }, '') - : false + return acc + }, '') + : false - const sep = (i + 1) % 5 === 0 ? '.' : ' ' + const sep = (i + 1) % 5 === 0 ? '.' : ' ' - while (slicedName.length < maxStatLength) { - slicedName = slicedName + sep - } + while (slicedName.length < maxStatLength) { + slicedName = slicedName + sep + } - let ammount = val[i].val ? val[i].val.toString() : '' - while (ammount.length < 5) { - ammount = ammount + ' ' - } + let ammount = val[i].val ? val[i].val.toString() : '' + while (ammount.length < 5) { + ammount = ammount + ' ' + } - if (toColor) { - row.push(C(toColor) + slicedName + ' ' + ammount + C()) - } else { - row.push(slicedName + ' ' + ammount) - } - }) + if (toColor) { + row.push(C(toColor) + slicedName + ' ' + ammount + C()) + } else { + row.push(slicedName + ' ' + ammount) + } + }) // ROW TITLE --------------------------------------------------- let currRowTitle = @@ -311,7 +314,7 @@ function preProcessDailyStats(obj) { if (i === index) { return { ...x, - val: z.val + x.val + val: z.val + x.val, } } return z @@ -333,12 +336,15 @@ function preProcessDailyStats(obj) { } }) } +function getDailyStat(day) { + return preProcessDailyStats(dailyStats[getDayIndex(day)]) +} pCols( [ - preProcessDailyStats(dailyStats[getDayIndex()]), - preProcessDailyStats(dailyStats[getDayIndex(-1)]), - preProcessDailyStats(dailyStats[getDayIndex(-2)]), + ...[...Array(colsToPrint).keys()].map((x) => { + return getDailyStat(-x) + }), ], null, true @@ -361,15 +367,18 @@ function preProcessUIdTestSolving(obj, minLength) { return Object.keys(obj).length.toString() } } +function getUserIdTestSolving(day) { + return [ + getDayName(day), + preProcessUIdTestSolving(userIdTestSolving[getDayIndex(day)]), + ] +} pCols( [ - ['Today', preProcessUIdTestSolving(userIdTestSolving[getDayIndex()])], - ['Yesterday', preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-1)])], - [ - 'Before yesterday', - preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-2)]), - ], + ...[...Array(colsToPrint).keys()].map((x) => { + return getUserIdTestSolving(-x) + }), ], null, false, @@ -379,24 +388,19 @@ pCols( // ------------------------------------------------------------------------------ printHeader('User id requests') const clientIdTestSolving = readJSON(`${dir}stats/uvstats`) +function getUserIdRequests(day) { + return [ + getDayName(day), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)]), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)], 5), + ] +} pCols( [ - [ - 'Today', - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()]), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()], 5), - ], - [ - 'Yesterday', - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)]), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)], 5), - ], - [ - 'Before Yesterday', - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)]), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)], 5), - ], + ...[...Array(colsToPrint).keys()].map((x) => { + return getUserIdRequests(-x) + }), ], ['', 'All', 'More than 5'], false, @@ -406,15 +410,18 @@ pCols( // ------------------------------------------------------------------------------ printHeader('Daily data count') const dailyDataCount = readFile(`${dir}stats/dailyDataCount`) +function getDailyDataCount(count) { + return [...Array(count).keys()].map((x) => { + return JSON.parse(head(tail(dailyDataCount, x + 1), 1)) + }) +} -printLastDataCount([ - JSON.parse(head(tail(dailyDataCount, 1), 1)), - JSON.parse(head(tail(dailyDataCount, 2), 1)), - JSON.parse(head(tail(dailyDataCount, 3), 1)), -]) +printLastDataCount(getDailyDataCount(colsToPrint)) function printLastDataCount(data) { - const res = [['Today'], ['Yesterday'], ['Before yesterday']] + const res = [...Array(colsToPrint).keys()].map((x) => { + return [getDayName(-x)] + }) data.forEach((dataCount, i) => { res[i].push(dataCount.userCount.toString()) res[i].push(dataCount.subjectCount.toString()) @@ -425,26 +432,24 @@ function printLastDataCount(data) { } // ------------------------------------------------------------------------------ printHeader('Daily script install / update check count') -const todaysLogs = readFile(`${dir}stats/vlogs/log`) -const yesterdaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex(-1)}`) || '' -const beforeYesterdaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex(-2)}`) || '' +function getDailyScriptStat(day) { + const log = + !day || day === 0 + ? readFile(`${dir}stats/vlogs/log`) + : readFile(`${dir}stats/vlogs/${getDayIndex(day)}`) -const installs = [ - [ - 'Today', - countLinesMatching(todaysLogs, '?install').toString(), - countLinesMatching(todaysLogs, '?up').toString(), - ], - [ - 'Yesterday', - countLinesMatching(yesterdaysLogs, '?install').toString(), - countLinesMatching(yesterdaysLogs, '?up').toString(), - ], - [ - 'Before yesterday', - countLinesMatching(beforeYesterdaysLogs, '?install').toString(), - countLinesMatching(beforeYesterdaysLogs, '?up').toString(), - ], -] + if (!log) { + return [getDayName(day), 0, 0] + } + return [ + getDayName(day), + countLinesMatching(log, '?install').toString(), + countLinesMatching(log, '?up').toString(), + ] +} + +const installs = [...Array(colsToPrint).keys()].map((x) => { + return getDailyScriptStat(-x) +}) pCols(installs, ['', 'Installs', 'Updates'], false, 'green') diff --git a/src/modules/api/submodules/chat.ts b/src/modules/api/submodules/chat.ts index 81434dc..6432615 100644 --- a/src/modules/api/submodules/chat.ts +++ b/src/modules/api/submodules/chat.ts @@ -46,19 +46,9 @@ function setup(data: SubmoduleData): void { io.on('connection', (socket: ExtendedSocket) => { const userid = socket.user.id - logger.Log(`Chat connect: ${userid}`, logger.GetColor('green')) socket.on('join', function (/*data*/) { socket.join(userid.toString()) - let currUser: any = dbtools.Select(userDB, 'users', { - id: userid, - }) - if (!currUser || currUser.length === 0) { - // TODO: handle somehow? - socket.disconnect() - } else { - currUser = currUser[0] - } const groups = dbtools .runStatement( diff --git a/src/server.ts b/src/server.ts index e6d2e3f..55ad0e0 100755 --- a/src/server.ts +++ b/src/server.ts @@ -261,7 +261,7 @@ function setLogTimer() { 1 ) logger.DebugLog(`Next daily action: ${night}`, 'daily', 1) - const msToMidnight = night.getTime() - now.getTime() + 1000 + const msToMidnight = night.getTime() - now.getTime() + 10000 logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1) logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)