diff --git a/scripts/serverStats.js b/scripts/serverStats.js index 68cd387..905feef 100644 --- a/scripts/serverStats.js +++ b/scripts/serverStats.js @@ -1,6 +1,9 @@ +/* eslint-disable no-inner-declarations */ // ----------------------------------------------------------------- // CONFIG // ----------------------------------------------------------------- +// TODO: ep for these values +// js script to get from a single & multiple server & sum them const cols = process.stdout.columns // const rows = process.stdout.rows @@ -9,73 +12,74 @@ const maxStatLength = colWidth // Math.floor(cols / 4) const statNameSpacing = 4 const beforeRowSpace = 13 const colsToPrint = - Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || 1 + Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || + 1 const coloredWords = { - red: ['lred', 'thanks'], - cyan: [ - 'getveteranpw', - 'pwrequest', - 'getpw', - 'availablepws', - 'login', - 'logout', - ], - green: [ - 'manual', - 'todos', - 'allquestions', - 'subjectbrowser', - 'contribute', - 'feedback', - 'ranklist', - 'allqr', - 'possibleAnswers', - 'faq', - '/script', - 'listUserDir', - 'forumEntries', - 'contacts.json', - 'patreon', - 'donate', - 'userfiles', - 'hasNewMsg' - ], - blue: [ - 'isadding', - 'react', - 'ask', - 'newUserDir', - 'updateQuestion', - 'uploadUserFile', - 'votetodo', - 'registerscript', - 'install', - ], - magenta: ['addPost', 'comment', 'postfeedback', 'quickvote'], + red: ['lred', 'thanks'], + cyan: [ + 'getveteranpw', + 'pwrequest', + 'getpw', + 'availablepws', + 'login', + 'logout', + ], + green: [ + 'manual', + 'todos', + 'allquestions', + 'subjectbrowser', + 'contribute', + 'feedback', + 'ranklist', + 'allqr', + 'possibleAnswers', + 'faq', + '/script', + 'listUserDir', + 'forumEntries', + 'contacts.json', + 'patreon', + 'donate', + 'userfiles', + 'hasNewMsg', + ], + blue: [ + 'isadding', + 'react', + 'ask', + 'newUserDir', + 'updateQuestion', + 'uploadUserFile', + 'votetodo', + 'registerscript', + 'install', + ], + magenta: ['addPost', 'comment', 'postfeedback', 'quickvote'], } const filterFromDailyStats = [ - 'savedQuestions', - 'sio/f', - 'sound/', - '/img/', - '.php', - '/wordpress/', - '/wp/', - '/wp-includes/', - 'favicon', - 'robots.txt', - 'ads.txt', - '/f/', - '.git', - 'apple-touch-icon', - '/.env', - '/userFiles/', - '.min.js', - '.xml', - '.aspx', - '/questionDbs/', - '/chatFiles/', - 'rss' + 'savedQuestions', + 'sio/f', + 'sound/', + '/img/', + '.php', + '/wordpress/', + '/wp/', + '/wp-includes/', + 'favicon', + 'robots.txt', + 'ads.txt', + '/f/', + '.git', + 'apple-touch-icon', + '/.env', + '/userFiles/', + '.min.js', + '.xml', + '.aspx', + '/questionDbs/', + '/chatFiles/', + 'rss', ] // ----------------------------------------------------------------- @@ -83,405 +87,413 @@ const filterFromDailyStats = [ const fs = require('fs') // eslint-disable-line const dir = process.argv[2] -const startDay = !isNaN(parseInt(process.argv[3])) && parseInt(process.argv[3]) > 0 ? 0 : parseInt(process.argv[3]) +const startDay = + !isNaN(parseInt(process.argv[3])) && parseInt(process.argv[3]) > 0 + ? 0 + : parseInt(process.argv[3]) if (!dir) { - console.log('No params') - process.exit() + console.log('No params') + process.exit() } function getDayIndex(offset) { - let os = offset - if (!offset) { - os = 0 - } - - if (!isNaN(startDay)) { + let os = offset if (!offset) { - os = startDay - } else { - os = startDay + offset + os = 0 } - } - const date = new Date() - if (os) { - date.setDate(date.getDate() + os) - } - return ( - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - ) + if (!isNaN(startDay)) { + if (!offset) { + os = startDay + } else { + os = startDay + offset + } + } + + const date = new Date() + if (os) { + date.setDate(date.getDate() + os) + } + return ( + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + ) } function hr(char) { - console.log(C('blue') + getLetterNTimes(char || '=', cols) + C()) + console.log(C('blue') + getLetterNTimes(char || '=', cols) + C()) } function printHeader(text) { - hr() - console.log(C('green') + text + C()) - hr() + hr() + console.log(C('green') + text + C()) + hr() } function C(color) { - if (color !== undefined) { - color = color.toLowerCase() - } + if (color !== undefined) { + color = color.toLowerCase() + } - if (color === 'redbg') { - return '\x1b[41m' - } - if (color === 'bluebg') { - return '\x1b[44m' - } - if (color === 'green') { - return '\x1b[32m' - } - if (color === 'red') { - return '\x1b[31m' - } - if (color === 'yellow') { - return '\x1b[33m' - } - if (color === 'blue') { - return '\x1b[34m' - } - if (color === 'magenta') { - return '\x1b[35m' - } - if (color === 'cyan') { - return '\x1b[36m' - } - return '\x1b[0m' + if (color === 'redbg') { + return '\x1b[41m' + } + if (color === 'bluebg') { + return '\x1b[44m' + } + if (color === 'green') { + return '\x1b[32m' + } + if (color === 'red') { + return '\x1b[31m' + } + if (color === 'yellow') { + return '\x1b[33m' + } + if (color === 'blue') { + return '\x1b[34m' + } + if (color === 'magenta') { + return '\x1b[35m' + } + if (color === 'cyan') { + return '\x1b[36m' + } + return '\x1b[0m' } function readJSON(name) { - return JSON.parse(fs.readFileSync(name, 'utf8')) + return JSON.parse(fs.readFileSync(name, 'utf8')) } function tail(text, number) { - const splitedText = text.split('\n') - return splitedText.slice(Math.max(splitedText.length - number, 1)).join('\n') + const splitedText = text.split('\n') + return splitedText + .slice(Math.max(splitedText.length - number, 1)) + .join('\n') } function head(text, number) { - return text.split('\n').slice(0, number).join('\n') + return text.split('\n').slice(0, number).join('\n') } function countLinesMatching(text, toMatch) { - let count = 0 - text.split('\n').forEach((line) => { - if (line.includes(toMatch.toLowerCase())) { - count++ - } - }) - return count + let count = 0 + text.split('\n').forEach((line) => { + if (line.includes(toMatch.toLowerCase())) { + count++ + } + }) + return count } function getDayName(day) { - let d = day - if (!isNaN(startDay)) { - d += startDay - } - switch (d) { - case 0: - case undefined: - return 'Today' - case -1: - return 'Yesterday' - case -2: - return 'Before yesterday' - default: - const now = new Date() - now.setDate(now.getDate() + d); - return now.toDateString() - } + let d = day + if (!isNaN(startDay)) { + d += startDay + } + switch (d) { + case 0: + case undefined: + return 'Today' + case -1: + return 'Yesterday' + case -2: + return 'Before yesterday' + default: + const now = new Date() + now.setDate(now.getDate() + d) + return now.toDateString() + } } function readFile(name) { - if (fs.existsSync(name)) { - return fs.readFileSync(name, 'utf8') - } + if (fs.existsSync(name)) { + return fs.readFileSync(name, 'utf8') + } } function getLetterNTimes(letter, number) { - let res = '' - while (res.length < number) { - res += letter - } - return res + let res = '' + while (res.length < number) { + res += letter + } + return res } function pCols(cols, rowTitles, colorNames, firstRowColor) { - // console.log(cols) - let maxLength = 0 - cols.reverse().forEach((col, i) => { - if (i >= colsToPrint) { - return - } - if (col.length > maxLength) { - maxLength = col.length - } - }) - - cols.reverse() - - for (let i = 0; i < maxLength; i++) { - const row = [] - - cols.forEach((val, colIndex) => { - if (colIndex >= colsToPrint) { - return - } - if (!val[i]) { - row.push(getLetterNTimes(' ', maxStatLength + statNameSpacing + 2)) - return - } - - const keyName = val[i].name || val[i] - - let slicedName = keyName.slice(0, maxStatLength) - const toColor = colorNames - ? Object.keys(coloredWords).reduce((acc, key) => { - const colorArray = coloredWords[key] - - const includes = colorArray.some((colorableIdName) => { - return keyName - .toLowerCase() - .includes(colorableIdName.toLowerCase()) - }) - - if (includes) { - return key - } - - return acc - }, '') - : false - - const sep = (i + 1) % 5 === 0 ? '.' : ' ' - - while (slicedName.length < maxStatLength) { - slicedName = slicedName + sep - } - - let ammount = val[i].val ? val[i].val.toLocaleString() : '' - while (ammount.length < 5) { - ammount = ammount + ' ' - } - - if (toColor) { - row.push(C(toColor) + slicedName + ' ' + ammount + C()) - } else { - row.push(slicedName + ' ' + ammount) - } + // console.log(cols) + let maxLength = 0 + cols.reverse().forEach((col, i) => { + if (i >= colsToPrint) { + return + } + if (col.length > maxLength) { + maxLength = col.length + } }) - // ROW TITLE --------------------------------------------------- - let currRowTitle = - rowTitles && rowTitles[i] - ? rowTitles[i] - : getLetterNTimes(' ', beforeRowSpace) + cols.reverse() - while (currRowTitle.length < beforeRowSpace) { - currRowTitle = currRowTitle + ' ' + for (let i = 0; i < maxLength; i++) { + const row = [] + + cols.forEach((val, colIndex) => { + if (colIndex >= colsToPrint) { + return + } + if (!val[i]) { + row.push( + getLetterNTimes(' ', maxStatLength + statNameSpacing + 2) + ) + return + } + + const keyName = val[i].name || val[i] + + let slicedName = keyName.slice(0, maxStatLength) + const toColor = colorNames + ? Object.keys(coloredWords).reduce((acc, key) => { + const colorArray = coloredWords[key] + + const includes = colorArray.some((colorableIdName) => { + return keyName + .toLowerCase() + .includes(colorableIdName.toLowerCase()) + }) + + if (includes) { + return key + } + + return acc + }, '') + : false + + const sep = (i + 1) % 5 === 0 ? '.' : ' ' + + while (slicedName.length < maxStatLength) { + slicedName = slicedName + sep + } + + let ammount = val[i].val ? val[i].val.toLocaleString() : '' + while (ammount.length < 5) { + ammount = ammount + ' ' + } + + if (toColor) { + row.push(C(toColor) + slicedName + ' ' + ammount + C()) + } else { + row.push(slicedName + ' ' + ammount) + } + }) + + // ROW TITLE --------------------------------------------------- + let currRowTitle = + rowTitles && rowTitles[i] + ? rowTitles[i] + : getLetterNTimes(' ', beforeRowSpace) + + while (currRowTitle.length < beforeRowSpace) { + currRowTitle = currRowTitle + ' ' + } + currRowTitle = C('blue') + currRowTitle + C() + // COLORING ---------------------------------------------------- + let res = '' + if (firstRowColor && i === 0) { + res = + currRowTitle + + C('green') + + row.join(getLetterNTimes(' ', statNameSpacing)) + + C() + } else { + res = currRowTitle + row.join(getLetterNTimes(' ', statNameSpacing)) + } + // SHOW DIFF --------------------------------------------------- + console.log(res) } - currRowTitle = C('blue') + currRowTitle + C() - // COLORING ---------------------------------------------------- - let res = '' - if (firstRowColor && i === 0) { - res = - currRowTitle + - C('green') + - row.join(getLetterNTimes(' ', statNameSpacing)) + - C() +} + +function preProcessUIdTestSolving(obj, minLength) { + if (!obj) { + return '0' + } + if (minLength) { + return Object.keys(obj) + .filter((key) => { + return obj[key] > minLength + }) + .length.toString() } else { - res = currRowTitle + row.join(getLetterNTimes(' ', statNameSpacing)) + return Object.keys(obj).length.toString() } - // SHOW DIFF --------------------------------------------------- - console.log(res) - } } // ------------------------------------------------------------------------------ printHeader('Daily stats') try { - const dailyStats = readJSON(`${dir}stats/vstats`) - function preProcessDailyStats(obj) { - const formatted = Object.keys(obj).reduce((acc, key) => { - const includes = filterFromDailyStats.some((keyword) => { - return key.toLowerCase().includes(keyword.toLowerCase()) - }) - if (!includes) { - acc.push({ name: key.replace(/\.html/g, ''), val: obj[key] }) - } - return acc - }, []) - - const merged = formatted.reduce((acc, x) => { - const index = acc.findIndex((y) => { - return x.name === y.name - }) - - if (index !== -1) { - acc = acc.map((z, i) => { - if (i === index) { - return { - ...x, - val: z.val + x.val, + const dailyStats = readJSON(`${dir}stats/vstats`) + function preProcessDailyStats(obj) { + const formatted = Object.keys(obj).reduce((acc, key) => { + const includes = filterFromDailyStats.some((keyword) => { + return key.toLowerCase().includes(keyword.toLowerCase()) + }) + if (!includes) { + acc.push({ name: key.replace(/\.html/g, ''), val: obj[key] }) + } + return acc + }, []) + + const merged = formatted.reduce((acc, x) => { + const index = acc.findIndex((y) => { + return x.name === y.name + }) + + if (index !== -1) { + acc = acc.map((z, i) => { + if (i === index) { + return { + ...x, + val: z.val + x.val, + } + } + return z + }) + } else { + acc.push(x) + } + + return acc + }, []) + + return merged.sort((a, b) => { + if (a.name > b.name) { + return 1 + } else if (a.name < b.name) { + return -1 + } else { + return 0 } - } - return z }) - } else { - acc.push(x) - } + } + function getDailyStat(day) { + return preProcessDailyStats(dailyStats[getDayIndex(day)]) + } - return acc - }, []) - - return merged.sort((a, b) => { - if (a.name > b.name) { - return 1 - } else if (a.name < b.name) { - return -1 - } else { - return 0 - } - }) - } - function getDailyStat(day) { - return preProcessDailyStats(dailyStats[getDayIndex(day)]) - } - - pCols( - [ - ...[...Array(colsToPrint).keys()].map((x) => { - return getDailyStat(-x) - }), - ], - null, - true - ) + pCols( + [ + ...[...Array(colsToPrint).keys()].map((x) => { + return getDailyStat(-x) + }), + ], + null, + true + ) } catch (e) { - console.error(e) + console.error(e) } // ------------------------------------------------------------------------------ printHeader('User id test solving') try { - const userIdTestSolving = readJSON(`${dir}stats/idvstats`) - function preProcessUIdTestSolving(obj, minLength) { - if (!obj) { - return '0' + const userIdTestSolving = readJSON(`${dir}stats/idvstats`) + function getUserIdTestSolving(day) { + return [ + getDayName(day), + preProcessUIdTestSolving(userIdTestSolving[getDayIndex(day)]), + ] } - if (minLength) { - return Object.keys(obj) - .filter((key) => { - return obj[key] > minLength - }) - .length.toString() - } else { - return Object.keys(obj).length.toString() - } - } - function getUserIdTestSolving(day) { - return [ - getDayName(day), - preProcessUIdTestSolving(userIdTestSolving[getDayIndex(day)]), - ] - } - pCols( - [ - ...[...Array(colsToPrint).keys()].map((x) => { - return getUserIdTestSolving(-x) - }), - ], - null, - false, - 'green' - ) -} catch(e) { - console.error(e) + pCols( + [ + ...[...Array(colsToPrint).keys()].map((x) => { + return getUserIdTestSolving(-x) + }), + ], + null, + false, + 'green' + ) +} catch (e) { + console.error(e) } // ------------------------------------------------------------------------------ printHeader('User id requests') try { - const clientIdTestSolving = readJSON(`${dir}stats/uvstats`) - function getUserIdRequests(day) { - return [ - getDayName(day), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)]), - preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)], 5), - ] - } + const clientIdTestSolving = readJSON(`${dir}stats/uvstats`) + function getUserIdRequests(day) { + return [ + getDayName(day), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)]), + preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)], 5), + ] + } - pCols( - [ - ...[...Array(colsToPrint).keys()].map((x) => { - return getUserIdRequests(-x) - }), - ], - ['', 'All', 'More than 5'], - false, - 'green' - ) + pCols( + [ + ...[...Array(colsToPrint).keys()].map((x) => { + return getUserIdRequests(-x) + }), + ], + ['', 'All', 'More than 5'], + false, + 'green' + ) - // ------------------------------------------------------------------------------ - printHeader('Daily data count') - const dailyDataCount = readFile(`${dir}stats/dailyDataCount`) - function getDailyDataCount(count) { - return [...Array(count).keys()].map((x) => { - return JSON.parse(head(tail(dailyDataCount, x + 1), 1)) - }) - } + // ------------------------------------------------------------------------------ + printHeader('Daily data count') + const dailyDataCount = readFile(`${dir}stats/dailyDataCount`) + function getDailyDataCount(count) { + return [...Array(count).keys()].map((x) => { + return JSON.parse(head(tail(dailyDataCount, x + 1), 1)) + }) + } - printLastDataCount(getDailyDataCount(colsToPrint)) + printLastDataCount(getDailyDataCount(colsToPrint)) - function printLastDataCount(data) { - const res = [...Array(colsToPrint).keys()].map((x) => { - return [getDayName(-x)] - }) - data.forEach((dataCount, i) => { - res[i].push(dataCount.userCount.toLocaleString()) - res[i].push(dataCount.subjectCount.toLocaleString()) - res[i].push(dataCount.questionCount.toLocaleString()) - }) + function printLastDataCount(data) { + const res = [...Array(colsToPrint).keys()].map((x) => { + return [getDayName(-x)] + }) + data.forEach((dataCount, i) => { + res[i].push(dataCount.userCount.toLocaleString()) + res[i].push(dataCount.subjectCount.toLocaleString()) + res[i].push(dataCount.questionCount.toLocaleString()) + }) - pCols(res, ['', 'Users', 'Subjects', 'Questions'], false, 'green') - } -} catch(e) { - console.error(e) + pCols(res, ['', 'Users', 'Subjects', 'Questions'], false, 'green') + } +} catch (e) { + console.error(e) } // ------------------------------------------------------------------------------ printHeader('Daily script install / update check count') try { - function getDailyScriptStat(day) { - const log = - !day || day === 0 - ? readFile(`${dir}stats/vlogs/log`) - : readFile(`${dir}stats/vlogs/${getDayIndex(day)}`) + function getDailyScriptStat(day) { + const log = + !day || day === 0 + ? readFile(`${dir}stats/vlogs/log`) + : readFile(`${dir}stats/vlogs/${getDayIndex(day)}`) - if (!log) { - return [getDayName(day), 0, 0] + if (!log) { + return [getDayName(day), 0, 0] + } + return [ + getDayName(day), + countLinesMatching(log, '?install').toLocaleString(), + countLinesMatching(log, '?up').toLocaleString(), + ] } - return [ - getDayName(day), - countLinesMatching(log, '?install').toLocaleString(), - countLinesMatching(log, '?up').toLocaleString(), - ] - } - const installs = [...Array(colsToPrint).keys()].map((x) => { - return getDailyScriptStat(-x) - }) + const installs = [...Array(colsToPrint).keys()].map((x) => { + return getDailyScriptStat(-x) + }) - pCols(installs, ['', 'Installs', 'Updates'], false, 'green') -} catch(e) { - console.error(e) + pCols(installs, ['', 'Installs', 'Updates'], false, 'green') +} catch (e) { + console.error(e) } diff --git a/src/constants.json b/src/constants.json new file mode 100644 index 0000000..c701ad0 --- /dev/null +++ b/src/constants.json @@ -0,0 +1,9 @@ +{ + "serverPath": "dist/server.js", + "qminingPageDir": "submodules/qmining-page", + "qminingIndexPath": "nextStatic/qminingPagePublic/index.html", + "dataEditorPageDir": "submodules/qmining-data-editor", + "dataEditorIndexPath": "nextStatic/dataEditorPublic/index.html", + "moodleTestUserscriptDir": "submodules/moodle-test-userscript", + "moodleTestUserscriptPath": "submodules/moodle-test-userscript/stable.user.js" +} diff --git a/src/middlewares/auth.middleware.ts b/src/middlewares/auth.middleware.ts index 2e76021..86e4f4e 100644 --- a/src/middlewares/auth.middleware.ts +++ b/src/middlewares/auth.middleware.ts @@ -23,7 +23,6 @@ import type { Request, User } from '../types/basicTypes' import type { Database } from 'better-sqlite3' import logger from '../utils/logger' -import utils from '../utils/utils' import dbtools from '../utils/dbtools' interface Options { @@ -133,7 +132,7 @@ export default function (options: Options): RequestHandler { userDB, 'sessions', { - lastAccess: utils.GetDateString(), + lastAccess: new Date().getTime(), }, { id: sessionID, @@ -144,7 +143,7 @@ export default function (options: Options): RequestHandler { userDB, 'users', { - lastAccess: utils.GetDateString(), + lastAccess: new Date().getTime(), }, { id: user.id, diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index c398002..92c4c88 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -159,7 +159,6 @@ function GetApp(): ModuleType { initWorkerPool(() => questionDbs) const submoduleDatas = setupSubModules(app, { - questionDbs: questionDbs, getQuestionDbs: () => { return questionDbs }, @@ -172,7 +171,6 @@ function GetApp(): ModuleType { // ------------------------------------------------------------------------------------------- publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) app.use(express.static(pdir)) }) @@ -224,7 +222,6 @@ function setupSubModules( const submodulePath = submoduleDir + file try { - logger.Log(`Loading submodule '${file}' for '${moduleName}'...`) const mod = require(submodulePath).default // eslint-disable-line const loadedModData = mod.setup({ app: parentApp, diff --git a/src/modules/api/submodules/chat.ts b/src/modules/api/submodules/chat.ts index 1fec8a3..3264232 100644 --- a/src/modules/api/submodules/chat.ts +++ b/src/modules/api/submodules/chat.ts @@ -49,7 +49,9 @@ function setup(data: SubmoduleData): void { const publicDir = publicdirs[0] const uloadFiles = publicDir + 'chatFiles' - logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`) + logger.Log( + `Chat: Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}` + ) // https://socket.io/docs/v4/handling-cors/#Configuration const io = new socket(httpsServer || httpServer, { cors: { diff --git a/src/modules/api/submodules/p2p.ts b/src/modules/api/submodules/p2p.ts index 69b749d..cac5aa2 100644 --- a/src/modules/api/submodules/p2p.ts +++ b/src/modules/api/submodules/p2p.ts @@ -19,7 +19,6 @@ ------------------------------------------------------------------------- */ import { Response } from 'express' -import * as child_process from 'child_process' import http from 'http' import logger from '../../../utils/logger' @@ -31,9 +30,10 @@ import { Subject, QuestionDb, User, + DataFile, } from '../../../types/basicTypes' import utils from '../../../utils/utils' -import { backupData /*writeData*/ } from '../../../utils/actions' +import { backupData, writeData } from '../../../utils/actions' import { WorkerResult } from '../../../utils/classes' import dbtools from '../../../utils/dbtools' import { @@ -59,6 +59,7 @@ import { SelfInfoSchema, validateJSON, } from '../../../types/typeSchemas' +import constants from '../../../constants.json' // TODO: remove FINALIZE-s and TOTEST-s @@ -72,14 +73,19 @@ interface MergeResult { interface RemotePeerInfo { selfInfo: PeerInfo myPeers: PeerInfo[] - revision?: string + serverRevision?: string + scriptRevision?: string + qminingPageRevision?: string + dataEditorRevision?: string + serverBuildTime?: number + qminingPageBuildTime?: number + dataEditorBuildTime?: number + scriptVersion?: string qdbInfo?: { - dbName: string - subjs: { - name: string - count: number - }[] - }[] + questionDbCount: number + subjectCount: number + questionCount: number + } } interface RequestResult { @@ -110,6 +116,7 @@ function get(options: http.RequestOptions): Promise> { try { resolve({ data: JSON.parse(body) }) } catch (e) { + console.log(body) resolve({ error: e, options: options }) } }) @@ -135,7 +142,7 @@ export function getNewDataSince(subjects: Subject[], date: number): Subject[] { return { ...subject, Questions: subject.Questions.filter((question) => { - return (question.data.date || 0) > date + return (question.data.date || 0) >= date }).map((question) => removeCacheFromQuestion(question)), } }) @@ -269,13 +276,21 @@ async function sendNewDataToWorkers( function writeNewData( newQuestionDbs: QuestionDb[], - changedQuestionDbs: QuestionDb[] + changedQuestionDbs: QuestionDb[], + dbsFilePath: string, + publicDir: string ) { - const qdbsToWrite = [...newQuestionDbs, ...changedQuestionDbs] + const qdbsToWrite = [...changedQuestionDbs, ...newQuestionDbs] + const qdbsFile: DataFile[] = qdbsToWrite.map((qdb) => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { data, index, ...restOfQdb } = qdb + return restOfQdb + }) + + utils.WriteFile(JSON.stringify(qdbsFile, null, 2), dbsFilePath) qdbsToWrite.forEach((qdb) => { try { - // FINALIZE: write to file - // writeData(qdb.data, qdb.path) + writeData(qdb.data, publicDir + qdb.path) } catch (e) { logger.Log(`Error writing ${qdb.name} qdb to file!`, 'redbg') console.error(e) @@ -319,18 +334,16 @@ function setup(data: SubmoduleData): Submodule { app, userDB, publicdirs, - moduleSpecificData: { questionDbs, setQuestionDbs, getQuestionDbs }, - // publicdirs, + moduleSpecificData: { setQuestionDbs, getQuestionDbs, dbsFile }, } = data const publicDir = publicdirs[0] + let syncInProgress = false // --------------------------------------------------------------------------------------- // SETUP // --------------------------------------------------------------------------------------- - // const publicDir = publicdirs[0] - if (!utils.FileExists(peersFile)) { logger.Log( `Warning: peers file was missing, so it was created`, @@ -440,11 +453,6 @@ function setup(data: SubmoduleData): Submodule { `Warning: peers file is empty. You probably want to fill it`, 'yellowbg' ) - } else { - logger.Log('Loaded peers: ' + peers.length) - peers.forEach((peer, i) => { - logger.Log(`\t${i}\t"${peer.name}": ${peerToString(peer)}`) - }) } // --------------------------------------------------------------------------------------- @@ -456,32 +464,35 @@ function setup(data: SubmoduleData): Submodule { selfInfo: selfInfo, myPeers: peers, } - - try { - // FIXME: dont log if fails - result.revision = child_process - .execSync('git rev-parse HEAD', { - cwd: __dirname, - stdio: [0, 'pipe', null], - }) - .toString() - .trim() - } catch (e) { - result.revision = 'Failed to get revision' - } + result.serverRevision = utils.getGitRevision(__dirname) + result.scriptRevision = utils.getGitRevision( + constants.moodleTestUserscriptDir + ) + result.qminingPageRevision = utils.getGitRevision( + constants.qminingPageDir + ) + result.dataEditorRevision = utils.getGitRevision( + constants.dataEditorPageDir + ) + result.qminingPageBuildTime = utils + .statFile(constants.qminingIndexPath) + ?.mtime.getTime() + result.serverBuildTime = utils + .statFile(constants.serverPath) + ?.mtime.getTime() + result.dataEditorBuildTime = utils + .statFile(constants.dataEditorIndexPath) + ?.mtime.getTime() + result.scriptVersion = utils.getScriptVersion() if (includeQdbInfo) { - result.qdbInfo = getQuestionDbs().map((qdb) => { - return { - dbName: qdb.name, - subjs: qdb.data.map((subj) => { - return { - name: subj.Name, - count: subj.Questions.length, - } - }), - } - }) + const questionDbCount = getQuestionDbs().length + const { subjCount, questionCount } = countOfQdbs(getQuestionDbs()) + result.qdbInfo = { + questionDbCount: questionDbCount, + subjectCount: subjCount, + questionCount: questionCount, + } } return result @@ -506,7 +517,7 @@ function setup(data: SubmoduleData): Submodule { return { ...qdb, index: availableIndexes[i], - path: `${publicDir}questionDbs/${qdb.name}.json'`, + path: `questionDbs/${qdb.name}.json'`, } }) } @@ -543,7 +554,6 @@ function setup(data: SubmoduleData): Submodule { } async function syncData() { - // TOTEST: try with 0 date to merge full dbs if (peers.length === 0) { logger.Log( `There are no peers specified in ${peersFile}, aborting sync`, @@ -560,31 +570,36 @@ function setup(data: SubmoduleData): Submodule { }${logger.C()} peers` ) - const lastSync = new Date('2022-03-12').getTime() // FINALIZE date: this is only for testing // selfInfo.lastSync + const lastSync = new Date('2012-03-12').getTime() // FINALIZE date: this is only for testing // selfInfo.lastSync logger.Log( `\tLast sync date: ${logger.C('blue')}${new Date( lastSync ).toLocaleString()}${logger.C()}` ) const syncStart = new Date().getTime() - const requests = peers.map((peer) => { - const lastSyncWithPeer = new Date('2022-03-12').getTime() // FINALIZE same as above // peer.lastSync || 0 + const lastSyncInfos = peers.map((peer) => { + return [ + peerToString(peer), + new Date(peer.lastSync).toLocaleString(), + ] + }) + logger.Log(`\tLast sync with peers:`) + logger.logTable([['', 'Date'], ...lastSyncInfos], { + colWidth: [15], + rowPrefix: '\t', + }) + + const requests = peers.map((peer) => { + const lastSyncWithPeer = new Date('2012-03-12').getTime() // FINALIZE same as above // peer.lastSync || 0 - logger.Log( - `\tLast sync with ${logger.C('blue')}${peerToString( - peer - )}${logger.C()}: ${logger.C('blue')}${new Date( - lastSyncWithPeer - ).toLocaleString()}${logger.C()}` - ) return new Promise>( (resolve) => { get({ host: peer.host, port: peer.port, - path: `/getnewdatasince?host=${selfInfo.host}${ - lastSync ? `&since=${lastSyncWithPeer}` : '' - }`, + path: `/getnewdatasince?host=${encodeURIComponent( + peerToString(selfInfo) + )}${lastSync ? `&since=${lastSyncWithPeer}` : ''}`, }).then((res) => { resolve({ ...res, data: { ...res.data, peer: peer } }) }) @@ -621,6 +636,7 @@ function setup(data: SubmoduleData): Submodule { } } + const recievedDataCounts: (number | string)[][] = [] const resultDataWithoutEmptyDbs = resultDataWithoutErrors.filter( (res) => { const qdbCount = res.questionDbs.length @@ -628,22 +644,26 @@ function setup(data: SubmoduleData): Submodule { res.questionDbs ) - logger.Log( - `\t"${logger.C('blue')}${peerToString( - res.peer - )}${logger.C()}" sent "${logger.C( - 'green' - )}${qdbCount}${logger.C()}" question DB-s with "${logger.C( - 'green' - )}${subjCount.toLocaleString()}${logger.C()}" subjects, and "${logger.C( - 'green' - )}${questionCount.toLocaleString()}${logger.C()}" questions` - ) + recievedDataCounts.push([ + peerToString(res.peer), + qdbCount, + subjCount, + questionCount, + ]) return questionCount > 0 } ) + logger.Log(`\tRecieved data from peers:`) + logger.logTable( + [['', 'QDBs', 'Subjs', 'Questions'], ...recievedDataCounts], + { + colWidth: [15], + rowPrefix: '\t', + } + ) + const resultData = resultDataWithoutEmptyDbs.map((res) => { return { ...res, @@ -665,17 +685,14 @@ function setup(data: SubmoduleData): Submodule { ) }) if (thirdPartyPeers.length > 0) { - logger.Log( - `\tPeers reported ${logger.C('green')}${ - thirdPartyPeers.length - }${logger.C()} third party peer(s) not connected to this server.` - ) utils.WriteFile( JSON.stringify(thirdPartyPeers, null, 2), thirdPartyPeersFile ) logger.Log( - `\tSee ${logger.C( + `\tPeers reported ${logger.C('green')}${ + thirdPartyPeers.length + }${logger.C()} third party peer(s) not connected to this server. See ${logger.C( 'blue' )}${thirdPartyPeersFile}${logger.C()} for details` ) @@ -690,6 +707,7 @@ function setup(data: SubmoduleData): Submodule { newQuestions?: number } } = {} + // ------------------------------------------------------------------------------------------------------- // new users handlin TOTEST: test // ------------------------------------------------------------------------------------------------------- @@ -700,25 +718,23 @@ function setup(data: SubmoduleData): Submodule { const decryptedUsers: User[] = JSON.parse( decrypt(privateKey, res.encryptedUsers) ) - let newUserCount = 0 decryptedUsers.forEach((remoteUser) => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const { id, ...remoteUserWithoutId } = remoteUser const localUser = dbtools.Select(userDB, 'users', { pw: remoteUser.pw, }) - if (!localUser) { + if (localUser.length === 0) { // FIXME: users will not have consistend id across servers. This may be // harmless, will see dbtools.Insert(userDB, 'users', { ...(remoteUserWithoutId as Omit), sourceHost: peerToString(res.peer), }) - newUserCount += 1 } }) resultsCount[peerToString(res.peer)] = { - newUsers: newUserCount, + newUsers: decryptedUsers.length, } } }) @@ -746,7 +762,6 @@ function setup(data: SubmoduleData): Submodule { // ------------------------------------------------------------------------------------------------------- // backup // ------------------------------------------------------------------------------------------------------- - const { subjCount: oldSubjCount, questionCount: oldQuestionCount } = countOfQdbs(getQuestionDbs()) const oldQuestionDbCount = getQuestionDbs().length @@ -757,7 +772,6 @@ function setup(data: SubmoduleData): Submodule { // ------------------------------------------------------------------------------------------------------- // adding questions to db // ------------------------------------------------------------------------------------------------------- - for (let i = 0; i < resultData.length; i++) { const { questionDbs: remoteQuestionDbs, peer } = resultData[i] logger.Log( @@ -786,7 +800,9 @@ function setup(data: SubmoduleData): Submodule { newQuestionDbs, getQuestionDbs().filter((qdb) => { return changedQdbIndexes.includes(qdb.index) - }) + }), + dbsFile, + publicDir ) setQuestionDbs([...mergedQuestionDbs, ...newQuestionDbs]) @@ -876,7 +892,7 @@ function setup(data: SubmoduleData): Submodule { newQuestionCount, ], ], - { colWidth: [15] } + { colWidth: [15], rowPrefix: '\t' } ) logger.Log( @@ -922,8 +938,8 @@ function setup(data: SubmoduleData): Submodule { const remoteHost = req.query.host const questionDbsWithNewQuestions = Number.isNaN(since) - ? questionDbs - : questionDbs + ? getQuestionDbs() + : getQuestionDbs() .map((qdb) => { return { ...qdb, @@ -949,26 +965,54 @@ function setup(data: SubmoduleData): Submodule { remoteInfo: getSelfInfo(), } + let hostToLog = req.hostname if (remoteHost) { - const remoteHostInfo = peers.find((peer) => { - return peer.host === remoteHost + const remotePeerInfo = peers.find((peer) => { + return peerToString(peer) === remoteHost }) - const remotePublicKey = remoteHostInfo?.publicKey - if (remotePublicKey) { - // FIXME: sign data? - const newUsers = getNewUsersSince(since) - result.encryptedUsers = encrypt( - remotePublicKey, - JSON.stringify(newUsers) - ) - } else if (remoteHostInfo) { + if (remotePeerInfo) { + hostToLog = peerToString(remotePeerInfo) + const remotePublicKey = remotePeerInfo?.publicKey + if (remotePublicKey) { + // FIXME: sign data? + const newUsers = getNewUsersSince(since) + result.encryptedUsers = encrypt( + remotePublicKey, + JSON.stringify(newUsers) + ) + logger.Log( + `Sending new users to "${remoteHost}" (encrypted)`, + 'green' + ) + } else if (remotePeerInfo) { + logger.Log( + `Warning: "${hostToLog}" has no public key saved!`, + 'yellowbg' + ) + } + } else { logger.Log( - `Warning: ${remoteHostInfo.host}:${remoteHostInfo.port} has no publick key saved!`, + 'Couldn\'t find remote peer info based on remoteHost: "' + + remoteHost + + '". This could mean that the host uses this server as peer, but this server does not ' + + 'use it as a peer.', 'yellowbg' ) } } + const dateToLog = Number.isNaN(since) + ? 'all time' + : new Date(since).toLocaleString() + + logger.Log( + `Sending new data to ${logger.C( + 'blue' + )}${hostToLog}${logger.C()} since ${logger.C( + 'blue' + )}${dateToLog}${logger.C()} ` + ) + res.json(result) }) @@ -984,6 +1028,15 @@ function setup(data: SubmoduleData): Submodule { // return // } + // FIXME: /syncResult EP if this EP times out, but we still need the result + if (syncInProgress) { + res.json({ + error: 'A sync is already in progress!', + }) + return + } + + syncInProgress = true setPendingJobsAlertCount(5000) syncData() .then((syncResult) => { @@ -992,6 +1045,7 @@ function setup(data: SubmoduleData): Submodule { ...syncResult, }) setPendingJobsAlertCount() + syncInProgress = false }) .catch((e) => { console.error(e) @@ -1000,10 +1054,17 @@ function setup(data: SubmoduleData): Submodule { msg: e.message, }) setPendingJobsAlertCount() + syncInProgress = false }) }) - logger.Log('P2P functionality set up. Peers: ' + peers.length, 'blue') + logger.Log( + 'P2P functionality set up. Peers (' + + peers.length + + '): ' + + peers.map((peer) => peerToString(peer)).join(', '), + 'blue' + ) return {} } diff --git a/src/modules/api/submodules/qminingapi.ts b/src/modules/api/submodules/qminingapi.ts index e21ba92..6022b37 100644 --- a/src/modules/api/submodules/qminingapi.ts +++ b/src/modules/api/submodules/qminingapi.ts @@ -59,6 +59,7 @@ import { getSubjNameWithoutYear, SearchResultQuestion, } from '../../../utils/qdbUtils' +import constants from '../../../constants.json' interface SavedQuestionData { fname: string @@ -79,7 +80,6 @@ interface SavedQuestionData { const line = '====================================================' // lol const registeredScriptsFile = 'stats/registeredScripts.json' const testUsersFile = 'data/testUsers.json' -const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js' const askedQuestionFile = 'stats/askedQuestions' const recievedQuestionFile = 'stats/recievedQuestions' const savedQuestionsFileName = 'savedQuestions.json' @@ -358,7 +358,9 @@ function saveQuestion( } function loadSupportedSites() { - const script = utils.ReadFile(userScriptFile).split('\n') + const script = utils + .ReadFile(constants.moodleTestUserscriptPath) + .split('\n') let i = 0 let stayIn = true @@ -386,18 +388,6 @@ function loadSupportedSites() { return sites } -function LoadVersion() { - const scriptContent = utils.ReadFile(userScriptFile) - - let temp: string | string[] = scriptContent.split('\n').find((x) => { - return x.includes('@version') - }) - temp = temp.split(' ') - temp = temp[temp.length - 1] - - return temp -} - function LoadMOTD(motdFile: string) { return utils.ReadFile(motdFile) } @@ -470,14 +460,14 @@ function setup(data: SubmoduleData): Submodule { app, userDB, /* url */ publicdirs, - moduleSpecificData: { questionDbs: questionDbs, dbsFile: dbsFile }, + moduleSpecificData: { getQuestionDbs, setQuestionDbs, dbsFile }, } = data const publicDir = publicdirs[0] const motdFile = publicDir + 'motd' const savedQuestionsDir = publicDir + 'savedQuestions' - let version = LoadVersion() + let version = utils.getScriptVersion() let supportedSites = loadSupportedSites() let motd = LoadMOTD(motdFile) let testUsers: number[] = LoadTestUsers() @@ -498,10 +488,10 @@ function setup(data: SubmoduleData): Submodule { }, }, { - fname: userScriptFile, + fname: constants.moodleTestUserscriptPath, logMsg: 'User script file changed', action: () => { - version = LoadVersion() + version = utils.getScriptVersion() supportedSites = loadSupportedSites() }, }, @@ -510,7 +500,7 @@ function setup(data: SubmoduleData): Submodule { app.get('/getDbs', (req: Request, res: Response) => { logger.LogReq(req) res.json( - questionDbs.map((qdb) => { + getQuestionDbs().map((qdb) => { return { path: qdb.path.replace(publicDir, ''), name: qdb.name, @@ -528,7 +518,7 @@ function setup(data: SubmoduleData): Submodule { res.setHeader('content-type', 'text/plain; charset=utf-8') if (db) { - const requestedDb = questionDbs.find((qdb) => { + const requestedDb = getQuestionDbs().find((qdb) => { return qdb.name === db }) @@ -543,7 +533,7 @@ function setup(data: SubmoduleData): Submodule { stringifiedData += dataToString(requestedDb.data) stringifiedData += '\n' + line + line + '\n' } else { - stringifiedData = questionDbs + stringifiedData = getQuestionDbs() .map((qdb) => { let result = '' result += '\n' + line @@ -576,7 +566,7 @@ function setup(data: SubmoduleData): Submodule { try { let maxIndex = -1 - const suitedQuestionDbs = questionDbs.filter((qdb) => { + const suitedQuestionDbs = getQuestionDbs().filter((qdb) => { if (maxIndex < qdb.index) { maxIndex = qdb.index } @@ -584,14 +574,14 @@ function setup(data: SubmoduleData): Submodule { }, []) if (suitedQuestionDbs.length === 0) { - if (!dbExists(location, questionDbs)) { + if (!dbExists(location, getQuestionDbs())) { suitedQuestionDbs.push( getNewQdb( location, maxIndex, dbsFile, publicDir, - questionDbs + getQuestionDbs() ) ) } else { @@ -684,7 +674,7 @@ function setup(data: SubmoduleData): Submodule { question: question, subj: subj, testUrl: testUrl, - questionDbs: questionDbs, + questionDbs: getQuestionDbs(), }) }) @@ -730,13 +720,13 @@ function setup(data: SubmoduleData): Submodule { logger.LogReq(req) if (req.query.detailed === 'all') { res.json({ - detailed: getDetailedRes(questionDbs), - simple: getSimplreRes(questionDbs), + detailed: getDetailedRes(getQuestionDbs()), + simple: getSimplreRes(getQuestionDbs()), }) } else if (req.query.detailed) { - res.json(getDetailedRes(questionDbs)) + res.json(getDetailedRes(getQuestionDbs())) } else { - res.json(getSimplreRes(questionDbs)) + res.json(getSimplreRes(getQuestionDbs())) } }) @@ -758,7 +748,7 @@ function setup(data: SubmoduleData): Submodule { } if (req.query.subjinfo) { - result.subjinfo = getSimplreRes(questionDbs) + result.subjinfo = getSimplreRes(getQuestionDbs()) } if (req.query.version) { result.version = version @@ -907,10 +897,10 @@ function setup(data: SubmoduleData): Submodule { return } - const dbIndex = questionDbs.findIndex((qdb) => { + const dbIndex = getQuestionDbs().findIndex((qdb) => { return qdb.name === selectedDb.name }) - const currDb = questionDbs[dbIndex] + const currDb = getQuestionDbs()[dbIndex] if (dbIndex === -1) { res.json({ @@ -937,7 +927,12 @@ function setup(data: SubmoduleData): Submodule { return } if (resultDb) { - questionDbs[dbIndex] = resultDb + setQuestionDbs( + getQuestionDbs().map((qdb, i) => { + if (i === dbIndex) return resultDb + return qdb + }) + ) } if (editType === 'delete') { @@ -1025,6 +1020,10 @@ function setup(data: SubmoduleData): Submodule { // TODO: dont allow multiple instances // TODO: get status of it cleaning logger.LogReq(req) + res.json({ + error: 'Not implemented / tested!', + }) + return const user: User = req.session.user const status: string = req.query.status @@ -1057,7 +1056,8 @@ function setup(data: SubmoduleData): Submodule { questionCleaner = fork( `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, - ['-s', `${process.cwd()}/${questionDbs[0].path}`] + ['-s', `${process.cwd()}/${getQuestionDbs()[0].path}`] // TODO: this only cleans index + // #0? ) questionCleaner.on('exit', function (code: number) { console.log('EXIT', code) @@ -1073,11 +1073,11 @@ function setup(data: SubmoduleData): Submodule { return { dailyAction: () => { - backupData(questionDbs) - ExportDailyDataCount(questionDbs, userDB) + backupData(getQuestionDbs()) + ExportDailyDataCount(getQuestionDbs(), userDB) }, load: () => { - backupData(questionDbs) + backupData(getQuestionDbs()) filesToWatch.forEach((ftw) => { if (utils.FileExists(ftw.fname)) { diff --git a/src/modules/api/submodules/userManagement.ts b/src/modules/api/submodules/userManagement.ts index 26f2724..a4e3471 100644 --- a/src/modules/api/submodules/userManagement.ts +++ b/src/modules/api/submodules/userManagement.ts @@ -63,6 +63,19 @@ function BackupDB(usersDbBackupPath: string, userDB: Database) { }) } +// TODO: figure out if this is needed +// const validationTokenNameFile = 'data/validationTokenName' +// function readValidationTokenName() { +// if (utils.FileExists(validationTokenNameFile)) { +// return utils.ReadFile(validationTokenNameFile) +// } else { +// throw new Error( +// `Validation token file does not exist! Should be: "${validationTokenNameFile}", content should be: "name for uuidv5 (any text)"` +// ) +// } +// } +const validationTokenName = 'qmining' // readValidationTokenName() + function setup(data: SubmoduleData): Submodule { const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] @@ -70,6 +83,13 @@ function setup(data: SubmoduleData): Submodule { domain = domain.join('.') // "frylabs.net" logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) + logger.Log( + `User count: ${dbtools + .TableInfo(userDB, 'users') + .dataCount.toLocaleString()} users`, + 'blue' + ) + app.get('/avaiblePWS', (req: Request, res: any) => { logger.LogReq(req) @@ -280,7 +300,6 @@ function setup(data: SubmoduleData): Submodule { }) }) - const name = 'qmining' app.get( '/validationtoken', (req: Request<{ token: string; userid: string }>, res: any) => { @@ -299,7 +318,7 @@ function setup(data: SubmoduleData): Submodule { msg: 'couldnt find user', }) } - const key = v5(name, specifiedUser[0].pw) + const key = v5(validationTokenName, specifiedUser[0].pw) const isValid = key === token res.json({ @@ -307,7 +326,7 @@ function setup(data: SubmoduleData): Submodule { isValid: isValid, }) } else { - const key = v5(name, user.pw) + const key = v5(validationTokenName, user.pw) res.json({ result: 'success', diff --git a/src/modules/api/usersDBStruct.ts b/src/modules/api/usersDBStruct.ts index fcc8ace..a346039 100644 --- a/src/modules/api/usersDBStruct.ts +++ b/src/modules/api/usersDBStruct.ts @@ -39,14 +39,14 @@ const DBStruct = { defaultZero: true, }, created: { - type: 'text', + type: 'number', notNull: true, }, lastLogin: { - type: 'text', + type: 'number', }, lastAccess: { - type: 'text', + type: 'number', }, avaiblePWRequests: { type: 'number', @@ -59,6 +59,9 @@ const DBStruct = { createdBy: { type: 'number', }, + sourceHost: { + type: 'text', + }, }, }, sessions: { @@ -80,11 +83,11 @@ const DBStruct = { notNull: true, }, createDate: { - type: 'text', + type: 'number', notNull: true, }, lastAccess: { - type: 'text', + type: 'number', }, isScript: { type: 'number', diff --git a/src/modules/dataEditor/dataEditor.ts b/src/modules/dataEditor/dataEditor.ts index c777f9b..762fc0f 100644 --- a/src/modules/dataEditor/dataEditor.ts +++ b/src/modules/dataEditor/dataEditor.ts @@ -64,7 +64,6 @@ function GetApp(): ModuleType { next() }) publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) app.use(express.static(pdir)) }) app.use(express.static(nextdir)) diff --git a/src/modules/main/main.ts b/src/modules/main/main.ts index 402b05a..b7853bb 100644 --- a/src/modules/main/main.ts +++ b/src/modules/main/main.ts @@ -23,7 +23,6 @@ import express, { RequestHandler } from 'express' const app = express() // other requires -import logger from '../../utils/logger' import { SetupData } from '../../server' import { ModuleType } from '../../types/basicTypes' @@ -35,7 +34,6 @@ function GetApp(): ModuleType { app.set('view engine', 'ejs') app.set('views', ['./src/modules/main/views', './src/sharedViews']) publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) app.use(express.static(pdir)) }) diff --git a/src/modules/qmining/qmining.ts b/src/modules/qmining/qmining.ts index ea8856d..afb2907 100644 --- a/src/modules/qmining/qmining.ts +++ b/src/modules/qmining/qmining.ts @@ -67,7 +67,6 @@ function GetApp(): ModuleType { next() }) publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) app.use(express.static(pdir)) }) app.use(express.static(nextdir)) diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index 7f184da..6ab57e8 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -114,7 +114,6 @@ export interface Request extends express.Request { export interface ModuleSpecificData { // TODO: rename to something more meaningfull - questionDbs: QuestionDb[] setQuestionDbs: (newVal: QuestionDb[]) => void getQuestionDbs: () => QuestionDb[] dbsFile: string @@ -175,6 +174,6 @@ export interface PeerInfo { port: number publicKey: string contact: string - lastSync?: Date + lastSync?: number note?: string } diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 9dc9683..4890641 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -504,7 +504,7 @@ export function loadJSON( if (!utils.FileExists(dataPath)) { logger.Log( - `${dataPath} data file did not exist, created empty one!`, + `${dataPath} data file does not exist, created empty one!`, 'yellowbg' ) utils.WriteFile(JSON.stringify([]), dataPath) @@ -513,7 +513,6 @@ export function loadJSON( try { acc.push({ ...dataFile, - path: dataPath, index: index, data: loadData(dataPath), }) @@ -529,8 +528,8 @@ export function loadJSON( const { subjCount, questionCount } = countOfQdbs(res) logger.Log( - `Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`, - logger.GetColor('green') + `Loaded ${subjCount.toLocaleString()} subjects with ${questionCount.toLocaleString()} questions from ${res.length.toLocaleString()} question db-s`, + 'blue' ) return res @@ -565,10 +564,7 @@ export function backupData(questionDbs: Array): void { // logger.Log(`Backing up ${data.name}...`) writeData( data.data, - `${path}${data.name}_${utils.GetDateString( - undefined, - true - )}.json` + `${path}${data.name}_${utils.GetDateString()}.json` ) // logger.Log('Done') } catch (err) { diff --git a/src/utils/logger.ts b/src/utils/logger.ts index b181bfc..da7becd 100755 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -421,12 +421,13 @@ function C(color?: string): string { function logTable( table: (string | number)[][], - options: { colWidth?: number[] } = {} + options: { colWidth?: number[]; rowPrefix?: string } = {} ): void { + const { colWidth, rowPrefix } = options + table.forEach((row, i) => { const rowString: string[] = [] row.forEach((cell, j) => { - const { colWidth } = options const cellColor = j === 0 || i === 0 ? 'blue' : 'green' let cellVal = '' if (!isNaN(+cell)) { @@ -447,7 +448,7 @@ function logTable( rowString.push(C(cellColor) + cellVal + C()) }) - Log(rowString.join('\t')) + Log((rowPrefix || '') + rowString.join('\t')) }) } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 7ff44b5..d177c74 100755 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -37,12 +37,16 @@ export default { renameFile: renameFile, deleteDir: deleteDir, formatBytes: formatBytes, + getGitRevision: getGitRevision, + getScriptVersion: getScriptVersion, } +import * as child_process from 'child_process' import fs from 'fs' import { v4 as uuidv4 } from 'uuid' import logger from '../utils/logger' +import constants from '../constants.json' import { Request } from '../types/basicTypes' interface URLFormatOptions { @@ -309,3 +313,29 @@ function formatBytes(number: number, unit: 'MB' | 'GB' = 'MB'): string { } return `${number} byte` } + +function getGitRevision(dir: string): string { + try { + return child_process + .execSync('git rev-parse HEAD', { + cwd: dir, + stdio: [0, 'pipe', null], + }) + .toString() + .trim() + } catch (e) { + return 'Failed to get revision' + } +} + +function getScriptVersion(): string { + const scriptContent = ReadFile(constants.moodleTestUserscriptPath) + + let temp: string | string[] = scriptContent.split('\n').find((x) => { + return x.includes('@version') + }) + temp = temp.split(' ') + temp = temp[temp.length - 1] + + return temp +} diff --git a/tsconfig.json b/tsconfig.json index 05c3795..ee3c2f9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,29 +1,30 @@ { - "compilerOptions": { - "noUnusedLocals": true, - "noUnusedParameters": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - "suppressImplicitAnyIndexErrors": true, - "moduleResolution": "node", - "esModuleInterop": true, - "target": "es6", - "allowJs": true, - "module": "commonjs", - "removeComments": true, - "preserveConstEnums": true, - "sourceMap": true, - "outDir": "dist", - "noImplicitAny": true, - "lib": ["dom", "ES2020"] - }, - "files": ["src/server.ts"], - "include": ["src/**/*"], - "exclude": [ - "src/tests/", - "node_modules", - "submodules", - "devel", - "src/standaloneUtils" - ] + "compilerOptions": { + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "suppressImplicitAnyIndexErrors": true, + "moduleResolution": "node", + "esModuleInterop": true, + "target": "es6", + "allowJs": true, + "module": "commonjs", + "removeComments": true, + "preserveConstEnums": true, + "sourceMap": true, + "outDir": "dist", + "noImplicitAny": true, + "lib": ["dom", "ES2020"], + "resolveJsonModule": true + }, + "files": ["src/server.ts"], + "include": ["src/**/*"], + "exclude": [ + "src/tests/", + "node_modules", + "submodules", + "devel", + "src/standaloneUtils" + ] }