albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

p2p fixes

Browse source
This commit is contained in:
mrfry 2023-03-26 19:11:07 +02:00
parent 2edc87d5dd
commit 16d6f04936
17 changed files with 707 additions and 582 deletions
Download patch file Download diff file Expand all files Collapse all files

780
scripts/serverStats.js
View file

@ -1,6 +1,9 @@
/* eslint-disable no-inner-declarations */
// ----------------------------------------------------------------- // -----------------------------------------------------------------
// CONFIG // CONFIG
// ----------------------------------------------------------------- // -----------------------------------------------------------------
// TODO: ep for these values
// js script to get from a single & multiple server & sum them
const cols = process.stdout.columns const cols = process.stdout.columns
// const rows = process.stdout.rows // const rows = process.stdout.rows
@ -9,73 +12,74 @@ const maxStatLength = colWidth // Math.floor(cols / 4)
const statNameSpacing = 4 const statNameSpacing = 4
const beforeRowSpace = 13 const beforeRowSpace = 13
const colsToPrint = const colsToPrint =
Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || 1 Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) ||
1
const coloredWords = { const coloredWords = {
red: ['lred', 'thanks'], red: ['lred', 'thanks'],
cyan: [ cyan: [
'getveteranpw', 'getveteranpw',
'pwrequest', 'pwrequest',
'getpw', 'getpw',
'availablepws', 'availablepws',
'login', 'login',
'logout', 'logout',
], ],
green: [ green: [
'manual', 'manual',
'todos', 'todos',
'allquestions', 'allquestions',
'subjectbrowser', 'subjectbrowser',
'contribute', 'contribute',
'feedback', 'feedback',
'ranklist', 'ranklist',
'allqr', 'allqr',
'possibleAnswers', 'possibleAnswers',
'faq', 'faq',
'/script', '/script',
'listUserDir', 'listUserDir',
'forumEntries', 'forumEntries',
'contacts.json', 'contacts.json',
'patreon', 'patreon',
'donate', 'donate',
'userfiles', 'userfiles',
'hasNewMsg' 'hasNewMsg',
], ],
blue: [ blue: [
'isadding', 'isadding',
'react', 'react',
'ask', 'ask',
'newUserDir', 'newUserDir',
'updateQuestion', 'updateQuestion',
'uploadUserFile', 'uploadUserFile',
'votetodo', 'votetodo',
'registerscript', 'registerscript',
'install', 'install',
], ],
magenta: ['addPost', 'comment', 'postfeedback', 'quickvote'], magenta: ['addPost', 'comment', 'postfeedback', 'quickvote'],
} }
const filterFromDailyStats = [ const filterFromDailyStats = [
'savedQuestions', 'savedQuestions',
'sio/f', 'sio/f',
'sound/', 'sound/',
'/img/', '/img/',
'.php', '.php',
'/wordpress/', '/wordpress/',
'/wp/', '/wp/',
'/wp-includes/', '/wp-includes/',
'favicon', 'favicon',
'robots.txt', 'robots.txt',
'ads.txt', 'ads.txt',
'/f/', '/f/',
'.git', '.git',
'apple-touch-icon', 'apple-touch-icon',
'/.env', '/.env',
'/userFiles/', '/userFiles/',
'.min.js', '.min.js',
'.xml', '.xml',
'.aspx', '.aspx',
'/questionDbs/', '/questionDbs/',
'/chatFiles/', '/chatFiles/',
'rss' 'rss',
] ]
// ----------------------------------------------------------------- // -----------------------------------------------------------------
@ -83,405 +87,413 @@ const filterFromDailyStats = [
const fs = require('fs') // eslint-disable-line const fs = require('fs') // eslint-disable-line
const dir = process.argv[2] const dir = process.argv[2]
const startDay = !isNaN(parseInt(process.argv[3])) && parseInt(process.argv[3]) > 0 ? 0 : parseInt(process.argv[3]) const startDay =
!isNaN(parseInt(process.argv[3])) && parseInt(process.argv[3]) > 0
? 0
: parseInt(process.argv[3])
if (!dir) { if (!dir) {
console.log('No params') console.log('No params')
process.exit() process.exit()
} }
function getDayIndex(offset) { function getDayIndex(offset) {
let os = offset let os = offset
if (!offset) {
os = 0
}
if (!isNaN(startDay)) {
if (!offset) { if (!offset) {
os = startDay os = 0
} else {
os = startDay + offset
} }
}
const date = new Date() if (!isNaN(startDay)) {
if (os) { if (!offset) {
date.setDate(date.getDate() + os) os = startDay
} } else {
return ( os = startDay + offset
date.getFullYear() + }
'-' + }
('0' + (date.getMonth() + 1)).slice(-2) +
'-' + const date = new Date()
('0' + date.getDate()).slice(-2) if (os) {
) date.setDate(date.getDate() + os)
}
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} }
function hr(char) { function hr(char) {
console.log(C('blue') + getLetterNTimes(char || '=', cols) + C()) console.log(C('blue') + getLetterNTimes(char || '=', cols) + C())
} }
function printHeader(text) { function printHeader(text) {
hr() hr()
console.log(C('green') + text + C()) console.log(C('green') + text + C())
hr() hr()
} }
function C(color) { function C(color) {
if (color !== undefined) { if (color !== undefined) {
color = color.toLowerCase() color = color.toLowerCase()
} }
if (color === 'redbg') { if (color === 'redbg') {
return '\x1b[41m' return '\x1b[41m'
} }
if (color === 'bluebg') { if (color === 'bluebg') {
return '\x1b[44m' return '\x1b[44m'
} }
if (color === 'green') { if (color === 'green') {
return '\x1b[32m' return '\x1b[32m'
} }
if (color === 'red') { if (color === 'red') {
return '\x1b[31m' return '\x1b[31m'
} }
if (color === 'yellow') { if (color === 'yellow') {
return '\x1b[33m' return '\x1b[33m'
} }
if (color === 'blue') { if (color === 'blue') {
return '\x1b[34m' return '\x1b[34m'
} }
if (color === 'magenta') { if (color === 'magenta') {
return '\x1b[35m' return '\x1b[35m'
} }
if (color === 'cyan') { if (color === 'cyan') {
return '\x1b[36m' return '\x1b[36m'
} }
return '\x1b[0m' return '\x1b[0m'
} }
function readJSON(name) { function readJSON(name) {
return JSON.parse(fs.readFileSync(name, 'utf8')) return JSON.parse(fs.readFileSync(name, 'utf8'))
} }
function tail(text, number) { function tail(text, number) {
const splitedText = text.split('\n') const splitedText = text.split('\n')
return splitedText.slice(Math.max(splitedText.length - number, 1)).join('\n') return splitedText
.slice(Math.max(splitedText.length - number, 1))
.join('\n')
} }
function head(text, number) { function head(text, number) {
return text.split('\n').slice(0, number).join('\n') return text.split('\n').slice(0, number).join('\n')
} }
function countLinesMatching(text, toMatch) { function countLinesMatching(text, toMatch) {
let count = 0 let count = 0
text.split('\n').forEach((line) => { text.split('\n').forEach((line) => {
if (line.includes(toMatch.toLowerCase())) { if (line.includes(toMatch.toLowerCase())) {
count++ count++
} }
}) })
return count return count
} }
function getDayName(day) { function getDayName(day) {
let d = day let d = day
if (!isNaN(startDay)) { if (!isNaN(startDay)) {
d += startDay d += startDay
} }
switch (d) { switch (d) {
case 0: case 0:
case undefined: case undefined:
return 'Today' return 'Today'
case -1: case -1:
return 'Yesterday' return 'Yesterday'
case -2: case -2:
return 'Before yesterday' return 'Before yesterday'
default: default:
const now = new Date() const now = new Date()
now.setDate(now.getDate() + d); now.setDate(now.getDate() + d)
return now.toDateString() return now.toDateString()
} }
} }
function readFile(name) { function readFile(name) {
if (fs.existsSync(name)) { if (fs.existsSync(name)) {
return fs.readFileSync(name, 'utf8') return fs.readFileSync(name, 'utf8')
} }
} }
function getLetterNTimes(letter, number) { function getLetterNTimes(letter, number) {
let res = '' let res = ''
while (res.length < number) { while (res.length < number) {
res += letter res += letter
} }
return res return res
} }
function pCols(cols, rowTitles, colorNames, firstRowColor) { function pCols(cols, rowTitles, colorNames, firstRowColor) {
// console.log(cols) // console.log(cols)
let maxLength = 0 let maxLength = 0
cols.reverse().forEach((col, i) => { cols.reverse().forEach((col, i) => {
if (i >= colsToPrint) { if (i >= colsToPrint) {
return return
} }
if (col.length > maxLength) { if (col.length > maxLength) {
maxLength = col.length maxLength = col.length
} }
})
cols.reverse()
for (let i = 0; i < maxLength; i++) {
const row = []
cols.forEach((val, colIndex) => {
if (colIndex >= colsToPrint) {
return
}
if (!val[i]) {
row.push(getLetterNTimes(' ', maxStatLength + statNameSpacing + 2))
return
}
const keyName = val[i].name || val[i]
let slicedName = keyName.slice(0, maxStatLength)
const toColor = colorNames
? Object.keys(coloredWords).reduce((acc, key) => {
const colorArray = coloredWords[key]
const includes = colorArray.some((colorableIdName) => {
return keyName
.toLowerCase()
.includes(colorableIdName.toLowerCase())
})
if (includes) {
return key
}
return acc
}, '')
: false
const sep = (i + 1) % 5 === 0 ? '.' : ' '
while (slicedName.length < maxStatLength) {
slicedName = slicedName + sep
}
let ammount = val[i].val ? val[i].val.toLocaleString() : ''
while (ammount.length < 5) {
ammount = ammount + ' '
}
if (toColor) {
row.push(C(toColor) + slicedName + ' ' + ammount + C())
} else {
row.push(slicedName + ' ' + ammount)
}
}) })
// ROW TITLE --------------------------------------------------- cols.reverse()
let currRowTitle =
rowTitles && rowTitles[i]
? rowTitles[i]
: getLetterNTimes(' ', beforeRowSpace)
while (currRowTitle.length < beforeRowSpace) { for (let i = 0; i < maxLength; i++) {
currRowTitle = currRowTitle + ' ' const row = []
cols.forEach((val, colIndex) => {
if (colIndex >= colsToPrint) {
return
}
if (!val[i]) {
row.push(
getLetterNTimes(' ', maxStatLength + statNameSpacing + 2)
)
return
}
const keyName = val[i].name || val[i]
let slicedName = keyName.slice(0, maxStatLength)
const toColor = colorNames
? Object.keys(coloredWords).reduce((acc, key) => {
const colorArray = coloredWords[key]
const includes = colorArray.some((colorableIdName) => {
return keyName
.toLowerCase()
.includes(colorableIdName.toLowerCase())
})
if (includes) {
return key
}
return acc
}, '')
: false
const sep = (i + 1) % 5 === 0 ? '.' : ' '
while (slicedName.length < maxStatLength) {
slicedName = slicedName + sep
}
let ammount = val[i].val ? val[i].val.toLocaleString() : ''
while (ammount.length < 5) {
ammount = ammount + ' '
}
if (toColor) {
row.push(C(toColor) + slicedName + ' ' + ammount + C())
} else {
row.push(slicedName + ' ' + ammount)
}
})
// ROW TITLE ---------------------------------------------------
let currRowTitle =
rowTitles && rowTitles[i]
? rowTitles[i]
: getLetterNTimes(' ', beforeRowSpace)
while (currRowTitle.length < beforeRowSpace) {
currRowTitle = currRowTitle + ' '
}
currRowTitle = C('blue') + currRowTitle + C()
// COLORING ----------------------------------------------------
let res = ''
if (firstRowColor && i === 0) {
res =
currRowTitle +
C('green') +
row.join(getLetterNTimes(' ', statNameSpacing)) +
C()
} else {
res = currRowTitle + row.join(getLetterNTimes(' ', statNameSpacing))
}
// SHOW DIFF ---------------------------------------------------
console.log(res)
} }
currRowTitle = C('blue') + currRowTitle + C() }
// COLORING ----------------------------------------------------
let res = '' function preProcessUIdTestSolving(obj, minLength) {
if (firstRowColor && i === 0) { if (!obj) {
res = return '0'
currRowTitle + }
C('green') + if (minLength) {
row.join(getLetterNTimes(' ', statNameSpacing)) + return Object.keys(obj)
C() .filter((key) => {
return obj[key] > minLength
})
.length.toString()
} else { } else {
res = currRowTitle + row.join(getLetterNTimes(' ', statNameSpacing)) return Object.keys(obj).length.toString()
} }
// SHOW DIFF ---------------------------------------------------
console.log(res)
}
} }
// ------------------------------------------------------------------------------ // ------------------------------------------------------------------------------
printHeader('Daily stats') printHeader('Daily stats')
try { try {
const dailyStats = readJSON(`${dir}stats/vstats`) const dailyStats = readJSON(`${dir}stats/vstats`)
function preProcessDailyStats(obj) { function preProcessDailyStats(obj) {
const formatted = Object.keys(obj).reduce((acc, key) => { const formatted = Object.keys(obj).reduce((acc, key) => {
const includes = filterFromDailyStats.some((keyword) => { const includes = filterFromDailyStats.some((keyword) => {
return key.toLowerCase().includes(keyword.toLowerCase()) return key.toLowerCase().includes(keyword.toLowerCase())
}) })
if (!includes) { if (!includes) {
acc.push({ name: key.replace(/\.html/g, ''), val: obj[key] }) acc.push({ name: key.replace(/\.html/g, ''), val: obj[key] })
} }
return acc return acc
}, []) }, [])
const merged = formatted.reduce((acc, x) => { const merged = formatted.reduce((acc, x) => {
const index = acc.findIndex((y) => { const index = acc.findIndex((y) => {
return x.name === y.name return x.name === y.name
}) })
if (index !== -1) { if (index !== -1) {
acc = acc.map((z, i) => { acc = acc.map((z, i) => {
if (i === index) { if (i === index) {
return { return {
...x, ...x,
val: z.val + x.val, val: z.val + x.val,
}
}
return z
})
} else {
acc.push(x)
}
return acc
}, [])
return merged.sort((a, b) => {
if (a.name > b.name) {
return 1
} else if (a.name < b.name) {
return -1
} else {
return 0
} }
}
return z
}) })
} else { }
acc.push(x) function getDailyStat(day) {
} return preProcessDailyStats(dailyStats[getDayIndex(day)])
}
return acc pCols(
}, []) [
...[...Array(colsToPrint).keys()].map((x) => {
return merged.sort((a, b) => { return getDailyStat(-x)
if (a.name > b.name) { }),
return 1 ],
} else if (a.name < b.name) { null,
return -1 true
} else { )
return 0
}
})
}
function getDailyStat(day) {
return preProcessDailyStats(dailyStats[getDayIndex(day)])
}
pCols(
[
...[...Array(colsToPrint).keys()].map((x) => {
return getDailyStat(-x)
}),
],
null,
true
)
} catch (e) { } catch (e) {
console.error(e) console.error(e)
} }
// ------------------------------------------------------------------------------ // ------------------------------------------------------------------------------
printHeader('User id test solving') printHeader('User id test solving')
try { try {
const userIdTestSolving = readJSON(`${dir}stats/idvstats`) const userIdTestSolving = readJSON(`${dir}stats/idvstats`)
function preProcessUIdTestSolving(obj, minLength) { function getUserIdTestSolving(day) {
if (!obj) { return [
return '0' getDayName(day),
preProcessUIdTestSolving(userIdTestSolving[getDayIndex(day)]),
]
} }
if (minLength) {
return Object.keys(obj)
.filter((key) => {
return obj[key] > minLength
})
.length.toString()
} else {
return Object.keys(obj).length.toString()
}
}
function getUserIdTestSolving(day) {
return [
getDayName(day),
preProcessUIdTestSolving(userIdTestSolving[getDayIndex(day)]),
]
}
pCols( pCols(
[ [
...[...Array(colsToPrint).keys()].map((x) => { ...[...Array(colsToPrint).keys()].map((x) => {
return getUserIdTestSolving(-x) return getUserIdTestSolving(-x)
}), }),
], ],
null, null,
false, false,
'green' 'green'
) )
} catch(e) { } catch (e) {
console.error(e) console.error(e)
} }
// ------------------------------------------------------------------------------ // ------------------------------------------------------------------------------
printHeader('User id requests') printHeader('User id requests')
try { try {
const clientIdTestSolving = readJSON(`${dir}stats/uvstats`) const clientIdTestSolving = readJSON(`${dir}stats/uvstats`)
function getUserIdRequests(day) { function getUserIdRequests(day) {
return [ return [
getDayName(day), getDayName(day),
preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)]), preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)]),
preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)], 5), preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)], 5),
] ]
} }
pCols( pCols(
[ [
...[...Array(colsToPrint).keys()].map((x) => { ...[...Array(colsToPrint).keys()].map((x) => {
return getUserIdRequests(-x) return getUserIdRequests(-x)
}), }),
], ],
['', 'All', 'More than 5'], ['', 'All', 'More than 5'],
false, false,
'green' 'green'
) )
// ------------------------------------------------------------------------------ // ------------------------------------------------------------------------------
printHeader('Daily data count') printHeader('Daily data count')
const dailyDataCount = readFile(`${dir}stats/dailyDataCount`) const dailyDataCount = readFile(`${dir}stats/dailyDataCount`)
function getDailyDataCount(count) { function getDailyDataCount(count) {
return [...Array(count).keys()].map((x) => { return [...Array(count).keys()].map((x) => {
return JSON.parse(head(tail(dailyDataCount, x + 1), 1)) return JSON.parse(head(tail(dailyDataCount, x + 1), 1))
}) })
} }
printLastDataCount(getDailyDataCount(colsToPrint)) printLastDataCount(getDailyDataCount(colsToPrint))
function printLastDataCount(data) { function printLastDataCount(data) {
const res = [...Array(colsToPrint).keys()].map((x) => { const res = [...Array(colsToPrint).keys()].map((x) => {
return [getDayName(-x)] return [getDayName(-x)]
}) })
data.forEach((dataCount, i) => { data.forEach((dataCount, i) => {
res[i].push(dataCount.userCount.toLocaleString()) res[i].push(dataCount.userCount.toLocaleString())
res[i].push(dataCount.subjectCount.toLocaleString()) res[i].push(dataCount.subjectCount.toLocaleString())
res[i].push(dataCount.questionCount.toLocaleString()) res[i].push(dataCount.questionCount.toLocaleString())
}) })
pCols(res, ['', 'Users', 'Subjects', 'Questions'], false, 'green') pCols(res, ['', 'Users', 'Subjects', 'Questions'], false, 'green')
} }
} catch(e) { } catch (e) {
console.error(e) console.error(e)
} }
// ------------------------------------------------------------------------------ // ------------------------------------------------------------------------------
printHeader('Daily script install / update check count') printHeader('Daily script install / update check count')
try { try {
function getDailyScriptStat(day) { function getDailyScriptStat(day) {
const log = const log =
!day || day === 0 !day || day === 0
? readFile(`${dir}stats/vlogs/log`) ? readFile(`${dir}stats/vlogs/log`)
: readFile(`${dir}stats/vlogs/${getDayIndex(day)}`) : readFile(`${dir}stats/vlogs/${getDayIndex(day)}`)
if (!log) { if (!log) {
return [getDayName(day), 0, 0] return [getDayName(day), 0, 0]
}
return [
getDayName(day),
countLinesMatching(log, '?install').toLocaleString(),
countLinesMatching(log, '?up').toLocaleString(),
]
} }
return [
getDayName(day),
countLinesMatching(log, '?install').toLocaleString(),
countLinesMatching(log, '?up').toLocaleString(),
]
}
const installs = [...Array(colsToPrint).keys()].map((x) => { const installs = [...Array(colsToPrint).keys()].map((x) => {
return getDailyScriptStat(-x) return getDailyScriptStat(-x)
}) })
pCols(installs, ['', 'Installs', 'Updates'], false, 'green') pCols(installs, ['', 'Installs', 'Updates'], false, 'green')
} catch(e) { } catch (e) {
console.error(e) console.error(e)
} }

9
src/constants.json Normal file
View file

@ -0,0 +1,9 @@
{
"serverPath": "dist/server.js",
"qminingPageDir": "submodules/qmining-page",
"qminingIndexPath": "nextStatic/qminingPagePublic/index.html",
"dataEditorPageDir": "submodules/qmining-data-editor",
"dataEditorIndexPath": "nextStatic/dataEditorPublic/index.html",
"moodleTestUserscriptDir": "submodules/moodle-test-userscript",
"moodleTestUserscriptPath": "submodules/moodle-test-userscript/stable.user.js"
}

5
src/middlewares/auth.middleware.ts
View file

@ -23,7 +23,6 @@ import type { Request, User } from '../types/basicTypes'
import type { Database } from 'better-sqlite3' import type { Database } from 'better-sqlite3'
import logger from '../utils/logger' import logger from '../utils/logger'
import utils from '../utils/utils'
import dbtools from '../utils/dbtools' import dbtools from '../utils/dbtools'
interface Options { interface Options {
@ -133,7 +132,7 @@ export default function (options: Options): RequestHandler {
userDB, userDB,
'sessions', 'sessions',
{ {
lastAccess: utils.GetDateString(), lastAccess: new Date().getTime(),
}, },
{ {
id: sessionID, id: sessionID,
@ -144,7 +143,7 @@ export default function (options: Options): RequestHandler {
userDB, userDB,
'users', 'users',
{ {
lastAccess: utils.GetDateString(), lastAccess: new Date().getTime(),
}, },
{ {
id: user.id, id: user.id,

3
src/modules/api/api.ts
View file

@ -159,7 +159,6 @@ function GetApp(): ModuleType {
initWorkerPool(() => questionDbs) initWorkerPool(() => questionDbs)
const submoduleDatas = setupSubModules(app, { const submoduleDatas = setupSubModules(app, {
questionDbs: questionDbs,
getQuestionDbs: () => { getQuestionDbs: () => {
return questionDbs return questionDbs
}, },
@ -172,7 +171,6 @@ function GetApp(): ModuleType {
// ------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------
publicdirs.forEach((pdir) => { publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir)) app.use(express.static(pdir))
}) })
@ -224,7 +222,6 @@ function setupSubModules(
const submodulePath = submoduleDir + file const submodulePath = submoduleDir + file
try { try {
logger.Log(`Loading submodule '${file}' for '${moduleName}'...`)
const mod = require(submodulePath).default // eslint-disable-line const mod = require(submodulePath).default // eslint-disable-line
const loadedModData = mod.setup({ const loadedModData = mod.setup({
app: parentApp, app: parentApp,

4
src/modules/api/submodules/chat.ts
View file

@ -49,7 +49,9 @@ function setup(data: SubmoduleData): void {
const publicDir = publicdirs[0] const publicDir = publicdirs[0]
const uloadFiles = publicDir + 'chatFiles' const uloadFiles = publicDir + 'chatFiles'
logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`) logger.Log(
`Chat: Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`
)
// https://socket.io/docs/v4/handling-cors/#Configuration // https://socket.io/docs/v4/handling-cors/#Configuration
const io = new socket(httpsServer || httpServer, { const io = new socket(httpsServer || httpServer, {
cors: { cors: {

265
src/modules/api/submodules/p2p.ts
View file

@ -19,7 +19,6 @@
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
import { Response } from 'express' import { Response } from 'express'
import * as child_process from 'child_process'
import http from 'http' import http from 'http'
import logger from '../../../utils/logger' import logger from '../../../utils/logger'
@ -31,9 +30,10 @@ import {
Subject, Subject,
QuestionDb, QuestionDb,
User, User,
DataFile,
} from '../../../types/basicTypes' } from '../../../types/basicTypes'
import utils from '../../../utils/utils' import utils from '../../../utils/utils'
import { backupData /*writeData*/ } from '../../../utils/actions' import { backupData, writeData } from '../../../utils/actions'
import { WorkerResult } from '../../../utils/classes' import { WorkerResult } from '../../../utils/classes'
import dbtools from '../../../utils/dbtools' import dbtools from '../../../utils/dbtools'
import { import {
@ -59,6 +59,7 @@ import {
SelfInfoSchema, SelfInfoSchema,
validateJSON, validateJSON,
} from '../../../types/typeSchemas' } from '../../../types/typeSchemas'
import constants from '../../../constants.json'
// TODO: remove FINALIZE-s and TOTEST-s // TODO: remove FINALIZE-s and TOTEST-s
@ -72,14 +73,19 @@ interface MergeResult {
interface RemotePeerInfo { interface RemotePeerInfo {
selfInfo: PeerInfo selfInfo: PeerInfo
myPeers: PeerInfo[] myPeers: PeerInfo[]
revision?: string serverRevision?: string
scriptRevision?: string
qminingPageRevision?: string
dataEditorRevision?: string
serverBuildTime?: number
qminingPageBuildTime?: number
dataEditorBuildTime?: number
scriptVersion?: string
qdbInfo?: { qdbInfo?: {
dbName: string questionDbCount: number
subjs: { subjectCount: number
name: string questionCount: number
count: number }
}[]
}[]
} }
interface RequestResult<T> { interface RequestResult<T> {
@ -110,6 +116,7 @@ function get<T>(options: http.RequestOptions): Promise<RequestResult<T>> {
try { try {
resolve({ data: JSON.parse(body) }) resolve({ data: JSON.parse(body) })
} catch (e) { } catch (e) {
console.log(body)
resolve({ error: e, options: options }) resolve({ error: e, options: options })
} }
}) })
@ -135,7 +142,7 @@ export function getNewDataSince(subjects: Subject[], date: number): Subject[] {
return { return {
...subject, ...subject,
Questions: subject.Questions.filter((question) => { Questions: subject.Questions.filter((question) => {
return (question.data.date || 0) > date return (question.data.date || 0) >= date
}).map((question) => removeCacheFromQuestion(question)), }).map((question) => removeCacheFromQuestion(question)),
} }
}) })
@ -269,13 +276,21 @@ async function sendNewDataToWorkers(
function writeNewData( function writeNewData(
newQuestionDbs: QuestionDb[], newQuestionDbs: QuestionDb[],
changedQuestionDbs: QuestionDb[] changedQuestionDbs: QuestionDb[],
dbsFilePath: string,
publicDir: string
) { ) {
const qdbsToWrite = [...newQuestionDbs, ...changedQuestionDbs] const qdbsToWrite = [...changedQuestionDbs, ...newQuestionDbs]
const qdbsFile: DataFile[] = qdbsToWrite.map((qdb) => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { data, index, ...restOfQdb } = qdb
return restOfQdb
})
utils.WriteFile(JSON.stringify(qdbsFile, null, 2), dbsFilePath)
qdbsToWrite.forEach((qdb) => { qdbsToWrite.forEach((qdb) => {
try { try {
// FINALIZE: write to file writeData(qdb.data, publicDir + qdb.path)
// writeData(qdb.data, qdb.path)
} catch (e) { } catch (e) {
logger.Log(`Error writing ${qdb.name} qdb to file!`, 'redbg') logger.Log(`Error writing ${qdb.name} qdb to file!`, 'redbg')
console.error(e) console.error(e)
@ -319,18 +334,16 @@ function setup(data: SubmoduleData): Submodule {
app, app,
userDB, userDB,
publicdirs, publicdirs,
moduleSpecificData: { questionDbs, setQuestionDbs, getQuestionDbs }, moduleSpecificData: { setQuestionDbs, getQuestionDbs, dbsFile },
// publicdirs,
} = data } = data
const publicDir = publicdirs[0] const publicDir = publicdirs[0]
let syncInProgress = false
// --------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------
// SETUP // SETUP
// --------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------
// const publicDir = publicdirs[0]
if (!utils.FileExists(peersFile)) { if (!utils.FileExists(peersFile)) {
logger.Log( logger.Log(
`Warning: peers file was missing, so it was created`, `Warning: peers file was missing, so it was created`,
@ -440,11 +453,6 @@ function setup(data: SubmoduleData): Submodule {
`Warning: peers file is empty. You probably want to fill it`, `Warning: peers file is empty. You probably want to fill it`,
'yellowbg' 'yellowbg'
) )
} else {
logger.Log('Loaded peers: ' + peers.length)
peers.forEach((peer, i) => {
logger.Log(`\t${i}\t"${peer.name}": ${peerToString(peer)}`)
})
} }
// --------------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------------
@ -456,32 +464,35 @@ function setup(data: SubmoduleData): Submodule {
selfInfo: selfInfo, selfInfo: selfInfo,
myPeers: peers, myPeers: peers,
} }
result.serverRevision = utils.getGitRevision(__dirname)
try { result.scriptRevision = utils.getGitRevision(
// FIXME: dont log if fails constants.moodleTestUserscriptDir
result.revision = child_process )
.execSync('git rev-parse HEAD', { result.qminingPageRevision = utils.getGitRevision(
cwd: __dirname, constants.qminingPageDir
stdio: [0, 'pipe', null], )
}) result.dataEditorRevision = utils.getGitRevision(
.toString() constants.dataEditorPageDir
.trim() )
} catch (e) { result.qminingPageBuildTime = utils
result.revision = 'Failed to get revision' .statFile(constants.qminingIndexPath)
} ?.mtime.getTime()
result.serverBuildTime = utils
.statFile(constants.serverPath)
?.mtime.getTime()
result.dataEditorBuildTime = utils
.statFile(constants.dataEditorIndexPath)
?.mtime.getTime()
result.scriptVersion = utils.getScriptVersion()
if (includeQdbInfo) { if (includeQdbInfo) {
result.qdbInfo = getQuestionDbs().map((qdb) => { const questionDbCount = getQuestionDbs().length
return { const { subjCount, questionCount } = countOfQdbs(getQuestionDbs())
dbName: qdb.name, result.qdbInfo = {
subjs: qdb.data.map((subj) => { questionDbCount: questionDbCount,
return { subjectCount: subjCount,
name: subj.Name, questionCount: questionCount,
count: subj.Questions.length, }
}
}),
}
})
} }
return result return result
@ -506,7 +517,7 @@ function setup(data: SubmoduleData): Submodule {
return { return {
...qdb, ...qdb,
index: availableIndexes[i], index: availableIndexes[i],
path: `${publicDir}questionDbs/${qdb.name}.json'`, path: `questionDbs/${qdb.name}.json'`,
} }
}) })
} }
@ -543,7 +554,6 @@ function setup(data: SubmoduleData): Submodule {
} }
async function syncData() { async function syncData() {
// TOTEST: try with 0 date to merge full dbs
if (peers.length === 0) { if (peers.length === 0) {
logger.Log( logger.Log(
`There are no peers specified in ${peersFile}, aborting sync`, `There are no peers specified in ${peersFile}, aborting sync`,
@ -560,31 +570,36 @@ function setup(data: SubmoduleData): Submodule {
}${logger.C()} peers` }${logger.C()} peers`
) )
const lastSync = new Date('2022-03-12').getTime() // FINALIZE date: this is only for testing // selfInfo.lastSync const lastSync = new Date('2012-03-12').getTime() // FINALIZE date: this is only for testing // selfInfo.lastSync
logger.Log( logger.Log(
`\tLast sync date: ${logger.C('blue')}${new Date( `\tLast sync date: ${logger.C('blue')}${new Date(
lastSync lastSync
).toLocaleString()}${logger.C()}` ).toLocaleString()}${logger.C()}`
) )
const syncStart = new Date().getTime() const syncStart = new Date().getTime()
const requests = peers.map((peer) => { const lastSyncInfos = peers.map((peer) => {
const lastSyncWithPeer = new Date('2022-03-12').getTime() // FINALIZE same as above // peer.lastSync || 0 return [
peerToString(peer),
new Date(peer.lastSync).toLocaleString(),
]
})
logger.Log(`\tLast sync with peers:`)
logger.logTable([['', 'Date'], ...lastSyncInfos], {
colWidth: [15],
rowPrefix: '\t',
})
const requests = peers.map((peer) => {
const lastSyncWithPeer = new Date('2012-03-12').getTime() // FINALIZE same as above // peer.lastSync || 0
logger.Log(
`\tLast sync with ${logger.C('blue')}${peerToString(
peer
)}${logger.C()}: ${logger.C('blue')}${new Date(
lastSyncWithPeer
).toLocaleString()}${logger.C()}`
)
return new Promise<RequestResult<SyncDataRes & { peer: PeerInfo }>>( return new Promise<RequestResult<SyncDataRes & { peer: PeerInfo }>>(
(resolve) => { (resolve) => {
get<SyncDataRes>({ get<SyncDataRes>({
host: peer.host, host: peer.host,
port: peer.port, port: peer.port,
path: `/getnewdatasince?host=${selfInfo.host}${ path: `/getnewdatasince?host=${encodeURIComponent(
lastSync ? `&since=${lastSyncWithPeer}` : '' peerToString(selfInfo)
}`, )}${lastSync ? `&since=${lastSyncWithPeer}` : ''}`,
}).then((res) => { }).then((res) => {
resolve({ ...res, data: { ...res.data, peer: peer } }) resolve({ ...res, data: { ...res.data, peer: peer } })
}) })
@ -621,6 +636,7 @@ function setup(data: SubmoduleData): Submodule {
} }
} }
const recievedDataCounts: (number | string)[][] = []
const resultDataWithoutEmptyDbs = resultDataWithoutErrors.filter( const resultDataWithoutEmptyDbs = resultDataWithoutErrors.filter(
(res) => { (res) => {
const qdbCount = res.questionDbs.length const qdbCount = res.questionDbs.length
@ -628,22 +644,26 @@ function setup(data: SubmoduleData): Submodule {
res.questionDbs res.questionDbs
) )
logger.Log( recievedDataCounts.push([
`\t"${logger.C('blue')}${peerToString( peerToString(res.peer),
res.peer qdbCount,
)}${logger.C()}" sent "${logger.C( subjCount,
'green' questionCount,
)}${qdbCount}${logger.C()}" question DB-s with "${logger.C( ])
'green'
)}${subjCount.toLocaleString()}${logger.C()}" subjects, and "${logger.C(
'green'
)}${questionCount.toLocaleString()}${logger.C()}" questions`
)
return questionCount > 0 return questionCount > 0
} }
) )
logger.Log(`\tRecieved data from peers:`)
logger.logTable(
[['', 'QDBs', 'Subjs', 'Questions'], ...recievedDataCounts],
{
colWidth: [15],
rowPrefix: '\t',
}
)
const resultData = resultDataWithoutEmptyDbs.map((res) => { const resultData = resultDataWithoutEmptyDbs.map((res) => {
return { return {
...res, ...res,
@ -665,17 +685,14 @@ function setup(data: SubmoduleData): Submodule {
) )
}) })
if (thirdPartyPeers.length > 0) { if (thirdPartyPeers.length > 0) {
logger.Log(
`\tPeers reported ${logger.C('green')}${
thirdPartyPeers.length
}${logger.C()} third party peer(s) not connected to this server.`
)
utils.WriteFile( utils.WriteFile(
JSON.stringify(thirdPartyPeers, null, 2), JSON.stringify(thirdPartyPeers, null, 2),
thirdPartyPeersFile thirdPartyPeersFile
) )
logger.Log( logger.Log(
`\tSee ${logger.C( `\tPeers reported ${logger.C('green')}${
thirdPartyPeers.length
}${logger.C()} third party peer(s) not connected to this server. See ${logger.C(
'blue' 'blue'
)}${thirdPartyPeersFile}${logger.C()} for details` )}${thirdPartyPeersFile}${logger.C()} for details`
) )
@ -690,6 +707,7 @@ function setup(data: SubmoduleData): Submodule {
newQuestions?: number newQuestions?: number
} }
} = {} } = {}
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
// new users handlin TOTEST: test // new users handlin TOTEST: test
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
@ -700,25 +718,23 @@ function setup(data: SubmoduleData): Submodule {
const decryptedUsers: User[] = JSON.parse( const decryptedUsers: User[] = JSON.parse(
decrypt(privateKey, res.encryptedUsers) decrypt(privateKey, res.encryptedUsers)
) )
let newUserCount = 0
decryptedUsers.forEach((remoteUser) => { decryptedUsers.forEach((remoteUser) => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
const { id, ...remoteUserWithoutId } = remoteUser const { id, ...remoteUserWithoutId } = remoteUser
const localUser = dbtools.Select(userDB, 'users', { const localUser = dbtools.Select(userDB, 'users', {
pw: remoteUser.pw, pw: remoteUser.pw,
}) })
if (!localUser) { if (localUser.length === 0) {
// FIXME: users will not have consistend id across servers. This may be // FIXME: users will not have consistend id across servers. This may be
// harmless, will see // harmless, will see
dbtools.Insert(userDB, 'users', { dbtools.Insert(userDB, 'users', {
...(remoteUserWithoutId as Omit<User, 'id'>), ...(remoteUserWithoutId as Omit<User, 'id'>),
sourceHost: peerToString(res.peer), sourceHost: peerToString(res.peer),
}) })
newUserCount += 1
} }
}) })
resultsCount[peerToString(res.peer)] = { resultsCount[peerToString(res.peer)] = {
newUsers: newUserCount, newUsers: decryptedUsers.length,
} }
} }
}) })
@ -746,7 +762,6 @@ function setup(data: SubmoduleData): Submodule {
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
// backup // backup
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
const { subjCount: oldSubjCount, questionCount: oldQuestionCount } = const { subjCount: oldSubjCount, questionCount: oldQuestionCount } =
countOfQdbs(getQuestionDbs()) countOfQdbs(getQuestionDbs())
const oldQuestionDbCount = getQuestionDbs().length const oldQuestionDbCount = getQuestionDbs().length
@ -757,7 +772,6 @@ function setup(data: SubmoduleData): Submodule {
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
// adding questions to db // adding questions to db
// ------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------
for (let i = 0; i < resultData.length; i++) { for (let i = 0; i < resultData.length; i++) {
const { questionDbs: remoteQuestionDbs, peer } = resultData[i] const { questionDbs: remoteQuestionDbs, peer } = resultData[i]
logger.Log( logger.Log(
@ -786,7 +800,9 @@ function setup(data: SubmoduleData): Submodule {
newQuestionDbs, newQuestionDbs,
getQuestionDbs().filter((qdb) => { getQuestionDbs().filter((qdb) => {
return changedQdbIndexes.includes(qdb.index) return changedQdbIndexes.includes(qdb.index)
}) }),
dbsFile,
publicDir
) )
setQuestionDbs([...mergedQuestionDbs, ...newQuestionDbs]) setQuestionDbs([...mergedQuestionDbs, ...newQuestionDbs])
@ -876,7 +892,7 @@ function setup(data: SubmoduleData): Submodule {
newQuestionCount, newQuestionCount,
], ],
], ],
{ colWidth: [15] } { colWidth: [15], rowPrefix: '\t' }
) )
logger.Log( logger.Log(
@ -922,8 +938,8 @@ function setup(data: SubmoduleData): Submodule {
const remoteHost = req.query.host const remoteHost = req.query.host
const questionDbsWithNewQuestions = Number.isNaN(since) const questionDbsWithNewQuestions = Number.isNaN(since)
? questionDbs ? getQuestionDbs()
: questionDbs : getQuestionDbs()
.map((qdb) => { .map((qdb) => {
return { return {
...qdb, ...qdb,
@ -949,26 +965,54 @@ function setup(data: SubmoduleData): Submodule {
remoteInfo: getSelfInfo(), remoteInfo: getSelfInfo(),
} }
let hostToLog = req.hostname
if (remoteHost) { if (remoteHost) {
const remoteHostInfo = peers.find((peer) => { const remotePeerInfo = peers.find((peer) => {
return peer.host === remoteHost return peerToString(peer) === remoteHost
}) })
const remotePublicKey = remoteHostInfo?.publicKey if (remotePeerInfo) {
if (remotePublicKey) { hostToLog = peerToString(remotePeerInfo)
// FIXME: sign data? const remotePublicKey = remotePeerInfo?.publicKey
const newUsers = getNewUsersSince(since) if (remotePublicKey) {
result.encryptedUsers = encrypt( // FIXME: sign data?
remotePublicKey, const newUsers = getNewUsersSince(since)
JSON.stringify(newUsers) result.encryptedUsers = encrypt(
) remotePublicKey,
} else if (remoteHostInfo) { JSON.stringify(newUsers)
)
logger.Log(
`Sending new users to "${remoteHost}" (encrypted)`,
'green'
)
} else if (remotePeerInfo) {
logger.Log(
`Warning: "${hostToLog}" has no public key saved!`,
'yellowbg'
)
}
} else {
logger.Log( logger.Log(
`Warning: ${remoteHostInfo.host}:${remoteHostInfo.port} has no publick key saved!`, 'Couldn\'t find remote peer info based on remoteHost: "' +
remoteHost +
'". This could mean that the host uses this server as peer, but this server does not ' +
'use it as a peer.',
'yellowbg' 'yellowbg'
) )
} }
} }
const dateToLog = Number.isNaN(since)
? 'all time'
: new Date(since).toLocaleString()
logger.Log(
`Sending new data to ${logger.C(
'blue'
)}${hostToLog}${logger.C()} since ${logger.C(
'blue'
)}${dateToLog}${logger.C()} `
)
res.json(result) res.json(result)
}) })
@ -984,6 +1028,15 @@ function setup(data: SubmoduleData): Submodule {
// return // return
// } // }
// FIXME: /syncResult EP if this EP times out, but we still need the result
if (syncInProgress) {
res.json({
error: 'A sync is already in progress!',
})
return
}
syncInProgress = true
setPendingJobsAlertCount(5000) setPendingJobsAlertCount(5000)
syncData() syncData()
.then((syncResult) => { .then((syncResult) => {
@ -992,6 +1045,7 @@ function setup(data: SubmoduleData): Submodule {
...syncResult, ...syncResult,
}) })
setPendingJobsAlertCount() setPendingJobsAlertCount()
syncInProgress = false
}) })
.catch((e) => { .catch((e) => {
console.error(e) console.error(e)
@ -1000,10 +1054,17 @@ function setup(data: SubmoduleData): Submodule {
msg: e.message, msg: e.message,
}) })
setPendingJobsAlertCount() setPendingJobsAlertCount()
syncInProgress = false
}) })
}) })
logger.Log('P2P functionality set up. Peers: ' + peers.length, 'blue') logger.Log(
'P2P functionality set up. Peers (' +
peers.length +
'): ' +
peers.map((peer) => peerToString(peer)).join(', '),
'blue'
)
return {} return {}
} }

74
src/modules/api/submodules/qminingapi.ts
View file

@ -59,6 +59,7 @@ import {
getSubjNameWithoutYear, getSubjNameWithoutYear,
SearchResultQuestion, SearchResultQuestion,
} from '../../../utils/qdbUtils' } from '../../../utils/qdbUtils'
import constants from '../../../constants.json'
interface SavedQuestionData { interface SavedQuestionData {
fname: string fname: string
@ -79,7 +80,6 @@ interface SavedQuestionData {
const line = '====================================================' // lol const line = '====================================================' // lol
const registeredScriptsFile = 'stats/registeredScripts.json' const registeredScriptsFile = 'stats/registeredScripts.json'
const testUsersFile = 'data/testUsers.json' const testUsersFile = 'data/testUsers.json'
const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js'
const askedQuestionFile = 'stats/askedQuestions' const askedQuestionFile = 'stats/askedQuestions'
const recievedQuestionFile = 'stats/recievedQuestions' const recievedQuestionFile = 'stats/recievedQuestions'
const savedQuestionsFileName = 'savedQuestions.json' const savedQuestionsFileName = 'savedQuestions.json'
@ -358,7 +358,9 @@ function saveQuestion(
} }
function loadSupportedSites() { function loadSupportedSites() {
const script = utils.ReadFile(userScriptFile).split('\n') const script = utils
.ReadFile(constants.moodleTestUserscriptPath)
.split('\n')
let i = 0 let i = 0
let stayIn = true let stayIn = true
@ -386,18 +388,6 @@ function loadSupportedSites() {
return sites return sites
} }
function LoadVersion() {
const scriptContent = utils.ReadFile(userScriptFile)
let temp: string | string[] = scriptContent.split('\n').find((x) => {
return x.includes('@version')
})
temp = temp.split(' ')
temp = temp[temp.length - 1]
return temp
}
function LoadMOTD(motdFile: string) { function LoadMOTD(motdFile: string) {
return utils.ReadFile(motdFile) return utils.ReadFile(motdFile)
} }
@ -470,14 +460,14 @@ function setup(data: SubmoduleData): Submodule {
app, app,
userDB, userDB,
/* url */ publicdirs, /* url */ publicdirs,
moduleSpecificData: { questionDbs: questionDbs, dbsFile: dbsFile }, moduleSpecificData: { getQuestionDbs, setQuestionDbs, dbsFile },
} = data } = data
const publicDir = publicdirs[0] const publicDir = publicdirs[0]
const motdFile = publicDir + 'motd' const motdFile = publicDir + 'motd'
const savedQuestionsDir = publicDir + 'savedQuestions' const savedQuestionsDir = publicDir + 'savedQuestions'
let version = LoadVersion() let version = utils.getScriptVersion()
let supportedSites = loadSupportedSites() let supportedSites = loadSupportedSites()
let motd = LoadMOTD(motdFile) let motd = LoadMOTD(motdFile)
let testUsers: number[] = LoadTestUsers() let testUsers: number[] = LoadTestUsers()
@ -498,10 +488,10 @@ function setup(data: SubmoduleData): Submodule {
}, },
}, },
{ {
fname: userScriptFile, fname: constants.moodleTestUserscriptPath,
logMsg: 'User script file changed', logMsg: 'User script file changed',
action: () => { action: () => {
version = LoadVersion() version = utils.getScriptVersion()
supportedSites = loadSupportedSites() supportedSites = loadSupportedSites()
}, },
}, },
@ -510,7 +500,7 @@ function setup(data: SubmoduleData): Submodule {
app.get('/getDbs', (req: Request, res: Response) => { app.get('/getDbs', (req: Request, res: Response) => {
logger.LogReq(req) logger.LogReq(req)
res.json( res.json(
questionDbs.map((qdb) => { getQuestionDbs().map((qdb) => {
return { return {
path: qdb.path.replace(publicDir, ''), path: qdb.path.replace(publicDir, ''),
name: qdb.name, name: qdb.name,
@ -528,7 +518,7 @@ function setup(data: SubmoduleData): Submodule {
res.setHeader('content-type', 'text/plain; charset=utf-8') res.setHeader('content-type', 'text/plain; charset=utf-8')
if (db) { if (db) {
const requestedDb = questionDbs.find((qdb) => { const requestedDb = getQuestionDbs().find((qdb) => {
return qdb.name === db return qdb.name === db
}) })
@ -543,7 +533,7 @@ function setup(data: SubmoduleData): Submodule {
stringifiedData += dataToString(requestedDb.data) stringifiedData += dataToString(requestedDb.data)
stringifiedData += '\n' + line + line + '\n' stringifiedData += '\n' + line + line + '\n'
} else { } else {
stringifiedData = questionDbs stringifiedData = getQuestionDbs()
.map((qdb) => { .map((qdb) => {
let result = '' let result = ''
result += '\n' + line result += '\n' + line
@ -576,7 +566,7 @@ function setup(data: SubmoduleData): Submodule {
try { try {
let maxIndex = -1 let maxIndex = -1
const suitedQuestionDbs = questionDbs.filter((qdb) => { const suitedQuestionDbs = getQuestionDbs().filter((qdb) => {
if (maxIndex < qdb.index) { if (maxIndex < qdb.index) {
maxIndex = qdb.index maxIndex = qdb.index
} }
@ -584,14 +574,14 @@ function setup(data: SubmoduleData): Submodule {
}, []) }, [])
if (suitedQuestionDbs.length === 0) { if (suitedQuestionDbs.length === 0) {
if (!dbExists(location, questionDbs)) { if (!dbExists(location, getQuestionDbs())) {
suitedQuestionDbs.push( suitedQuestionDbs.push(
getNewQdb( getNewQdb(
location, location,
maxIndex, maxIndex,
dbsFile, dbsFile,
publicDir, publicDir,
questionDbs getQuestionDbs()
) )
) )
} else { } else {
@ -684,7 +674,7 @@ function setup(data: SubmoduleData): Submodule {
question: question, question: question,
subj: subj, subj: subj,
testUrl: testUrl, testUrl: testUrl,
questionDbs: questionDbs, questionDbs: getQuestionDbs(),
}) })
}) })
@ -730,13 +720,13 @@ function setup(data: SubmoduleData): Submodule {
logger.LogReq(req) logger.LogReq(req)
if (req.query.detailed === 'all') { if (req.query.detailed === 'all') {
res.json({ res.json({
detailed: getDetailedRes(questionDbs), detailed: getDetailedRes(getQuestionDbs()),
simple: getSimplreRes(questionDbs), simple: getSimplreRes(getQuestionDbs()),
}) })
} else if (req.query.detailed) { } else if (req.query.detailed) {
res.json(getDetailedRes(questionDbs)) res.json(getDetailedRes(getQuestionDbs()))
} else { } else {
res.json(getSimplreRes(questionDbs)) res.json(getSimplreRes(getQuestionDbs()))
} }
}) })
@ -758,7 +748,7 @@ function setup(data: SubmoduleData): Submodule {
} }
if (req.query.subjinfo) { if (req.query.subjinfo) {
result.subjinfo = getSimplreRes(questionDbs) result.subjinfo = getSimplreRes(getQuestionDbs())
} }
if (req.query.version) { if (req.query.version) {
result.version = version result.version = version
@ -907,10 +897,10 @@ function setup(data: SubmoduleData): Submodule {
return return
} }
const dbIndex = questionDbs.findIndex((qdb) => { const dbIndex = getQuestionDbs().findIndex((qdb) => {
return qdb.name === selectedDb.name return qdb.name === selectedDb.name
}) })
const currDb = questionDbs[dbIndex] const currDb = getQuestionDbs()[dbIndex]
if (dbIndex === -1) { if (dbIndex === -1) {
res.json({ res.json({
@ -937,7 +927,12 @@ function setup(data: SubmoduleData): Submodule {
return return
} }
if (resultDb) { if (resultDb) {
questionDbs[dbIndex] = resultDb setQuestionDbs(
getQuestionDbs().map((qdb, i) => {
if (i === dbIndex) return resultDb
return qdb
})
)
} }
if (editType === 'delete') { if (editType === 'delete') {
@ -1025,6 +1020,10 @@ function setup(data: SubmoduleData): Submodule {
// TODO: dont allow multiple instances // TODO: dont allow multiple instances
// TODO: get status of it cleaning // TODO: get status of it cleaning
logger.LogReq(req) logger.LogReq(req)
res.json({
error: 'Not implemented / tested!',
})
return
const user: User = req.session.user const user: User = req.session.user
const status: string = req.query.status const status: string = req.query.status
@ -1057,7 +1056,8 @@ function setup(data: SubmoduleData): Submodule {
questionCleaner = fork( questionCleaner = fork(
`${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`,
['-s', `${process.cwd()}/${questionDbs[0].path}`] ['-s', `${process.cwd()}/${getQuestionDbs()[0].path}`] // TODO: this only cleans index
// #0?
) )
questionCleaner.on('exit', function (code: number) { questionCleaner.on('exit', function (code: number) {
console.log('EXIT', code) console.log('EXIT', code)
@ -1073,11 +1073,11 @@ function setup(data: SubmoduleData): Submodule {
return { return {
dailyAction: () => { dailyAction: () => {
backupData(questionDbs) backupData(getQuestionDbs())
ExportDailyDataCount(questionDbs, userDB) ExportDailyDataCount(getQuestionDbs(), userDB)
}, },
load: () => { load: () => {
backupData(questionDbs) backupData(getQuestionDbs())
filesToWatch.forEach((ftw) => { filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) { if (utils.FileExists(ftw.fname)) {

25
src/modules/api/submodules/userManagement.ts
View file

@ -63,6 +63,19 @@ function BackupDB(usersDbBackupPath: string, userDB: Database) {
}) })
} }
// TODO: figure out if this is needed
// const validationTokenNameFile = 'data/validationTokenName'
// function readValidationTokenName() {
// if (utils.FileExists(validationTokenNameFile)) {
// return utils.ReadFile(validationTokenNameFile)
// } else {
// throw new Error(
// `Validation token file does not exist! Should be: "${validationTokenNameFile}", content should be: "name for uuidv5 (any text)"`
// )
// }
// }
const validationTokenName = 'qmining' // readValidationTokenName()
function setup(data: SubmoduleData): Submodule { function setup(data: SubmoduleData): Submodule {
const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
@ -70,6 +83,13 @@ function setup(data: SubmoduleData): Submodule {
domain = domain.join('.') // "frylabs.net" domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
logger.Log(
`User count: ${dbtools
.TableInfo(userDB, 'users')
.dataCount.toLocaleString()} users`,
'blue'
)
app.get('/avaiblePWS', (req: Request, res: any) => { app.get('/avaiblePWS', (req: Request, res: any) => {
logger.LogReq(req) logger.LogReq(req)
@ -280,7 +300,6 @@ function setup(data: SubmoduleData): Submodule {
}) })
}) })
const name = 'qmining'
app.get( app.get(
'/validationtoken', '/validationtoken',
(req: Request<{ token: string; userid: string }>, res: any) => { (req: Request<{ token: string; userid: string }>, res: any) => {
@ -299,7 +318,7 @@ function setup(data: SubmoduleData): Submodule {
msg: 'couldnt find user', msg: 'couldnt find user',
}) })
} }
const key = v5(name, specifiedUser[0].pw) const key = v5(validationTokenName, specifiedUser[0].pw)
const isValid = key === token const isValid = key === token
res.json({ res.json({
@ -307,7 +326,7 @@ function setup(data: SubmoduleData): Submodule {
isValid: isValid, isValid: isValid,
}) })
} else { } else {
const key = v5(name, user.pw) const key = v5(validationTokenName, user.pw)
res.json({ res.json({
result: 'success', result: 'success',

13
src/modules/api/usersDBStruct.ts
View file

@ -39,14 +39,14 @@ const DBStruct = {
defaultZero: true, defaultZero: true,
}, },
created: { created: {
type: 'text', type: 'number',
notNull: true, notNull: true,
}, },
lastLogin: { lastLogin: {
type: 'text', type: 'number',
}, },
lastAccess: { lastAccess: {
type: 'text', type: 'number',
}, },
avaiblePWRequests: { avaiblePWRequests: {
type: 'number', type: 'number',
@ -59,6 +59,9 @@ const DBStruct = {
createdBy: { createdBy: {
type: 'number', type: 'number',
}, },
sourceHost: {
type: 'text',
},
}, },
}, },
sessions: { sessions: {
@ -80,11 +83,11 @@ const DBStruct = {
notNull: true, notNull: true,
}, },
createDate: { createDate: {
type: 'text', type: 'number',
notNull: true, notNull: true,
}, },
lastAccess: { lastAccess: {
type: 'text', type: 'number',
}, },
isScript: { isScript: {
type: 'number', type: 'number',

1
src/modules/dataEditor/dataEditor.ts
View file

@ -64,7 +64,6 @@ function GetApp(): ModuleType {
next() next()
}) })
publicdirs.forEach((pdir) => { publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir)) app.use(express.static(pdir))
}) })
app.use(express.static(nextdir)) app.use(express.static(nextdir))

2
src/modules/main/main.ts
View file

@ -23,7 +23,6 @@ import express, { RequestHandler } from 'express'
const app = express() const app = express()
// other requires // other requires
import logger from '../../utils/logger'
import { SetupData } from '../../server' import { SetupData } from '../../server'
import { ModuleType } from '../../types/basicTypes' import { ModuleType } from '../../types/basicTypes'
@ -35,7 +34,6 @@ function GetApp(): ModuleType {
app.set('view engine', 'ejs') app.set('view engine', 'ejs')
app.set('views', ['./src/modules/main/views', './src/sharedViews']) app.set('views', ['./src/modules/main/views', './src/sharedViews'])
publicdirs.forEach((pdir) => { publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir)) app.use(express.static(pdir))
}) })

1
src/modules/qmining/qmining.ts
View file

@ -67,7 +67,6 @@ function GetApp(): ModuleType {
next() next()
}) })
publicdirs.forEach((pdir) => { publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir)) app.use(express.static(pdir))
}) })
app.use(express.static(nextdir)) app.use(express.static(nextdir))

3
src/types/basicTypes.ts
View file

@ -114,7 +114,6 @@ export interface Request<T = any> extends express.Request {
export interface ModuleSpecificData { export interface ModuleSpecificData {
// TODO: rename to something more meaningfull // TODO: rename to something more meaningfull
questionDbs: QuestionDb[]
setQuestionDbs: (newVal: QuestionDb[]) => void setQuestionDbs: (newVal: QuestionDb[]) => void
getQuestionDbs: () => QuestionDb[] getQuestionDbs: () => QuestionDb[]
dbsFile: string dbsFile: string
@ -175,6 +174,6 @@ export interface PeerInfo {
port: number port: number
publicKey: string publicKey: string
contact: string contact: string
lastSync?: Date lastSync?: number
note?: string note?: string
} }

12
src/utils/actions.ts
View file

@ -504,7 +504,7 @@ export function loadJSON(
if (!utils.FileExists(dataPath)) { if (!utils.FileExists(dataPath)) {
logger.Log( logger.Log(
`${dataPath} data file did not exist, created empty one!`, `${dataPath} data file does not exist, created empty one!`,
'yellowbg' 'yellowbg'
) )
utils.WriteFile(JSON.stringify([]), dataPath) utils.WriteFile(JSON.stringify([]), dataPath)
@ -513,7 +513,6 @@ export function loadJSON(
try { try {
acc.push({ acc.push({
...dataFile, ...dataFile,
path: dataPath,
index: index, index: index,
data: loadData(dataPath), data: loadData(dataPath),
}) })
@ -529,8 +528,8 @@ export function loadJSON(
const { subjCount, questionCount } = countOfQdbs(res) const { subjCount, questionCount } = countOfQdbs(res)
logger.Log( logger.Log(
`Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`, `Loaded ${subjCount.toLocaleString()} subjects with ${questionCount.toLocaleString()} questions from ${res.length.toLocaleString()} question db-s`,
logger.GetColor('green') 'blue'
) )
return res return res
@ -565,10 +564,7 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
// logger.Log(`Backing up ${data.name}...`) // logger.Log(`Backing up ${data.name}...`)
writeData( writeData(
data.data, data.data,
`${path}${data.name}_${utils.GetDateString( `${path}${data.name}_${utils.GetDateString()}.json`
undefined,
true
)}.json`
) )
// logger.Log('Done') // logger.Log('Done')
} catch (err) { } catch (err) {

7
src/utils/logger.ts
View file

@ -421,12 +421,13 @@ function C(color?: string): string {
function logTable( function logTable(
table: (string | number)[][], table: (string | number)[][],
options: { colWidth?: number[] } = {} options: { colWidth?: number[]; rowPrefix?: string } = {}
): void { ): void {
const { colWidth, rowPrefix } = options
table.forEach((row, i) => { table.forEach((row, i) => {
const rowString: string[] = [] const rowString: string[] = []
row.forEach((cell, j) => { row.forEach((cell, j) => {
const { colWidth } = options
const cellColor = j === 0 || i === 0 ? 'blue' : 'green' const cellColor = j === 0 || i === 0 ? 'blue' : 'green'
let cellVal = '' let cellVal = ''
if (!isNaN(+cell)) { if (!isNaN(+cell)) {
@ -447,7 +448,7 @@ function logTable(
rowString.push(C(cellColor) + cellVal + C()) rowString.push(C(cellColor) + cellVal + C())
}) })
Log(rowString.join('\t')) Log((rowPrefix || '') + rowString.join('\t'))
}) })
} }

30
src/utils/utils.ts
View file

@ -37,12 +37,16 @@ export default {
renameFile: renameFile, renameFile: renameFile,
deleteDir: deleteDir, deleteDir: deleteDir,
formatBytes: formatBytes, formatBytes: formatBytes,
getGitRevision: getGitRevision,
getScriptVersion: getScriptVersion,
} }
import * as child_process from 'child_process'
import fs from 'fs' import fs from 'fs'
import { v4 as uuidv4 } from 'uuid' import { v4 as uuidv4 } from 'uuid'
import logger from '../utils/logger' import logger from '../utils/logger'
import constants from '../constants.json'
import { Request } from '../types/basicTypes' import { Request } from '../types/basicTypes'
interface URLFormatOptions { interface URLFormatOptions {
@ -309,3 +313,29 @@ function formatBytes(number: number, unit: 'MB' | 'GB' = 'MB'): string {
} }
return `${number} byte` return `${number} byte`
} }
function getGitRevision(dir: string): string {
try {
return child_process
.execSync('git rev-parse HEAD', {
cwd: dir,
stdio: [0, 'pipe', null],
})
.toString()
.trim()
} catch (e) {
return 'Failed to get revision'
}
}
function getScriptVersion(): string {
const scriptContent = ReadFile(constants.moodleTestUserscriptPath)
let temp: string | string[] = scriptContent.split('\n').find((x) => {
return x.includes('@version')
})
temp = temp.split(' ')
temp = temp[temp.length - 1]
return temp
}

55
tsconfig.json
View file

@ -1,29 +1,30 @@
{ {
"compilerOptions": { "compilerOptions": {
"noUnusedLocals": true, "noUnusedLocals": true,
"noUnusedParameters": true, "noUnusedParameters": true,
"noImplicitReturns": true, "noImplicitReturns": true,
"noFallthroughCasesInSwitch": true, "noFallthroughCasesInSwitch": true,
"suppressImplicitAnyIndexErrors": true, "suppressImplicitAnyIndexErrors": true,
"moduleResolution": "node", "moduleResolution": "node",
"esModuleInterop": true, "esModuleInterop": true,
"target": "es6", "target": "es6",
"allowJs": true, "allowJs": true,
"module": "commonjs", "module": "commonjs",
"removeComments": true, "removeComments": true,
"preserveConstEnums": true, "preserveConstEnums": true,
"sourceMap": true, "sourceMap": true,
"outDir": "dist", "outDir": "dist",
"noImplicitAny": true, "noImplicitAny": true,
"lib": ["dom", "ES2020"] "lib": ["dom", "ES2020"],
}, "resolveJsonModule": true
"files": ["src/server.ts"], },
"include": ["src/**/*"], "files": ["src/server.ts"],
"exclude": [ "include": ["src/**/*"],
"src/tests/", "exclude": [
"node_modules", "src/tests/",
"submodules", "node_modules",
"devel", "submodules",
"src/standaloneUtils" "devel",
] "src/standaloneUtils"
]
} }