added bit more advanced file existance checking, removed vhosts in favor or routes

This commit is contained in:
mrfry 2023-03-29 19:10:44 +02:00
parent ba89f4a342
commit 113a114821
24 changed files with 2720 additions and 2474 deletions

View file

@ -526,7 +526,7 @@ function handleWorkerData() {
} catch (e) {
console.error(e)
parentPort.postMessage({
msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`,
msg: `From thread #${workerIndex}: unhandled error occured!`,
workerIndex: workerIndex,
e: e,
})

246
src/utils/files.ts Normal file
View file

@ -0,0 +1,246 @@
import { Schema } from 'jsonschema'
import {
TestUsersSchema,
isJsonValidAndLogError,
PeersInfoSchema,
PeerInfoSchema,
ModulesSchema,
} from '../types/typeSchemas'
import logger from './logger'
import utils from './utils'
// FIXME: remove all file exists checks from everywhere for files that are created / checked here
type FileDescriptor = {
path: string
schema?: Schema
defaultValue?: string
shouldBe?: string
description?: string
}
export const validateFiles = (): boolean => {
let everythingValid = true
Object.entries(files).forEach(([key, file]: [string, FileDescriptor]) => {
let fileExists = utils.FileExists(file.path)
if (file.defaultValue != null && !fileExists) {
utils.WriteFile(file.defaultValue, file.path)
fileExists = true
}
if (file.shouldBe && !fileExists) {
const errMsg = [`File "${file.path}" does not exist! (${key})`]
if (file.shouldBe) {
errMsg.push(`Should be: ${file.shouldBe}`)
}
logger.Log(errMsg.join(' '), 'redbg')
everythingValid = false
return
}
if (file.schema && fileExists) {
const val = utils.ReadJSON(file.path)
if (!isJsonValidAndLogError(val, file.schema, file.path)) {
everythingValid = false
return
}
}
})
return everythingValid
}
export const files = {
// --------------------------------------------------------------------------------
// server / modules files
// --------------------------------------------------------------------------------
serverPath: {
path: 'dist/server.js',
shouldBe:
'server main entry file, created after running "npm run build"',
},
qminingPageDir: {
path: 'submodules/qmining-page',
shouldBe:
'qmining page submodule directory, created by pulling submodules / setup script',
},
qminingIndexPath: {
path: 'nextStatic/qminingPagePublic/index.html',
shouldBe:
'qmining page-s build index.html, created by "npm run build" in qmining page submodule dir',
},
dataEditorPageDir: {
path: 'submodules/qmining-data-editor',
shouldBe:
'qmining data editor page submodule directory, created by pulling submodules / setup script',
},
dataEditorIndexPath: {
path: 'nextStatic/dataEditorPublic/index.html',
shouldBe:
'qmining data editor-s build index.html, created by "npm run build" in qmining data editor submodule dir',
},
moodleTestUserscriptDir: {
path: 'submodules/moodle-test-userscript',
shouldBe:
'moodle test userscript submodule directory, created by pulling submodules / setup script',
},
moodleTestUserscriptPath: {
path: 'submodules/moodle-test-userscript/stable.user.js',
shouldBe:
'moodle test userscript file, created by pulling submodules / setup script',
},
domainFile: {
path: 'data/domain',
shouldBe:
'server domain for cookies and stuff, for ex.: "frylabs.net", no "http://" and things like that, just the domain',
},
// --------------------------------------------------------------------------------
// stats files
// --------------------------------------------------------------------------------
registeredScriptsFile: {
path: 'stats/registeredScripts.json',
defaultValue: JSON.stringify([]),
},
askedQuestionFile: {
path: 'stats/askedQuestions',
description: 'text file of recieved data on /ask',
},
recievedQuestionFile: {
path: 'stats/recievedQuestions',
description: 'text file of recieved data on /isAdding',
},
dailyDataCountFile: {
path: 'stats/dailyDataCount',
description: 'text file of daily data count',
},
dataEditsLog: {
path: 'stats/dataEdits',
description: 'text file of data edit logs',
},
// --------------------------------------------------------------------------------
// https files
// --------------------------------------------------------------------------------
privkeyFile: {
path: '/etc/letsencrypt/live/frylabs.net/privkey.pem',
description: 'private key file for https',
},
fullchainFile: {
path: '/etc/letsencrypt/live/frylabs.net/fullchain.pem',
description: 'full chain key file for https',
},
chainFile: {
path: '/etc/letsencrypt/live/frylabs.net/chain.pem',
description: 'chain key file for https',
},
// --------------------------------------------------------------------------------
// api files
// --------------------------------------------------------------------------------
rootRedirectToFile: {
path: 'data/apiRootRedirectTo',
description: 'url to redirect users trying to acces root api path',
},
modulesFile: {
path: './src/modules.json',
shouldBe: 'module files for server',
schema: ModulesSchema,
},
extraModulesFile: {
path: './src/extraModules/extraModules.json',
description: 'extra private modules for server, not tracked by git',
schema: ModulesSchema,
},
statExcludeFile: {
path: './data/statExclude.json',
shouldBe:
'array of strings which if included in requests url-s then the request itself is not counted in stats',
defaultValue: JSON.stringify([]),
schema: { type: 'array', items: { type: 'string' } },
},
usersDBPath: {
path: './data/dbs/users.db',
shouldBe: 'users sqlite db file',
},
// --------------------------------------------------------------------------------
// qmining api
// --------------------------------------------------------------------------------
testUsersFile: {
path: 'data/testUsers.json',
defaultValue: JSON.stringify({ userIds: [] }),
schema: TestUsersSchema,
description: 'test users, which are excluded from stats',
},
// --------------------------------------------------------------------------------
// log files
// --------------------------------------------------------------------------------
vlogDir: {
path: 'stats/vlogs/',
description: 'verbose logs directory',
},
logDir: {
path: 'stats/logs/',
description: 'basic logs directory',
},
statFile: {
path: 'stats/stats',
defaultValue: JSON.stringify({}),
description: 'json of visited paths all time',
},
vStatFile: {
path: 'stats/vstats',
defaultValue: JSON.stringify({}),
description: 'json of visited paths by day',
},
uStatsFile: {
path: 'stats/ustats',
defaultValue: JSON.stringify({}),
description: 'json of visits per user',
},
uvStatsFile: {
path: 'stats/uvstats',
defaultValue: JSON.stringify({}),
description: 'json of visits per user by day',
},
nologFile: {
path: './data/nolog',
defaultValue: '',
description:
'text file of users seperated by new lines to ignore in logging / stats',
},
// --------------------------------------------------------------------------------
// peer files
// --------------------------------------------------------------------------------
peersPath: {
path: 'data/p2p/',
description: 'p2p files directory',
},
peersFile: {
path: 'data/p2p/peers.json',
description: 'json of list of peers',
defaultValue: JSON.stringify([]),
schema: PeersInfoSchema,
},
selfInfoFile: {
path: 'data/p2p/selfInfo.json',
description: 'json of info of this servers peer functionality',
defaultValue: JSON.stringify({}),
schema: PeerInfoSchema,
},
thirdPartyPeersFile: {
path: 'data/p2p/thirdPartyPeers.json',
description: 'json of third party peers reported by other peers',
},
keyFile: {
path: 'data/p2p/key',
},
} as const satisfies Record<string, FileDescriptor>
export const paths = Object.entries(files).reduce(
(acc, [key, value]) => ({ ...acc, [key]: value.path }),
{}
) as Record<keyof typeof files, string>

View file

@ -26,14 +26,7 @@ const DELIM = C('green') + '|' + C()
// import express from 'express'
import utils from '../utils/utils'
import { Request } from '../types/basicTypes'
const vlogDir = 'stats/vlogs/'
const logDir = 'stats/logs/'
const statFile = 'stats/stats'
const vStatFile = 'stats/vstats'
const uStatsFile = 'stats/ustats'
const uvStatsFile = 'stats/uvstats'
const nologFile = './data/nolog'
import { paths } from './files'
const colors = ['green', 'red', 'yellow', 'blue', 'magenta', 'cyan']
const logFileName = 'log'
@ -93,7 +86,7 @@ function Log(msg: string | object, color?: string): void {
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
paths.logDir + logFileName
)
}
@ -156,7 +149,10 @@ function LogReq(
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
logEntry +=
GetRandomColor(req.originalUrl.split('?')[0]) +
req.originalUrl +
C()
if (statusCode !== undefined) {
logEntry += dl + statusCode
@ -168,7 +164,7 @@ function LogReq(
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
utils.AppendToFile(defLogs, paths.vlogDir + logFileName)
}
} catch (err) {
console.error(err)
@ -187,17 +183,19 @@ function parseNoLogFile(newData: string) {
}
function setNoLogReadInterval() {
utils.WatchFile(nologFile, (newData: string) => {
utils.WatchFile(paths.nologFile, (newData: string) => {
parseNoLogFile(newData)
Log('No Log user ID-s changed: ' + noLogIds.join(', '))
})
parseNoLogFile(utils.ReadFile(nologFile))
parseNoLogFile(utils.ReadFile(paths.nologFile))
}
function Load(): void {
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
if (utils.FileExists(paths.uStatsFile)) {
uvData = JSON.parse(utils.ReadFile(paths.uStatsFile))
}
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
@ -207,7 +205,9 @@ function Load(): void {
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
if (utils.FileExists(paths.uvStatsFile)) {
udvData = JSON.parse(utils.ReadFile(paths.uvStatsFile))
}
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
@ -217,7 +217,9 @@ function Load(): void {
}
try {
vData = utils.ReadJSON(statFile)
if (utils.FileExists(paths.statFile)) {
vData = utils.ReadJSON(paths.statFile)
}
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
@ -227,7 +229,9 @@ function Load(): void {
}
try {
dvData = utils.ReadJSON(vStatFile)
if (utils.FileExists(paths.vStatFile)) {
dvData = utils.ReadJSON(paths.vStatFile)
}
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
@ -329,26 +333,26 @@ function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
utils.WriteFile(JSON.stringify(uvData), paths.uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
utils.WriteFile(JSON.stringify(udvData), paths.uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
utils.WriteFile(JSON.stringify(vData), paths.statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
utils.WriteFile(JSON.stringify(dvData), paths.vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log(
@ -463,8 +467,6 @@ export default {
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
setLoggingDisabled: setLoggingDisabled,
logTable: logTable,
}

View file

@ -46,7 +46,7 @@ import fs from 'fs'
import { v4 as uuidv4 } from 'uuid'
import logger from '../utils/logger'
import constants from '../constants.json'
import { paths } from './files'
import { Request } from '../types/basicTypes'
interface URLFormatOptions {
@ -115,7 +115,7 @@ function ReadDir(path: string, listHidden?: boolean): Array<string> {
}
}
function ReadJSON(name: string): any {
function ReadJSON<T = any>(name: string): T {
try {
return JSON.parse(ReadFile(name))
} catch (err) {
@ -329,7 +329,7 @@ function getGitRevision(dir: string): string {
}
function getScriptVersion(): string {
const scriptContent = ReadFile(constants.moodleTestUserscriptPath)
const scriptContent = ReadFile(paths.moodleTestUserscriptPath)
let temp: string | string[] = scriptContent.split('\n').find((x) => {
return x.includes('@version')

View file

@ -186,7 +186,7 @@ export function initWorkerPool(
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
'yellowbg'
)
}