mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Typescript fine tuning
This commit is contained in:
mrfry
parent
c0deaf5e16
commit
2532e45228
5 changed files with 12 additions and 552 deletions
|
|
@ -29,7 +29,7 @@ const app = express()
|
|||
// other requires
|
||||
const logger = require('../../utils/logger.js')
|
||||
const utils = require('../../utils/utils.js')
|
||||
const actions = require('../../utils/actions.js')
|
||||
const actions = require('../../utils/actions')
|
||||
const dbtools = require('../../utils/dbtools.js')
|
||||
const auth = require('../../middlewares/auth.middleware.js')
|
||||
const { dataToString, searchData } = require('../../utils/classes.js')
|
||||
|
|
|
|||
272
src/server.js
272
src/server.js
|
|
@ -1,272 +0,0 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
console.log('Node version: ' + process.version)
|
||||
console.log('Current working directory: ' + process.cwd())
|
||||
|
||||
const startHTTPS = true
|
||||
const isRoot = process.getuid && process.getuid() === 0
|
||||
|
||||
const port = isRoot ? 80 : 8080
|
||||
const httpsport = isRoot ? 443 : 5001
|
||||
|
||||
const express = require('express')
|
||||
const vhost = require('vhost')
|
||||
const logger = require('./utils/logger.js')
|
||||
const utils = require('./utils/utils.js')
|
||||
const http = require('http')
|
||||
const https = require('https')
|
||||
const cors = require('cors')
|
||||
const cookieParser = require('cookie-parser')
|
||||
const { v4: uuidv4 } = require('uuid')
|
||||
|
||||
const dbtools = require('./utils/dbtools.js')
|
||||
const reqlogger = require('./middlewares/reqlogger.middleware.js')
|
||||
const extraModulesFile = './src/extraModules.json'
|
||||
const modulesFile = './src/modules.json'
|
||||
const usersDBPath = 'data/dbs/users.db'
|
||||
const idStats = require('./utils/ids.js')
|
||||
idStats.Load()
|
||||
|
||||
if (!utils.FileExists(usersDBPath)) {
|
||||
throw new Error('No user DB exists yet! please run utils/dbSetup.js first!')
|
||||
}
|
||||
const userDB = dbtools.GetDB(usersDBPath)
|
||||
|
||||
let modules = JSON.parse(utils.ReadFile(modulesFile))
|
||||
|
||||
logger.Load()
|
||||
const debugLevel = parseInt(process.env.NS_LOGLEVEL) || 0
|
||||
logger.Log('Loglevel is: ' + debugLevel)
|
||||
|
||||
try {
|
||||
if (utils.FileExists(extraModulesFile)) {
|
||||
const extraModules = JSON.parse(utils.ReadFile(extraModulesFile))
|
||||
modules = {
|
||||
...extraModules,
|
||||
...modules,
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.Log('Failed to read extra modules file')
|
||||
console.error(err)
|
||||
}
|
||||
|
||||
// Setting up exits
|
||||
// process.on('exit', () => exit('exit'))
|
||||
process.on('SIGINT', () => exit('SIGINT'))
|
||||
process.on('SIGTERM', () => exit('SIGTERM'))
|
||||
|
||||
function exit(reason) {
|
||||
console.log()
|
||||
logger.Log(`Exiting, reason: ${reason}`)
|
||||
Object.keys(modules).forEach((key) => {
|
||||
const module = modules[key]
|
||||
if (module.cleanup) {
|
||||
try {
|
||||
module.cleanup()
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
`Error in ${key} cleanup! Details in STDERR`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
logger.Log('Closing Auth DB')
|
||||
userDB.close()
|
||||
|
||||
process.exit()
|
||||
}
|
||||
|
||||
const app = express()
|
||||
|
||||
if (!process.env.NS_DEVEL) {
|
||||
app.use(function(req, res, next) {
|
||||
if (req.secure) {
|
||||
next()
|
||||
} else {
|
||||
logger.DebugLog(
|
||||
`HTTPS ${req.method} redirect to: ${'https://' +
|
||||
req.headers.host +
|
||||
req.url}`,
|
||||
'https',
|
||||
1
|
||||
)
|
||||
if (req.method === 'POST') {
|
||||
res.redirect(307, 'https://' + req.headers.host + req.url)
|
||||
} else {
|
||||
res.redirect('https://' + req.headers.host + req.url)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
// https://github.com/expressjs/cors#configuration-options
|
||||
app.use(
|
||||
cors({
|
||||
credentials: true,
|
||||
origin: true,
|
||||
// origin: [ /\.frylabs\.net$/ ]
|
||||
})
|
||||
)
|
||||
const cookieSecret = uuidv4()
|
||||
app.use(cookieParser(cookieSecret))
|
||||
app.use(
|
||||
reqlogger({
|
||||
loggableKeywords: ['stable.user.js'],
|
||||
loggableModules: ['dataeditor'],
|
||||
})
|
||||
)
|
||||
|
||||
Object.keys(modules).forEach(function(key) {
|
||||
let module = modules[key]
|
||||
try {
|
||||
let mod = require(module.path)
|
||||
logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow'))
|
||||
|
||||
module.publicdirs.forEach((pdir) => {
|
||||
utils.CreatePath(pdir)
|
||||
})
|
||||
|
||||
if (mod.setup) {
|
||||
mod.setup({
|
||||
url: 'https://' + module.urls[0],
|
||||
userDB: userDB,
|
||||
publicdirs: module.publicdirs,
|
||||
nextdir: module.nextdir,
|
||||
})
|
||||
}
|
||||
|
||||
const modApp = mod.getApp()
|
||||
module.app = modApp.app
|
||||
module.dailyAction = modApp.dailyAction
|
||||
module.cleanup = modApp.cleanup
|
||||
|
||||
module.urls.forEach((url) => {
|
||||
app.use(vhost(url, module.app))
|
||||
})
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
}
|
||||
})
|
||||
|
||||
// https://certbot.eff.org/
|
||||
const privkeyFile = '/etc/letsencrypt/live/frylabs.net/privkey.pem'
|
||||
const fullchainFile = '/etc/letsencrypt/live/frylabs.net/fullchain.pem'
|
||||
const chainFile = '/etc/letsencrypt/live/frylabs.net/chain.pem'
|
||||
|
||||
var certsLoaded = false
|
||||
if (
|
||||
startHTTPS &&
|
||||
utils.FileExists(privkeyFile) &&
|
||||
utils.FileExists(fullchainFile) &&
|
||||
utils.FileExists(chainFile)
|
||||
) {
|
||||
try {
|
||||
const key = utils.ReadFile(privkeyFile)
|
||||
const cert = utils.ReadFile(fullchainFile)
|
||||
const ca = utils.ReadFile(chainFile)
|
||||
var certs = {
|
||||
key: key,
|
||||
cert: cert,
|
||||
ca: ca,
|
||||
}
|
||||
certsLoaded = true
|
||||
} catch (err) {
|
||||
logger.Log('Error loading cert files!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
setLogTimer()
|
||||
function setLogTimer() {
|
||||
const now = new Date()
|
||||
const night = new Date(
|
||||
now.getFullYear(),
|
||||
now.getMonth(),
|
||||
now.getDate() + 1,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
)
|
||||
logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
|
||||
const msToMidnight = night.getTime() - now.getTime()
|
||||
logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
|
||||
logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)
|
||||
|
||||
if (msToMidnight < 0) {
|
||||
logger.Log(
|
||||
`Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
setTimeout(function() {
|
||||
LogTimerAction()
|
||||
setLogTimer()
|
||||
}, msToMidnight)
|
||||
}
|
||||
|
||||
function LogTimerAction() {
|
||||
logger.DebugLog(`Running Log Timer Action`, 'daily', 1)
|
||||
Object.keys(modules).forEach((key) => {
|
||||
const module = modules[key]
|
||||
logger.DebugLog(`Ckecking ${key}`, 'daily', 1)
|
||||
if (module.dailyAction) {
|
||||
try {
|
||||
logger.Log(`Running daily action of ${key}`)
|
||||
module.dailyAction()
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
`Error in ${key} daily action! Details in STDERR`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.err(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const line =
|
||||
'==================================================================================================================================================='
|
||||
logger.Log(line)
|
||||
logger.setNewLogfileName()
|
||||
}
|
||||
|
||||
logger.Log('Node version: ' + process.version)
|
||||
logger.Log('Current working directory: ' + process.cwd())
|
||||
logger.Log('Listening on port: ' + port)
|
||||
if (isRoot) {
|
||||
logger.Log('Running as root', logger.GetColor('red'))
|
||||
}
|
||||
|
||||
const httpServer = http.createServer(app)
|
||||
httpServer.listen(port)
|
||||
if (certsLoaded) {
|
||||
const httpsServer = https.createServer(certs, app)
|
||||
httpsServer.listen(httpsport)
|
||||
logger.Log('Listening on port: ' + httpsport + ' (https)')
|
||||
} else {
|
||||
logger.Log('Https not avaible')
|
||||
}
|
||||
|
||||
// app.listen(port)
|
||||
|
|
@ -1,252 +0,0 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
module.exports = {
|
||||
ProcessIncomingRequest: ProcessIncomingRequest,
|
||||
LoadJSON: LoadJSON,
|
||||
backupData: backupData,
|
||||
}
|
||||
|
||||
const recDataFile = './stats/recdata'
|
||||
const dataLockFile = './data/lockData'
|
||||
|
||||
const logger = require('../utils/logger.js')
|
||||
const { searchData, createQuestion } = require('../utils/classes.js')
|
||||
const idStats = require('../utils/ids.js')
|
||||
const utils = require('../utils/utils.js')
|
||||
const { addQuestion, getSubjNameWithoutYear } = require('./classes.js')
|
||||
// if a recievend question doesnt match at least this % to any other question in the db it gets
|
||||
// added to db
|
||||
const minMatchToAmmountToAdd = 90
|
||||
|
||||
const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
|
||||
var currWrites = 0
|
||||
|
||||
// TODO: function that runs ProcessINcReq for every questiondb
|
||||
|
||||
function ProcessIncomingRequest(recievedData, questionDbs, dryRun, user) {
|
||||
return Promise.all(
|
||||
questionDbs.map((qdb) => {
|
||||
return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user)
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
function ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) {
|
||||
return new Promise((resolve, reject) => {
|
||||
logger.DebugLog('Processing incoming request', 'actions', 1)
|
||||
|
||||
if (recievedData === undefined) {
|
||||
logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
|
||||
reject(new Error('Recieved data is undefined!'))
|
||||
}
|
||||
|
||||
try {
|
||||
let towrite = logger.GetDateString() + '\n'
|
||||
towrite +=
|
||||
'------------------------------------------------------------------------------\n'
|
||||
if (typeof recievedData === 'object') {
|
||||
towrite += JSON.stringify(recievedData)
|
||||
} else {
|
||||
towrite += recievedData
|
||||
}
|
||||
towrite +=
|
||||
'\n------------------------------------------------------------------------------\n'
|
||||
utils.AppendToFile(towrite, recDataFile)
|
||||
logger.DebugLog('recDataFile written', 'actions', 1)
|
||||
} catch (err) {
|
||||
logger.log('Error writing recieved data.')
|
||||
}
|
||||
|
||||
if (utils.FileExists(dataLockFile)) {
|
||||
logger.Log(
|
||||
'Data lock file exists, skipping recieved data processing',
|
||||
logger.GetColor('red')
|
||||
)
|
||||
resolve(-1)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
// recievedData: { version: "", id: "", subj: "" quiz: {} }
|
||||
let data = recievedData
|
||||
const recievedQuestions = []
|
||||
// FIXME: if is for backwards compatibility, remove this sometime in the future
|
||||
if (typeof data !== 'object') {
|
||||
data = JSON.parse(recievedData)
|
||||
}
|
||||
|
||||
logger.DebugLog('recievedData JSON parsed', 'actions', 1)
|
||||
logger.DebugLog(data, 'actions', 3)
|
||||
let allQLength = data.quiz.length
|
||||
const questionSearchPromises = []
|
||||
data.quiz.forEach((question) => {
|
||||
logger.DebugLog('Question:', 'actions', 2)
|
||||
logger.DebugLog(question, 'actions', 2)
|
||||
let currentQuestion = createQuestion(
|
||||
question.Q,
|
||||
question.A,
|
||||
question.data
|
||||
)
|
||||
logger.DebugLog(
|
||||
'Searching for question in subj ' + data.subj,
|
||||
'actions',
|
||||
3
|
||||
)
|
||||
logger.DebugLog(currentQuestion, 'actions', 3)
|
||||
recievedQuestions.push(currentQuestion)
|
||||
questionSearchPromises.push(
|
||||
searchData(qdb.data, currentQuestion, data.subj)
|
||||
)
|
||||
})
|
||||
|
||||
Promise.all(questionSearchPromises)
|
||||
.then((results) => {
|
||||
const allQuestions = [] // all new questions here that do not have result
|
||||
results.forEach((result, i) => {
|
||||
let add = result.every((res) => {
|
||||
return res.match < minMatchToAmmountToAdd
|
||||
})
|
||||
if (add) {
|
||||
allQuestions.push(recievedQuestions[i])
|
||||
}
|
||||
})
|
||||
|
||||
try {
|
||||
let color = logger.GetColor('green')
|
||||
let msg = ''
|
||||
if (allQuestions.length > 0) {
|
||||
color = logger.GetColor('blue')
|
||||
msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
|
||||
allQuestions.forEach((currentQuestion) => {
|
||||
const sName = getSubjNameWithoutYear(data.subj)
|
||||
logger.DebugLog(
|
||||
'Adding question with subjName: ' + sName + ' :',
|
||||
'actions',
|
||||
3
|
||||
)
|
||||
logger.DebugLog(currentQuestion, 'actions', 3)
|
||||
addQuestion(qdb.data, sName, currentQuestion)
|
||||
})
|
||||
|
||||
currWrites++
|
||||
logger.DebugLog(
|
||||
'currWrites for data.json: ' + currWrites,
|
||||
'actions',
|
||||
1
|
||||
)
|
||||
if (currWrites >= writeAfter && !dryRun) {
|
||||
currWrites = 0
|
||||
logger.DebugLog('Writing data.json', 'actions', 1)
|
||||
utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
|
||||
logger.Log('\tData file written', color)
|
||||
} else if (dryRun) {
|
||||
logger.Log('\tDry run')
|
||||
}
|
||||
} else {
|
||||
msg += `No new data ( ${allQLength} )`
|
||||
}
|
||||
|
||||
let subjRow = '\t' + data.subj
|
||||
if (data.id) {
|
||||
subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')'
|
||||
}
|
||||
idStats.LogId(user.id, data.subj, allQuestions.length, allQLength)
|
||||
logger.Log(subjRow)
|
||||
if (data.version !== undefined) {
|
||||
msg += '. Version: ' + data.version
|
||||
}
|
||||
|
||||
logger.Log('\t' + msg, color)
|
||||
logger.DebugLog('New Questions:', 'actions', 2)
|
||||
logger.DebugLog(allQuestions, 'actions', 2)
|
||||
|
||||
logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
|
||||
resolve(allQuestions.length)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
logger.Log(
|
||||
'Error while processing processData worker result!',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
reject(
|
||||
new Error('Error while processing processData worker result!')
|
||||
)
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.Log(
|
||||
'Error while searching for questions in ProcessIncomingRequest!',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
})
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
logger.Log(
|
||||
'There was en error handling incoming quiz data, see stderr',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
reject(new Error('Couldnt parse JSON data'))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function LoadJSON(dataFiles) {
|
||||
return dataFiles.reduce((acc, dataFile) => {
|
||||
if (!utils.FileExists(dataFile.path)) {
|
||||
utils.WriteFile(JSON.stringify([]), dataFile.path)
|
||||
}
|
||||
|
||||
try {
|
||||
acc.push({
|
||||
...dataFile,
|
||||
data: JSON.parse(utils.ReadFile(dataFile.path)),
|
||||
})
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
logger.Log(
|
||||
"data is undefined! Couldn't load data!",
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
}
|
||||
|
||||
function backupData(questionDbs) {
|
||||
questionDbs.forEach((data) => {
|
||||
const path = './publicDirs/qminingPublic/backs/'
|
||||
utils.CreatePath(path)
|
||||
try {
|
||||
logger.Log(`Backing up ${data.name}...`)
|
||||
utils.WriteFile(
|
||||
JSON.stringify(data.data),
|
||||
`${path}${data.name}_${utils.GetDateString(true)}.json`
|
||||
)
|
||||
logger.Log('Done')
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
`Error backing up data file ${data.name}!`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -26,7 +26,11 @@ module.exports = {
|
|||
const recDataFile = './stats/recdata'
|
||||
const dataLockFile = './data/lockData'
|
||||
|
||||
const logger = require('../utils/logger.js')
|
||||
import { Db } from '../types/basicTypes'
|
||||
|
||||
import logger from '../utils/logger.js'
|
||||
|
||||
// const logger = require('../utils/logger.js')
|
||||
const { searchData, createQuestion } = require('../utils/classes.js')
|
||||
const idStats = require('../utils/ids.js')
|
||||
const utils = require('../utils/utils.js')
|
||||
|
|
@ -38,33 +42,10 @@ const minMatchToAmmountToAdd = 90
|
|||
const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
|
||||
var currWrites = 0
|
||||
|
||||
interface QuestionData {
|
||||
type: String
|
||||
images?: Array<String>
|
||||
hashedImages?: Array<String>
|
||||
}
|
||||
|
||||
interface Question {
|
||||
Q: String
|
||||
A: String
|
||||
data: QuestionData
|
||||
}
|
||||
|
||||
interface Subject {
|
||||
Name: String
|
||||
Questions: Array<Question>
|
||||
}
|
||||
|
||||
interface Db {
|
||||
name: String
|
||||
path: String
|
||||
data: Array<Subject>
|
||||
}
|
||||
|
||||
function ProcessIncomingRequest(
|
||||
recievedData,
|
||||
questionDbs: Array<Db>,
|
||||
dryRun,
|
||||
dryRun: boolean,
|
||||
user
|
||||
) {
|
||||
return Promise.all(
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"target": "es6",
|
||||
"allowJs": true,
|
||||
"module": "system",
|
||||
"module": "commonjs",
|
||||
"removeComments": true,
|
||||
"preserveConstEnums": true,
|
||||
"sourceMap": true,
|
||||
|
|
@ -12,7 +15,7 @@
|
|||
]
|
||||
},
|
||||
"files": [
|
||||
"src/server.js"
|
||||
"src/server.ts"
|
||||
],
|
||||
"include": [
|
||||
"src/**/*"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue