diff --git a/src/modules/api/api.js b/src/modules/api/api.js index 517ffcf..ef15be1 100644 --- a/src/modules/api/api.js +++ b/src/modules/api/api.js @@ -29,7 +29,7 @@ const app = express() // other requires const logger = require('../../utils/logger.js') const utils = require('../../utils/utils.js') -const actions = require('../../utils/actions.js') +const actions = require('../../utils/actions') const dbtools = require('../../utils/dbtools.js') const auth = require('../../middlewares/auth.middleware.js') const { dataToString, searchData } = require('../../utils/classes.js') diff --git a/src/server.js b/src/server.js deleted file mode 100755 index e8598bf..0000000 --- a/src/server.js +++ /dev/null @@ -1,272 +0,0 @@ -/* ---------------------------------------------------------------------------- - - Question Server - GitLab: - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - ------------------------------------------------------------------------- */ - -console.log('Node version: ' + process.version) -console.log('Current working directory: ' + process.cwd()) - -const startHTTPS = true -const isRoot = process.getuid && process.getuid() === 0 - -const port = isRoot ? 80 : 8080 -const httpsport = isRoot ? 443 : 5001 - -const express = require('express') -const vhost = require('vhost') -const logger = require('./utils/logger.js') -const utils = require('./utils/utils.js') -const http = require('http') -const https = require('https') -const cors = require('cors') -const cookieParser = require('cookie-parser') -const { v4: uuidv4 } = require('uuid') - -const dbtools = require('./utils/dbtools.js') -const reqlogger = require('./middlewares/reqlogger.middleware.js') -const extraModulesFile = './src/extraModules.json' -const modulesFile = './src/modules.json' -const usersDBPath = 'data/dbs/users.db' -const idStats = require('./utils/ids.js') -idStats.Load() - -if (!utils.FileExists(usersDBPath)) { - throw new Error('No user DB exists yet! please run utils/dbSetup.js first!') -} -const userDB = dbtools.GetDB(usersDBPath) - -let modules = JSON.parse(utils.ReadFile(modulesFile)) - -logger.Load() -const debugLevel = parseInt(process.env.NS_LOGLEVEL) || 0 -logger.Log('Loglevel is: ' + debugLevel) - -try { - if (utils.FileExists(extraModulesFile)) { - const extraModules = JSON.parse(utils.ReadFile(extraModulesFile)) - modules = { - ...extraModules, - ...modules, - } - } -} catch (err) { - logger.Log('Failed to read extra modules file') - console.error(err) -} - -// Setting up exits -// process.on('exit', () => exit('exit')) -process.on('SIGINT', () => exit('SIGINT')) -process.on('SIGTERM', () => exit('SIGTERM')) - -function exit(reason) { - console.log() - logger.Log(`Exiting, reason: ${reason}`) - Object.keys(modules).forEach((key) => { - const module = modules[key] - if (module.cleanup) { - try { - module.cleanup() - } catch (err) { - logger.Log( - `Error in ${key} cleanup! Details in STDERR`, - logger.GetColor('redbg') - ) - console.error(err) - } - } - }) - - logger.Log('Closing Auth DB') - userDB.close() - - process.exit() -} - -const app = express() - -if (!process.env.NS_DEVEL) { - app.use(function(req, res, next) { - if (req.secure) { - next() - } else { - logger.DebugLog( - `HTTPS ${req.method} redirect to: ${'https://' + - req.headers.host + - req.url}`, - 'https', - 1 - ) - if (req.method === 'POST') { - res.redirect(307, 'https://' + req.headers.host + req.url) - } else { - res.redirect('https://' + req.headers.host + req.url) - } - } - }) -} -// https://github.com/expressjs/cors#configuration-options -app.use( - cors({ - credentials: true, - origin: true, - // origin: [ /\.frylabs\.net$/ ] - }) -) -const cookieSecret = uuidv4() -app.use(cookieParser(cookieSecret)) -app.use( - reqlogger({ - loggableKeywords: ['stable.user.js'], - loggableModules: ['dataeditor'], - }) -) - -Object.keys(modules).forEach(function(key) { - let module = modules[key] - try { - let mod = require(module.path) - logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow')) - - module.publicdirs.forEach((pdir) => { - utils.CreatePath(pdir) - }) - - if (mod.setup) { - mod.setup({ - url: 'https://' + module.urls[0], - userDB: userDB, - publicdirs: module.publicdirs, - nextdir: module.nextdir, - }) - } - - const modApp = mod.getApp() - module.app = modApp.app - module.dailyAction = modApp.dailyAction - module.cleanup = modApp.cleanup - - module.urls.forEach((url) => { - app.use(vhost(url, module.app)) - }) - } catch (err) { - console.error(err) - } -}) - -// https://certbot.eff.org/ -const privkeyFile = '/etc/letsencrypt/live/frylabs.net/privkey.pem' -const fullchainFile = '/etc/letsencrypt/live/frylabs.net/fullchain.pem' -const chainFile = '/etc/letsencrypt/live/frylabs.net/chain.pem' - -var certsLoaded = false -if ( - startHTTPS && - utils.FileExists(privkeyFile) && - utils.FileExists(fullchainFile) && - utils.FileExists(chainFile) -) { - try { - const key = utils.ReadFile(privkeyFile) - const cert = utils.ReadFile(fullchainFile) - const ca = utils.ReadFile(chainFile) - var certs = { - key: key, - cert: cert, - ca: ca, - } - certsLoaded = true - } catch (err) { - logger.Log('Error loading cert files!', logger.GetColor('redbg')) - console.error(err) - } -} - -setLogTimer() -function setLogTimer() { - const now = new Date() - const night = new Date( - now.getFullYear(), - now.getMonth(), - now.getDate() + 1, - 0, - 0, - 0 - ) - logger.DebugLog(`Next daily action: ${night}`, 'daily', 1) - const msToMidnight = night.getTime() - now.getTime() - logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1) - logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1) - - if (msToMidnight < 0) { - logger.Log( - `Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`, - logger.GetColor('redbg') - ) - return - } - - setTimeout(function() { - LogTimerAction() - setLogTimer() - }, msToMidnight) -} - -function LogTimerAction() { - logger.DebugLog(`Running Log Timer Action`, 'daily', 1) - Object.keys(modules).forEach((key) => { - const module = modules[key] - logger.DebugLog(`Ckecking ${key}`, 'daily', 1) - if (module.dailyAction) { - try { - logger.Log(`Running daily action of ${key}`) - module.dailyAction() - } catch (err) { - logger.Log( - `Error in ${key} daily action! Details in STDERR`, - logger.GetColor('redbg') - ) - console.err(err) - } - } - }) - - const line = - '===================================================================================================================================================' - logger.Log(line) - logger.setNewLogfileName() -} - -logger.Log('Node version: ' + process.version) -logger.Log('Current working directory: ' + process.cwd()) -logger.Log('Listening on port: ' + port) -if (isRoot) { - logger.Log('Running as root', logger.GetColor('red')) -} - -const httpServer = http.createServer(app) -httpServer.listen(port) -if (certsLoaded) { - const httpsServer = https.createServer(certs, app) - httpsServer.listen(httpsport) - logger.Log('Listening on port: ' + httpsport + ' (https)') -} else { - logger.Log('Https not avaible') -} - -// app.listen(port) diff --git a/src/utils/actions.js b/src/utils/actions.js deleted file mode 100755 index 713552a..0000000 --- a/src/utils/actions.js +++ /dev/null @@ -1,252 +0,0 @@ -/* ---------------------------------------------------------------------------- -Question Server - GitLab: - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - ------------------------------------------------------------------------- */ - -module.exports = { - ProcessIncomingRequest: ProcessIncomingRequest, - LoadJSON: LoadJSON, - backupData: backupData, -} - -const recDataFile = './stats/recdata' -const dataLockFile = './data/lockData' - -const logger = require('../utils/logger.js') -const { searchData, createQuestion } = require('../utils/classes.js') -const idStats = require('../utils/ids.js') -const utils = require('../utils/utils.js') -const { addQuestion, getSubjNameWithoutYear } = require('./classes.js') -// if a recievend question doesnt match at least this % to any other question in the db it gets -// added to db -const minMatchToAmmountToAdd = 90 - -const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate -var currWrites = 0 - -// TODO: function that runs ProcessINcReq for every questiondb - -function ProcessIncomingRequest(recievedData, questionDbs, dryRun, user) { - return Promise.all( - questionDbs.map((qdb) => { - return ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) - }) - ) -} - -function ProcessIncomingRequestUsingDb(recievedData, qdb, dryRun, user) { - return new Promise((resolve, reject) => { - logger.DebugLog('Processing incoming request', 'actions', 1) - - if (recievedData === undefined) { - logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) - reject(new Error('Recieved data is undefined!')) - } - - try { - let towrite = logger.GetDateString() + '\n' - towrite += - '------------------------------------------------------------------------------\n' - if (typeof recievedData === 'object') { - towrite += JSON.stringify(recievedData) - } else { - towrite += recievedData - } - towrite += - '\n------------------------------------------------------------------------------\n' - utils.AppendToFile(towrite, recDataFile) - logger.DebugLog('recDataFile written', 'actions', 1) - } catch (err) { - logger.log('Error writing recieved data.') - } - - if (utils.FileExists(dataLockFile)) { - logger.Log( - 'Data lock file exists, skipping recieved data processing', - logger.GetColor('red') - ) - resolve(-1) - return - } - - try { - // recievedData: { version: "", id: "", subj: "" quiz: {} } - let data = recievedData - const recievedQuestions = [] - // FIXME: if is for backwards compatibility, remove this sometime in the future - if (typeof data !== 'object') { - data = JSON.parse(recievedData) - } - - logger.DebugLog('recievedData JSON parsed', 'actions', 1) - logger.DebugLog(data, 'actions', 3) - let allQLength = data.quiz.length - const questionSearchPromises = [] - data.quiz.forEach((question) => { - logger.DebugLog('Question:', 'actions', 2) - logger.DebugLog(question, 'actions', 2) - let currentQuestion = createQuestion( - question.Q, - question.A, - question.data - ) - logger.DebugLog( - 'Searching for question in subj ' + data.subj, - 'actions', - 3 - ) - logger.DebugLog(currentQuestion, 'actions', 3) - recievedQuestions.push(currentQuestion) - questionSearchPromises.push( - searchData(qdb.data, currentQuestion, data.subj) - ) - }) - - Promise.all(questionSearchPromises) - .then((results) => { - const allQuestions = [] // all new questions here that do not have result - results.forEach((result, i) => { - let add = result.every((res) => { - return res.match < minMatchToAmmountToAdd - }) - if (add) { - allQuestions.push(recievedQuestions[i]) - } - }) - - try { - let color = logger.GetColor('green') - let msg = '' - if (allQuestions.length > 0) { - color = logger.GetColor('blue') - msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` - allQuestions.forEach((currentQuestion) => { - const sName = getSubjNameWithoutYear(data.subj) - logger.DebugLog( - 'Adding question with subjName: ' + sName + ' :', - 'actions', - 3 - ) - logger.DebugLog(currentQuestion, 'actions', 3) - addQuestion(qdb.data, sName, currentQuestion) - }) - - currWrites++ - logger.DebugLog( - 'currWrites for data.json: ' + currWrites, - 'actions', - 1 - ) - if (currWrites >= writeAfter && !dryRun) { - currWrites = 0 - logger.DebugLog('Writing data.json', 'actions', 1) - utils.WriteFile(JSON.stringify(qdb.data), qdb.path) - logger.Log('\tData file written', color) - } else if (dryRun) { - logger.Log('\tDry run') - } - } else { - msg += `No new data ( ${allQLength} )` - } - - let subjRow = '\t' + data.subj - if (data.id) { - subjRow += ' ( CID: ' + logger.logHashed(data.id) + ')' - } - idStats.LogId(user.id, data.subj, allQuestions.length, allQLength) - logger.Log(subjRow) - if (data.version !== undefined) { - msg += '. Version: ' + data.version - } - - logger.Log('\t' + msg, color) - logger.DebugLog('New Questions:', 'actions', 2) - logger.DebugLog(allQuestions, 'actions', 2) - - logger.DebugLog('ProcessIncomingRequest done', 'actions', 1) - resolve(allQuestions.length) - } catch (error) { - console.error(error) - logger.Log( - 'Error while processing processData worker result!', - logger.GetColor('redbg') - ) - reject( - new Error('Error while processing processData worker result!') - ) - } - }) - .catch((err) => { - logger.Log( - 'Error while searching for questions in ProcessIncomingRequest!', - logger.GetColor('redbg') - ) - console.error(err) - }) - } catch (err) { - console.error(err) - logger.Log( - 'There was en error handling incoming quiz data, see stderr', - logger.GetColor('redbg') - ) - reject(new Error('Couldnt parse JSON data')) - } - }) -} - -function LoadJSON(dataFiles) { - return dataFiles.reduce((acc, dataFile) => { - if (!utils.FileExists(dataFile.path)) { - utils.WriteFile(JSON.stringify([]), dataFile.path) - } - - try { - acc.push({ - ...dataFile, - data: JSON.parse(utils.ReadFile(dataFile.path)), - }) - } catch (err) { - console.error(err) - logger.Log( - "data is undefined! Couldn't load data!", - logger.GetColor('redbg') - ) - } - return acc - }, []) -} - -function backupData(questionDbs) { - questionDbs.forEach((data) => { - const path = './publicDirs/qminingPublic/backs/' - utils.CreatePath(path) - try { - logger.Log(`Backing up ${data.name}...`) - utils.WriteFile( - JSON.stringify(data.data), - `${path}${data.name}_${utils.GetDateString(true)}.json` - ) - logger.Log('Done') - } catch (err) { - logger.Log( - `Error backing up data file ${data.name}!`, - logger.GetColor('redbg') - ) - console.error(err) - } - }) -} diff --git a/src/utils/actions.ts b/src/utils/actions.ts index c3d5c80..1a3535f 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -26,7 +26,11 @@ module.exports = { const recDataFile = './stats/recdata' const dataLockFile = './data/lockData' -const logger = require('../utils/logger.js') +import { Db } from '../types/basicTypes' + +import logger from '../utils/logger.js' + +// const logger = require('../utils/logger.js') const { searchData, createQuestion } = require('../utils/classes.js') const idStats = require('../utils/ids.js') const utils = require('../utils/utils.js') @@ -38,33 +42,10 @@ const minMatchToAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate var currWrites = 0 -interface QuestionData { - type: String - images?: Array - hashedImages?: Array -} - -interface Question { - Q: String - A: String - data: QuestionData -} - -interface Subject { - Name: String - Questions: Array -} - -interface Db { - name: String - path: String - data: Array -} - function ProcessIncomingRequest( recievedData, questionDbs: Array, - dryRun, + dryRun: boolean, user ) { return Promise.all( diff --git a/tsconfig.json b/tsconfig.json index 80b665b..e3cb5cd 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,10 @@ { "compilerOptions": { + "moduleResolution": "node", + "esModuleInterop": true, + "target": "es6", "allowJs": true, - "module": "system", + "module": "commonjs", "removeComments": true, "preserveConstEnums": true, "sourceMap": true, @@ -12,7 +15,7 @@ ] }, "files": [ - "src/server.js" + "src/server.ts" ], "include": [ "src/**/*"