From 96b413a3659f06eae09f18fd0973791b56cbed8a Mon Sep 17 00:00:00 2001 From: mrfry Date: Sat, 10 Dec 2022 15:34:54 +0100 Subject: [PATCH] prettier 4 tabwidth --- .prettierrc.js | 2 +- src/middlewares/auth.middleware.ts | 244 +-- src/middlewares/reqlogger.middleware.ts | 114 +- src/middlewares/socketAuth.middleware.ts | 80 +- src/modules/api/api.ts | 308 +-- src/modules/api/msgsDbStruct.ts | 58 +- src/modules/api/submodules/chat.ts | 423 ++-- src/modules/api/submodules/feedback.ts | 12 +- src/modules/api/submodules/forum.ts | 1030 ++++----- src/modules/api/submodules/qminingapi.ts | 1877 +++++++++-------- src/modules/api/submodules/quickvote.ts | 151 +- src/modules/api/submodules/ranklist.ts | 181 +- src/modules/api/submodules/todos.ts | 152 +- src/modules/api/submodules/userFiles.ts | 576 ++--- src/modules/api/submodules/userManagement.ts | 587 +++--- src/modules/api/usersDBStruct.ts | 142 +- src/modules/dataEditor/dataEditor.ts | 140 +- src/modules/main/main.ts | 72 +- src/modules/qmining/qmining.ts | 391 ++-- src/server.ts | 390 ++-- src/standaloneUtils/createJSON.js | 42 +- src/standaloneUtils/dataSimplifyer.js | 32 +- src/standaloneUtils/dbSetup.js | 46 +- src/standaloneUtils/invalidQuestionRemover.js | 54 +- src/standaloneUtils/manualSearch.js | 78 +- src/standaloneUtils/rmDuplicates.js | 684 +++--- src/standaloneUtils/sqliteTest.js | 130 +- src/tests/addQuestion.test.ts | 64 +- src/tests/base64ToText.test.ts | 36 +- src/tests/oldQuestionRemoving.test.ts | 282 +-- src/tests/possibleAnswerPenalty.test.ts | 466 ++-- src/tests/shouldLog.test.ts | 12 +- src/types/basicTypes.ts | 172 +- src/utils/actions.ts | 1404 ++++++------ src/utils/classes.ts | 1448 ++++++------- src/utils/dbtools.ts | 468 ++-- src/utils/ids.ts | 178 +- src/utils/logger.ts | 572 ++--- src/utils/tesseract.ts | 58 +- src/utils/utils.ts | 409 ++-- src/utils/workerPool.ts | 372 ++-- submodules/qmining-page | 2 +- 42 files changed, 7034 insertions(+), 6905 deletions(-) diff --git a/.prettierrc.js b/.prettierrc.js index d0c6251..8a9603c 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -1,6 +1,6 @@ module.exports = { trailingComma: 'es5', - tabWidth: 2, + tabWidth: 4, semi: false, singleQuote: true, arrowParens: 'always', diff --git a/src/middlewares/auth.middleware.ts b/src/middlewares/auth.middleware.ts index fa997dc..4d4f1c0 100644 --- a/src/middlewares/auth.middleware.ts +++ b/src/middlewares/auth.middleware.ts @@ -27,148 +27,148 @@ import utils from '../utils/utils' import dbtools from '../utils/dbtools' interface Options { - userDB: Database - jsonResponse: boolean - exceptions: Array + userDB: Database + jsonResponse: boolean + exceptions: Array } export const testUser = { - id: 19, - avaiblePWRequests: 645, - pwRequestCount: 19, - created: new Date(), - pw: 'secret', - loginCount: 3, - createdBy: 1, + id: 19, + avaiblePWRequests: 645, + pwRequestCount: 19, + created: new Date(), + pw: 'secret', + loginCount: 3, + createdBy: 1, } function renderLogin(_req: Request, res: Response, jsonResponse: boolean) { - res.status(401) // Unauthorized - if (jsonResponse) { - res.json({ - result: 'nouser', - msg: 'You are not logged in', - }) - } else { - res.render('login', { - devel: process.env.NS_DEVEL, - }) - } + res.status(401) // Unauthorized + if (jsonResponse) { + res.json({ + result: 'nouser', + msg: 'You are not logged in', + }) + } else { + res.render('login', { + devel: process.env.NS_DEVEL, + }) + } } export default function (options: Options): RequestHandler { - const { - userDB, - jsonResponse, - exceptions, - }: { - userDB: Database - jsonResponse: boolean - exceptions: string[] - } = options + const { + userDB, + jsonResponse, + exceptions, + }: { + userDB: Database + jsonResponse: boolean + exceptions: string[] + } = options - return function (req: Request, res: Response, next: NextFunction) { - const sessionID = req.cookies.sessionID - const isException = exceptions.some((exc) => { - return req.url.split('?')[0] === exc - }) + return function (req: Request, res: Response, next: NextFunction) { + const sessionID = req.cookies.sessionID + const isException = exceptions.some((exc) => { + return req.url.split('?')[0] === exc + }) - if (process.env.NS_NOUSER) { - req.session = { - user: testUser, - sessionID: sessionID || 11111111111, - isException: false, - } - next() - return - } + if (process.env.NS_NOUSER) { + req.session = { + user: testUser, + sessionID: sessionID || 11111111111, + isException: false, + } + next() + return + } - // FIXME Allowing all urls with _next in it, but not in params - if ( - req.url.split('?')[0].includes('_next') || - req.url.split('?')[0].includes('well-known/acme-challenge') - ) { - req.session = { isException: true } - next() - return - } + // FIXME Allowing all urls with _next in it, but not in params + if ( + req.url.split('?')[0].includes('_next') || + req.url.split('?')[0].includes('well-known/acme-challenge') + ) { + req.session = { isException: true } + next() + return + } + + if (!sessionID) { + if (isException) { + logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1) + req.session = { isException: true } + next() + return + } + logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1) + renderLogin(req, res, jsonResponse) + return + } + + const user = GetUserBySessionID(userDB, sessionID) + + if (!user) { + if (isException) { + logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1) + req.session = { isException: true } + next() + return + } + logger.DebugLog(`No user:${req.url}`, 'auth', 1) + renderLogin(req, res, jsonResponse) + return + } + + req.session = { + user: user, + sessionID: sessionID, + isException: isException, + } + + logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1) + + dbtools.Update( + userDB, + 'sessions', + { + lastAccess: utils.GetDateString(), + }, + { + id: sessionID, + } + ) + + dbtools.Update( + userDB, + 'users', + { + lastAccess: utils.GetDateString(), + }, + { + id: user.id, + } + ) - if (!sessionID) { - if (isException) { - logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1) - req.session = { isException: true } next() - return - } - logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1) - renderLogin(req, res, jsonResponse) - return } - - const user = GetUserBySessionID(userDB, sessionID) - - if (!user) { - if (isException) { - logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1) - req.session = { isException: true } - next() - return - } - logger.DebugLog(`No user:${req.url}`, 'auth', 1) - renderLogin(req, res, jsonResponse) - return - } - - req.session = { - user: user, - sessionID: sessionID, - isException: isException, - } - - logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1) - - dbtools.Update( - userDB, - 'sessions', - { - lastAccess: utils.GetDateString(), - }, - { - id: sessionID, - } - ) - - dbtools.Update( - userDB, - 'users', - { - lastAccess: utils.GetDateString(), - }, - { - id: user.id, - } - ) - - next() - } } function GetUserBySessionID(db: Database, sessionID: string) { - logger.DebugLog(`Getting user from db`, 'auth', 2) + logger.DebugLog(`Getting user from db`, 'auth', 2) - const session = dbtools.Select(db, 'sessions', { - id: sessionID, - })[0] + const session = dbtools.Select(db, 'sessions', { + id: sessionID, + })[0] - if (!session) { - return - } + if (!session) { + return + } - const user = dbtools.Select(db, 'users', { - id: session.userID, - })[0] + const user = dbtools.Select(db, 'users', { + id: session.userID, + })[0] - if (user) { - return user - } + if (user) { + return user + } } diff --git a/src/middlewares/reqlogger.middleware.ts b/src/middlewares/reqlogger.middleware.ts index 8c7fc82..6470635 100644 --- a/src/middlewares/reqlogger.middleware.ts +++ b/src/middlewares/reqlogger.middleware.ts @@ -23,68 +23,70 @@ import type { Response, NextFunction } from 'express' import type { Request } from '../types/basicTypes' interface Options { - loggableKeywords: Array - loggableModules: Array - exceptions: Array - excludeFromStats: Array + loggableKeywords: Array + loggableModules: Array + exceptions: Array + excludeFromStats: Array } export default function (options: Options): any { - const loggableKeywords = options ? options.loggableKeywords : undefined - const loggableModules = options ? options.loggableModules : undefined - const exceptions = options.exceptions || [] - const excludeFromStats = options.excludeFromStats || [] + const loggableKeywords = options ? options.loggableKeywords : undefined + const loggableModules = options ? options.loggableModules : undefined + const exceptions = options.exceptions || [] + const excludeFromStats = options.excludeFromStats || [] - return function (req: Request, res: Response, next: NextFunction) { - res.on('finish', function () { - // TODO: test this - const isException = exceptions.some((ex) => { - return req.url.includes(ex) - }) + return function (req: Request, res: Response, next: NextFunction) { + res.on('finish', function () { + // TODO: test this + const isException = exceptions.some((ex) => { + return req.url.includes(ex) + }) - if (isException) { - return - } + if (isException) { + return + } - let hostname = 'NOHOST' - if (req.hostname) { - hostname = req.hostname.replace('www.', '').split('.')[0] - } else { - logger.Log('Hostname is undefined!', logger.GetColor('redbg')) - console.log(req.body) - console.log(req.query) - console.log(req.headers) - } + let hostname = 'NOHOST' + if (req.hostname) { + hostname = req.hostname.replace('www.', '').split('.')[0] + } else { + logger.Log('Hostname is undefined!', logger.GetColor('redbg')) + console.log(req.body) + console.log(req.query) + console.log(req.headers) + } - const hasLoggableKeyword = - loggableKeywords && - loggableKeywords.some((keyword) => { - return req.url.includes(keyword) + const hasLoggableKeyword = + loggableKeywords && + loggableKeywords.some((keyword) => { + return req.url.includes(keyword) + }) + const hasLoggableModule = + loggableModules && + loggableModules.some((keyword) => { + return hostname.includes(keyword) + }) + const toLog = hasLoggableModule || hasLoggableKeyword + + logger.LogReq(req, true, res.statusCode) + if (toLog) { + logger.LogReq(req) + } + + const shouldLogStat = !excludeFromStats.some((ex) => { + return req.url.includes(ex) + }) + + if (res.statusCode !== 404 && shouldLogStat) { + logger.LogStat( + req.url, + hostname, + req.session && req.session.user + ? req.session.user.id + : 'NOUSER' + ) + } }) - const hasLoggableModule = - loggableModules && - loggableModules.some((keyword) => { - return hostname.includes(keyword) - }) - const toLog = hasLoggableModule || hasLoggableKeyword - - logger.LogReq(req, true, res.statusCode) - if (toLog) { - logger.LogReq(req) - } - - const shouldLogStat = !excludeFromStats.some((ex) => { - return req.url.includes(ex) - }) - - if (res.statusCode !== 404 && shouldLogStat) { - logger.LogStat( - req.url, - hostname, - req.session && req.session.user ? req.session.user.id : 'NOUSER' - ) - } - }) - next() - } + next() + } } diff --git a/src/middlewares/socketAuth.middleware.ts b/src/middlewares/socketAuth.middleware.ts index ca36bb1..cf02b99 100644 --- a/src/middlewares/socketAuth.middleware.ts +++ b/src/middlewares/socketAuth.middleware.ts @@ -27,60 +27,60 @@ import { Socket } from '../types/basicTypes' import { testUser } from './auth.middleware' interface Options { - userDB: any + userDB: any } export default function SocketAuth(options: Options): any { - const { userDB } = options + const { userDB } = options - return (socket: Socket, next: (arg0?: any) => void) => { - try { - const cookies = cookie.parse(socket.handshake.headers.cookie || '') - const sessionID = cookies.sessionID + return (socket: Socket, next: (arg0?: any) => void) => { + try { + const cookies = cookie.parse(socket.handshake.headers.cookie || '') + const sessionID = cookies.sessionID - if (process.env.NS_NOUSER) { - socket.user = testUser - next() - return - } + if (process.env.NS_NOUSER) { + socket.user = testUser + next() + return + } - if (!sessionID) { - next(new Error('Not authenticated, please log in')) - return - } + if (!sessionID) { + next(new Error('Not authenticated, please log in')) + return + } - const user = GetUserBySessionID(userDB, sessionID) + const user = GetUserBySessionID(userDB, sessionID) - if (!user) { - next(new Error('Not authenticated, please log in')) - return - } - socket.user = user - next() - } catch (e) { - next(new Error('Authentication server error')) - console.error('Authentication server error') - console.error(e) + if (!user) { + next(new Error('Not authenticated, please log in')) + return + } + socket.user = user + next() + } catch (e) { + next(new Error('Authentication server error')) + console.error('Authentication server error') + console.error(e) + } } - } } function GetUserBySessionID(db: any, sessionID: string) { - logger.DebugLog(`Getting user from db`, 'auth', 2) + logger.DebugLog(`Getting user from db`, 'auth', 2) - const session = dbtools.Select(db, 'sessions', { - id: sessionID, - })[0] + const session = dbtools.Select(db, 'sessions', { + id: sessionID, + })[0] - if (!session) { - return - } + if (!session) { + return + } - const user = dbtools.Select(db, 'users', { - id: session.userID, - })[0] + const user = dbtools.Select(db, 'users', { + id: session.userID, + })[0] - if (user) { - return user - } + if (user) { + return user + } } diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index 3143ee1..ebf05e9 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -46,188 +46,188 @@ let httpServer: http.Server let httpsServer: https.Server function GetApp(): ModuleType { - const app = express() + const app = express() - const publicDir = publicdirs[0] - if (!publicDir) { - throw new Error(`No public dir! ( API )`) - } - - let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] - domain.shift() // [ "frylabs", "net" ] - domain = domain.join('.') // "frylabs.net" - logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) - - // ------------------------------------------------------------------------------------------- - - app.use( - express.urlencoded({ - limit: '10mb', - extended: true, - }) as RequestHandler - ) - app.use( - express.json({ - limit: '10mb', - }) as RequestHandler - ) - app.set('view engine', 'ejs') - app.set('views', ['./src/modules/api/views', './src/sharedViews']) - app.use( - auth({ - userDB: userDB, - jsonResponse: true, - exceptions: [ - '/register', - '/favicon.ico', - '/login', - '/postfeedback', - '/fosuploader', - '/badtestsender', - ], - }) - ) - app.use( - fileUpload({ - limits: { fileSize: 50 * 1024 * 1024 }, - }) - ) - // ------------------------------------------------------------------------------------------- - - let rootRedirectURL = '' - - function reloadRootRedirectURL() { - if (utils.FileExists(rootRedirectToFile)) { - rootRedirectURL = utils.ReadFile(rootRedirectToFile) + const publicDir = publicdirs[0] + if (!publicDir) { + throw new Error(`No public dir! ( API )`) } - } - const filesToWatch = [ - { - fname: rootRedirectToFile, - logMsg: 'Root redirect URL changed', - action: reloadRootRedirectURL, - }, - ] + let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] + domain.shift() // [ "frylabs", "net" ] + domain = domain.join('.') // "frylabs.net" + logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) - function Load() { - filesToWatch.forEach((ftw) => { - if (utils.FileExists(ftw.fname)) { - utils.WatchFile(ftw.fname, () => { - logger.Log(ftw.logMsg) - ftw.action() + // ------------------------------------------------------------------------------------------- + + app.use( + express.urlencoded({ + limit: '10mb', + extended: true, + }) as RequestHandler + ) + app.use( + express.json({ + limit: '10mb', + }) as RequestHandler + ) + app.set('view engine', 'ejs') + app.set('views', ['./src/modules/api/views', './src/sharedViews']) + app.use( + auth({ + userDB: userDB, + jsonResponse: true, + exceptions: [ + '/register', + '/favicon.ico', + '/login', + '/postfeedback', + '/fosuploader', + '/badtestsender', + ], }) - ftw.action() - } else { - logger.Log( - `File ${ftw.fname} does not exists to watch!`, - logger.GetColor('redbg') - ) - } - }) - } + ) + app.use( + fileUpload({ + limits: { fileSize: 50 * 1024 * 1024 }, + }) + ) + // ------------------------------------------------------------------------------------------- - Load() + let rootRedirectURL = '' - // -------------------------------------------------------------- - - app.get('/', function (req: Request, res: any) { - logger.LogReq(req) - if (reloadRootRedirectURL) { - res.redirect(rootRedirectURL) - } else { - res.json({ msg: 'hi c:' }) + function reloadRootRedirectURL() { + if (utils.FileExists(rootRedirectToFile)) { + rootRedirectURL = utils.ReadFile(rootRedirectToFile) + } } - }) - // ------------------------------------------------------------------------------------------- + const filesToWatch = [ + { + fname: rootRedirectToFile, + logMsg: 'Root redirect URL changed', + action: reloadRootRedirectURL, + }, + ] - const submoduleDatas = setupSubModules(app) + function Load() { + filesToWatch.forEach((ftw) => { + if (utils.FileExists(ftw.fname)) { + utils.WatchFile(ftw.fname, () => { + logger.Log(ftw.logMsg) + ftw.action() + }) + ftw.action() + } else { + logger.Log( + `File ${ftw.fname} does not exists to watch!`, + logger.GetColor('redbg') + ) + } + }) + } - // ------------------------------------------------------------------------------------------- + Load() - publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) - app.use(express.static(pdir)) - }) + // -------------------------------------------------------------- - // ------------------------------------------------------------------------------------------- + app.get('/', function (req: Request, res: any) { + logger.LogReq(req) + if (reloadRootRedirectURL) { + res.redirect(rootRedirectURL) + } else { + res.json({ msg: 'hi c:' }) + } + }) - app.get('*', function (_req: Request, res: any) { - res.status(404).render('404') - }) + // ------------------------------------------------------------------------------------------- - app.post('*', function (_req: Request, res: any) { - res.status(404).render('404') - }) + const submoduleDatas = setupSubModules(app) + + // ------------------------------------------------------------------------------------------- + + publicdirs.forEach((pdir) => { + logger.Log(`Using public dir: ${pdir}`) + app.use(express.static(pdir)) + }) + + // ------------------------------------------------------------------------------------------- + + app.get('*', function (_req: Request, res: any) { + res.status(404).render('404') + }) + + app.post('*', function (_req: Request, res: any) { + res.status(404).render('404') + }) + + function DailyAction() { + submoduleDatas.forEach((data) => { + if (data.dailyAction) { + data.dailyAction() + } + }) + } - function DailyAction() { submoduleDatas.forEach((data) => { - if (data.dailyAction) { - data.dailyAction() - } + if (data.load) { + data.load() + } }) - } - submoduleDatas.forEach((data) => { - if (data.load) { - data.load() + return { + dailyAction: DailyAction, + app: app, } - }) - - return { - dailyAction: DailyAction, - app: app, - } } function setupSubModules( - parentApp: express.Application, - moduleSpecificData?: any + parentApp: express.Application, + moduleSpecificData?: any ): Submodule[] { - const submoduleDir = './submodules/' - const absolutePath = __dirname + '/' + submoduleDir - if (!utils.FileExists(absolutePath)) { - return null - } - const files = utils.ReadDir(absolutePath) - const moduleDatas: Submodule[] = [] - files.forEach((file) => { - if (!file.endsWith('.js')) { - return + const submoduleDir = './submodules/' + const absolutePath = __dirname + '/' + submoduleDir + if (!utils.FileExists(absolutePath)) { + return null } - const submodulePath = submoduleDir + file + const files = utils.ReadDir(absolutePath) + const moduleDatas: Submodule[] = [] + files.forEach((file) => { + if (!file.endsWith('.js')) { + return + } + const submodulePath = submoduleDir + file - try { - logger.Log(`Loading submodule '${file}' for '${moduleName}'...`) - const mod = require(submodulePath).default // eslint-disable-line - const loadedModData = mod.setup({ - app: parentApp, - userDB: userDB, - url: url, - publicdirs: publicdirs, - moduleSpecificData: moduleSpecificData, - httpServer: httpServer, - httpsServer: httpsServer, - }) - moduleDatas.push(loadedModData || {}) - } catch (e) { - logger.Log(`Error loading submodule from ${submodulePath}`) - console.error(e) - } - }) + try { + logger.Log(`Loading submodule '${file}' for '${moduleName}'...`) + const mod = require(submodulePath).default // eslint-disable-line + const loadedModData = mod.setup({ + app: parentApp, + userDB: userDB, + url: url, + publicdirs: publicdirs, + moduleSpecificData: moduleSpecificData, + httpServer: httpServer, + httpsServer: httpsServer, + }) + moduleDatas.push(loadedModData || {}) + } catch (e) { + logger.Log(`Error loading submodule from ${submodulePath}`) + console.error(e) + } + }) - return moduleDatas + return moduleDatas } export default { - name: moduleName, - getApp: GetApp, - setup: (data: SetupData): void => { - userDB = data.userDB - url = data.url - publicdirs = data.publicdirs - httpServer = data.httpServer - httpsServer = data.httpsServer - }, + name: moduleName, + getApp: GetApp, + setup: (data: SetupData): void => { + userDB = data.userDB + url = data.url + publicdirs = data.publicdirs + httpServer = data.httpServer + httpsServer = data.httpsServer + }, } diff --git a/src/modules/api/msgsDbStruct.ts b/src/modules/api/msgsDbStruct.ts index bb8f208..b6e4138 100644 --- a/src/modules/api/msgsDbStruct.ts +++ b/src/modules/api/msgsDbStruct.ts @@ -19,36 +19,36 @@ ------------------------------------------------------------------------- */ const DbStruct = { - msgs: { - tableStruct: { - id: { - type: 'integer', - primary: true, - autoIncrement: true, - }, - sender: { - type: 'integer', - notNull: true, - }, - reciever: { - type: 'integer', - notNull: true, - }, - msg: { - type: 'text', - }, - type: { - type: 'text', - }, - date: { - type: 'integer', - }, - unread: { - type: 'integer', - defaultZero: true, - }, + msgs: { + tableStruct: { + id: { + type: 'integer', + primary: true, + autoIncrement: true, + }, + sender: { + type: 'integer', + notNull: true, + }, + reciever: { + type: 'integer', + notNull: true, + }, + msg: { + type: 'text', + }, + type: { + type: 'text', + }, + date: { + type: 'integer', + }, + unread: { + type: 'integer', + defaultZero: true, + }, + }, }, - }, } export default DbStruct diff --git a/src/modules/api/submodules/chat.ts b/src/modules/api/submodules/chat.ts index 67cb1a7..1fec8a3 100644 --- a/src/modules/api/submodules/chat.ts +++ b/src/modules/api/submodules/chat.ts @@ -30,65 +30,65 @@ const msgDbPath = './data/dbs/msgs.db' const msgPaginationLimit = 15 interface ExtendedSocket extends Socket { - user: User + user: User } interface Message { - id: number - sender: number - reciever: number - msg: string - type: string - date: number - unread: number + id: number + sender: number + reciever: number + msg: string + type: string + date: number + unread: number } function setup(data: SubmoduleData): void { - const { app, httpServer, httpsServer, userDB, publicdirs } = data - const msgDB = dbtools.GetDB(msgDbPath) + const { app, httpServer, httpsServer, userDB, publicdirs } = data + const msgDB = dbtools.GetDB(msgDbPath) - const publicDir = publicdirs[0] - const uloadFiles = publicDir + 'chatFiles' - logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`) - // https://socket.io/docs/v4/handling-cors/#Configuration - const io = new socket(httpsServer || httpServer, { - cors: { - credentials: true, - origin: true, - }, - }) + const publicDir = publicdirs[0] + const uloadFiles = publicDir + 'chatFiles' + logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`) + // https://socket.io/docs/v4/handling-cors/#Configuration + const io = new socket(httpsServer || httpServer, { + cors: { + credentials: true, + origin: true, + }, + }) - function chatMessageRead({ - sender, - reciever, - }: { - sender: number - reciever: number - }) { - dbtools.runStatement( - msgDB, - `update msgs + function chatMessageRead({ + sender, + reciever, + }: { + sender: number + reciever: number + }) { + dbtools.runStatement( + msgDB, + `update msgs set unread = 0 where sender = ${sender} and reciever = ${reciever}`, - 'run' - ) - io.sockets.in(sender.toString()).emit('chat message read', { - userReadMsg: reciever, - }) - } + 'run' + ) + io.sockets.in(sender.toString()).emit('chat message read', { + userReadMsg: reciever, + }) + } - io.use(socketAuth({ userDB: userDB })) + io.use(socketAuth({ userDB: userDB })) - io.on('connection', (socket: ExtendedSocket) => { - const userid = socket.user.id + io.on('connection', (socket: ExtendedSocket) => { + const userid = socket.user.id - socket.on('join', function (/*data*/) { - socket.join(userid.toString()) + socket.on('join', function (/*data*/) { + socket.join(userid.toString()) - const groups: number[] = dbtools - .runStatement( - msgDB, - `select * from + const groups: number[] = dbtools + .runStatement( + msgDB, + `select * from ( select sender as a from msgs @@ -99,172 +99,177 @@ function setup(data: SubmoduleData): void { where sender = ${userid} or reciever = ${userid} )t order by t.a asc` - ) - .reduce((acc: number[], x: { a: number }) => { - if (x.a !== userid) acc.push(x.a) - return acc - }, []) + ) + .reduce((acc: number[], x: { a: number }) => { + if (x.a !== userid) acc.push(x.a) + return acc + }, []) - socket.emit('prev messages', { - prevMsgs: groups.map((to) => { - const first: Message = dbtools.runStatement( - msgDB, - `select * from msgs + socket.emit('prev messages', { + prevMsgs: groups.map((to) => { + const first: Message = dbtools.runStatement( + msgDB, + `select * from msgs where sender = ${userid} and reciever = ${to} or sender = ${to} and reciever = ${userid} order by date desc limit 1` - )[0] - return first - }), - }) + )[0] + return first + }), + }) - socket.on('get chat messages', (data) => { - const { chatPartner, from } = data + socket.on('get chat messages', (data) => { + const { chatPartner, from } = data - const msgs = dbtools.runStatement( - msgDB, - `select * from msgs + const msgs = dbtools.runStatement( + msgDB, + `select * from msgs where (sender = ${userid} and reciever = ${chatPartner} or sender = ${chatPartner} and reciever = ${userid}) ${from ? `and date < ${from}` : ''} order by date desc limit ${msgPaginationLimit}` - ) + ) - socket.emit('get chat messages', { - requestsdMsgs: msgs, - hasMore: msgs.length === msgPaginationLimit, + socket.emit('get chat messages', { + requestsdMsgs: msgs, + hasMore: msgs.length === msgPaginationLimit, + }) + + // Read update + chatMessageRead({ sender: chatPartner, reciever: userid }) + }) + + socket.on('chat message read', (data) => { + const { chatPartner } = data + chatMessageRead({ sender: chatPartner, reciever: userid }) + }) + + socket.on( + 'chat message', + (message: { reciever: string; msg: string; type: string }) => { + const { reciever, msg, type } = message + if (!reciever || !msg || !msg.trim() || !type) { + return + } + const recieverUser = dbtools.Select(userDB, 'users', { + id: reciever, + })[0] + if (!recieverUser) { + socket.emit('chat message', { + success: false, + date: new Date().getTime(), + sender: reciever, + reciever: userid, + type: 'text', + msg: `A #${reciever} számú felhasználó nem létezik`, + }) + return + } + + const msgObj = { + sender: userid, + reciever: parseInt(reciever), + msg: dbtools.sanitizeQuery(msg), + type: type || 'text', + date: new Date().getTime(), + unread: 1, + } + dbtools.Insert(msgDB, 'msgs', msgObj) + if (userid !== parseInt(reciever)) { + io.sockets + .in(reciever.toString()) + .emit('chat message', msgObj) + } + } + ) }) - // Read update - chatMessageRead({ sender: chatPartner, reciever: userid }) - }) - - socket.on('chat message read', (data) => { - const { chatPartner } = data - chatMessageRead({ sender: chatPartner, reciever: userid }) - }) - - socket.on( - 'chat message', - (message: { reciever: string; msg: string; type: string }) => { - const { reciever, msg, type } = message - if (!reciever || !msg || !msg.trim() || !type) { - return - } - const recieverUser = dbtools.Select(userDB, 'users', { - id: reciever, - })[0] - if (!recieverUser) { - socket.emit('chat message', { - success: false, - date: new Date().getTime(), - sender: reciever, - reciever: userid, - type: 'text', - msg: `A #${reciever} számú felhasználó nem létezik`, - }) - return - } - - const msgObj = { - sender: userid, - reciever: parseInt(reciever), - msg: dbtools.sanitizeQuery(msg), - type: type || 'text', - date: new Date().getTime(), - unread: 1, - } - dbtools.Insert(msgDB, 'msgs', msgObj) - if (userid !== parseInt(reciever)) { - io.sockets.in(reciever.toString()).emit('chat message', msgObj) - } - } - ) + // socket.on('disconnect', () => {}) + // socket.on('close', () => {}) }) - // socket.on('disconnect', () => {}) - // socket.on('close', () => {}) - }) + app.post('/postchatfile', function (req: Request, res) { + logger.LogReq(req) + utils + .uploadFile(req, uloadFiles) + .then((result) => { + res.json({ + success: true, + path: result.filePath.replace(publicDir, ''), + }) + }) + .catch(() => { + res.json({ success: false, msg: 'error during uploading' }) + return + }) + }) - app.post('/postchatfile', function (req: Request, res) { - logger.LogReq(req) - utils - .uploadFile(req, uloadFiles) - .then((result) => { - res.json({ - success: true, - path: result.filePath.replace(publicDir, ''), - }) - }) - .catch(() => { - res.json({ success: false, msg: 'error during uploading' }) - return - }) - }) + app.post('/postfeedbackfile', function (req: Request, res) { + logger.LogReq(req) + const user: User = req.session.user - app.post('/postfeedbackfile', function (req: Request, res) { - logger.LogReq(req) - const user: User = req.session.user + utils + .uploadFile(req, uloadFiles) + .then(({ filePath }) => { + const fileName = filePath.replace(publicDir, '') + const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => { + return fileName.toLowerCase().includes(ext) + }) + const msgObj = { + sender: user.id, + reciever: 1, + msg: fileName, + type: isImage ? 'img' : 'file', + date: new Date().getTime(), + unread: 1, + } + dbtools.Insert(msgDB, 'msgs', msgObj) - utils - .uploadFile(req, uloadFiles) - .then(({ filePath }) => { - const fileName = filePath.replace(publicDir, '') - const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => { - return fileName.toLowerCase().includes(ext) - }) - const msgObj = { - sender: user.id, - reciever: 1, - msg: fileName, - type: isImage ? 'img' : 'file', - date: new Date().getTime(), - unread: 1, + res.json({ success: true }) + io.sockets.in('1').emit('chat message', msgObj) + }) + .catch(() => { + res.json({ success: false, msg: 'error during uploading' }) + return + }) + }) + + app.post( + '/postfeedback', + function (req: Request<{ content: string }>, res) { + logger.LogReq(req) + const user: User = req.session.user + const { content } = req.body + if (!content || !content.trim()) { + res.json({ success: false }) + return + } + + const msgObj = { + sender: user.id, + reciever: 1, + msg: dbtools.sanitizeQuery(req.body.content), + type: 'text', + date: new Date().getTime(), + unread: 1, + } + dbtools.Insert(msgDB, 'msgs', msgObj) + + res.json({ success: true }) + io.sockets.in('1').emit('chat message', msgObj) } - dbtools.Insert(msgDB, 'msgs', msgObj) + ) - res.json({ success: true }) - io.sockets.in('1').emit('chat message', msgObj) - }) - .catch(() => { - res.json({ success: false, msg: 'error during uploading' }) - return - }) - }) + app.get('/hasNewMsg', (req: Request, res) => { + const user: User = req.session.user + const userid: number = user.id - app.post('/postfeedback', function (req: Request<{ content: string }>, res) { - logger.LogReq(req) - const user: User = req.session.user - const { content } = req.body - if (!content || !content.trim()) { - res.json({ success: false }) - return - } - - const msgObj = { - sender: user.id, - reciever: 1, - msg: dbtools.sanitizeQuery(req.body.content), - type: 'text', - date: new Date().getTime(), - unread: 1, - } - dbtools.Insert(msgDB, 'msgs', msgObj) - - res.json({ success: true }) - io.sockets.in('1').emit('chat message', msgObj) - }) - - app.get('/hasNewMsg', (req: Request, res) => { - const user: User = req.session.user - const userid: number = user.id - - const groups = dbtools - .runStatement( - msgDB, - `select * from + const groups = dbtools + .runStatement( + msgDB, + `select * from ( select sender as a from msgs @@ -275,35 +280,35 @@ function setup(data: SubmoduleData): void { where sender = ${userid} or reciever = ${userid} )t order by t.a asc` - ) - .reduce((acc: number[], x: { a: number }) => { - if (x.a !== userid) acc.push(x.a) - return acc - }, []) + ) + .reduce((acc: number[], x: { a: number }) => { + if (x.a !== userid) acc.push(x.a) + return acc + }, []) - const prevMsgs = groups.map((to: number) => { - const first = dbtools.runStatement( - msgDB, - `select * from msgs + const prevMsgs = groups.map((to: number) => { + const first = dbtools.runStatement( + msgDB, + `select * from msgs where sender = ${userid} and reciever = ${to} or sender = ${to} and reciever = ${userid} order by date desc limit 1` - )[0] - return first - }) + )[0] + return first + }) - res.json({ - unreads: prevMsgs.reduce((acc: number[], msg: Message) => { - if (msg && msg.unread === 1 && msg.sender !== userid) { - acc.push(msg.sender) - } - return acc - }, []), + res.json({ + unreads: prevMsgs.reduce((acc: number[], msg: Message) => { + if (msg && msg.unread === 1 && msg.sender !== userid) { + acc.push(msg.sender) + } + return acc + }, []), + }) }) - }) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/feedback.ts b/src/modules/api/submodules/feedback.ts index 36eeecd..f0aada2 100644 --- a/src/modules/api/submodules/feedback.ts +++ b/src/modules/api/submodules/feedback.ts @@ -26,15 +26,15 @@ import { Request, SubmoduleData } from '../../../types/basicTypes' const uloadFiles = 'data/f' function setup(data: SubmoduleData): void { - const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data + const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data - app.route('/fosuploader').post(function (req: Request, res: Response) { - utils.uploadFile(req, uloadFiles).then(({ fileName }) => { - res.redirect('/f/' + fileName) + app.route('/fosuploader').post(function (req: Request, res: Response) { + utils.uploadFile(req, uloadFiles).then(({ fileName }) => { + res.redirect('/f/' + fileName) + }) }) - }) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/forum.ts b/src/modules/api/submodules/forum.ts index f80a6ae..b6e9dad 100644 --- a/src/modules/api/submodules/forum.ts +++ b/src/modules/api/submodules/forum.ts @@ -25,572 +25,592 @@ import utils from '../../../utils/utils' import { Request, SubmoduleData, User } from '../../../types/basicTypes' interface Comment { - date: string - user: number - content: string - admin: boolean - subComments?: Comment[] - reacts?: { - [key: string]: number[] - } + date: string + user: number + content: string + admin: boolean + subComments?: Comment[] + reacts?: { + [key: string]: number[] + } } interface ForumEntry { - date: string - user: number - title: string - content: string - admin: boolean - comments: Comment[] - mediaPath?: string - reacts: { - [key: string]: number[] - } + date: string + user: number + title: string + content: string + admin: boolean + comments: Comment[] + mediaPath?: string + reacts: { + [key: string]: number[] + } } interface ForumContents { - user: number - title: string - admin: boolean - commentCount: number + user: number + title: string + admin: boolean + commentCount: number } const adminUsersFile = 'data/admins.json' const forumContentsFileName = '.contents.json' function countComments(comments: Comment[]) { - return comments.reduce((acc, comment) => { - if (comment.subComments) { - acc += countComments(comment.subComments) + 1 - } else { - acc += 1 - } - return acc - }, 0) + return comments.reduce((acc, comment) => { + if (comment.subComments) { + acc += countComments(comment.subComments) + 1 + } else { + acc += 1 + } + return acc + }, 0) } function addComment(obj: Comment[], path: number[], comment: Comment) { - if (path.length === 0) { - obj.push(comment) - } else { - const i = path.pop() - if (!obj[i].subComments) { - obj[i].subComments = [] + if (path.length === 0) { + obj.push(comment) + } else { + const i = path.pop() + if (!obj[i].subComments) { + obj[i].subComments = [] + } + addComment(obj[i].subComments, path, comment) } - addComment(obj[i].subComments, path, comment) - } } function deleteComment( - obj: Comment[], - path: number[], - userid: number + obj: Comment[], + path: number[], + userid: number ): boolean { - if (path.length === 1) { - if (obj[path[0]].user === userid) { - obj.splice(path[0], 1) - return true + if (path.length === 1) { + if (obj[path[0]].user === userid) { + obj.splice(path[0], 1) + return true + } else { + return false + } } else { - return false + const i = path.pop() + deleteComment(obj[i].subComments, path, userid) + return true } - } else { - const i = path.pop() - deleteComment(obj[i].subComments, path, userid) - return true - } } function addReaction( - obj: Comment[], - path: number[], - { - reaction, - isDelete, - uid, - }: { reaction: string; isDelete: boolean; uid: number } + obj: Comment[], + path: number[], + { + reaction, + isDelete, + uid, + }: { reaction: string; isDelete: boolean; uid: number } ) { - if (path.length === 1) { - const index = path[0] - if (!obj[index].reacts) { - obj[index].reacts = {} - } - if (isDelete) { - if (obj[index].reacts[reaction]) { - obj[index].reacts[reaction] = obj[index].reacts[reaction].filter( - (currUid: number) => { - return uid !== currUid - } - ) - if (obj[index].reacts[reaction].length === 0) { - delete obj[index].reacts[reaction] + if (path.length === 1) { + const index = path[0] + if (!obj[index].reacts) { + obj[index].reacts = {} + } + if (isDelete) { + if (obj[index].reacts[reaction]) { + obj[index].reacts[reaction] = obj[index].reacts[ + reaction + ].filter((currUid: number) => { + return uid !== currUid + }) + if (obj[index].reacts[reaction].length === 0) { + delete obj[index].reacts[reaction] + } + } + } else { + if (!obj[index].reacts[reaction]) { + obj[index].reacts[reaction] = [uid] + } else { + if (!obj[index].reacts[reaction].includes(uid)) { + obj[index].reacts[reaction].push(uid) + } + } } - } } else { - if (!obj[index].reacts[reaction]) { - obj[index].reacts[reaction] = [uid] - } else { - if (!obj[index].reacts[reaction].includes(uid)) { - obj[index].reacts[reaction].push(uid) - } - } + const i = path.pop() + addReaction(obj[i].subComments, path, { + reaction: reaction, + isDelete: isDelete, + uid: uid, + }) } - } else { - const i = path.pop() - addReaction(obj[i].subComments, path, { - reaction: reaction, - isDelete: isDelete, - uid: uid, - }) - } } function getForumData( - forumName: string, - forumDir: string + forumName: string, + forumDir: string ): { forumPath: string; contentFilePath: string; contents: ForumContents[] } { - const safeForumName = forumName.replace(/\./g, '').replace(/\/+/g, '') - const forumPath = forumDir + '/' + safeForumName - const contentFilePath = forumPath + '/' + forumContentsFileName + const safeForumName = forumName.replace(/\./g, '').replace(/\/+/g, '') + const forumPath = forumDir + '/' + safeForumName + const contentFilePath = forumPath + '/' + forumContentsFileName - if (!utils.FileExists(forumPath)) { - utils.CreatePath(forumPath, true) - } + if (!utils.FileExists(forumPath)) { + utils.CreatePath(forumPath, true) + } - if (!utils.FileExists(contentFilePath)) { - utils.WriteFile('{}', contentFilePath) - } - const contents: ForumContents[] = utils.ReadJSON(contentFilePath) - return { - forumPath: forumPath, - contentFilePath: contentFilePath, - contents: contents, - } + if (!utils.FileExists(contentFilePath)) { + utils.WriteFile('{}', contentFilePath) + } + const contents: ForumContents[] = utils.ReadJSON(contentFilePath) + return { + forumPath: forumPath, + contentFilePath: contentFilePath, + contents: contents, + } } function getPostData( - postKey: string, - contents: ForumContents[], - forumPath: string + postKey: string, + contents: ForumContents[], + forumPath: string ): { - postData: ForumEntry - postPath: string + postData: ForumEntry + postPath: string } { - const safePostKey = postKey.replace(/\./g, '').replace(/\/+/g, '') - const postPath = forumPath + '/' + safePostKey - if (!contents[postKey] || !utils.FileExists(postPath)) { - return null - } - return { postData: utils.ReadJSON(postPath), postPath: postPath } + const safePostKey = postKey.replace(/\./g, '').replace(/\/+/g, '') + const postPath = forumPath + '/' + safePostKey + if (!contents[postKey] || !utils.FileExists(postPath)) { + return null + } + return { postData: utils.ReadJSON(postPath), postPath: postPath } } function setup(data: SubmoduleData): void { - const { app, /* userDB, url, */ publicdirs /*, moduleSpecificData */ } = data + const { app, /* userDB, url, */ publicdirs /*, moduleSpecificData */ } = + data - const publicDir = publicdirs[0] + const publicDir = publicdirs[0] - const forumDir = publicDir + 'forum' - const forumFiles = publicDir + 'forumFiles' + const forumDir = publicDir + 'forum' + const forumFiles = publicDir + 'forumFiles' - if (!utils.FileExists(forumDir)) { - utils.CreatePath(forumDir, true) - } - - app.get('/forumEntries', (req: Request, res) => { - logger.LogReq(req) - const forumName: string = req.query.forumName - if (!forumName) { - res.json({ - success: false, - msg: `Forum name is not specified!`, - }) - return + if (!utils.FileExists(forumDir)) { + utils.CreatePath(forumDir, true) } - const { forumPath, contents } = getForumData(forumName, forumDir) - - const from = req.query.from || Object.keys(contents).reverse()[0] - const count = parseInt(req.query.count ? req.query.count.toString() : '5') - const getContent = req.query.getContent - - let entries = {} - let i = 0 - let passed = false - let lastKey = undefined - Object.keys(contents) - .reverse() - .some((key) => { - const entry = getContent - ? utils.ReadJSON(forumPath + '/' + key) - : contents[key] - if (key === from) { - passed = true - } - if (i < count && passed) { - entries = { - ...entries, - [key]: entry, - } - } - if (i === count) { - lastKey = key - return true - } - if (passed) { - i++ - } - return false - }) - - res.json({ - entries: entries, - nextKey: lastKey, - }) - }) - - app.get('/forumEntry', (req: Request, res) => { - logger.LogReq(req) - const forumName: string = req.query.forumName - const postKey: string = req.query.postKey - if (!forumName || !postKey) { - res.json({ - success: false, - msg: `ForumName or postKey is not specified!`, - }) - return - } - - const { forumPath } = getForumData(forumName, forumDir) - - res.json({ - entry: utils.ReadJSON( - forumPath + '/' + postKey.replace(/\./g, '').replace(/\/+/g, '') - ), - success: true, - }) - }) - - app.get('/forumRanklist', (req: Request, res) => { - const forumName: string = req.query.forumName - if (!forumName) { - res.json({ success: false, msg: 'forumName required' }) - return - } - - const { forumPath, contents } = getForumData(forumName, forumDir) - - const forumEntries = Object.keys(contents).map((key) => { - const entry = utils.ReadJSON(forumPath + '/' + key) - return entry - }) - - const leaderBoard = forumEntries.reduce((acc, forumEntry) => { - const { user, reacts } = forumEntry - const ups = reacts?.['thumbs up']?.length || 0 - const downs = reacts?.['thumbs down']?.length || 0 - - if (!acc[user]) { - acc[user] = { - up: ups, - down: downs, - } - } else { - acc[user] = { - up: acc[user].up + ups, - down: acc[user].down + downs, - } - } - return acc - }, {}) - - res.json({ - success: true, - leaderBoard: Object.keys(leaderBoard) - .map((key) => { - const val = leaderBoard[key] - return { - ...val, - user: key, - sum: val.up - val.down, - } - }) - .sort((a, b) => { - return b.sum - a.sum - }), - }) - }) - - app.post('/postMeme', (req: Request, res) => { - utils - .uploadFile(req, forumFiles) - .then(() => { - res.json({ success: true }) - }) - .catch(() => { - res.json({ success: false, msg: 'error during uploading' }) - return - }) - }) - - app.post( - '/addPost', - ( - req: Request<{ - forumName: string - content: string - title: string - image?: string - }>, - res - ) => { - logger.LogReq(req) - - const { title, content, forumName, image } = req.body - if (!forumName) { - res.json({ - success: false, - msg: `Forum name is not specified!`, - }) - return - } - const { forumPath, contents, contentFilePath } = getForumData( - forumName, - forumDir - ) - const user: User = req.session.user - const admins: string[] = utils.FileExists(adminUsersFile) - ? utils.ReadJSON(adminUsersFile) - : [] - - const newPostKey = uuidv4() - const postData = { - date: utils.GetDateString(), - user: user.id, - title: title, - admin: admins.includes(user.id.toString()), - commentCount: 0, - mediaPath: image, - } - - contents[newPostKey] = postData - utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath) - utils.WriteFile( - JSON.stringify({ ...postData, content: content }, null, 2), - forumPath + '/' + newPostKey - ) - - res.json({ - success: true, - newPostKey: newPostKey, - newEntry: { ...postData, content: content }, - }) - } - ) - - app.post( - '/rmPost', - ( - req: Request<{ - postKey: string - forumName: string - }>, - res - ) => { - logger.LogReq(req) - const { postKey } = req.body - const forumName = req.body.forumName - if (!forumName) { - res.json({ - success: false, - msg: `Forum name is not specified!`, - }) - return - } - const { forumPath, contentFilePath, contents } = getForumData( - forumName, - forumDir - ) - const user: User = req.session.user - - if (contents[postKey] && contents[postKey].user === user.id) { - utils.deleteFile(forumPath + '/' + postKey) - delete contents[postKey] - utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath) - - res.json({ success: true, deletedId: postKey }) - } else { - res.json({ - success: false, - msg: 'Entry does not exists, or you are not authorized to delete this post', - }) - return - } - } - ) - - app.post( - '/comment', - ( - req: Request<{ - type: string - path: number[] - postKey: string - forumName: string - content: string - }>, - res - ) => { - logger.LogReq(req) - - const forumName = req.body.forumName - if (!forumName) { - res.json({ - success: false, - msg: `Forum name is not specified!`, - }) - return - } - const { forumPath, contentFilePath, contents } = getForumData( - forumName, - forumDir - ) - const user: User = req.session.user - const admins: string[] = utils.FileExists(adminUsersFile) - ? utils.ReadJSON(adminUsersFile) - : [] - const { type, path, postKey } = req.body - if (!postKey || !type || !path) { - res.json({ - success: false, - msg: 'type or path or postKey is undefined', - }) - return - } - - const { postPath, postData } = getPostData(postKey, contents, forumPath) - if (!postData) { - res.json({ - success: false, - msg: `Post entry '${postKey}' from forum '${forumName}' does not exits!`, - }) - } - - if (type === 'add') { - const { content } = req.body - const comment = { - date: utils.GetDateString(), - user: user.id, - content: content, - admin: admins.includes(user.id.toString()), - } - if (!postData.comments) { - postData.comments = [] - } - addComment(postData.comments, path, comment) - } else if (type === 'delete') { - if (postData.comments) { - const success = deleteComment(postData.comments, path, user.id) - if (!success) { + app.get('/forumEntries', (req: Request, res) => { + logger.LogReq(req) + const forumName: string = req.query.forumName + if (!forumName) { res.json({ - success: false, - msg: 'you cant delete other users comments', - postData: postData, + success: false, + msg: `Forum name is not specified!`, }) return - } } - } else { - res.json({ success: false, msg: 'no such type' }) - return - } - contents[postKey].commentCount = countComments(postData.comments) + const { forumPath, contents } = getForumData(forumName, forumDir) - utils.WriteFile(JSON.stringify(postData, null, 2), postPath) - utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath) - res.json({ success: true, postKey: postKey, postData: postData }) - } - ) + const from = req.query.from || Object.keys(contents).reverse()[0] + const count = parseInt( + req.query.count ? req.query.count.toString() : '5' + ) + const getContent = req.query.getContent - app.post( - '/react', - ( - req: Request<{ - forumName: string - postKey: string - path: number[] - reaction: string - isDelete: boolean - }>, - res - ) => { - logger.LogReq(req) - - const forumName = req.body.forumName - if (!forumName) { - res.json({ - success: false, - msg: `Forum name is not specified!`, - }) - return - } - const { forumPath, contents } = getForumData(forumName, forumDir) - const user: User = req.session.user - - const { postKey, path, reaction, isDelete } = req.body - if (!postKey || !reaction) { - res.json({ success: false, msg: 'no postkey or reaction' }) - return - } - - const { postPath, postData } = getPostData(postKey, contents, forumPath) - if (!postData) { - res.json({ - success: false, - msg: `Post entry '${postKey}' from forum '${forumName}' does not exits!`, - }) - } - - if (!path || path.length === 0) { - if (postData) { - if (isDelete) { - if (postData.reacts) { - postData.reacts[reaction] = postData.reacts[reaction].filter( - (uid) => { - return uid !== user.id + let entries = {} + let i = 0 + let passed = false + let lastKey = undefined + Object.keys(contents) + .reverse() + .some((key) => { + const entry = getContent + ? utils.ReadJSON(forumPath + '/' + key) + : contents[key] + if (key === from) { + passed = true + } + if (i < count && passed) { + entries = { + ...entries, + [key]: entry, + } + } + if (i === count) { + lastKey = key + return true + } + if (passed) { + i++ + } + return false + }) + + res.json({ + entries: entries, + nextKey: lastKey, + }) + }) + + app.get('/forumEntry', (req: Request, res) => { + logger.LogReq(req) + const forumName: string = req.query.forumName + const postKey: string = req.query.postKey + if (!forumName || !postKey) { + res.json({ + success: false, + msg: `ForumName or postKey is not specified!`, + }) + return + } + + const { forumPath } = getForumData(forumName, forumDir) + + res.json({ + entry: utils.ReadJSON( + forumPath + '/' + postKey.replace(/\./g, '').replace(/\/+/g, '') + ), + success: true, + }) + }) + + app.get('/forumRanklist', (req: Request, res) => { + const forumName: string = req.query.forumName + if (!forumName) { + res.json({ success: false, msg: 'forumName required' }) + return + } + + const { forumPath, contents } = getForumData(forumName, forumDir) + + const forumEntries = Object.keys(contents).map((key) => { + const entry = utils.ReadJSON(forumPath + '/' + key) + return entry + }) + + const leaderBoard = forumEntries.reduce((acc, forumEntry) => { + const { user, reacts } = forumEntry + const ups = reacts?.['thumbs up']?.length || 0 + const downs = reacts?.['thumbs down']?.length || 0 + + if (!acc[user]) { + acc[user] = { + up: ups, + down: downs, } - ) - if (postData.reacts[reaction].length === 0) { - delete postData.reacts[reaction] - } - } - } else { - if (!postData.reacts) { - postData.reacts = { [reaction]: [user.id] } } else { - if (Array.isArray(postData.reacts[reaction])) { - if (!postData.reacts[reaction].includes(user.id)) { - postData.reacts[reaction].push(user.id) + acc[user] = { + up: acc[user].up + ups, + down: acc[user].down + downs, } - } else { - postData.reacts[reaction] = [user.id] - } } - } - } - } else { - addReaction(postData.comments, path, { - reaction: reaction, - isDelete: isDelete, - uid: user.id, - }) - } + return acc + }, {}) - utils.WriteFile(JSON.stringify(postData, null, 2), postPath) - res.json({ success: true, postData: postData }) - } - ) + res.json({ + success: true, + leaderBoard: Object.keys(leaderBoard) + .map((key) => { + const val = leaderBoard[key] + return { + ...val, + user: key, + sum: val.up - val.down, + } + }) + .sort((a, b) => { + return b.sum - a.sum + }), + }) + }) + + app.post('/postMeme', (req: Request, res) => { + utils + .uploadFile(req, forumFiles) + .then(() => { + res.json({ success: true }) + }) + .catch(() => { + res.json({ success: false, msg: 'error during uploading' }) + return + }) + }) + + app.post( + '/addPost', + ( + req: Request<{ + forumName: string + content: string + title: string + image?: string + }>, + res + ) => { + logger.LogReq(req) + + const { title, content, forumName, image } = req.body + if (!forumName) { + res.json({ + success: false, + msg: `Forum name is not specified!`, + }) + return + } + const { forumPath, contents, contentFilePath } = getForumData( + forumName, + forumDir + ) + const user: User = req.session.user + const admins: string[] = utils.FileExists(adminUsersFile) + ? utils.ReadJSON(adminUsersFile) + : [] + + const newPostKey = uuidv4() + const postData = { + date: utils.GetDateString(), + user: user.id, + title: title, + admin: admins.includes(user.id.toString()), + commentCount: 0, + mediaPath: image, + } + + contents[newPostKey] = postData + utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath) + utils.WriteFile( + JSON.stringify({ ...postData, content: content }, null, 2), + forumPath + '/' + newPostKey + ) + + res.json({ + success: true, + newPostKey: newPostKey, + newEntry: { ...postData, content: content }, + }) + } + ) + + app.post( + '/rmPost', + ( + req: Request<{ + postKey: string + forumName: string + }>, + res + ) => { + logger.LogReq(req) + const { postKey } = req.body + const forumName = req.body.forumName + if (!forumName) { + res.json({ + success: false, + msg: `Forum name is not specified!`, + }) + return + } + const { forumPath, contentFilePath, contents } = getForumData( + forumName, + forumDir + ) + const user: User = req.session.user + + if (contents[postKey] && contents[postKey].user === user.id) { + utils.deleteFile(forumPath + '/' + postKey) + delete contents[postKey] + utils.WriteFile( + JSON.stringify(contents, null, 2), + contentFilePath + ) + + res.json({ success: true, deletedId: postKey }) + } else { + res.json({ + success: false, + msg: 'Entry does not exists, or you are not authorized to delete this post', + }) + return + } + } + ) + + app.post( + '/comment', + ( + req: Request<{ + type: string + path: number[] + postKey: string + forumName: string + content: string + }>, + res + ) => { + logger.LogReq(req) + + const forumName = req.body.forumName + if (!forumName) { + res.json({ + success: false, + msg: `Forum name is not specified!`, + }) + return + } + const { forumPath, contentFilePath, contents } = getForumData( + forumName, + forumDir + ) + const user: User = req.session.user + const admins: string[] = utils.FileExists(adminUsersFile) + ? utils.ReadJSON(adminUsersFile) + : [] + const { type, path, postKey } = req.body + if (!postKey || !type || !path) { + res.json({ + success: false, + msg: 'type or path or postKey is undefined', + }) + return + } + + const { postPath, postData } = getPostData( + postKey, + contents, + forumPath + ) + if (!postData) { + res.json({ + success: false, + msg: `Post entry '${postKey}' from forum '${forumName}' does not exits!`, + }) + } + + if (type === 'add') { + const { content } = req.body + const comment = { + date: utils.GetDateString(), + user: user.id, + content: content, + admin: admins.includes(user.id.toString()), + } + if (!postData.comments) { + postData.comments = [] + } + addComment(postData.comments, path, comment) + } else if (type === 'delete') { + if (postData.comments) { + const success = deleteComment( + postData.comments, + path, + user.id + ) + if (!success) { + res.json({ + success: false, + msg: 'you cant delete other users comments', + postData: postData, + }) + return + } + } + } else { + res.json({ success: false, msg: 'no such type' }) + return + } + + contents[postKey].commentCount = countComments(postData.comments) + + utils.WriteFile(JSON.stringify(postData, null, 2), postPath) + utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath) + res.json({ success: true, postKey: postKey, postData: postData }) + } + ) + + app.post( + '/react', + ( + req: Request<{ + forumName: string + postKey: string + path: number[] + reaction: string + isDelete: boolean + }>, + res + ) => { + logger.LogReq(req) + + const forumName = req.body.forumName + if (!forumName) { + res.json({ + success: false, + msg: `Forum name is not specified!`, + }) + return + } + const { forumPath, contents } = getForumData(forumName, forumDir) + const user: User = req.session.user + + const { postKey, path, reaction, isDelete } = req.body + if (!postKey || !reaction) { + res.json({ success: false, msg: 'no postkey or reaction' }) + return + } + + const { postPath, postData } = getPostData( + postKey, + contents, + forumPath + ) + if (!postData) { + res.json({ + success: false, + msg: `Post entry '${postKey}' from forum '${forumName}' does not exits!`, + }) + } + + if (!path || path.length === 0) { + if (postData) { + if (isDelete) { + if (postData.reacts) { + postData.reacts[reaction] = postData.reacts[ + reaction + ].filter((uid) => { + return uid !== user.id + }) + if (postData.reacts[reaction].length === 0) { + delete postData.reacts[reaction] + } + } + } else { + if (!postData.reacts) { + postData.reacts = { [reaction]: [user.id] } + } else { + if (Array.isArray(postData.reacts[reaction])) { + if ( + !postData.reacts[reaction].includes(user.id) + ) { + postData.reacts[reaction].push(user.id) + } + } else { + postData.reacts[reaction] = [user.id] + } + } + } + } + } else { + addReaction(postData.comments, path, { + reaction: reaction, + isDelete: isDelete, + uid: user.id, + }) + } + + utils.WriteFile(JSON.stringify(postData, null, 2), postPath) + res.json({ success: true, postData: postData }) + } + ) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/qminingapi.ts b/src/modules/api/submodules/qminingapi.ts index 1d0efb9..c3a07c2 100644 --- a/src/modules/api/submodules/qminingapi.ts +++ b/src/modules/api/submodules/qminingapi.ts @@ -26,49 +26,49 @@ import type { Database } from 'better-sqlite3' import logger from '../../../utils/logger' import utils from '../../../utils/utils' import { - User, - DataFile, - Request, - QuestionDb, - SubmoduleData, - Question, - QuestionFromScript, - DbSearchResult, - RegisteredUserEntry, - Submodule, + User, + DataFile, + Request, + QuestionDb, + SubmoduleData, + Question, + QuestionFromScript, + DbSearchResult, + RegisteredUserEntry, + Submodule, } from '../../../types/basicTypes' import { - processIncomingRequest, - logResult, - shouldSaveDataFile, - Result, - backupData, - shouldSearchDataFile, - loadJSON, - writeData, - editDb, - RecievedData, + processIncomingRequest, + logResult, + shouldSaveDataFile, + Result, + backupData, + shouldSearchDataFile, + loadJSON, + writeData, + editDb, + RecievedData, } from '../../../utils/actions' import { - dataToString, - getSubjNameWithoutYear, - WorkerResult, - SearchResultQuestion, - // compareQuestionObj, + dataToString, + getSubjNameWithoutYear, + WorkerResult, + SearchResultQuestion, + // compareQuestionObj, } from '../../../utils/classes' import { - doALongTask, - msgAllWorker, - initWorkerPool, + doALongTask, + msgAllWorker, + initWorkerPool, } from '../../../utils/workerPool' import dbtools from '../../../utils/dbtools' interface SavedQuestionData { - fname: string - subj: string - userid: number - testUrl: string - date: string | Date + fname: string + subj: string + userid: number + testUrl: string + date: string | Date } // interface SavedQuestion { @@ -91,1002 +91,1015 @@ const dailyDataCountFile = 'stats/dailyDataCount' const dataEditsLog = 'stats/dataEdits' function getSubjCount(qdbs: QuestionDb[]): number { - return qdbs.reduce((acc, qdb) => { - return acc + qdb.data.length - }, 0) + return qdbs.reduce((acc, qdb) => { + return acc + qdb.data.length + }, 0) } function getQuestionCount(qdbs: QuestionDb[]): number { - return qdbs.reduce((acc, qdb) => { - return ( - acc + - qdb.data.reduce((qacc, subject) => { - return qacc + subject.Questions.length - }, 0) - ) - }, 0) + return qdbs.reduce((acc, qdb) => { + return ( + acc + + qdb.data.reduce((qacc, subject) => { + return qacc + subject.Questions.length + }, 0) + ) + }, 0) } function ExportDailyDataCount(questionDbs: QuestionDb[], userDB: Database) { - logger.Log('Saving daily data count ...') - utils.AppendToFile( - JSON.stringify({ - date: utils.GetDateString(), - subjectCount: getSubjCount(questionDbs), - questionCount: getQuestionCount(questionDbs), - questionDbsCount: questionDbs.length, - userCount: dbtools.TableInfo(userDB, 'users').dataCount, - }), - dailyDataCountFile - ) + logger.Log('Saving daily data count ...') + utils.AppendToFile( + JSON.stringify({ + date: utils.GetDateString(), + subjectCount: getSubjCount(questionDbs), + questionCount: getQuestionCount(questionDbs), + questionDbsCount: questionDbs.length, + userCount: dbtools.TableInfo(userDB, 'users').dataCount, + }), + dailyDataCountFile + ) } function getDbIndexesToSearchIn( - testUrl: string, - questionDbs: Array, - trueIfAlways?: boolean + testUrl: string, + questionDbs: Array, + trueIfAlways?: boolean ): number[] { - return testUrl - ? questionDbs.reduce((acc, qdb, i) => { - if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) { - acc.push(i) - } - return acc - }, []) - : [] + return testUrl + ? questionDbs.reduce((acc, qdb, i) => { + if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) { + acc.push(i) + } + return acc + }, []) + : [] } function getSimplreRes(questionDbs: QuestionDb[]): { - subjects: number - questions: number + subjects: number + questions: number } { - return { - subjects: getSubjCount(questionDbs), - questions: getQuestionCount(questionDbs), - } + return { + subjects: getSubjCount(questionDbs), + questions: getQuestionCount(questionDbs), + } } function getDetailedRes(questionDbs: QuestionDb[]) { - return questionDbs.map((qdb) => { - return { - dbName: qdb.name, - subjs: qdb.data.map((subj) => { + return questionDbs.map((qdb) => { return { - name: subj.Name, - count: subj.Questions.length, + dbName: qdb.name, + subjs: qdb.data.map((subj) => { + return { + name: subj.Name, + count: subj.Questions.length, + } + }), } - }), - } - }) + }) } function getMotd(version: string, motd: string) { - if (version) { - if (version.startsWith('2.0.')) { - if (utils.FileExists(oldMotdFile)) { - return utils.ReadFile(oldMotdFile) - } + if (version) { + if (version.startsWith('2.0.')) { + if (utils.FileExists(oldMotdFile)) { + return utils.ReadFile(oldMotdFile) + } + } } - } - return motd + return motd } function searchInDbs( - question: Question, - subj: string, - searchIn: number[], - testUrl?: string + question: Question, + subj: string, + searchIn: number[], + testUrl?: string ): Promise { - // searchIn could be [0], [1], ... to search every db in different thread. Put this into a - // forEach(qdbs) to achieve this - return new Promise((resolve) => { - doALongTask({ - type: 'work', - data: { - searchIn: searchIn, - testUrl: testUrl, - question: question, - subjName: subj, - searchInAllIfNoResult: true, - }, - }) - .then((taskResult: WorkerResult) => { - try { - logger.DebugLog(taskResult, 'ask', 2) - resolve({ - question: question, - result: taskResult.result as SearchResultQuestion[], - success: true, - }) - } catch (err) { - console.error(err) - logger.Log( - 'Error while sending ask results', - logger.GetColor('redbg') - ) - } - }) - .catch((err) => { - logger.Log('Search Data error!', logger.GetColor('redbg')) - console.error(err) - resolve({ - question: question, - message: `There was an error processing the question: ${err.message}`, - result: [], - success: false, + // searchIn could be [0], [1], ... to search every db in different thread. Put this into a + // forEach(qdbs) to achieve this + return new Promise((resolve) => { + doALongTask({ + type: 'work', + data: { + searchIn: searchIn, + testUrl: testUrl, + question: question, + subjName: subj, + searchInAllIfNoResult: true, + }, }) - }) - }) + .then((taskResult: WorkerResult) => { + try { + logger.DebugLog(taskResult, 'ask', 2) + resolve({ + question: question, + result: taskResult.result as SearchResultQuestion[], + success: true, + }) + } catch (err) { + console.error(err) + logger.Log( + 'Error while sending ask results', + logger.GetColor('redbg') + ) + } + }) + .catch((err) => { + logger.Log('Search Data error!', logger.GetColor('redbg')) + console.error(err) + resolve({ + question: question, + message: `There was an error processing the question: ${err.message}`, + result: [], + success: false, + }) + }) + }) } function getResult(data: { - question: Question - subj: string - questionDbs: Array - testUrl: string + question: Question + subj: string + questionDbs: Array + testUrl: string }): Promise { - const { question, subj, questionDbs, testUrl } = data - return new Promise((resolve) => { - const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false) + const { question, subj, questionDbs, testUrl } = data + return new Promise((resolve) => { + const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false) - searchInDbs(question, subj, searchIn, testUrl).then( - (res: DbSearchResult) => { - if (res.result.length === 0) { - logger.DebugLog( - `No result while searching specific question db ${testUrl}`, - 'ask', - 1 - ) - const searchInMore = getDbIndexesToSearchIn( - testUrl, - questionDbs, - true - ).filter((x) => { - return !searchIn.includes(x) - }) - searchInDbs(question, subj, searchInMore, testUrl).then((res) => { - resolve(res) - }) - } else { - resolve(res) - } - } - ) - }) + searchInDbs(question, subj, searchIn, testUrl).then( + (res: DbSearchResult) => { + if (res.result.length === 0) { + logger.DebugLog( + `No result while searching specific question db ${testUrl}`, + 'ask', + 1 + ) + const searchInMore = getDbIndexesToSearchIn( + testUrl, + questionDbs, + true + ).filter((x) => { + return !searchIn.includes(x) + }) + searchInDbs(question, subj, searchInMore, testUrl).then( + (res) => { + resolve(res) + } + ) + } else { + resolve(res) + } + } + ) + }) } function dbExists(location: string, qdbs: Array) { - return qdbs.some((qdb) => { - return qdb.name === location - }) + return qdbs.some((qdb) => { + return qdb.name === location + }) } function writeAskData(body: QuestionFromScript) { - try { - let towrite = utils.GetDateString() + '\n' - towrite += - '------------------------------------------------------------------------------\n' - towrite += JSON.stringify(body) - towrite += - '\n------------------------------------------------------------------------------\n' - utils.AppendToFile(towrite, askedQuestionFile) - } catch (err) { - logger.Log('Error writing revieved /ask POST data') - console.error(err) - } + try { + let towrite = utils.GetDateString() + '\n' + towrite += + '------------------------------------------------------------------------------\n' + towrite += JSON.stringify(body) + towrite += + '\n------------------------------------------------------------------------------\n' + utils.AppendToFile(towrite, askedQuestionFile) + } catch (err) { + logger.Log('Error writing revieved /ask POST data') + console.error(err) + } } function writeIsAddingData(body: RecievedData) { - try { - let towrite = utils.GetDateString() + '\n' - towrite += - '------------------------------------------------------------------------------\n' - towrite += JSON.stringify(body) - towrite += - '\n------------------------------------------------------------------------------\n' - utils.AppendToFile(towrite, recievedQuestionFile) - } catch (err) { - logger.Log('Error writing revieved /ask POST data') - console.error(err) - } + try { + let towrite = utils.GetDateString() + '\n' + towrite += + '------------------------------------------------------------------------------\n' + towrite += JSON.stringify(body) + towrite += + '\n------------------------------------------------------------------------------\n' + utils.AppendToFile(towrite, recievedQuestionFile) + } catch (err) { + logger.Log('Error writing revieved /ask POST data') + console.error(err) + } } function saveQuestion( - questions: Question[], - subj: string, - testUrl: string, - userid: number, - savedQuestionsDir: string + questions: Question[], + subj: string, + testUrl: string, + userid: number, + savedQuestionsDir: string ) { - // TODO: clear folder every now and then, check if saved questions exist - if (!subj) { - logger.Log('No subj name to save test question') - return - } - const questionsToSave = { - questions: questions, - subj: subj, - userid: userid, - testUrl: testUrl, - date: new Date(), - } - const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json` - const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-') - const subjPath = `${savedQuestionsDir}/${subject}` - const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}` + // TODO: clear folder every now and then, check if saved questions exist + if (!subj) { + logger.Log('No subj name to save test question') + return + } + const questionsToSave = { + questions: questions, + subj: subj, + userid: userid, + testUrl: testUrl, + date: new Date(), + } + const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json` + const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-') + const subjPath = `${savedQuestionsDir}/${subject}` + const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}` - utils.CreatePath(subjPath, true) - if (!utils.FileExists(savedSubjQuestionsFilePath)) { - utils.WriteFile('[]', savedSubjQuestionsFilePath) - } + utils.CreatePath(subjPath, true) + if (!utils.FileExists(savedSubjQuestionsFilePath)) { + utils.WriteFile('[]', savedSubjQuestionsFilePath) + } - const savedQuestions: SavedQuestionData[] = utils.ReadJSON( - savedSubjQuestionsFilePath - ) + const savedQuestions: SavedQuestionData[] = utils.ReadJSON( + savedSubjQuestionsFilePath + ) - const testExists = false - // TODO: do this on another thread? - // const testExists = savedQuestions.some((savedQuestion) => { - // const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`) + const testExists = false + // TODO: do this on another thread? + // const testExists = savedQuestions.some((savedQuestion) => { + // const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`) - // return data.questions.some((dQuestion) => { - // return questions.some((question) => { - // const percent = compareQuestionObj( - // createQuestion(question), - // '', - // createQuestion(dQuestion), - // '' - // ) + // return data.questions.some((dQuestion) => { + // return questions.some((question) => { + // const percent = compareQuestionObj( + // createQuestion(question), + // '', + // createQuestion(dQuestion), + // '' + // ) - // return percent.avg === 100 - // }) - // }) - // }) + // return percent.avg === 100 + // }) + // }) + // }) - if (testExists) { - return - } + if (testExists) { + return + } - savedQuestions.push({ - fname: fname, - subj: subj, - userid: userid, - testUrl: testUrl, - date: new Date(), - }) - utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath) - utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`) + savedQuestions.push({ + fname: fname, + subj: subj, + userid: userid, + testUrl: testUrl, + date: new Date(), + }) + utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath) + utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`) } function loadSupportedSites() { - const script = utils.ReadFile(userScriptFile).split('\n') + const script = utils.ReadFile(userScriptFile).split('\n') - let i = 0 - let stayIn = true - let inHeader = false - let inMatch = false - const sites = [] - while (i < script.length && stayIn) { - if (inHeader) { - if (script[i].includes('@match')) { - inMatch = true - } - if (inMatch && !script[i].includes('match')) { - stayIn = false - inMatch = false - } - if (inMatch) { - sites.push(script[i].split(' ').pop()) - } - } else { - inHeader = script[i].includes('==UserScript==') + let i = 0 + let stayIn = true + let inHeader = false + let inMatch = false + const sites = [] + while (i < script.length && stayIn) { + if (inHeader) { + if (script[i].includes('@match')) { + inMatch = true + } + if (inMatch && !script[i].includes('match')) { + stayIn = false + inMatch = false + } + if (inMatch) { + sites.push(script[i].split(' ').pop()) + } + } else { + inHeader = script[i].includes('==UserScript==') + } + i++ } - i++ - } - return sites + return sites } function LoadVersion() { - const scriptContent = utils.ReadFile(userScriptFile) + const scriptContent = utils.ReadFile(userScriptFile) - let temp: string | string[] = scriptContent.split('\n').find((x) => { - return x.includes('@version') - }) - temp = temp.split(' ') - temp = temp[temp.length - 1] + let temp: string | string[] = scriptContent.split('\n').find((x) => { + return x.includes('@version') + }) + temp = temp.split(' ') + temp = temp[temp.length - 1] - return temp + return temp } function LoadMOTD(motdFile: string) { - return utils.ReadFile(motdFile) + return utils.ReadFile(motdFile) } function LoadTestUsers() { - let testUsers = utils.ReadJSON(testUsersFile) - if (testUsers) { - testUsers = testUsers.userIds - } - return testUsers + let testUsers = utils.ReadJSON(testUsersFile) + if (testUsers) { + testUsers = testUsers.userIds + } + return testUsers } function getNewQdb( - location: string, - maxIndex: number, - dbsFile: string, - publicDir: string, - questionDbs: QuestionDb[] + location: string, + maxIndex: number, + dbsFile: string, + publicDir: string, + questionDbs: QuestionDb[] ) { - logger.Log( - `No suitable questiondbs found for ${location}, creating a new one...` - ) - const newDb: DataFile = { - path: `questionDbs/${location}.json`, - name: location, - shouldSearch: { - location: { - val: location, - }, - }, - shouldSave: { - location: { - val: location, - }, - }, - } - - utils.WriteFile( - JSON.stringify( - [ - ...utils.ReadJSON(dbsFile), - newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime - ], - null, - 2 - ), - dbsFile - ) - - // "loading" new db - const loadedNewDb: QuestionDb = { - ...newDb, - data: [], - path: publicDir + newDb.path, - index: maxIndex, - } - utils.WriteFile('[]', loadedNewDb.path) - - questionDbs.push(loadedNewDb) - msgAllWorker({ - data: loadedNewDb, - type: 'newdb', - }) - - return loadedNewDb -} - -function setup(data: SubmoduleData): Submodule { - const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data - - const publicDir = publicdirs[0] - const motdFile = publicDir + 'motd' - const dbsFile = publicDir + 'questionDbs.json' - const savedQuestionsDir = publicDir + 'savedQuestions' - - let version = LoadVersion() - let supportedSites = loadSupportedSites() - let motd = LoadMOTD(motdFile) - let testUsers: number[] = LoadTestUsers() - - const dataFiles: Array = utils.ReadJSON(dbsFile) - const questionDbs: Array = loadJSON(dataFiles, publicDir) - initWorkerPool(questionDbs) - - const filesToWatch = [ - { - fname: motdFile, - logMsg: 'Motd updated', - action: () => { - motd = LoadMOTD(motdFile) - }, - }, - { - fname: testUsersFile, - logMsg: 'Test Users file changed', - action: () => { - testUsers = LoadTestUsers() - }, - }, - { - fname: userScriptFile, - logMsg: 'User script file changed', - action: () => { - version = LoadVersion() - supportedSites = loadSupportedSites() - }, - }, - ] - - app.get('/getDbs', (req: Request, res: Response) => { - logger.LogReq(req) - res.json( - questionDbs.map((qdb) => { - return { - path: qdb.path.replace(publicDir, ''), - name: qdb.name, - locked: qdb.locked, - } - }) + logger.Log( + `No suitable questiondbs found for ${location}, creating a new one...` ) - }) - - app.get('/allqr.txt', function (req: Request, res: Response) { - logger.LogReq(req) - const db: string = req.query.db - let stringifiedData = '' - - res.setHeader('content-type', 'text/plain; charset=utf-8') - - if (db) { - const requestedDb = questionDbs.find((qdb) => { - return qdb.name === db - }) - - if (!requestedDb) { - res.end(`No such db ${db}`) - return - } - - stringifiedData = '\n' + line - stringifiedData += ` Questions in ${requestedDb.name}: ` - stringifiedData += line + '\n' - stringifiedData += dataToString(requestedDb.data) - stringifiedData += '\n' + line + line + '\n' - } else { - stringifiedData = questionDbs - .map((qdb) => { - let result = '' - result += '\n' + line - result += ` Questions in ${qdb.name}: ` - result += line + '\n' - result += dataToString(qdb.data) - result += '\n' + line + line + '\n' - return result - }) - .join('\n\n') - } - res.end(stringifiedData) - }) - - app.post('/isAdding', function (req: Request, res: Response) { - logger.LogReq(req) - const user: User = req.session.user - const dryRun = testUsers.includes(user.id) - if (!req.body.location) { - logger.Log( - '\tbody.location is missing, client version:' + req.body.version - ) - res.json({ msg: 'body.location is missing' }) - return - } - - writeIsAddingData(req.body) - - const location = req.body.location.split('/')[2] - - try { - let maxIndex = -1 - const suitedQuestionDbs = questionDbs.filter((qdb) => { - if (maxIndex < qdb.index) { - maxIndex = qdb.index - } - return shouldSaveDataFile(qdb, req.body) - }, []) - - if (suitedQuestionDbs.length === 0) { - if (!dbExists(location, questionDbs)) { - suitedQuestionDbs.push( - getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) - ) - } else { - logger.Log( - `Tried to add existing db named ${location}!`, - logger.GetColor('red') - ) - } - } - if (suitedQuestionDbs.length === 0) { - res.json({ - status: 'fail', - msg: 'No suitable dbs to add questions to', - }) - return - } - - processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user) - .then((resultArray: Array) => { - logResult(req.body, resultArray, user.id, dryRun) - - const totalNewQuestions = resultArray.reduce((acc, sres) => { - return acc + sres.newQuestions.length - }, 0) - - res.json({ - success: resultArray.length > 0, - newQuestions: resultArray, - totalNewQuestions: totalNewQuestions, - }) - - if (totalNewQuestions > 0) { - resultArray.forEach((result) => { - msgAllWorker({ - // TODO: recognize base64 image - type: 'newQuestions', - data: result, - }) - }) - } - }) - .catch((err) => { - logger.Log( - 'Error during processing incoming request', - logger.GetColor('redbg') - ) - console.error(err) - res.json({ - success: false, - }) - }) - } catch (err) { - logger.Log( - 'Error during getting incoming request processor promises ', - logger.GetColor('redbg') - ) - console.error(err) - res.json({ - success: false, - }) - } - }) - - app.post('/ask', function (req: Request, res) { - const user: User = req.session.user - - if (!req.body.questions) { - res.json({ - message: `ask something! { questions:'' ,subject:'', location:'' }`, - result: [], - success: false, - }) - return - } - const subj: string = req.body.subj || '' - // TODO: test if testUrl is undefined (it shouldnt) - const testUrl: string = req.body.testUrl - ? req.body.testUrl.split('/')[2] - : undefined - - writeAskData(req.body) - - // every question in a different thread - const resultPromises: Promise[] = req.body.questions.map( - (question: Question) => { - return getResult({ - question: question, - subj: subj, - testUrl: testUrl, - questionDbs: questionDbs, - }) - } - ) - - Promise.all(resultPromises).then((results: DbSearchResult[]) => { - const response = results.map((result: DbSearchResult) => { - return { - answers: result.result, - question: result.question, - } - }) - res.json(response) - - const saveableQuestions = response.reduce((acc, res) => { - const save = res.answers.every((answer) => { - return answer.match < 90 - }) - - if (save) { - acc.push(res.question) - } - return acc - }, []) - - if (saveableQuestions.length > 0) { - saveQuestion( - saveableQuestions, - subj, - testUrl, - user.id, - savedQuestionsDir - ) - } - }) - }) - - app.get('/supportedSites', function (req: Request, res: Response) { - logger.LogReq(req) - - res.json(supportedSites) - }) - - app.get('/datacount', function (req: Request, res: Response) { - logger.LogReq(req) - if (req.query.detailed === 'all') { - res.json({ - detailed: getDetailedRes(questionDbs), - simple: getSimplreRes(questionDbs), - }) - } else if (req.query.detailed) { - res.json(getDetailedRes(questionDbs)) - } else { - res.json(getSimplreRes(questionDbs)) - } - }) - - app.get('/infos', function (req: Request, res) { - const user: User = req.session.user - - const result: { - result: string - uid: number - version?: string - subjinfo?: { - subjects: number - questions: number - } - motd?: string - } = { - result: 'success', - uid: user.id, - } - - if (req.query.subjinfo) { - result.subjinfo = getSimplreRes(questionDbs) - } - if (req.query.version) { - result.version = version - } - if (req.query.motd) { - result.motd = getMotd(req.query.cversion, motd) - } - res.json(result) - }) - - app.post('/registerscript', function (req: Request, res) { - logger.LogReq(req) - - if (!utils.FileExists(registeredScriptsFile)) { - utils.WriteFile('[]', registeredScriptsFile) - } - - const ua: string = req.headers['user-agent'] - const registeredScripts: RegisteredUserEntry[] = utils.ReadJSON( - registeredScriptsFile - ) - const { cid, uid, version, installSource, date } = req.body - - const index = registeredScripts.findIndex((registration) => { - return registration.cid === cid - }) - - if (index === -1) { - const x: RegisteredUserEntry = { - cid: cid, - version: version, - installSource: installSource, - date: date, - userAgent: ua, - } - - if (uid) { - x.uid = uid - x.loginDate = date - } - registeredScripts.push(x) - } else { - const currRegistration = registeredScripts[index] - - if (!currRegistration.uid && uid) { - registeredScripts[index] = { - ...registeredScripts[index], - uid: uid, - loginDate: date, - } - } else { - logger.DebugLog( - `cid: ${cid}, uid: ${uid} tried to register multiple times`, - 'register', - 1 - ) - } + const newDb: DataFile = { + path: `questionDbs/${location}.json`, + name: location, + shouldSearch: { + location: { + val: location, + }, + }, + shouldSave: { + location: { + val: location, + }, + }, } utils.WriteFile( - JSON.stringify(registeredScripts, null, 2), - registeredScriptsFile + JSON.stringify( + [ + ...utils.ReadJSON(dbsFile), + newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime + ], + null, + 2 + ), + dbsFile ) - res.json({ msg: 'done' }) - }) + // "loading" new db + const loadedNewDb: QuestionDb = { + ...newDb, + data: [], + path: publicDir + newDb.path, + index: maxIndex, + } + utils.WriteFile('[]', loadedNewDb.path) - app.get('/possibleAnswers', (req: Request, res: Response) => { - logger.LogReq(req) - const files = utils.ReadDir(savedQuestionsDir) - - files.sort(function (a, b) { - return ( - fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() - - fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime() - ) + questionDbs.push(loadedNewDb) + msgAllWorker({ + data: loadedNewDb, + type: 'newdb', }) - res.json({ - savedQuestionsFileName: savedQuestionsFileName, - subjects: files.map((subj) => { - return { - name: subj, - path: `savedQuestions/${subj}/`, - } - }), - }) - }) + return loadedNewDb +} - app.post('/rmPossibleAnswer', (req: Request, res: Response) => { - logger.LogReq(req) - const user: User = req.session.user +function setup(data: SubmoduleData): Submodule { + const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data - const subj = req.body.subj - const file = req.body.file - const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}` - const savedQuestions: SavedQuestionData[] = - utils.ReadJSON(savedQuestionsPath) - let path = `${savedQuestionsDir}/${subj}/${file}` - while (path.includes('..')) { - path = path.replace(/\.\./g, '.') - } + const publicDir = publicdirs[0] + const motdFile = publicDir + 'motd' + const dbsFile = publicDir + 'questionDbs.json' + const savedQuestionsDir = publicDir + 'savedQuestions' - if (utils.FileExists(path)) { - utils.deleteFile(path) + let version = LoadVersion() + let supportedSites = loadSupportedSites() + let motd = LoadMOTD(motdFile) + let testUsers: number[] = LoadTestUsers() - utils.WriteFile( - JSON.stringify( - savedQuestions.filter((sq) => { - return sq.fname !== file - }) - ), - savedQuestionsPath - ) + const dataFiles: Array = utils.ReadJSON(dbsFile) + const questionDbs: Array = loadJSON(dataFiles, publicDir) + initWorkerPool(questionDbs) - logger.Log( - `User #${user.id} deleted '${file}' from subject '${subj}'`, - logger.GetColor('cyan') - ) - res.json({ - res: 'ok', - }) - } else { - logger.Log( - `User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`, - logger.GetColor('red') - ) - res.json({ - res: 'fail', - }) - } - }) - - app.post('/updateQuestion', (req: Request, res) => { - logger.LogReq(req) - const user: User = req.session.user - const date = utils.GetDateString() - - const editType = req.body.type - const selectedDb = req.body.selectedDb - if (!editType || !selectedDb) { - res.json({ - status: 'fail', - msg: 'No .editType or .selectedDb !', - }) - return - } - - const dbIndex = questionDbs.findIndex((qdb) => { - return qdb.name === selectedDb.name - }) - const currDb = questionDbs[dbIndex] - - if (dbIndex === -1) { - res.json({ - status: 'fail', - msg: `No question db named like ${selectedDb.name}!`, - }) - return - } - - // ----------------- - const { - success, - msg, - resultDb, - deletedQuestion, - newVal, - oldVal, - deletedQuestions, - changedQuestions, - } = editDb(currDb, req.body) - - if (!success) { - res.json({ success: success, msg: msg }) - return - } - if (resultDb) { - questionDbs[dbIndex] = resultDb - } - - if (editType === 'delete') { - const { index, subjName } = req.body - logger.Log( - `User #${user.id} deleted a question from '${subjName}'`, - logger.GetColor('cyan') - ) - utils.AppendToFile( - `${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`, - dataEditsLog - ) - utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog) - } - - if (editType === 'edit') { - const { index, subjName } = req.body - logger.Log( - `User #${user.id} edited a question in '${subjName}'`, - logger.GetColor('cyan') - ) - utils.AppendToFile( - `${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`, - dataEditsLog - ) - utils.AppendToFile( - JSON.stringify( - { - newVal: newVal, - oldVal: oldVal, - }, - null, - 2 - ), - dataEditsLog - ) - } - - if (editType === 'subjEdit') { - const { subjName } = req.body - logger.Log( - `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, - logger.GetColor('cyan') - ) - utils.AppendToFile( - `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, - dataEditsLog - ) - utils.AppendToFile( - JSON.stringify( - { - deletedQuestions: deletedQuestions, - changedQuestions: changedQuestions, - }, - null, - 2 - ), - dataEditsLog - ) - } - // ------------------ - - if (success) { - writeData(currDb.data, currDb.path) - msgAllWorker({ - type: 'dbEdit', - data: { - dbIndex: dbIndex, - edits: req.body, + const filesToWatch = [ + { + fname: motdFile, + logMsg: 'Motd updated', + action: () => { + motd = LoadMOTD(motdFile) + }, }, - }) - } + { + fname: testUsersFile, + logMsg: 'Test Users file changed', + action: () => { + testUsers = LoadTestUsers() + }, + }, + { + fname: userScriptFile, + logMsg: 'User script file changed', + action: () => { + version = LoadVersion() + supportedSites = loadSupportedSites() + }, + }, + ] - res.json({ - success: true, - msg: 'OK', - }) - }) - - let questionCleaner: ChildProcess = null - app.get('/clearQuestions', (req: Request, res) => { - // TODO: dont allow multiple instances - // TODO: get status of it cleaning - logger.LogReq(req) - const user: User = req.session.user - const status: string = req.query.status - - if (status) { - if (!questionCleaner) { - res.json({ - msg: 'question cleaner not running', - success: false, - }) - return - } - questionCleaner.once('message', function (response) { - res.json({ - msg: response, - success: true, - }) - }) - - questionCleaner.send({ data: 'asd' }) - return - } - - if (questionCleaner) { - res.json({ - msg: 'question cleaner already running', - success: false, - }) - return - } - - questionCleaner = fork( - `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, - ['-s', `${process.cwd()}/${questionDbs[0].path}`] - ) - questionCleaner.on('exit', function (code: number) { - console.log('EXIT', code) - questionCleaner = null + app.get('/getDbs', (req: Request, res: Response) => { + logger.LogReq(req) + res.json( + questionDbs.map((qdb) => { + return { + path: qdb.path.replace(publicDir, ''), + name: qdb.name, + locked: qdb.locked, + } + }) + ) }) - res.json({ - user: user, - success: true, - msg: 'OK', - }) - }) + app.get('/allqr.txt', function (req: Request, res: Response) { + logger.LogReq(req) + const db: string = req.query.db + let stringifiedData = '' - return { - dailyAction: () => { - backupData(questionDbs) - ExportDailyDataCount(questionDbs, userDB) - }, - load: () => { - backupData(questionDbs) + res.setHeader('content-type', 'text/plain; charset=utf-8') - filesToWatch.forEach((ftw) => { - if (utils.FileExists(ftw.fname)) { - utils.WatchFile(ftw.fname, () => { - logger.Log(ftw.logMsg) - ftw.action() - }) - ftw.action() + if (db) { + const requestedDb = questionDbs.find((qdb) => { + return qdb.name === db + }) + + if (!requestedDb) { + res.end(`No such db ${db}`) + return + } + + stringifiedData = '\n' + line + stringifiedData += ` Questions in ${requestedDb.name}: ` + stringifiedData += line + '\n' + stringifiedData += dataToString(requestedDb.data) + stringifiedData += '\n' + line + line + '\n' } else { - logger.Log( - `File ${ftw.fname} does not exists to watch!`, - logger.GetColor('redbg') - ) + stringifiedData = questionDbs + .map((qdb) => { + let result = '' + result += '\n' + line + result += ` Questions in ${qdb.name}: ` + result += line + '\n' + result += dataToString(qdb.data) + result += '\n' + line + line + '\n' + return result + }) + .join('\n\n') } - }) - }, - } + res.end(stringifiedData) + }) + + app.post('/isAdding', function (req: Request, res: Response) { + logger.LogReq(req) + const user: User = req.session.user + const dryRun = testUsers.includes(user.id) + if (!req.body.location) { + logger.Log( + '\tbody.location is missing, client version:' + req.body.version + ) + res.json({ msg: 'body.location is missing' }) + return + } + + writeIsAddingData(req.body) + + const location = req.body.location.split('/')[2] + + try { + let maxIndex = -1 + const suitedQuestionDbs = questionDbs.filter((qdb) => { + if (maxIndex < qdb.index) { + maxIndex = qdb.index + } + return shouldSaveDataFile(qdb, req.body) + }, []) + + if (suitedQuestionDbs.length === 0) { + if (!dbExists(location, questionDbs)) { + suitedQuestionDbs.push( + getNewQdb( + location, + maxIndex, + dbsFile, + publicDir, + questionDbs + ) + ) + } else { + logger.Log( + `Tried to add existing db named ${location}!`, + logger.GetColor('red') + ) + } + } + if (suitedQuestionDbs.length === 0) { + res.json({ + status: 'fail', + msg: 'No suitable dbs to add questions to', + }) + return + } + + processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user) + .then((resultArray: Array) => { + logResult(req.body, resultArray, user.id, dryRun) + + const totalNewQuestions = resultArray.reduce( + (acc, sres) => { + return acc + sres.newQuestions.length + }, + 0 + ) + + res.json({ + success: resultArray.length > 0, + newQuestions: resultArray, + totalNewQuestions: totalNewQuestions, + }) + + if (totalNewQuestions > 0) { + resultArray.forEach((result) => { + msgAllWorker({ + // TODO: recognize base64 image + type: 'newQuestions', + data: result, + }) + }) + } + }) + .catch((err) => { + logger.Log( + 'Error during processing incoming request', + logger.GetColor('redbg') + ) + console.error(err) + res.json({ + success: false, + }) + }) + } catch (err) { + logger.Log( + 'Error during getting incoming request processor promises ', + logger.GetColor('redbg') + ) + console.error(err) + res.json({ + success: false, + }) + } + }) + + app.post('/ask', function (req: Request, res) { + const user: User = req.session.user + + if (!req.body.questions) { + res.json({ + message: `ask something! { questions:'' ,subject:'', location:'' }`, + result: [], + success: false, + }) + return + } + const subj: string = req.body.subj || '' + // TODO: test if testUrl is undefined (it shouldnt) + const testUrl: string = req.body.testUrl + ? req.body.testUrl.split('/')[2] + : undefined + + writeAskData(req.body) + + // every question in a different thread + const resultPromises: Promise[] = + req.body.questions.map((question: Question) => { + return getResult({ + question: question, + subj: subj, + testUrl: testUrl, + questionDbs: questionDbs, + }) + }) + + Promise.all(resultPromises).then((results: DbSearchResult[]) => { + const response = results.map((result: DbSearchResult) => { + return { + answers: result.result, + question: result.question, + } + }) + res.json(response) + + const saveableQuestions = response.reduce((acc, res) => { + const save = res.answers.every((answer) => { + return answer.match < 90 + }) + + if (save) { + acc.push(res.question) + } + return acc + }, []) + + if (saveableQuestions.length > 0) { + saveQuestion( + saveableQuestions, + subj, + testUrl, + user.id, + savedQuestionsDir + ) + } + }) + }) + + app.get('/supportedSites', function (req: Request, res: Response) { + logger.LogReq(req) + + res.json(supportedSites) + }) + + app.get('/datacount', function (req: Request, res: Response) { + logger.LogReq(req) + if (req.query.detailed === 'all') { + res.json({ + detailed: getDetailedRes(questionDbs), + simple: getSimplreRes(questionDbs), + }) + } else if (req.query.detailed) { + res.json(getDetailedRes(questionDbs)) + } else { + res.json(getSimplreRes(questionDbs)) + } + }) + + app.get('/infos', function (req: Request, res) { + const user: User = req.session.user + + const result: { + result: string + uid: number + version?: string + subjinfo?: { + subjects: number + questions: number + } + motd?: string + } = { + result: 'success', + uid: user.id, + } + + if (req.query.subjinfo) { + result.subjinfo = getSimplreRes(questionDbs) + } + if (req.query.version) { + result.version = version + } + if (req.query.motd) { + result.motd = getMotd(req.query.cversion, motd) + } + res.json(result) + }) + + app.post('/registerscript', function (req: Request, res) { + logger.LogReq(req) + + if (!utils.FileExists(registeredScriptsFile)) { + utils.WriteFile('[]', registeredScriptsFile) + } + + const ua: string = req.headers['user-agent'] + const registeredScripts: RegisteredUserEntry[] = utils.ReadJSON( + registeredScriptsFile + ) + const { cid, uid, version, installSource, date } = req.body + + const index = registeredScripts.findIndex((registration) => { + return registration.cid === cid + }) + + if (index === -1) { + const x: RegisteredUserEntry = { + cid: cid, + version: version, + installSource: installSource, + date: date, + userAgent: ua, + } + + if (uid) { + x.uid = uid + x.loginDate = date + } + registeredScripts.push(x) + } else { + const currRegistration = registeredScripts[index] + + if (!currRegistration.uid && uid) { + registeredScripts[index] = { + ...registeredScripts[index], + uid: uid, + loginDate: date, + } + } else { + logger.DebugLog( + `cid: ${cid}, uid: ${uid} tried to register multiple times`, + 'register', + 1 + ) + } + } + + utils.WriteFile( + JSON.stringify(registeredScripts, null, 2), + registeredScriptsFile + ) + + res.json({ msg: 'done' }) + }) + + app.get('/possibleAnswers', (req: Request, res: Response) => { + logger.LogReq(req) + const files = utils.ReadDir(savedQuestionsDir) + + files.sort(function (a, b) { + return ( + fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() - + fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime() + ) + }) + + res.json({ + savedQuestionsFileName: savedQuestionsFileName, + subjects: files.map((subj) => { + return { + name: subj, + path: `savedQuestions/${subj}/`, + } + }), + }) + }) + + app.post('/rmPossibleAnswer', (req: Request, res: Response) => { + logger.LogReq(req) + const user: User = req.session.user + + const subj = req.body.subj + const file = req.body.file + const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}` + const savedQuestions: SavedQuestionData[] = + utils.ReadJSON(savedQuestionsPath) + let path = `${savedQuestionsDir}/${subj}/${file}` + while (path.includes('..')) { + path = path.replace(/\.\./g, '.') + } + + if (utils.FileExists(path)) { + utils.deleteFile(path) + + utils.WriteFile( + JSON.stringify( + savedQuestions.filter((sq) => { + return sq.fname !== file + }) + ), + savedQuestionsPath + ) + + logger.Log( + `User #${user.id} deleted '${file}' from subject '${subj}'`, + logger.GetColor('cyan') + ) + res.json({ + res: 'ok', + }) + } else { + logger.Log( + `User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`, + logger.GetColor('red') + ) + res.json({ + res: 'fail', + }) + } + }) + + app.post('/updateQuestion', (req: Request, res) => { + logger.LogReq(req) + const user: User = req.session.user + const date = utils.GetDateString() + + const editType = req.body.type + const selectedDb = req.body.selectedDb + if (!editType || !selectedDb) { + res.json({ + status: 'fail', + msg: 'No .editType or .selectedDb !', + }) + return + } + + const dbIndex = questionDbs.findIndex((qdb) => { + return qdb.name === selectedDb.name + }) + const currDb = questionDbs[dbIndex] + + if (dbIndex === -1) { + res.json({ + status: 'fail', + msg: `No question db named like ${selectedDb.name}!`, + }) + return + } + + // ----------------- + const { + success, + msg, + resultDb, + deletedQuestion, + newVal, + oldVal, + deletedQuestions, + changedQuestions, + } = editDb(currDb, req.body) + + if (!success) { + res.json({ success: success, msg: msg }) + return + } + if (resultDb) { + questionDbs[dbIndex] = resultDb + } + + if (editType === 'delete') { + const { index, subjName } = req.body + logger.Log( + `User #${user.id} deleted a question from '${subjName}'`, + logger.GetColor('cyan') + ) + utils.AppendToFile( + `${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`, + dataEditsLog + ) + utils.AppendToFile( + JSON.stringify(deletedQuestion, null, 2), + dataEditsLog + ) + } + + if (editType === 'edit') { + const { index, subjName } = req.body + logger.Log( + `User #${user.id} edited a question in '${subjName}'`, + logger.GetColor('cyan') + ) + utils.AppendToFile( + `${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`, + dataEditsLog + ) + utils.AppendToFile( + JSON.stringify( + { + newVal: newVal, + oldVal: oldVal, + }, + null, + 2 + ), + dataEditsLog + ) + } + + if (editType === 'subjEdit') { + const { subjName } = req.body + logger.Log( + `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, + logger.GetColor('cyan') + ) + utils.AppendToFile( + `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`, + dataEditsLog + ) + utils.AppendToFile( + JSON.stringify( + { + deletedQuestions: deletedQuestions, + changedQuestions: changedQuestions, + }, + null, + 2 + ), + dataEditsLog + ) + } + // ------------------ + + if (success) { + writeData(currDb.data, currDb.path) + msgAllWorker({ + type: 'dbEdit', + data: { + dbIndex: dbIndex, + edits: req.body, + }, + }) + } + + res.json({ + success: true, + msg: 'OK', + }) + }) + + let questionCleaner: ChildProcess = null + app.get('/clearQuestions', (req: Request, res) => { + // TODO: dont allow multiple instances + // TODO: get status of it cleaning + logger.LogReq(req) + const user: User = req.session.user + const status: string = req.query.status + + if (status) { + if (!questionCleaner) { + res.json({ + msg: 'question cleaner not running', + success: false, + }) + return + } + questionCleaner.once('message', function (response) { + res.json({ + msg: response, + success: true, + }) + }) + + questionCleaner.send({ data: 'asd' }) + return + } + + if (questionCleaner) { + res.json({ + msg: 'question cleaner already running', + success: false, + }) + return + } + + questionCleaner = fork( + `${process.cwd()}/src/standaloneUtils/rmDuplicates.js`, + ['-s', `${process.cwd()}/${questionDbs[0].path}`] + ) + questionCleaner.on('exit', function (code: number) { + console.log('EXIT', code) + questionCleaner = null + }) + + res.json({ + user: user, + success: true, + msg: 'OK', + }) + }) + + return { + dailyAction: () => { + backupData(questionDbs) + ExportDailyDataCount(questionDbs, userDB) + }, + load: () => { + backupData(questionDbs) + + filesToWatch.forEach((ftw) => { + if (utils.FileExists(ftw.fname)) { + utils.WatchFile(ftw.fname, () => { + logger.Log(ftw.logMsg) + ftw.action() + }) + ftw.action() + } else { + logger.Log( + `File ${ftw.fname} does not exists to watch!`, + logger.GetColor('redbg') + ) + } + }) + }, + } } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/quickvote.ts b/src/modules/api/submodules/quickvote.ts index 65485b7..1719ce0 100644 --- a/src/modules/api/submodules/quickvote.ts +++ b/src/modules/api/submodules/quickvote.ts @@ -26,98 +26,101 @@ import type { Response } from 'express' const quickVoteResultsDir = 'stats/qvote' const quickVotes = 'stats/qvote/votes.json' interface QuickVotes { - voteNames?: string[] + voteNames?: string[] } interface QuickVote { - votes: { - [key: string]: string - } - sum: { - [key: string]: number - } + votes: { + [key: string]: string + } + sum: { + [key: string]: number + } } function setup(data: SubmoduleData): void { - const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data + const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data - app.get('/quickvote', (req: Request, res: Response) => { - const key = req.query.key.toString() - const val: string = req.query.val - const user: User = req.session.user + app.get('/quickvote', (req: Request, res: Response) => { + const key = req.query.key.toString() + const val: string = req.query.val + const user: User = req.session.user - if (!key || !val) { - res.render('votethank', { - results: 'error', - msg: 'no key or val query param!', - }) - return - } + if (!key || !val) { + res.render('votethank', { + results: 'error', + msg: 'no key or val query param!', + }) + return + } - // FIXME: check vote type in file - let votes: QuickVotes = {} - if (utils.FileExists(quickVotes)) { - votes = utils.ReadJSON(quickVotes) - } else { - logger.Log( - `No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`, - logger.GetColor('blue') - ) - res.render('votethank', { - result: 'no such pool', - }) - return - } + // FIXME: check vote type in file + let votes: QuickVotes = {} + if (utils.FileExists(quickVotes)) { + votes = utils.ReadJSON(quickVotes) + } else { + logger.Log( + `No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`, + logger.GetColor('blue') + ) + res.render('votethank', { + result: 'no such pool', + }) + return + } - if (!votes.voteNames.includes(key)) { - logger.Log( - `No such vote "${key}" ( #${user.id}: ${key}-${val} )`, - logger.GetColor('blue') - ) - res.render('votethank', { - result: 'no such pool', - }) - return - } + if (!votes.voteNames.includes(key)) { + logger.Log( + `No such vote "${key}" ( #${user.id}: ${key}-${val} )`, + logger.GetColor('blue') + ) + res.render('votethank', { + result: 'no such pool', + }) + return + } - const voteFile = quickVoteResultsDir + '/' + key + '.json' + const voteFile = quickVoteResultsDir + '/' + key + '.json' - let voteData: QuickVote = { - votes: {}, - sum: {}, - } + let voteData: QuickVote = { + votes: {}, + sum: {}, + } - if (utils.FileExists(voteFile)) { - voteData = utils.ReadJSON(voteFile) - } else { - utils.CreatePath(quickVoteResultsDir) - } + if (utils.FileExists(voteFile)) { + voteData = utils.ReadJSON(voteFile) + } else { + utils.CreatePath(quickVoteResultsDir) + } - const prevVote = voteData.votes[user.id] + const prevVote = voteData.votes[user.id] - voteData.votes[user.id] = val - if (voteData.sum[val]) { - voteData.sum[val]++ - } else { - voteData.sum[val] = 1 - } - if (prevVote) { - if (voteData.sum[prevVote]) { - voteData.sum[prevVote] -= 1 - } - } + voteData.votes[user.id] = val + if (voteData.sum[val]) { + voteData.sum[val]++ + } else { + voteData.sum[val] = 1 + } + if (prevVote) { + if (voteData.sum[prevVote]) { + voteData.sum[prevVote] -= 1 + } + } - logger.Log(`Vote from #${user.id}: ${key}: ${val}`, logger.GetColor('blue')) - res.render('votethank', { - result: prevVote ? 'already voted' : 'success', - prevVote: prevVote, - msg: 'vote added', + logger.Log( + `Vote from #${user.id}: ${key}: ${val}`, + logger.GetColor('blue') + ) + res.render('votethank', { + result: prevVote ? 'already voted' : 'success', + prevVote: prevVote, + msg: 'vote added', + }) + + utils.WriteFile(JSON.stringify(voteData), voteFile) }) - - utils.WriteFile(JSON.stringify(voteData), voteFile) - }) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/ranklist.ts b/src/modules/api/submodules/ranklist.ts index fa01690..aa88d4d 100644 --- a/src/modules/api/submodules/ranklist.ts +++ b/src/modules/api/submodules/ranklist.ts @@ -23,121 +23,128 @@ import utils from '../../../utils/utils' import { Request, SubmoduleData, User } from '../../../types/basicTypes' interface Subjects { - [key: string]: number + [key: string]: number } interface IdStat { - count: number - newQuestions: number - allQuestions: number - subjs: Subjects + count: number + newQuestions: number + allQuestions: number + subjs: Subjects } interface IdStats { - [key: string]: IdStat + [key: string]: IdStat } interface IdStatWithUID extends IdStat { - userId: number + userId: number } const idStatFile = 'stats/idstats' const idvStatFile = 'stats/idvstats' function mergeObjSum(a: Subjects, b: Subjects) { - const res = { ...b } - Object.keys(a).forEach((key) => { - if (res[key]) { - res[key] += a[key] - } else { - res[key] = a[key] - } - }) + const res = { ...b } + Object.keys(a).forEach((key) => { + if (res[key]) { + res[key] += a[key] + } else { + res[key] = a[key] + } + }) - return res + return res } function setup(data: SubmoduleData): void { - const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data + const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data - app.get('/ranklist', (req: Request, res) => { - logger.LogReq(req) - let result: IdStats - const querySince: string = req.query.since - const user: User = req.session.user + app.get('/ranklist', (req: Request, res) => { + logger.LogReq(req) + let result: IdStats + const querySince: string = req.query.since + const user: User = req.session.user - if (!querySince) { - result = utils.ReadJSON(idStatFile) - } else { - try { - const since = new Date(querySince) - if (!(since instanceof Date) || isNaN(since.getTime())) { - throw new Error('Not a date') - } - const data = utils.ReadJSON(idvStatFile) - result = {} - - Object.keys(data).forEach((key) => { - const dailyStat = data[key] - - if (new Date(key) > since) { - Object.keys(dailyStat).forEach((userId) => { - const userStat = dailyStat[userId] - const uidRes = result[userId] - - if (!uidRes) { - result[userId] = userStat - } else { - result[userId] = { - count: uidRes.count + userStat.count, - newQuestions: uidRes.newQuestions + userStat.newQuestions, - allQuestions: uidRes.allQuestions + userStat.allQuestions, - subjs: mergeObjSum(uidRes.subjs, userStat.subjs), + if (!querySince) { + result = utils.ReadJSON(idStatFile) + } else { + try { + const since = new Date(querySince) + if (!(since instanceof Date) || isNaN(since.getTime())) { + throw new Error('Not a date') } - } + const data = utils.ReadJSON(idvStatFile) + result = {} + + Object.keys(data).forEach((key) => { + const dailyStat = data[key] + + if (new Date(key) > since) { + Object.keys(dailyStat).forEach((userId) => { + const userStat = dailyStat[userId] + const uidRes = result[userId] + + if (!uidRes) { + result[userId] = userStat + } else { + result[userId] = { + count: uidRes.count + userStat.count, + newQuestions: + uidRes.newQuestions + + userStat.newQuestions, + allQuestions: + uidRes.allQuestions + + userStat.allQuestions, + subjs: mergeObjSum( + uidRes.subjs, + userStat.subjs + ), + } + } + }) + } + }) + } catch (err) { + res.json({ + msg: 'invalid date format, or other error occured', + }) + } + } + + const list: Array = [] + const sum = { + count: 0, + newQuestions: 0, + allQuestions: 0, + } + Object.keys(result).forEach((key) => { + list.push({ + userId: parseInt(key), + ...result[key], }) - } + + sum.count = sum.count + result[key].count + sum.newQuestions = sum.newQuestions + result[key].newQuestions + sum.allQuestions = sum.allQuestions + result[key].allQuestions }) - } catch (err) { + + if (list.length === 0) { + res.json({ + msg: 'There are no users in the stats db :c', + }) + return + } + res.json({ - msg: 'invalid date format, or other error occured', + since: querySince, + sum: sum, + list: list, + selfuserId: user.id, }) - } - } - - const list: Array = [] - const sum = { - count: 0, - newQuestions: 0, - allQuestions: 0, - } - Object.keys(result).forEach((key) => { - list.push({ - userId: parseInt(key), - ...result[key], - }) - - sum.count = sum.count + result[key].count - sum.newQuestions = sum.newQuestions + result[key].newQuestions - sum.allQuestions = sum.allQuestions + result[key].allQuestions }) - - if (list.length === 0) { - res.json({ - msg: 'There are no users in the stats db :c', - }) - return - } - - res.json({ - since: querySince, - sum: sum, - list: list, - selfuserId: user.id, - }) - }) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/todos.ts b/src/modules/api/submodules/todos.ts index 53e5d11..1593305 100644 --- a/src/modules/api/submodules/todos.ts +++ b/src/modules/api/submodules/todos.ts @@ -25,110 +25,110 @@ import utils from '../../../utils/utils' import { Request, SubmoduleData } from '../../../types/basicTypes' interface Categories { - [key: string]: { - name: string - color: string - } + [key: string]: { + name: string + color: string + } } enum CardState { - TODO = 'todo', - INPROGRESS = 'inprogress', - TESTING = 'testing', - DONE = 'done', - INPROD = 'inprod', - NOTPOSSIBLE = 'notpossible', + TODO = 'todo', + INPROGRESS = 'inprogress', + TESTING = 'testing', + DONE = 'done', + INPROD = 'inprod', + NOTPOSSIBLE = 'notpossible', } interface Card { - id: number - name: string - description: string - category: string - points: number - state: CardState - votes: number[] + id: number + name: string + description: string + category: string + points: number + state: CardState + votes: number[] } type Columns = { - [key in CardState]: { - name: string - clickable: boolean - } + [key in CardState]: { + name: string + clickable: boolean + } } interface Groups { - [key: string]: { - name: string - description: string - } + [key: string]: { + name: string + description: string + } } interface Todos { - categories: Categories - cards: Card[] - columns: Columns - groups: Groups + categories: Categories + cards: Card[] + columns: Columns + groups: Groups } const todosFile = 'data/todos.json' function setup(data: SubmoduleData): void { - const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data + const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data - app.get('/voteTodo', (req: Request, res: Response) => { - logger.LogReq(req) - const userId = req.session.user.id - const id: string = req.query.id - const todos: Todos = utils.ReadJSON(todosFile) + app.get('/voteTodo', (req: Request, res: Response) => { + logger.LogReq(req) + const userId = req.session.user.id + const id: string = req.query.id + const todos: Todos = utils.ReadJSON(todosFile) - if (!id) { - res.json({ - msg: 'id query undefined', - result: 'not ok', - }) - } + if (!id) { + res.json({ + msg: 'id query undefined', + result: 'not ok', + }) + } - const cardIndex = todos.cards.findIndex((currcard) => { - return currcard.id === parseInt(id) + const cardIndex = todos.cards.findIndex((currcard) => { + return currcard.id === parseInt(id) + }) + if (cardIndex === -1) { + res.json({ + msg: 'card not found', + result: 'not ok', + }) + return + } + + const ind = todos.cards[cardIndex].votes.indexOf(userId) + if (ind === -1) { + todos.cards[cardIndex].votes.push(userId) + } else { + todos.cards[cardIndex].votes.splice(ind, 1) + } + + utils.WriteFile(JSON.stringify(todos, null, 2), todosFile) + res.json({ + todos: todos, + userId: userId, + msg: 'updated', + result: 'ok', + }) }) - if (cardIndex === -1) { - res.json({ - msg: 'card not found', - result: 'not ok', - }) - return - } - const ind = todos.cards[cardIndex].votes.indexOf(userId) - if (ind === -1) { - todos.cards[cardIndex].votes.push(userId) - } else { - todos.cards[cardIndex].votes.splice(ind, 1) - } + app.get('/todos', (req: Request, res: Response) => { + logger.LogReq(req) + const userId = req.session.user.id + const todos = utils.ReadJSON(todosFile) - utils.WriteFile(JSON.stringify(todos, null, 2), todosFile) - res.json({ - todos: todos, - userId: userId, - msg: 'updated', - result: 'ok', + res.json({ + todos: todos, + userId: userId, + result: 'ok', + }) }) - }) - - app.get('/todos', (req: Request, res: Response) => { - logger.LogReq(req) - const userId = req.session.user.id - const todos = utils.ReadJSON(todosFile) - - res.json({ - todos: todos, - userId: userId, - result: 'ok', - }) - }) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/userFiles.ts b/src/modules/api/submodules/userFiles.ts index ad1c661..b992645 100644 --- a/src/modules/api/submodules/userFiles.ts +++ b/src/modules/api/submodules/userFiles.ts @@ -27,320 +27,324 @@ import { Request, SubmoduleData, User } from '../../../types/basicTypes' const dataFileName = '.data.json' function listDir(publicDir: string, subdir: string, userFilesDir: string) { - const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '') - const dir = userFilesDir + '/' + safeSubdir - const usersFile = dir + '/' + dataFileName + const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '') + const dir = userFilesDir + '/' + safeSubdir + const usersFile = dir + '/' + dataFileName - if (!utils.FileExists(dir)) { - return { - success: false, - msg: `Directory ${subdir} does not exists`, + if (!utils.FileExists(dir)) { + return { + success: false, + msg: `Directory ${subdir} does not exists`, + } } - } - if (!utils.FileExists(usersFile)) { - utils.WriteFile('{}', usersFile) - } - const users = utils.ReadJSON(usersFile) - - if (!utils.FileExists(dir)) { - return { - success: false, - msg: `Path '${safeSubdir}' does not exists`, + if (!utils.FileExists(usersFile)) { + utils.WriteFile('{}', usersFile) } - } + const users = utils.ReadJSON(usersFile) - return { - success: true, - files: utils.ReadDir(dir).reduce((acc, file) => { - const stat = fs.lstatSync(dir + '/' + file) + if (!utils.FileExists(dir)) { + return { + success: false, + msg: `Path '${safeSubdir}' does not exists`, + } + } - if (stat.isDirectory()) { - return acc - } + return { + success: true, + files: utils.ReadDir(dir).reduce((acc, file) => { + const stat = fs.lstatSync(dir + '/' + file) - acc.push({ - name: file, - path: dir.replace(publicDir, '') + '/' + file, - size: stat.size, - date: stat.mtime.getTime(), - user: users && users[file] ? users[file].uid : -1, - views: - users && users[file] && users[file].views ? users[file].views : 0, - upvotes: - users && users[file] && users[file].upvotes - ? users[file].upvotes - : [], - downvotes: - users && users[file] && users[file].downvotes - ? users[file].downvotes - : [], - }) - return acc - }, []), - } + if (stat.isDirectory()) { + return acc + } + + acc.push({ + name: file, + path: dir.replace(publicDir, '') + '/' + file, + size: stat.size, + date: stat.mtime.getTime(), + user: users && users[file] ? users[file].uid : -1, + views: + users && users[file] && users[file].views + ? users[file].views + : 0, + upvotes: + users && users[file] && users[file].upvotes + ? users[file].upvotes + : [], + downvotes: + users && users[file] && users[file].downvotes + ? users[file].downvotes + : [], + }) + return acc + }, []), + } } function setup(data: SubmoduleData): void { - const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data + const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data - app.use((req: Request, _res, next) => { - // /userFiles/test/2021-04-28_10-59.png - try { - if (req.url.includes('/userFiles/')) { + app.use((req: Request, _res, next) => { + // /userFiles/test/2021-04-28_10-59.png + try { + if (req.url.includes('/userFiles/')) { + logger.LogReq(req) + const safePath = decodeURIComponent(req.url) + .split('?')[0] + .replace(/\.+/g, '.') + .replace(/\/+/g, '/') + const x = safePath.split('/') + const dir = x[2] + const fname = x.pop() + const dataFilePath = + userFilesDir + '/' + dir + '/' + dataFileName + + const data = utils.ReadJSON(dataFilePath) + + if (data[fname]) { + if (!data[fname].views) { + data[fname].views = 0 + } + data[fname].views = data[fname].views + 1 + + utils.WriteFile(JSON.stringify(data), dataFilePath) + } + } + } catch (e) { + console.error(e) + logger.Log( + `Error trying to update view count on ${req.url}`, + logger.GetColor('redbg') + ) + } + next() + }) + + const publicDir = publicdirs[0] + + const userFilesDir = publicDir + 'userFiles' + if (!utils.FileExists(userFilesDir)) { + utils.CreatePath(userFilesDir, true) + } + + app.get('/listUserDir', (req: Request, res) => { logger.LogReq(req) - const safePath = decodeURIComponent(req.url) - .split('?')[0] - .replace(/\.+/g, '.') - .replace(/\/+/g, '/') + + if (!utils.FileExists(userFilesDir)) { + utils.CreatePath(userFilesDir, true) + } + + const subdir: string = req.query.subdir + + if (subdir) { + const result = listDir(publicDir, subdir, userFilesDir) + res.json(result) + } else { + res.json({ + success: true, + dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => { + const stat = fs.lstatSync(userFilesDir + '/' + file) + + if (!stat.isDirectory()) { + return acc + } + + acc.push({ + name: file, + date: stat.mtime.getTime(), + size: utils.ReadDir(userFilesDir + '/' + file).length, + }) + return acc + }, []), + }) + } + }) + + app.post( + '/deleteUserFile', + (req: Request<{ dir: string; fname: string }>, res) => { + logger.LogReq(req) + const dir: string = req.body.dir + const fname: string = req.body.fname + if (!dir || !fname) { + res.json({ + success: false, + msg: `'dir' or 'fname' is undefined!`, + }) + return + } + const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '') + const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '') + const filePath = userFilesDir + '/' + safeDir + '/' + safeFname + + if (!utils.FileExists(filePath)) { + res.json({ + success: false, + msg: `path does not exists!`, + }) + return + } + utils.deleteFile(filePath) + const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName + const users = utils.ReadJSON(usersFile) + delete users[safeFname] + utils.WriteFile(JSON.stringify(users), usersFile) + + res.json({ + success: true, + }) + } + ) + + app.post('/newUserDir', (req: Request<{ name: string }>, res) => { + logger.LogReq(req) + + const name: string = req.body.name + if (!name) { + res.json({ + success: false, + msg: `name is undefined!`, + }) + return + } + const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '') + + if (utils.FileExists(userFilesDir + '/' + safeName)) { + res.json({ + success: false, + msg: `Dir ${name} already exists`, + }) + return + } + utils.CreatePath(userFilesDir + '/' + safeName, true) + + res.json({ + success: true, + }) + }) + + app.post('/uploadUserFile', (req: Request<{ dir: string }>, res) => { + logger.LogReq(req) + + const user: User = req.session.user + const dir = req.body.dir + if (!dir) { + res.json({ + success: false, + msg: `dir '${dir}' is undefined!`, + }) + return + } + const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/') + if (!utils.FileExists(userFilesDir + '/' + safeDir)) { + res.json({ + success: false, + msg: `dir '${dir}' does not exists!`, + }) + return + } + + utils + .uploadFile(req, userFilesDir + '/' + safeDir) + .then((body) => { + logger.Log( + `Successfull upload ${body.filePath}`, + logger.GetColor('blue') + ) + + const usersFile = + userFilesDir + '/' + safeDir + '/' + dataFileName + const users = utils.ReadJSON(usersFile) + users[body.fileName] = { uid: user.id } + utils.WriteFile(JSON.stringify(users), usersFile) + + res.json({ + success: true, + }) + }) + .catch(() => { + res.json({ success: false, msg: 'something bad happened :s' }) + }) + }) + + app.post('/voteFile', (req: Request<{ path: string; to: string }>, res) => { + logger.LogReq(req) + const user: User = req.session.user + // { path: 'userFiles/test/2021-04-28_10-59.png', to: 'up' } 19 + const { path, to } = req.body + const safePath = path.replace(/\.+/g, '.').replace(/\/+/g, '/') const x = safePath.split('/') - const dir = x[2] + const dir = x[1] const fname = x.pop() const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName const data = utils.ReadJSON(dataFilePath) if (data[fname]) { - if (!data[fname].views) { - data[fname].views = 0 - } - data[fname].views = data[fname].views + 1 + if (!data[fname].upvotes) { + data[fname].upvotes = [] + } + if (!data[fname].downvotes) { + data[fname].downvotes = [] + } - utils.WriteFile(JSON.stringify(data), dataFilePath) + const removeVote = (from: number[], uid: number) => { + if (!from.includes(uid)) { + return from + } + return from.reduce((acc, id) => { + if (id !== uid) { + acc = [...acc, id] + } + return acc + }, []) + } + + data[fname].downvotes = removeVote(data[fname].downvotes, user.id) + data[fname].upvotes = removeVote(data[fname].upvotes, user.id) + + if (to === 'up') { + data[fname].upvotes = [...data[fname].upvotes, user.id] + } else if (to === 'down') { + data[fname].downvotes = [...data[fname].downvotes, user.id] + } else if (to === 'clear') { + // ... already cleared + } + + utils.WriteFile(JSON.stringify(data), dataFilePath) } - } - } catch (e) { - console.error(e) - logger.Log( - `Error trying to update view count on ${req.url}`, - logger.GetColor('redbg') - ) - } - next() - }) - const publicDir = publicdirs[0] - - const userFilesDir = publicDir + 'userFiles' - if (!utils.FileExists(userFilesDir)) { - utils.CreatePath(userFilesDir, true) - } - - app.get('/listUserDir', (req: Request, res) => { - logger.LogReq(req) - - if (!utils.FileExists(userFilesDir)) { - utils.CreatePath(userFilesDir, true) - } - - const subdir: string = req.query.subdir - - if (subdir) { - const result = listDir(publicDir, subdir, userFilesDir) - res.json(result) - } else { - res.json({ - success: true, - dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => { - const stat = fs.lstatSync(userFilesDir + '/' + file) - - if (!stat.isDirectory()) { - return acc - } - - acc.push({ - name: file, - date: stat.mtime.getTime(), - size: utils.ReadDir(userFilesDir + '/' + file).length, - }) - return acc - }, []), - }) - } - }) - - app.post( - '/deleteUserFile', - (req: Request<{ dir: string; fname: string }>, res) => { - logger.LogReq(req) - const dir: string = req.body.dir - const fname: string = req.body.fname - if (!dir || !fname) { - res.json({ - success: false, - msg: `'dir' or 'fname' is undefined!`, - }) - return - } - const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '') - const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '') - const filePath = userFilesDir + '/' + safeDir + '/' + safeFname - - if (!utils.FileExists(filePath)) { - res.json({ - success: false, - msg: `path does not exists!`, - }) - return - } - utils.deleteFile(filePath) - const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName - const users = utils.ReadJSON(usersFile) - delete users[safeFname] - utils.WriteFile(JSON.stringify(users), usersFile) - - res.json({ - success: true, - }) - } - ) - - app.post('/newUserDir', (req: Request<{ name: string }>, res) => { - logger.LogReq(req) - - const name: string = req.body.name - if (!name) { - res.json({ - success: false, - msg: `name is undefined!`, - }) - return - } - const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '') - - if (utils.FileExists(userFilesDir + '/' + safeName)) { - res.json({ - success: false, - msg: `Dir ${name} already exists`, - }) - return - } - utils.CreatePath(userFilesDir + '/' + safeName, true) - - res.json({ - success: true, + const result = listDir(publicDir, dir, userFilesDir) + res.json(result) }) - }) - app.post('/uploadUserFile', (req: Request<{ dir: string }>, res) => { - logger.LogReq(req) + app.post('/deleteDir', (req: Request<{ name: string }>, res) => { + logger.LogReq(req) + const { name } = req.body - const user: User = req.session.user - const dir = req.body.dir - if (!dir) { - res.json({ - success: false, - msg: `dir '${dir}' is undefined!`, - }) - return - } - const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/') - if (!utils.FileExists(userFilesDir + '/' + safeDir)) { - res.json({ - success: false, - msg: `dir '${dir}' does not exists!`, - }) - return - } + const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '') - utils - .uploadFile(req, userFilesDir + '/' + safeDir) - .then((body) => { - logger.Log( - `Successfull upload ${body.filePath}`, - logger.GetColor('blue') - ) - - const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName - const users = utils.ReadJSON(usersFile) - users[body.fileName] = { uid: user.id } - utils.WriteFile(JSON.stringify(users), usersFile) - - res.json({ - success: true, - }) - }) - .catch(() => { - res.json({ success: false, msg: 'something bad happened :s' }) - }) - }) - - app.post('/voteFile', (req: Request<{ path: string; to: string }>, res) => { - logger.LogReq(req) - const user: User = req.session.user - // { path: 'userFiles/test/2021-04-28_10-59.png', to: 'up' } 19 - const { path, to } = req.body - const safePath = path.replace(/\.+/g, '.').replace(/\/+/g, '/') - const x = safePath.split('/') - const dir = x[1] - const fname = x.pop() - const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName - - const data = utils.ReadJSON(dataFilePath) - - if (data[fname]) { - if (!data[fname].upvotes) { - data[fname].upvotes = [] - } - if (!data[fname].downvotes) { - data[fname].downvotes = [] - } - - const removeVote = (from: number[], uid: number) => { - if (!from.includes(uid)) { - return from + if (!utils.FileExists(userFilesDir + '/' + safeName)) { + res.json({ + success: false, + msg: `Dir ${name} does not exist!`, + }) + return + } + utils.CreatePath(userFilesDir + '/' + safeName, true) + const result = listDir(publicDir, name, userFilesDir) + if (result.files.length === 0) { + utils.deleteDir(userFilesDir + '/' + safeName) + } else { + res.json({ succes: false, msg: `Dir ${name} is not empty!` }) + return } - return from.reduce((acc, id) => { - if (id !== uid) { - acc = [...acc, id] - } - return acc - }, []) - } - data[fname].downvotes = removeVote(data[fname].downvotes, user.id) - data[fname].upvotes = removeVote(data[fname].upvotes, user.id) - - if (to === 'up') { - data[fname].upvotes = [...data[fname].upvotes, user.id] - } else if (to === 'down') { - data[fname].downvotes = [...data[fname].downvotes, user.id] - } else if (to === 'clear') { - // ... already cleared - } - - utils.WriteFile(JSON.stringify(data), dataFilePath) - } - - const result = listDir(publicDir, dir, userFilesDir) - res.json(result) - }) - - app.post('/deleteDir', (req: Request<{ name: string }>, res) => { - logger.LogReq(req) - const { name } = req.body - - const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '') - - if (!utils.FileExists(userFilesDir + '/' + safeName)) { - res.json({ - success: false, - msg: `Dir ${name} does not exist!`, - }) - return - } - utils.CreatePath(userFilesDir + '/' + safeName, true) - const result = listDir(publicDir, name, userFilesDir) - if (result.files.length === 0) { - utils.deleteDir(userFilesDir + '/' + safeName) - } else { - res.json({ succes: false, msg: `Dir ${name} is not empty!` }) - return - } - - res.json({ succes: true }) - }) + res.json({ succes: true }) + }) } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/submodules/userManagement.ts b/src/modules/api/submodules/userManagement.ts index 287355b..65969eb 100644 --- a/src/modules/api/submodules/userManagement.ts +++ b/src/modules/api/submodules/userManagement.ts @@ -24,10 +24,10 @@ import type { Database } from 'better-sqlite3' import logger from '../../../utils/logger' import utils from '../../../utils/utils' import { - Request, - SubmoduleData, - User, - Submodule, + Request, + SubmoduleData, + User, + Submodule, } from '../../../types/basicTypes' import dbtools from '../../../utils/dbtools' @@ -38,321 +38,322 @@ const maxPWCount = 3 const daysAfterUserGetsPWs = 7 // days after user gets pw-s interface Session { - id: string - userId: number - createDate: string - lastAccess: string - isScript: number + id: string + userId: number + createDate: string + lastAccess: string + isScript: number } function BackupDB(usersDbBackupPath: string, userDB: Database) { - logger.Log('Backing up auth DB ...') - utils.CreatePath(usersDbBackupPath, true) - userDB - .backup( - `${usersDbBackupPath}/users.${utils - .GetDateString() - .replace(/ /g, '_')}.db` - ) - .then(() => { - logger.Log('Auth DB backup complete!') - }) - .catch((err: Error) => { - logger.Log('Auth DB backup failed!', logger.GetColor('redbg')) - console.error(err) - }) + logger.Log('Backing up auth DB ...') + utils.CreatePath(usersDbBackupPath, true) + userDB + .backup( + `${usersDbBackupPath}/users.${utils + .GetDateString() + .replace(/ /g, '_')}.db` + ) + .then(() => { + logger.Log('Auth DB backup complete!') + }) + .catch((err: Error) => { + logger.Log('Auth DB backup failed!', logger.GetColor('redbg')) + console.error(err) + }) } function setup(data: SubmoduleData): Submodule { - const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data - let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] - domain.shift() // [ "frylabs", "net" ] - domain = domain.join('.') // "frylabs.net" - logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) + const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data + let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] + domain.shift() // [ "frylabs", "net" ] + domain = domain.join('.') // "frylabs.net" + logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) - app.get('/avaiblePWS', (req: Request, res: any) => { - logger.LogReq(req) + app.get('/avaiblePWS', (req: Request, res: any) => { + logger.LogReq(req) - const user: User = req.session.user + const user: User = req.session.user - res.json({ - success: true, - userCreated: user.created, - availablePWS: user.avaiblePWRequests, - requestedPWS: user.pwRequestCount, - maxPWCount: maxPWCount, - daysAfterUserGetsPWs: daysAfterUserGetsPWs, - dayDiff: getDayDiff(user.created), - userCount: dbtools.TableInfo(userDB, 'users').dataCount, - }) - }) - - app.post('/getpw', function (req: Request, res: any) { - logger.LogReq(req) - - const requestingUser = req.session.user - - if (requestingUser.avaiblePWRequests <= 0) { - res.json({ - result: 'error', - success: false, - msg: 'Too many passwords requested or cant request password yet, try later', - }) - logger.Log( - `User #${requestingUser.id} requested too much passwords`, - logger.GetColor('cyan') - ) - return - } - - dbtools.Update( - userDB, - 'users', - { - avaiblePWRequests: requestingUser.avaiblePWRequests - 1, - pwRequestCount: requestingUser.pwRequestCount + 1, - }, - { - id: requestingUser.id, - } - ) - - const pw = uuidv4() - const insertRes = dbtools.Insert(userDB, 'users', { - pw: pw, - avaiblePWRequests: 0, - created: utils.GetDateString(), - createdBy: requestingUser.id, - }) - - logger.Log( - `User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`, - logger.GetColor('cyan') - ) - - res.json({ - pw: pw, - success: true, - userCreated: requestingUser.created, - availablePWS: requestingUser.avaiblePWRequests, - requestedPWS: requestingUser.pwRequestCount, - maxPWCount: maxPWCount, - daysAfterUserGetsPWs: daysAfterUserGetsPWs, - dayDiff: getDayDiff(requestingUser.created), - userCount: dbtools.TableInfo(userDB, 'users').dataCount, - }) - }) - - app.post('/login', (req: Request, res: any) => { - logger.LogReq(req) - const pw = req.body.pw - ? req.body.pw.replace(/'/g, '').replace(/"/g, '').replace(/;/g, '') - : false - const isScript = req.body.script - const user: User = dbtools.Select(userDB, 'users', { - pw: pw, - })[0] - - if (user) { - const sessionID = uuidv4() - - const existingSessions = dbtools - .Select(userDB, 'sessions', { - userID: user.id, - isScript: isScript ? 1 : 0, - }) - .sort((a: Session, b: Session) => { - return ( - new Date(a.lastAccess).getTime() - new Date(b.lastAccess).getTime() - ) + res.json({ + success: true, + userCreated: user.created, + availablePWS: user.avaiblePWRequests, + requestedPWS: user.pwRequestCount, + maxPWCount: maxPWCount, + daysAfterUserGetsPWs: daysAfterUserGetsPWs, + dayDiff: getDayDiff(user.created), + userCount: dbtools.TableInfo(userDB, 'users').dataCount, }) + }) - const diff = existingSessions.length - minimumAlowwedSessions - if (diff > 0) { - logger.Log( - `Multiple ${isScript ? 'script' : 'website'} sessions ( ${ - existingSessions.length - } ) for #${user.id}, deleting olds`, - logger.GetColor('cyan') + app.post('/getpw', function (req: Request, res: any) { + logger.LogReq(req) + + const requestingUser = req.session.user + + if (requestingUser.avaiblePWRequests <= 0) { + res.json({ + result: 'error', + success: false, + msg: 'Too many passwords requested or cant request password yet, try later', + }) + logger.Log( + `User #${requestingUser.id} requested too much passwords`, + logger.GetColor('cyan') + ) + return + } + + dbtools.Update( + userDB, + 'users', + { + avaiblePWRequests: requestingUser.avaiblePWRequests - 1, + pwRequestCount: requestingUser.pwRequestCount + 1, + }, + { + id: requestingUser.id, + } ) - for (let i = 0; i < diff; i++) { - const id = existingSessions[i].id - dbtools.Delete(userDB, 'sessions', { - id: id, - isScript: isScript ? 1 : 0, - }) - } - } - dbtools.Update( - userDB, - 'users', - { - loginCount: user.loginCount + 1, - lastLogin: utils.GetDateString(), - }, - { - id: user.id, - } - ) + const pw = uuidv4() + const insertRes = dbtools.Insert(userDB, 'users', { + pw: pw, + avaiblePWRequests: 0, + created: utils.GetDateString(), + createdBy: requestingUser.id, + }) - dbtools.Insert(userDB, 'sessions', { - id: sessionID, - userID: user.id, - isScript: isScript ? 1 : 0, - createDate: utils.GetDateString(), - }) - - // https://www.npmjs.com/package/cookie - res.cookie('sessionID', sessionID, { - domain: domain, - expires: new Date( - new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000 - ), - sameSite: 'none', - secure: true, - }) - res.cookie('sessionID', sessionID, { - expires: new Date( - new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000 - ), - sameSite: 'none', - secure: true, - }) - - res.json({ - result: 'success', - msg: 'you are now logged in', - }) - logger.Log( - `Successfull login to ${ - isScript ? 'script' : 'website' - } with user ID: #${user.id}`, - logger.GetColor('cyan') - ) - } else { - logger.Log( - `Login attempt with invalid pw: ${pw} to ${ - isScript ? 'script' : 'website' - }`, - logger.GetColor('cyan') - ) - res.json({ - result: 'error', - msg: 'Invalid password', - }) - } - }) - - app.get('/logout', (req: Request, res: any) => { - logger.LogReq(req) - const sessionID = req.cookies.sessionID - const user: User = req.session.user - const { all } = req.query - - if (!user) { - res.json({ - msg: 'You are not logged in', - success: false, - }) - return - } - - logger.Log( - `Successfull logout with user ID: #${user.id}`, - logger.GetColor('cyan') - ) - - if (all) { - dbtools.Delete(userDB, 'sessions', { - userID: user.id, - }) - } else { - dbtools.Delete(userDB, 'sessions', { - id: sessionID, - }) - } - - res.clearCookie('sessionID').json({ - msg: 'Successfull logout', - result: 'success', - }) - }) - - function getDayDiff(dateString: string | Date) { - const msdiff = new Date().getTime() - new Date(dateString).getTime() - return Math.floor(msdiff / (1000 * 3600 * 24)) - } - - function IncrementAvaiblePWs() { - // FIXME: check this if this is legit and works - logger.Log('Incrementing avaible PW-s ...') - const users: Array = dbtools.SelectAll(userDB, 'users') - const day = new Date().getDay() - - if (day === 1) { - users.forEach((user) => { - const dayDiff = getDayDiff(user.created) - if (dayDiff < daysAfterUserGetsPWs) { - logger.Log( - `User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`, + logger.Log( + `User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`, logger.GetColor('cyan') - ) - return - } + ) - if (user.avaiblePWRequests >= maxPWCount) { - return + res.json({ + pw: pw, + success: true, + userCreated: requestingUser.created, + availablePWS: requestingUser.avaiblePWRequests, + requestedPWS: requestingUser.pwRequestCount, + maxPWCount: maxPWCount, + daysAfterUserGetsPWs: daysAfterUserGetsPWs, + dayDiff: getDayDiff(requestingUser.created), + userCount: dbtools.TableInfo(userDB, 'users').dataCount, + }) + }) + + app.post('/login', (req: Request, res: any) => { + logger.LogReq(req) + const pw = req.body.pw + ? req.body.pw.replace(/'/g, '').replace(/"/g, '').replace(/;/g, '') + : false + const isScript = req.body.script + const user: User = dbtools.Select(userDB, 'users', { + pw: pw, + })[0] + + if (user) { + const sessionID = uuidv4() + + const existingSessions = dbtools + .Select(userDB, 'sessions', { + userID: user.id, + isScript: isScript ? 1 : 0, + }) + .sort((a: Session, b: Session) => { + return ( + new Date(a.lastAccess).getTime() - + new Date(b.lastAccess).getTime() + ) + }) + + const diff = existingSessions.length - minimumAlowwedSessions + if (diff > 0) { + logger.Log( + `Multiple ${isScript ? 'script' : 'website'} sessions ( ${ + existingSessions.length + } ) for #${user.id}, deleting olds`, + logger.GetColor('cyan') + ) + for (let i = 0; i < diff; i++) { + const id = existingSessions[i].id + dbtools.Delete(userDB, 'sessions', { + id: id, + isScript: isScript ? 1 : 0, + }) + } + } + + dbtools.Update( + userDB, + 'users', + { + loginCount: user.loginCount + 1, + lastLogin: utils.GetDateString(), + }, + { + id: user.id, + } + ) + + dbtools.Insert(userDB, 'sessions', { + id: sessionID, + userID: user.id, + isScript: isScript ? 1 : 0, + createDate: utils.GetDateString(), + }) + + // https://www.npmjs.com/package/cookie + res.cookie('sessionID', sessionID, { + domain: domain, + expires: new Date( + new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000 + ), + sameSite: 'none', + secure: true, + }) + res.cookie('sessionID', sessionID, { + expires: new Date( + new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000 + ), + sameSite: 'none', + secure: true, + }) + + res.json({ + result: 'success', + msg: 'you are now logged in', + }) + logger.Log( + `Successfull login to ${ + isScript ? 'script' : 'website' + } with user ID: #${user.id}`, + logger.GetColor('cyan') + ) + } else { + logger.Log( + `Login attempt with invalid pw: ${pw} to ${ + isScript ? 'script' : 'website' + }`, + logger.GetColor('cyan') + ) + res.json({ + result: 'error', + msg: 'Invalid password', + }) + } + }) + + app.get('/logout', (req: Request, res: any) => { + logger.LogReq(req) + const sessionID = req.cookies.sessionID + const user: User = req.session.user + const { all } = req.query + + if (!user) { + res.json({ + msg: 'You are not logged in', + success: false, + }) + return } logger.Log( - `Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`, - logger.GetColor('cyan') + `Successfull logout with user ID: #${user.id}`, + logger.GetColor('cyan') ) - dbtools.Update( - userDB, - 'users', - { - avaiblePWRequests: maxPWCount, - }, - { - id: user.id, - } - ) - }) + if (all) { + dbtools.Delete(userDB, 'sessions', { + userID: user.id, + }) + } else { + dbtools.Delete(userDB, 'sessions', { + id: sessionID, + }) + } + + res.clearCookie('sessionID').json({ + msg: 'Successfull logout', + result: 'success', + }) + }) + + function getDayDiff(dateString: string | Date) { + const msdiff = new Date().getTime() - new Date(dateString).getTime() + return Math.floor(msdiff / (1000 * 3600 * 24)) } - users.forEach((user) => { - const dayDiff = getDayDiff(user.created) - if (dayDiff === daysAfterUserGetsPWs) { - logger.Log( - `Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`, - logger.GetColor('cyan') - ) + function IncrementAvaiblePWs() { + // FIXME: check this if this is legit and works + logger.Log('Incrementing avaible PW-s ...') + const users: Array = dbtools.SelectAll(userDB, 'users') + const day = new Date().getDay() - dbtools.Update( - userDB, - 'users', - { - avaiblePWRequests: maxPWCount, - }, - { - id: user.id, - } - ) - } - }) - } + if (day === 1) { + users.forEach((user) => { + const dayDiff = getDayDiff(user.created) + if (dayDiff < daysAfterUserGetsPWs) { + logger.Log( + `User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`, + logger.GetColor('cyan') + ) + return + } - return { - dailyAction: () => { - BackupDB(usersDbBackupPath, userDB) - IncrementAvaiblePWs() - }, - } + if (user.avaiblePWRequests >= maxPWCount) { + return + } + + logger.Log( + `Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`, + logger.GetColor('cyan') + ) + + dbtools.Update( + userDB, + 'users', + { + avaiblePWRequests: maxPWCount, + }, + { + id: user.id, + } + ) + }) + } + + users.forEach((user) => { + const dayDiff = getDayDiff(user.created) + if (dayDiff === daysAfterUserGetsPWs) { + logger.Log( + `Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`, + logger.GetColor('cyan') + ) + + dbtools.Update( + userDB, + 'users', + { + avaiblePWRequests: maxPWCount, + }, + { + id: user.id, + } + ) + } + }) + } + + return { + dailyAction: () => { + BackupDB(usersDbBackupPath, userDB) + IncrementAvaiblePWs() + }, + } } export default { - setup: setup, + setup: setup, } diff --git a/src/modules/api/usersDBStruct.ts b/src/modules/api/usersDBStruct.ts index abcce7c..fcc8ace 100644 --- a/src/modules/api/usersDBStruct.ts +++ b/src/modules/api/usersDBStruct.ts @@ -19,78 +19,78 @@ ------------------------------------------------------------------------- */ const DBStruct = { - users: { - tableStruct: { - id: { - type: 'integer', - primary: true, - autoIncrement: true, - }, - pw: { - type: 'text', - notNull: true, - unique: true, - }, - notes: { - type: 'text', - }, - loginCount: { - type: 'number', - defaultZero: true, - }, - created: { - type: 'text', - notNull: true, - }, - lastLogin: { - type: 'text', - }, - lastAccess: { - type: 'text', - }, - avaiblePWRequests: { - type: 'number', - defaultZero: true, - }, - pwRequestCount: { - type: 'number', - defaultZero: true, - }, - createdBy: { - type: 'number', - }, + users: { + tableStruct: { + id: { + type: 'integer', + primary: true, + autoIncrement: true, + }, + pw: { + type: 'text', + notNull: true, + unique: true, + }, + notes: { + type: 'text', + }, + loginCount: { + type: 'number', + defaultZero: true, + }, + created: { + type: 'text', + notNull: true, + }, + lastLogin: { + type: 'text', + }, + lastAccess: { + type: 'text', + }, + avaiblePWRequests: { + type: 'number', + defaultZero: true, + }, + pwRequestCount: { + type: 'number', + defaultZero: true, + }, + createdBy: { + type: 'number', + }, + }, }, - }, - sessions: { - foreignKey: [ - { - keysFrom: ['userID'], - table: 'users', - keysTo: ['id'], - }, - ], - tableStruct: { - id: { - type: 'text', - primary: true, - notNull: true, - }, - userID: { - type: 'number', - notNull: true, - }, - createDate: { - type: 'text', - notNull: true, - }, - lastAccess: { - type: 'text', - }, - isScript: { - type: 'number', - notNull: true, - }, + sessions: { + foreignKey: [ + { + keysFrom: ['userID'], + table: 'users', + keysTo: ['id'], + }, + ], + tableStruct: { + id: { + type: 'text', + primary: true, + notNull: true, + }, + userID: { + type: 'number', + notNull: true, + }, + createDate: { + type: 'text', + notNull: true, + }, + lastAccess: { + type: 'text', + }, + isScript: { + type: 'number', + notNull: true, + }, + }, }, - }, } export default DBStruct diff --git a/src/modules/dataEditor/dataEditor.ts b/src/modules/dataEditor/dataEditor.ts index c03412e..c777f9b 100644 --- a/src/modules/dataEditor/dataEditor.ts +++ b/src/modules/dataEditor/dataEditor.ts @@ -36,85 +36,85 @@ let publicdirs: string[] = [] let nextdir = '' function GetApp(): ModuleType { - app.use( - express.urlencoded({ - limit: '5mb', - extended: true, - }) as RequestHandler - ) - app.use( - express.json({ - limit: '5mb', - }) as RequestHandler - ) - app.set('view engine', 'ejs') - app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews']) - app.use( - auth({ - userDB: userDB, - jsonResponse: false, - exceptions: ['/favicon.ico'], + app.use( + express.urlencoded({ + limit: '5mb', + extended: true, + }) as RequestHandler + ) + app.use( + express.json({ + limit: '5mb', + }) as RequestHandler + ) + app.set('view engine', 'ejs') + app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews']) + app.use( + auth({ + userDB: userDB, + jsonResponse: false, + exceptions: ['/favicon.ico'], + }) + ) + app.use((req: Request, _res, next) => { + const url = req.url.split('?')[0] + if (url.includes('.html') || url === '/') { + logger.LogReq(req) + } + next() }) - ) - app.use((req: Request, _res, next) => { - const url = req.url.split('?')[0] - if (url.includes('.html') || url === '/') { - logger.LogReq(req) + publicdirs.forEach((pdir) => { + logger.Log(`Using public dir: ${pdir}`) + app.use(express.static(pdir)) + }) + app.use(express.static(nextdir)) + + // -------------------------------------------------------------- + + function AddHtmlRoutes(files: string[]) { + const routes = files.reduce((acc, file) => { + if (file.includes('html')) { + acc.push(file.split('.')[0]) + return acc + } + return acc + }, []) + + routes.forEach((route) => { + logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1) + app.get(`/${route}`, function (_req: Request, res) { + res.redirect(`${route}.html`) + }) + }) } - next() - }) - publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) - app.use(express.static(pdir)) - }) - app.use(express.static(nextdir)) + AddHtmlRoutes(utils.ReadDir(nextdir)) - // -------------------------------------------------------------- + // -------------------------------------------------------------- - function AddHtmlRoutes(files: string[]) { - const routes = files.reduce((acc, file) => { - if (file.includes('html')) { - acc.push(file.split('.')[0]) - return acc - } - return acc - }, []) - - routes.forEach((route) => { - logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1) - app.get(`/${route}`, function (_req: Request, res) { - res.redirect(`${route}.html`) - }) + app.get('/', function (req: Request, res) { + res.end('hai') + logger.LogReq(req) }) - } - AddHtmlRoutes(utils.ReadDir(nextdir)) - // -------------------------------------------------------------- + app.get('*', function (_req: Request, res) { + res.status(404).render('404') + }) - app.get('/', function (req: Request, res) { - res.end('hai') - logger.LogReq(req) - }) + app.post('*', function (_req: Request, res) { + res.status(404).render('404') + }) - app.get('*', function (_req: Request, res) { - res.status(404).render('404') - }) - - app.post('*', function (_req: Request, res) { - res.status(404).render('404') - }) - - return { - app: app, - } + return { + app: app, + } } export default { - name: 'Data editor', - getApp: GetApp, - setup: (data: SetupData): void => { - userDB = data.userDB - publicdirs = data.publicdirs - nextdir = data.nextdir - }, + name: 'Data editor', + getApp: GetApp, + setup: (data: SetupData): void => { + userDB = data.userDB + publicdirs = data.publicdirs + nextdir = data.nextdir + }, } diff --git a/src/modules/main/main.ts b/src/modules/main/main.ts index 4dcbd66..402b05a 100644 --- a/src/modules/main/main.ts +++ b/src/modules/main/main.ts @@ -32,47 +32,47 @@ let publicdirs: string[] = [] let url = '' // http(s)//asd.basd function GetApp(): ModuleType { - app.set('view engine', 'ejs') - app.set('views', ['./src/modules/main/views', './src/sharedViews']) - publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) - app.use(express.static(pdir)) - }) - - app.use(express.json() as RequestHandler) - app.use( - express.urlencoded({ - limit: '5mb', - extended: true, - }) as RequestHandler - ) - - // -------------------------------------------------------------- - - app.get('/', function (_req, res) { - res.render('main', { - siteurl: url, + app.set('view engine', 'ejs') + app.set('views', ['./src/modules/main/views', './src/sharedViews']) + publicdirs.forEach((pdir) => { + logger.Log(`Using public dir: ${pdir}`) + app.use(express.static(pdir)) }) - }) - app.get('*', function (_req, res) { - res.status(404).render('404') - }) + app.use(express.json() as RequestHandler) + app.use( + express.urlencoded({ + limit: '5mb', + extended: true, + }) as RequestHandler + ) - app.post('*', function (_req, res) { - res.status(404).render('404') - }) + // -------------------------------------------------------------- - return { - app: app, - } + app.get('/', function (_req, res) { + res.render('main', { + siteurl: url, + }) + }) + + app.get('*', function (_req, res) { + res.status(404).render('404') + }) + + app.post('*', function (_req, res) { + res.status(404).render('404') + }) + + return { + app: app, + } } export default { - name: 'Main', - getApp: GetApp, - setup: (data: SetupData): void => { - url = data.url - publicdirs = data.publicdirs - }, + name: 'Main', + getApp: GetApp, + setup: (data: SetupData): void => { + url = data.url + publicdirs = data.publicdirs + }, } diff --git a/src/modules/qmining/qmining.ts b/src/modules/qmining/qmining.ts index 9e37c86..ea8856d 100644 --- a/src/modules/qmining/qmining.ts +++ b/src/modules/qmining/qmining.ts @@ -36,212 +36,219 @@ let userDB: Database let nextdir = '' function GetApp(): ModuleType { - app.use( - express.urlencoded({ - limit: '5mb', - extended: true, - }) as RequestHandler - ) - app.use( - express.json({ - limit: '5mb', - }) as RequestHandler - ) - app.set('view engine', 'ejs') - app.set('views', ['./src/modules/qmining/views', './src/sharedViews']) - app.use( - auth({ - userDB: userDB, - jsonResponse: false, - exceptions: ['/favicon.ico', '/img/frylabs-logo_large_transparent.png'], - }) - ) - app.use((req: Request, _res, next) => { - const url = req.url.split('?')[0] - if (url.includes('.html') || url === '/') { - logger.LogReq(req) - } - next() - }) - publicdirs.forEach((pdir) => { - logger.Log(`Using public dir: ${pdir}`) - app.use(express.static(pdir)) - }) - app.use(express.static(nextdir)) - const linksFile = 'data/links.json' - let links: { [key: string]: string } = {} - - function loadDonateURL() { - try { - links = utils.ReadJSON(linksFile) - } catch (err) { - logger.Log('Couldnt read donate URL file!', logger.GetColor('red')) - console.error(err) - } - } - - loadDonateURL() - - if (utils.FileExists(linksFile)) { - utils.WatchFile(linksFile, (newData: string) => { - logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`) - loadDonateURL() - }) - } else { - logger.Log('Couldnt read donate URL file!', logger.GetColor('red')) - } - - // -------------------------------------------------------------- - // REDIRECTS - // -------------------------------------------------------------- - - // to be backwards compatible - app.get('/ask', function (req: Request, res) { - logger.DebugLog(`Qmining module ask redirect`, 'ask', 1) - res.redirect( - `http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}` + app.use( + express.urlencoded({ + limit: '5mb', + extended: true, + }) as RequestHandler ) - }) - - const simpleRedirects = [ - { - from: '/dataeditor', - to: 'https://dataeditor.frylabs.net', - }, - { - from: '/install', - to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js', - }, - { - from: '/servergit', - to: 'https://gitlab.com/MrFry/mrfrys-node-server', - }, - { - from: '/scriptgit', - to: 'https://gitlab.com/MrFry/moodle-test-userscript', - }, - { - from: '/qminingSite', - to: 'https://gitlab.com/MrFry/qmining-page', - }, - { - from: '/classesgit', - to: 'https://gitlab.com/MrFry/question-classes', - }, - { - from: '/addQuestion', - to: 'https://dataeditor.frylabs.net', - }, - { - from: '/donate', - to: links.donate, - }, - { - from: '/menuClick', - to: '/', - }, - { - from: '/legacy', - to: '/allQuestions.html', - }, - { - from: '/subjectBrowser', - to: '/allQuestions.html', - }, - { - from: '/lred', - to: '/allQuestions', - }, - { - from: '/allqr', - to: 'https://api.frylabs.net/allqr.txt', - }, - { - from: '/allqr.txt', - to: 'https://api.frylabs.net/allqr.txt', - }, - { - from: '/infos', - to: 'https://api.frylabs.net/infos?version=true&motd=true&subjinfo=true', - nolog: true, - }, - { - from: '/irc', - to: '/chat', - }, - { - from: '/patreon', - to: links.patreon, - }, - ] - - simpleRedirects.forEach((redirect) => { - app.get(redirect.from, function (req: Request, res) { - if (!redirect.nolog) { - logger.LogReq(req) - } - logger.DebugLog(`Qmining module ${redirect.from} redirect`, 'infos', 1) - - let target = redirect.to - if (!redirect.to.includes('https://')) { - target += utils.formatUrl({ query: req.query }) - } - - res.redirect(target) + app.use( + express.json({ + limit: '5mb', + }) as RequestHandler + ) + app.set('view engine', 'ejs') + app.set('views', ['./src/modules/qmining/views', './src/sharedViews']) + app.use( + auth({ + userDB: userDB, + jsonResponse: false, + exceptions: [ + '/favicon.ico', + '/img/frylabs-logo_large_transparent.png', + ], + }) + ) + app.use((req: Request, _res, next) => { + const url = req.url.split('?')[0] + if (url.includes('.html') || url === '/') { + logger.LogReq(req) + } + next() }) - }) + publicdirs.forEach((pdir) => { + logger.Log(`Using public dir: ${pdir}`) + app.use(express.static(pdir)) + }) + app.use(express.static(nextdir)) + const linksFile = 'data/links.json' + let links: { [key: string]: string } = {} - // -------------------------------------------------------------- + function loadDonateURL() { + try { + links = utils.ReadJSON(linksFile) + } catch (err) { + logger.Log('Couldnt read donate URL file!', logger.GetColor('red')) + console.error(err) + } + } - function AddHtmlRoutes(files: string[]) { - const routes = files.reduce((acc, file) => { - if (file.includes('html')) { - acc.push(file.split('.')[0]) - return acc - } - return acc - }, []) + loadDonateURL() - routes.forEach((route: string) => { - logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1) - app.get(`/${route}`, function (req: Request, res) { + if (utils.FileExists(linksFile)) { + utils.WatchFile(linksFile, (newData: string) => { + logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`) + loadDonateURL() + }) + } else { + logger.Log('Couldnt read donate URL file!', logger.GetColor('red')) + } + + // -------------------------------------------------------------- + // REDIRECTS + // -------------------------------------------------------------- + + // to be backwards compatible + app.get('/ask', function (req: Request, res) { + logger.DebugLog(`Qmining module ask redirect`, 'ask', 1) res.redirect( - utils.formatUrl({ - pathname: `${route}.html`, - query: req.query, - }) + `http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}` ) - }) }) - } - AddHtmlRoutes(utils.ReadDir(nextdir)) - // -------------------------------------------------------------- + const simpleRedirects = [ + { + from: '/dataeditor', + to: 'https://dataeditor.frylabs.net', + }, + { + from: '/install', + to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js', + }, + { + from: '/servergit', + to: 'https://gitlab.com/MrFry/mrfrys-node-server', + }, + { + from: '/scriptgit', + to: 'https://gitlab.com/MrFry/moodle-test-userscript', + }, + { + from: '/qminingSite', + to: 'https://gitlab.com/MrFry/qmining-page', + }, + { + from: '/classesgit', + to: 'https://gitlab.com/MrFry/question-classes', + }, + { + from: '/addQuestion', + to: 'https://dataeditor.frylabs.net', + }, + { + from: '/donate', + to: links.donate, + }, + { + from: '/menuClick', + to: '/', + }, + { + from: '/legacy', + to: '/allQuestions.html', + }, + { + from: '/subjectBrowser', + to: '/allQuestions.html', + }, + { + from: '/lred', + to: '/allQuestions', + }, + { + from: '/allqr', + to: 'https://api.frylabs.net/allqr.txt', + }, + { + from: '/allqr.txt', + to: 'https://api.frylabs.net/allqr.txt', + }, + { + from: '/infos', + to: 'https://api.frylabs.net/infos?version=true&motd=true&subjinfo=true', + nolog: true, + }, + { + from: '/irc', + to: '/chat', + }, + { + from: '/patreon', + to: links.patreon, + }, + ] - app.get('/', function (req: Request, res) { - res.end('hai') - logger.LogReq(req) - }) + simpleRedirects.forEach((redirect) => { + app.get(redirect.from, function (req: Request, res) { + if (!redirect.nolog) { + logger.LogReq(req) + } + logger.DebugLog( + `Qmining module ${redirect.from} redirect`, + 'infos', + 1 + ) - app.get('*', function (_req: Request, res) { - res.status(404).render('404') - }) + let target = redirect.to + if (!redirect.to.includes('https://')) { + target += utils.formatUrl({ query: req.query }) + } - app.post('*', function (_req: Request, res) { - res.status(404).render('404') - }) + res.redirect(target) + }) + }) - return { - app: app, - } + // -------------------------------------------------------------- + + function AddHtmlRoutes(files: string[]) { + const routes = files.reduce((acc, file) => { + if (file.includes('html')) { + acc.push(file.split('.')[0]) + return acc + } + return acc + }, []) + + routes.forEach((route: string) => { + logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1) + app.get(`/${route}`, function (req: Request, res) { + res.redirect( + utils.formatUrl({ + pathname: `${route}.html`, + query: req.query, + }) + ) + }) + }) + } + AddHtmlRoutes(utils.ReadDir(nextdir)) + + // -------------------------------------------------------------- + + app.get('/', function (req: Request, res) { + res.end('hai') + logger.LogReq(req) + }) + + app.get('*', function (_req: Request, res) { + res.status(404).render('404') + }) + + app.post('*', function (_req: Request, res) { + res.status(404).render('404') + }) + + return { + app: app, + } } export default { - name: 'Qmining', - getApp: GetApp, - setup: (data: SetupData): void => { - userDB = data.userDB - publicdirs = data.publicdirs - nextdir = data.nextdir - }, + name: 'Qmining', + getApp: GetApp, + setup: (data: SetupData): void => { + userDB = data.userDB + publicdirs = data.publicdirs + nextdir = data.nextdir + }, } diff --git a/src/server.ts b/src/server.ts index 6d93e18..c13f28b 100755 --- a/src/server.ts +++ b/src/server.ts @@ -53,17 +53,17 @@ const logFile = logger.logDir + logger.logFileName const vlogFile = logger.vlogDir + logger.logFileName function moveLogIfNotFromToday(path: string, to: string) { - if (utils.FileExists(path)) { - const today = new Date() - const stat = utils.statFile(path) - if ( - today.getFullYear() !== stat.mtime.getFullYear() || - today.getMonth() !== stat.mtime.getMonth() || - today.getDate() !== stat.mtime.getDate() - ) { - utils.renameFile(path, to + utils.GetDateString(stat.mtime)) + if (utils.FileExists(path)) { + const today = new Date() + const stat = utils.statFile(path) + if ( + today.getFullYear() !== stat.mtime.getFullYear() || + today.getMonth() !== stat.mtime.getMonth() || + today.getDate() !== stat.mtime.getDate() + ) { + utils.renameFile(path, to + utils.GetDateString(stat.mtime)) + } } - } } moveLogIfNotFromToday(logFile, logger.logDir) moveLogIfNotFromToday(vlogFile, logger.vlogDir) @@ -72,32 +72,32 @@ idStats.Load() logger.Load() interface Modules { - [name: string]: Module + [name: string]: Module } interface Module { - path: string - publicdirs: Array - name: string - urls: Array - nextdir?: string - isNextJs?: boolean - app: express.Application - dailyAction: Function - cleanup: Function + path: string + publicdirs: Array + name: string + urls: Array + nextdir?: string + isNextJs?: boolean + app: express.Application + dailyAction: Function + cleanup: Function } export interface SetupData { - url: string - publicdirs: Array - userDB?: Database - nextdir?: string - httpServer: http.Server - httpsServer: https.Server + url: string + publicdirs: Array + userDB?: Database + nextdir?: string + httpServer: http.Server + httpsServer: https.Server } if (!utils.FileExists(usersDBPath)) { - throw new Error('No user DB exists yet! please run utils/dbSetup.js first!') + throw new Error('No user DB exists yet! please run utils/dbSetup.js first!') } const userDB = dbtools.GetDB(usersDBPath) let modules: Modules = utils.ReadJSON(modulesFile) @@ -108,43 +108,43 @@ logger.Log(`Log path: ${logFile}`) logger.Log(`vLog path: ${vlogFile}`) try { - if (utils.FileExists(extraModulesFile)) { - const extraModules = JSON.parse(utils.ReadFile(extraModulesFile)) - modules = { - ...extraModules, - ...modules, + if (utils.FileExists(extraModulesFile)) { + const extraModules = JSON.parse(utils.ReadFile(extraModulesFile)) + modules = { + ...extraModules, + ...modules, + } } - } } catch (err) { - logger.Log('Failed to read extra modules file') - console.error(err) + logger.Log('Failed to read extra modules file') + console.error(err) } process.on('SIGINT', () => exit('SIGINT')) process.on('SIGTERM', () => exit('SIGTERM')) function exit(reason: string) { - console.log() - logger.Log(`Exiting, reason: ${reason}`) - Object.keys(modules).forEach((key) => { - const module = modules[key] - if (module.cleanup) { - try { - module.cleanup() - } catch (err) { - logger.Log( - `Error in ${key} cleanup! Details in STDERR`, - logger.GetColor('redbg') - ) - console.error(err) - } - } - }) + console.log() + logger.Log(`Exiting, reason: ${reason}`) + Object.keys(modules).forEach((key) => { + const module = modules[key] + if (module.cleanup) { + try { + module.cleanup() + } catch (err) { + logger.Log( + `Error in ${key} cleanup! Details in STDERR`, + logger.GetColor('redbg') + ) + console.error(err) + } + } + }) - logger.Log('Closing Auth DB') - userDB.close() + logger.Log('Closing Auth DB') + userDB.close() - process.exit() + process.exit() } // https://certbot.eff.org/ @@ -156,201 +156,201 @@ let certsLoaded = false let certs: { key: string; cert: string; ca: string } if ( - startHTTPS && - utils.FileExists(privkeyFile) && - utils.FileExists(fullchainFile) && - utils.FileExists(chainFile) + startHTTPS && + utils.FileExists(privkeyFile) && + utils.FileExists(fullchainFile) && + utils.FileExists(chainFile) ) { - try { - const key = utils.ReadFile(privkeyFile) - const cert = utils.ReadFile(fullchainFile) - const ca = utils.ReadFile(chainFile) - certs = { - key: key, - cert: cert, - ca: ca, + try { + const key = utils.ReadFile(privkeyFile) + const cert = utils.ReadFile(fullchainFile) + const ca = utils.ReadFile(chainFile) + certs = { + key: key, + cert: cert, + ca: ca, + } + certsLoaded = true + } catch (err) { + logger.Log('Error loading cert files!', logger.GetColor('redbg')) + console.error(err) } - certsLoaded = true - } catch (err) { - logger.Log('Error loading cert files!', logger.GetColor('redbg')) - console.error(err) - } } const app = express() const httpServer = http.createServer(app) let httpsServer: https.Server if (certsLoaded) { - httpsServer = https.createServer(certs, app) - logger.Log('Listening on port: ' + httpsport + ' (https)') + httpsServer = https.createServer(certs, app) + logger.Log('Listening on port: ' + httpsport + ' (https)') } else { - logger.Log('Https not avaible') + logger.Log('Https not avaible') } if (!process.env.NS_DEVEL) { - app.use(function (req, res, next) { - if (req.secure) { - next() - } else { - logger.DebugLog( - `HTTPS ${req.method} redirect to: ${ - 'https://' + req.headers.host + req.url - }`, - 'https', - 1 - ) - if (req.method === 'POST') { - res.redirect(307, 'https://' + req.headers.host + req.url) - } else { - res.redirect('https://' + req.headers.host + req.url) - } - } - }) + app.use(function (req, res, next) { + if (req.secure) { + next() + } else { + logger.DebugLog( + `HTTPS ${req.method} redirect to: ${ + 'https://' + req.headers.host + req.url + }`, + 'https', + 1 + ) + if (req.method === 'POST') { + res.redirect(307, 'https://' + req.headers.host + req.url) + } else { + res.redirect('https://' + req.headers.host + req.url) + } + } + }) } // https://github.com/expressjs/cors#configuration-options app.use( - cors({ - credentials: true, - origin: true, - // origin: [ /\.frylabs\.net$/ ] - }) + cors({ + credentials: true, + origin: true, + // origin: [ /\.frylabs\.net$/ ] + }) ) const cookieSecret = uuidv4() app.use(cookieParser(cookieSecret)) if (!utils.FileExists(statExcludeFile)) { - utils.WriteFile('[]', statExcludeFile) + utils.WriteFile('[]', statExcludeFile) } const excludeFromStats = utils.ReadJSON(statExcludeFile) app.use( - reqlogger({ - loggableKeywords: ['news.json'], - loggableModules: [], - exceptions: ['_next/static'], - excludeFromStats: excludeFromStats, - }) + reqlogger({ + loggableKeywords: ['news.json'], + loggableModules: [], + exceptions: ['_next/static'], + excludeFromStats: excludeFromStats, + }) ) Object.keys(modules).forEach(function (key) { - const module = modules[key] - try { - const mod = require(module.path).default // eslint-disable-line - // const mod = require(module.path) - logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow')) + const module = modules[key] + try { + const mod = require(module.path).default // eslint-disable-line + // const mod = require(module.path) + logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow')) - module.publicdirs.forEach((pdir) => { - utils.CreatePath(pdir) - }) + module.publicdirs.forEach((pdir) => { + utils.CreatePath(pdir) + }) - if (mod.setup) { - mod.setup({ - url: 'https://' + module.urls[0], - userDB: userDB, - publicdirs: module.publicdirs, - nextdir: module.nextdir, - httpServer: httpServer, - httpsServer: httpsServer, - }) + if (mod.setup) { + mod.setup({ + url: 'https://' + module.urls[0], + userDB: userDB, + publicdirs: module.publicdirs, + nextdir: module.nextdir, + httpServer: httpServer, + httpsServer: httpsServer, + }) + } + + const modApp = mod.getApp() + module.app = modApp.app + module.dailyAction = modApp.dailyAction + module.cleanup = modApp.cleanup + module.urls.forEach((url) => { + app.use(vhost(url, module.app)) + }) + } catch (err) { + console.error(err) } - - const modApp = mod.getApp() - module.app = modApp.app - module.dailyAction = modApp.dailyAction - module.cleanup = modApp.cleanup - module.urls.forEach((url) => { - app.use(vhost(url, module.app)) - }) - } catch (err) { - console.error(err) - } }) setLogTimer() function setLogTimer() { - const now = new Date() - const night = new Date( - now.getFullYear(), - now.getMonth(), - now.getDate() + 1, - 0, - 0, - 1 - ) - logger.DebugLog(`Next daily action: ${night}`, 'daily', 1) - const msToMidnight = night.getTime() - now.getTime() + 10000 - logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1) - logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1) - - if (msToMidnight < 0) { - logger.Log( - `Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`, - logger.GetColor('redbg') + const now = new Date() + const night = new Date( + now.getFullYear(), + now.getMonth(), + now.getDate() + 1, + 0, + 0, + 1 ) - return - } + logger.DebugLog(`Next daily action: ${night}`, 'daily', 1) + const msToMidnight = night.getTime() - now.getTime() + 10000 + logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1) + logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1) - setTimeout(function () { - LogTimerAction() - rotateLog() - setLogTimer() - }, msToMidnight) + if (msToMidnight < 0) { + logger.Log( + `Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`, + logger.GetColor('redbg') + ) + return + } + + setTimeout(function () { + LogTimerAction() + rotateLog() + setLogTimer() + }, msToMidnight) } function rotateLog() { - const date = new Date() - date.setDate(date.getDate() - 1) - const fname = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) + const date = new Date() + date.setDate(date.getDate() - 1) + const fname = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) - if (utils.FileExists(logFile)) { - utils.CopyFile(logFile, logger.logDir + fname) - } - if (utils.FileExists(vlogFile)) { - utils.CopyFile(vlogFile, logger.vlogDir + fname) - } + if (utils.FileExists(logFile)) { + utils.CopyFile(logFile, logger.logDir + fname) + } + if (utils.FileExists(vlogFile)) { + utils.CopyFile(vlogFile, logger.vlogDir + fname) + } - utils.WriteFile(fname, logFile) - utils.WriteFile(fname, vlogFile) + utils.WriteFile(fname, logFile) + utils.WriteFile(fname, vlogFile) } function LogTimerAction() { - logger.DebugLog(`Running Log Timer Action`, 'daily', 1) - Object.keys(modules).forEach((key) => { - const module = modules[key] - logger.DebugLog(`Ckecking ${key}`, 'daily', 1) - if (module.dailyAction) { - try { - logger.Log(`Running daily action of ${key}`) - module.dailyAction() - } catch (err) { - logger.Log( - `Error in ${key} daily action! Details in STDERR`, - logger.GetColor('redbg') - ) - console.error(err) - } - } - }) + logger.DebugLog(`Running Log Timer Action`, 'daily', 1) + Object.keys(modules).forEach((key) => { + const module = modules[key] + logger.DebugLog(`Ckecking ${key}`, 'daily', 1) + if (module.dailyAction) { + try { + logger.Log(`Running daily action of ${key}`) + module.dailyAction() + } catch (err) { + logger.Log( + `Error in ${key} daily action! Details in STDERR`, + logger.GetColor('redbg') + ) + console.error(err) + } + } + }) - const line = - '===================================================================================================================================================' - logger.Log(line) + const line = + '===================================================================================================================================================' + logger.Log(line) } logger.Log('Node version: ' + process.version) logger.Log('Current working directory: ' + process.cwd()) logger.Log('Listening on port: ' + port) if (isRoot) { - logger.Log('Running as root', logger.GetColor('red')) + logger.Log('Running as root', logger.GetColor('red')) } httpServer.listen(port) if (httpsServer) { - httpsServer.listen(httpsport) + httpsServer.listen(httpsport) } diff --git a/src/standaloneUtils/createJSON.js b/src/standaloneUtils/createJSON.js index 81329c0..a18aaed 100644 --- a/src/standaloneUtils/createJSON.js +++ b/src/standaloneUtils/createJSON.js @@ -3,36 +3,36 @@ const fs = require('fs') const params = process.argv const file = params[2] -const data = fs.readFileSync(file,'utf8').split('\n') +const data = fs.readFileSync(file, 'utf8').split('\n') console.log(data) console.log("TODO: remove 'Q: ' and 'A: '") let currVal = {} const res = data.reduce((acc, val) => { - const formattedVal = val.replace(/\r/g, '').trim() + const formattedVal = val.replace(/\r/g, '').trim() - if (formattedVal.startsWith('#')) return acc - if (formattedVal.startsWith('Q')) { - currVal = { - Q: formattedVal - } - return acc - } - if (formattedVal.startsWith('A')) { - currVal.A = formattedVal - return [ - ...acc, - { - ...currVal, - data: { - type: 'simple' + if (formattedVal.startsWith('#')) return acc + if (formattedVal.startsWith('Q')) { + currVal = { + Q: formattedVal, } - } - ] - } + return acc + } + if (formattedVal.startsWith('A')) { + currVal.A = formattedVal + return [ + ...acc, + { + ...currVal, + data: { + type: 'simple', + }, + }, + ] + } - return acc + return acc }, []) console.log(res) diff --git a/src/standaloneUtils/dataSimplifyer.js b/src/standaloneUtils/dataSimplifyer.js index 5520cdb..8bf2670 100644 --- a/src/standaloneUtils/dataSimplifyer.js +++ b/src/standaloneUtils/dataSimplifyer.js @@ -7,23 +7,23 @@ const data = JSON.parse(fs.readFileSync(file, 'utf8')) const res = [] data.forEach((subj) => { - const questions = [] - subj.Questions.forEach((question) => { - const res = {} - if (question.Q) { - res.Q = simplifyString(question.Q) - } - if (question.A) { - res.A = simplifyString(question.A) - } - res.data = question.data + const questions = [] + subj.Questions.forEach((question) => { + const res = {} + if (question.Q) { + res.Q = simplifyString(question.Q) + } + if (question.A) { + res.A = simplifyString(question.A) + } + res.data = question.data - questions.push(res) - }) - res.push({ - Name: subj.Name, - Questions: questions, - }) + questions.push(res) + }) + res.push({ + Name: subj.Name, + Questions: questions, + }) }) fs.writeFileSync(file + '.res', JSON.stringify(res)) diff --git a/src/standaloneUtils/dbSetup.js b/src/standaloneUtils/dbSetup.js index feafed0..b98c925 100644 --- a/src/standaloneUtils/dbSetup.js +++ b/src/standaloneUtils/dbSetup.js @@ -4,38 +4,38 @@ const dbtools = require('../../dist/utils/dbtools.js').default // eslint-disable const { v4: uuidv4 } = require('uuid') // eslint-disable-line const dbStructPaths = [ - { structPath: '../modules/api/usersDBStruct.json', name: 'users.db' }, - { structPath: '../modules/api/msgsDbStruct.json', name: 'msgs.db' }, + { structPath: '../modules/api/usersDBStruct.json', name: 'users.db' }, + { structPath: '../modules/api/msgsDbStruct.json', name: 'msgs.db' }, ] dbStructPaths.forEach((data) => { - const { structPath, name } = data - createDB(structPath, name) + const { structPath, name } = data + createDB(structPath, name) }) function createDB(path, name) { - const dbStruct = utils.ReadJSON(path) - const db = dbtools.GetDB(`./${name}`) - db.pragma('synchronous = OFF') + const dbStruct = utils.ReadJSON(path) + const db = dbtools.GetDB(`./${name}`) + db.pragma('synchronous = OFF') - Object.keys(dbStruct).forEach((tableName) => { - const tableData = dbStruct[tableName] - dbtools.CreateTable( - db, - tableName, - tableData.tableStruct, - tableData.foreignKey - ) - }) - printDb(db, dbStruct) - db.close() + Object.keys(dbStruct).forEach((tableName) => { + const tableData = dbStruct[tableName] + dbtools.CreateTable( + db, + tableName, + tableData.tableStruct, + tableData.foreignKey + ) + }) + printDb(db, dbStruct) + db.close() - logger.Log('Done') + logger.Log('Done') } function printDb(db, dbStruct) { - Object.keys(dbStruct).forEach((key) => { - console.log(dbtools.TableInfo(db, key)) - console.log(dbtools.SelectAll(db, key)) - }) + Object.keys(dbStruct).forEach((key) => { + console.log(dbtools.TableInfo(db, key)) + console.log(dbtools.SelectAll(db, key)) + }) } diff --git a/src/standaloneUtils/invalidQuestionRemover.js b/src/standaloneUtils/invalidQuestionRemover.js index 0ca2cf8..5b68b16 100644 --- a/src/standaloneUtils/invalidQuestionRemover.js +++ b/src/standaloneUtils/invalidQuestionRemover.js @@ -1,13 +1,13 @@ const fs = require('fs') function GetParams() { - return process.argv.splice(2) + return process.argv.splice(2) } const params = GetParams() console.log(params) if (params.length === 0) { - console.error('No params! Need a path to a question database!') - process.exit() + console.error('No params! Need a path to a question database!') + process.exit() } const file = params[0] @@ -16,36 +16,36 @@ const res = [] let invalidQuestionCount = 0 data.forEach((subj) => { - const questions = [] - subj.Questions.forEach((question) => { - if (isInvalidQuestion(question)) { - console.log(`invalid question in ${subj.Name}:`) - console.log(question) - invalidQuestionCount++ - } else { - questions.push(question) - } - }) - res.push({ - Name: subj.Name, - Questions: questions, - }) + const questions = [] + subj.Questions.forEach((question) => { + if (isInvalidQuestion(question)) { + console.log(`invalid question in ${subj.Name}:`) + console.log(question) + invalidQuestionCount++ + } else { + questions.push(question) + } + }) + res.push({ + Name: subj.Name, + Questions: questions, + }) }) function isInvalidQuestion(q) { - if (q.Q === 'Ugrás...' || q.A === 'Ugrás...') { - return true - } + if (q.Q === 'Ugrás...' || q.A === 'Ugrás...') { + return true + } - if (!q.Q && !q.A) { - return true - } + if (!q.Q && !q.A) { + return true + } - if (!q.Q && q.data.type === 'simple') { - return true - } + if (!q.Q && q.data.type === 'simple') { + return true + } - return false + return false } console.log(`${invalidQuestionCount} invalid questions, writing results...`) diff --git a/src/standaloneUtils/manualSearch.js b/src/standaloneUtils/manualSearch.js index a3b6aef..fcc7994 100644 --- a/src/standaloneUtils/manualSearch.js +++ b/src/standaloneUtils/manualSearch.js @@ -21,16 +21,16 @@ const answer = params[2] console.time('SEARCH') const searchRes = search({ - qdb: loadData(path), - subjName: 'Elektronika', - question: { - Q: question, - A: answer, - data: { - type: 'simple', + qdb: loadData(path), + subjName: 'Elektronika', + question: { + Q: question, + A: answer, + data: { + type: 'simple', + }, }, - }, - searchTillMatchPercent: 80, + searchTillMatchPercent: 80, }) hr() console.log('Search result') @@ -39,9 +39,9 @@ showSearchResult(searchRes) hr() console.timeEnd('SEARCH') log( - `Searched for question: "${C('green')}${question}${C()}" answer: "${C( - 'green' - )}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"` + `Searched for question: "${C('green')}${question}${C()}" answer: "${C( + 'green' + )}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"` ) hr() @@ -50,44 +50,44 @@ hr() // --------------------------------------------------------------------------------- function showSearchResult(res) { - res.forEach((x) => { - console.log(`${C('green')}Q:${C()}`, x.q.Q) - console.log(`${C('green')}A:${C()}`, x.q.A) - console.log(`${C('green')}match:${C()}`, x.match) - console.log() - }) - console.log(`Result length: ${C('green')}${res.length}${C()}`) + res.forEach((x) => { + console.log(`${C('green')}Q:${C()}`, x.q.Q) + console.log(`${C('green')}A:${C()}`, x.q.A) + console.log(`${C('green')}match:${C()}`, x.match) + console.log() + }) + console.log(`Result length: ${C('green')}${res.length}${C()}`) } function search({ qdb, subjName, question, searchInAllIfNoResult }) { - return doSearch( - qdb, - subjName, - question, - null, - minpercent, - searchInAllIfNoResult - ) + return doSearch( + qdb, + subjName, + question, + null, + minpercent, + searchInAllIfNoResult + ) } function hr() { - let res = '' - for (let i = 0; i < process.stdout.columns; i++) { - res += '=' - } - log(`${C('cyan')}${res}${C()}`) + let res = '' + for (let i = 0; i < process.stdout.columns; i++) { + res += '=' + } + log(`${C('cyan')}${res}${C()}`) } function log(text) { - utils.AppendToFile(text, globalLog) - if (process.stdout.isTTY) { - process.stdout.clearLine() - process.stdout.cursorTo(0) - } + utils.AppendToFile(text, globalLog) + if (process.stdout.isTTY) { + process.stdout.clearLine() + process.stdout.cursorTo(0) + } - console.log(text) + console.log(text) } function C(color) { - return logger.C(color) + return logger.C(color) } diff --git a/src/standaloneUtils/rmDuplicates.js b/src/standaloneUtils/rmDuplicates.js index 2526bde..6a1af18 100644 --- a/src/standaloneUtils/rmDuplicates.js +++ b/src/standaloneUtils/rmDuplicates.js @@ -1,10 +1,10 @@ const utils = require('../../dist/utils/utils.js').default // eslint-disable-line const logger = require('../../dist/utils/logger.js').default // eslint-disable-line const { - addQuestion, - doSearch, - compareQuestionObj, - createQuestion, + addQuestion, + doSearch, + compareQuestionObj, + createQuestion, } = require('../../dist/utils/classes.js') // eslint-disable-line const { loadData, writeData } = require('../../dist/utils/actions.js') // eslint-disable-line const fs = require('fs') // eslint-disable-line @@ -35,7 +35,7 @@ const fs = require('fs') // eslint-disable-line const minpercent = 95 const line = - '====================================================================' + '====================================================================' const logPath = './duplicateRemovingLog/' const globalLog = './duplicateRemovingLog/log' utils.CreatePath(logPath) @@ -45,25 +45,25 @@ utils.WriteFile('', globalLog) let currentMaxIndex = -1 let currentIndex = -1 process.on('message', function () { - process.send({ - currentMaxIndex: currentMaxIndex, - currentIndex: currentIndex, - }) + process.send({ + currentMaxIndex: currentMaxIndex, + currentIndex: currentIndex, + }) }) // ---------------------------------------------- let params = process.argv.splice(2) let silenced = false if (params.includes('-s')) { - silenced = true + silenced = true } params = params.filter((x) => { - return !x.startsWith('-') + return !x.startsWith('-') }) console.log(params) if (params.length === 0) { - console.log('At least 1 parameter required (path to DB)') - process.exit(1) + console.log('At least 1 parameter required (path to DB)') + process.exit(1) } const pathA = params[0] @@ -71,61 +71,67 @@ const pathB = params[1] const stat = fs.lstatSync(pathA) if (stat.isDirectory()) { - if (pathB) { - log( - `Clearing possible questions from ${C( - 'green' - )}${pathA}${C()} based on ${C('green')}${pathB}${C()} db` - ) - const db = pathB ? loadData(pathB) : null + if (pathB) { + log( + `Clearing possible questions from ${C( + 'green' + )}${pathA}${C()} based on ${C('green')}${pathB}${C()} db` + ) + const db = pathB ? loadData(pathB) : null - clearPossibleAnswers(pathA, db) + clearPossibleAnswers(pathA, db) - log( - `Cleared possible questions from ${C('green')}${pathA}${C()} based on ${C( - 'green' - )}${pathB}${C()} db` - ) - } else { - log( - `Removing possible question duplicates from ${C('green')}${pathA}${C()}` - ) - removePossibleAnswersDuplicates(pathA) - log(`Removed possible question duplicates from ${C('green')}${pathA}${C()}`) - } + log( + `Cleared possible questions from ${C( + 'green' + )}${pathA}${C()} based on ${C('green')}${pathB}${C()} db` + ) + } else { + log( + `Removing possible question duplicates from ${C( + 'green' + )}${pathA}${C()}` + ) + removePossibleAnswersDuplicates(pathA) + log( + `Removed possible question duplicates from ${C( + 'green' + )}${pathA}${C()}` + ) + } } else { - console.time('load') - const dbA = loadData(pathA) - const dbB = pathB ? loadData(pathB) : null - console.timeEnd('load') + console.time('load') + const dbA = loadData(pathA) + const dbB = pathB ? loadData(pathB) : null + console.timeEnd('load') - console.time('rmduplicates') + console.time('rmduplicates') - if (!dbB) { - log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`) - const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1] - const res = rmDuplicates(dbA) - console.timeEnd('rmduplicates') - writeData(res, resultDbFileName + '.res') - log('File written') - log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`) - } else { - log( - `Removing questions found in ${C('green')}${pathB}${C()} from ${C( - 'green' - )}${pathA}${C()}` - ) - const res = difference({ dbA: dbA, dbB: dbB }) - console.timeEnd('rmduplicates') - const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1] - writeData(res, resultDbFileName + '.res') - log('File written') - log( - `Removed questions found in ${C('green')}${pathB}${C()} from ${C( - 'green' - )}${pathA}${C()}` - ) - } + if (!dbB) { + log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`) + const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1] + const res = rmDuplicates(dbA) + console.timeEnd('rmduplicates') + writeData(res, resultDbFileName + '.res') + log('File written') + log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`) + } else { + log( + `Removing questions found in ${C('green')}${pathB}${C()} from ${C( + 'green' + )}${pathA}${C()}` + ) + const res = difference({ dbA: dbA, dbB: dbB }) + console.timeEnd('rmduplicates') + const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1] + writeData(res, resultDbFileName + '.res') + log('File written') + log( + `Removed questions found in ${C('green')}${pathB}${C()} from ${C( + 'green' + )}${pathA}${C()}` + ) + } } // --------------------------------------------------------------------------------- @@ -135,141 +141,141 @@ if (stat.isDirectory()) { // TODO: dont check every file, only check per directorires // only compare questions of same subjects function removePossibleAnswersDuplicates(path) { - const dirs = fs.readdirSync(path) - let count = 0 - let currIndex = 1 - let delets = 0 + const dirs = fs.readdirSync(path) + let count = 0 + let currIndex = 1 + let delets = 0 - iterateDir(path, () => { - count++ - }) - - dirs.forEach((currDir) => { - const contents = fs.readdirSync(path + '/' + currDir) - - contents.forEach((currFile) => { - const currPath = path + '/' + currDir + '/' + currFile - if (currPath.includes('savedQuestions.json')) { - return - } - if (!utils.FileExists(currPath)) { - return - } - const dataA = utils.ReadJSON(currPath) - - currIndex++ - printProgressBar(currIndex, count - 1) - - contents.forEach((currFile2) => { - const currPath2 = path + '/' + currDir + '/' + currFile2 - if (currPath2.includes('savedQuestions.json')) { - return - } - if (!utils.FileExists(currPath2)) { - return - } - if (currPath === currPath2) { - return - } - const dataB = utils.ReadJSON(currPath2) - - dataA.questions.forEach((q1) => { - dataB.questions.some((q2) => { - const percent = compareQuestionObj( - createQuestion(q1), - '', - createQuestion(q2), - '' - ) - if (percent.avg === 100) { - utils.deleteFile(currPath2) - count-- - delets++ - return true - } - }) - }) - }) + iterateDir(path, () => { + count++ }) - }) - log(`${C('green')}Deleting empty directories ...${C()}`) - count = dirs.length - currIndex = 0 - let deletedDirCount = 0 - dirs.forEach((dir) => { - currIndex++ - const currDirContent = fs.readdirSync(path + '/' + dir) - if (currDirContent.length === 0) { - fs.rmdirSync(path + '/' + dir) - deletedDirCount++ - } - printProgressBar(currIndex, count) - }) + dirs.forEach((currDir) => { + const contents = fs.readdirSync(path + '/' + currDir) - log(`${C('green')}Updating savedQuestions.json ...${C()}`) - count = dirs.length - currIndex = 0 - dirs.forEach((dir) => { - currIndex++ - updateSavedQuestionsFile(path + '/' + dir) - printProgressBar(currIndex, count) - }) + contents.forEach((currFile) => { + const currPath = path + '/' + currDir + '/' + currFile + if (currPath.includes('savedQuestions.json')) { + return + } + if (!utils.FileExists(currPath)) { + return + } + const dataA = utils.ReadJSON(currPath) - log( - `Deleted ${C('green')}${delets}${C()} files, and ${C( - 'green' - )}${deletedDirCount}${C()} directories` - ) + currIndex++ + printProgressBar(currIndex, count - 1) + + contents.forEach((currFile2) => { + const currPath2 = path + '/' + currDir + '/' + currFile2 + if (currPath2.includes('savedQuestions.json')) { + return + } + if (!utils.FileExists(currPath2)) { + return + } + if (currPath === currPath2) { + return + } + const dataB = utils.ReadJSON(currPath2) + + dataA.questions.forEach((q1) => { + dataB.questions.some((q2) => { + const percent = compareQuestionObj( + createQuestion(q1), + '', + createQuestion(q2), + '' + ) + if (percent.avg === 100) { + utils.deleteFile(currPath2) + count-- + delets++ + return true + } + }) + }) + }) + }) + }) + + log(`${C('green')}Deleting empty directories ...${C()}`) + count = dirs.length + currIndex = 0 + let deletedDirCount = 0 + dirs.forEach((dir) => { + currIndex++ + const currDirContent = fs.readdirSync(path + '/' + dir) + if (currDirContent.length === 0) { + fs.rmdirSync(path + '/' + dir) + deletedDirCount++ + } + printProgressBar(currIndex, count) + }) + + log(`${C('green')}Updating savedQuestions.json ...${C()}`) + count = dirs.length + currIndex = 0 + dirs.forEach((dir) => { + currIndex++ + updateSavedQuestionsFile(path + '/' + dir) + printProgressBar(currIndex, count) + }) + + log( + `Deleted ${C('green')}${delets}${C()} files, and ${C( + 'green' + )}${deletedDirCount}${C()} directories` + ) } function clearPossibleAnswers(path, db) { - let count = 0 - let currIndex = 1 - let delets = 0 - iterateDir(path, () => { - count++ - }) - - iterateDir(path, (currPath) => { - currIndex++ - if (currPath.includes('savedQuestions.json')) { - return - } - const { subj, questions } = utils.ReadJSON(currPath) - - questions.forEach((question) => { - const searchRes = search({ - qdb: db, - subjName: subj, - question: question, - searchTillMatchPercent: 80, - }) - if (searchRes.length > 0) { - utils.deleteFile(currPath) - delets++ - } + let count = 0 + let currIndex = 1 + let delets = 0 + iterateDir(path, () => { + count++ }) - printProgressBar(currIndex, count) - }) - log(`Deleted ${C('green')}${delets}${C()} files`) + + iterateDir(path, (currPath) => { + currIndex++ + if (currPath.includes('savedQuestions.json')) { + return + } + const { subj, questions } = utils.ReadJSON(currPath) + + questions.forEach((question) => { + const searchRes = search({ + qdb: db, + subjName: subj, + question: question, + searchTillMatchPercent: 80, + }) + if (searchRes.length > 0) { + utils.deleteFile(currPath) + delets++ + } + }) + printProgressBar(currIndex, count) + }) + log(`Deleted ${C('green')}${delets}${C()} files`) } function updateSavedQuestionsFile(path) { - const filePath = path + '/' + 'savedQuestions.json' - if (!utils.FileExists(filePath)) { - log(`${filePath} does not exists!`) - return - } + const filePath = path + '/' + 'savedQuestions.json' + if (!utils.FileExists(filePath)) { + log(`${filePath} does not exists!`) + return + } - const savedQuestions = utils.ReadJSON(filePath) - const filtered = savedQuestions.filter((sq) => { - return utils.FileExists(path + '/' + sq.fname) - }) + const savedQuestions = utils.ReadJSON(filePath) + const filtered = savedQuestions.filter((sq) => { + return utils.FileExists(path + '/' + sq.fname) + }) - if (savedQuestions.length !== filtered.length) { - utils.WriteFile(JSON.stringify(filtered), filePath) - } + if (savedQuestions.length !== filtered.length) { + utils.WriteFile(JSON.stringify(filtered), filePath) + } } // --------------------------------------------------------------------------------- @@ -277,103 +283,105 @@ function updateSavedQuestionsFile(path) { // --------------------------------------------------------------------------------- function rmDuplicates(db) { - return difference({ dbA: db }) + return difference({ dbA: db }) } function difference({ dbA, dbB }) { - const doingDifference = !!dbB - // Stuff only from A - const resultDb = [] - let dbLength = 0 - let removedTotal = 0 - let processedQuestions = 0 + const doingDifference = !!dbB + // Stuff only from A + const resultDb = [] + let dbLength = 0 + let removedTotal = 0 + let processedQuestions = 0 - iterateSubjects(dbA, () => { - dbLength++ - }) - currentMaxIndex = dbLength - - const getResultDbLength = () => { - let resultDbLength = 0 - iterateSubjects(resultDb, () => { - resultDbLength++ + iterateSubjects(dbA, () => { + dbLength++ }) - return resultDbLength - } + currentMaxIndex = dbLength - for (let i = 0; i < dbA.length; i++) { - const subj = dbA[i] - const subjLogPath = logPath + subj.Name - utils.WriteFile('', subjLogPath) - let removedCount = 0 + const getResultDbLength = () => { + let resultDbLength = 0 + iterateSubjects(resultDb, () => { + resultDbLength++ + }) + return resultDbLength + } + + for (let i = 0; i < dbA.length; i++) { + const subj = dbA[i] + const subjLogPath = logPath + subj.Name + utils.WriteFile('', subjLogPath) + let removedCount = 0 + + hr() + log( + `${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${ + subj.Name + }, ${C('blue')}${subj.Questions.length}${C( + 'green' + )} questions${C()}` + ) + + printProgressBar(i + 1, dbA.length) + for (let j = 0; j < subj.Questions.length; j++) { + const question = subj.Questions[j] + const searchRes = search({ + qdb: doingDifference ? dbB : resultDb, + subjName: subj.Name, + question: question, + searchInAllIfNoResult: doingDifference, + searchTillMatchPercent: minpercent, + }) + + printProgressBar(processedQuestions, dbLength) + processedQuestions++ + currentIndex = processedQuestions + + const res = hasRequiredPercent(searchRes, minpercent) + + // no result: adding to difference + if (res.length === 0) { + // no result: adding to difference + addQuestion(resultDb, subj.Name, question) + } else { + // has result, not adding to difference + utils.AppendToFile( + line + + '\n' + + line + + '\n' + + JSON.stringify(question, null, 2) + + '\n' + + line + + JSON.stringify(res, null, 2) + + '\n', + subjLogPath + ) + removedCount++ + removedTotal++ + } + } + log( + `${C('yellow')}Removed ${C('red')}${removedCount}${C( + 'yellow' + )} questions${C()}` + ) + } hr() log( - `${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${subj.Name}, ${C( - 'blue' - )}${subj.Questions.length}${C('green')} questions${C()}` + `Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions` ) - - printProgressBar(i + 1, dbA.length) - for (let j = 0; j < subj.Questions.length; j++) { - const question = subj.Questions[j] - const searchRes = search({ - qdb: doingDifference ? dbB : resultDb, - subjName: subj.Name, - question: question, - searchInAllIfNoResult: doingDifference, - searchTillMatchPercent: minpercent, - }) - - printProgressBar(processedQuestions, dbLength) - processedQuestions++ - currentIndex = processedQuestions - - const res = hasRequiredPercent(searchRes, minpercent) - - // no result: adding to difference - if (res.length === 0) { - // no result: adding to difference - addQuestion(resultDb, subj.Name, question) - } else { - // has result, not adding to difference - utils.AppendToFile( - line + - '\n' + - line + - '\n' + - JSON.stringify(question, null, 2) + - '\n' + - line + - JSON.stringify(res, null, 2) + - '\n', - subjLogPath - ) - removedCount++ - removedTotal++ - } - } - log( - `${C('yellow')}Removed ${C('red')}${removedCount}${C( - 'yellow' - )} questions${C()}` - ) - } - - hr() - log( - `Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions` - ) - return resultDb + return resultDb } function hasRequiredPercent(result, minpercent) { - return result.reduce((acc, res) => { - if (res.match >= minpercent) { - acc.push(res) - } - return acc - }, []) + return result.reduce((acc, res) => { + if (res.match >= minpercent) { + acc.push(res) + } + return acc + }, []) } // --------------------------------------------------------------------------------- @@ -381,22 +389,22 @@ function hasRequiredPercent(result, minpercent) { // --------------------------------------------------------------------------------- function search({ qdb, subjName, question, searchInAllIfNoResult }) { - return doSearch( - qdb, - subjName, - question, - null, - minpercent, - searchInAllIfNoResult - ) + return doSearch( + qdb, + subjName, + question, + null, + minpercent, + searchInAllIfNoResult + ) } function iterateSubjects(db, fn) { - db.forEach((subj) => { - subj.Questions.forEach((question) => { - fn(subj, question) + db.forEach((subj) => { + subj.Questions.forEach((question) => { + fn(subj, question) + }) }) - }) } // --------------------------------------------------------------------------------- @@ -404,19 +412,19 @@ function iterateSubjects(db, fn) { // --------------------------------------------------------------------------------- function iterateDir(path, action) { - if (!utils.FileExists(path)) { - return - } + if (!utils.FileExists(path)) { + return + } - const stat = fs.lstatSync(path) - if (stat.isDirectory()) { - const content = fs.readdirSync(path) - content.forEach((currContent) => { - iterateDir(`${path}/${currContent}`, action) - }) - } else { - action(path) - } + const stat = fs.lstatSync(path) + if (stat.isDirectory()) { + const content = fs.readdirSync(path) + content.forEach((currContent) => { + iterateDir(`${path}/${currContent}`, action) + }) + } else { + action(path) + } } // --------------------------------------------------------------------------------- @@ -424,69 +432,69 @@ function iterateDir(path, action) { // --------------------------------------------------------------------------------- function hr() { - let res = '' - for (let i = 0; i < process.stdout.columns; i++) { - res += '=' - } - log(`${C('cyan')}${res}${C()}`) + let res = '' + for (let i = 0; i < process.stdout.columns; i++) { + res += '=' + } + log(`${C('cyan')}${res}${C()}`) } function log(text) { - utils.AppendToFile(text, globalLog) - if (silenced) return - if (process.stdout.isTTY) { - process.stdout.clearLine() - process.stdout.cursorTo(0) - } + utils.AppendToFile(text, globalLog) + if (silenced) return + if (process.stdout.isTTY) { + process.stdout.clearLine() + process.stdout.cursorTo(0) + } - console.log(text) + console.log(text) } function writeInSameLine(text, returnToLineStart) { - if (!process.stdout.isTTY) { - return - } - process.stdout.clearLine() - process.stdout.cursorTo(0) - process.stdout.write(text) - if (returnToLineStart) { - process.stdout.write('\r') - } else { - process.stdout.write('\n') - } + if (!process.stdout.isTTY) { + return + } + process.stdout.clearLine() + process.stdout.cursorTo(0) + process.stdout.write(text) + if (returnToLineStart) { + process.stdout.write('\r') + } else { + process.stdout.write('\n') + } } function printProgressBar(current, total) { - if (!process.stdout.isTTY || silenced) { - return - } - const width = process.stdout.columns - 30 + if (!process.stdout.isTTY || silenced) { + return + } + const width = process.stdout.columns - 30 - if (width <= 0) { - return - } + if (width <= 0) { + return + } - const x = width / total - const xCurrent = Math.floor(current * x) - const xTotal = Math.floor(total * x) + const x = width / total + const xCurrent = Math.floor(current * x) + const xTotal = Math.floor(total * x) - let line = '' - for (let i = 0; i < xCurrent; i++) { - line += '=' - } + let line = '' + for (let i = 0; i < xCurrent; i++) { + line += '=' + } - for (let i = 0; i < xTotal - xCurrent; i++) { - line += ' ' - } - const numbers = `${current} / ${total}` - writeInSameLine( - `${C('magenta')} [${line}]${C('green')} ${numbers}${C()}`, - current !== total - ) + for (let i = 0; i < xTotal - xCurrent; i++) { + line += ' ' + } + const numbers = `${current} / ${total}` + writeInSameLine( + `${C('magenta')} [${line}]${C('green')} ${numbers}${C()}`, + current !== total + ) } function C(color) { - return logger.C(color) + return logger.C(color) } process.exit() diff --git a/src/standaloneUtils/sqliteTest.js b/src/standaloneUtils/sqliteTest.js index 64af3ee..a912a5c 100644 --- a/src/standaloneUtils/sqliteTest.js +++ b/src/standaloneUtils/sqliteTest.js @@ -6,73 +6,73 @@ const dbtools = require('../utils/dbtools.js') Main() function Main() { - const cols = { - uname: { - type: 'text', - }, - pw: { - type: 'text', - }, - notes: { - type: 'text', - }, - } - const dbName = 'test' + const cols = { + uname: { + type: 'text', + }, + pw: { + type: 'text', + }, + notes: { + type: 'text', + }, + } + const dbName = 'test' - const db = dbtools.GetDB('./testdb.db') + const db = dbtools.GetDB('./testdb.db') - // Creating table - dbtools.CreateTable(db, dbName, cols) - console.log(dbtools.TableInfo(db, dbName)) - dbtools.SelectAll(db, dbName) - // inserting test val to table - dbtools.Insert(db, dbName, { - uname: 'mrfry', - pw: 'dsads', - }) - // Selecting a record - console.log( - dbtools.Select(db, dbName, { - uname: 'mrfry', + // Creating table + dbtools.CreateTable(db, dbName, cols) + console.log(dbtools.TableInfo(db, dbName)) + dbtools.SelectAll(db, dbName) + // inserting test val to table + dbtools.Insert(db, dbName, { + uname: 'mrfry', + pw: 'dsads', }) - ) - console.log(dbtools.TableInfo(db, dbName)) - console.log(dbtools.SelectAll(db, dbName)) - // Updating record - dbtools.Update( - db, - dbName, - { - pw: 'sspw', - }, - { - uname: 'mrfry', - } - ) - console.log(dbtools.SelectAll(db, dbName)) - // Updating record again - dbtools.Update( - db, - dbName, - { - notes: 'new note!', - }, - { - uname: 'mrfry', - } - ) - console.log(dbtools.SelectAll(db, dbName)) - // Adding new column and - dbtools.AddColumn(db, dbName, { - test: 'text', - }) - console.log(dbtools.TableInfo(db, dbName)) - console.log(dbtools.SelectAll(db, dbName)) - // Deleting stuff - dbtools.Delete(db, dbName, { - uname: 'mrfry', - }) - console.log(dbtools.SelectAll(db, dbName)) + // Selecting a record + console.log( + dbtools.Select(db, dbName, { + uname: 'mrfry', + }) + ) + console.log(dbtools.TableInfo(db, dbName)) + console.log(dbtools.SelectAll(db, dbName)) + // Updating record + dbtools.Update( + db, + dbName, + { + pw: 'sspw', + }, + { + uname: 'mrfry', + } + ) + console.log(dbtools.SelectAll(db, dbName)) + // Updating record again + dbtools.Update( + db, + dbName, + { + notes: 'new note!', + }, + { + uname: 'mrfry', + } + ) + console.log(dbtools.SelectAll(db, dbName)) + // Adding new column and + dbtools.AddColumn(db, dbName, { + test: 'text', + }) + console.log(dbtools.TableInfo(db, dbName)) + console.log(dbtools.SelectAll(db, dbName)) + // Deleting stuff + dbtools.Delete(db, dbName, { + uname: 'mrfry', + }) + console.log(dbtools.SelectAll(db, dbName)) - dbtools.CloseDB(db) + dbtools.CloseDB(db) } diff --git a/src/tests/addQuestion.test.ts b/src/tests/addQuestion.test.ts index 85c3316..063257a 100644 --- a/src/tests/addQuestion.test.ts +++ b/src/tests/addQuestion.test.ts @@ -4,45 +4,45 @@ import { Subject, Question } from '../types/basicTypes' const question: Question = createQuestion('asd', 'asd', { type: 'simple' }) test('Adds questions to empty db', () => { - const emptyDb: Subject[] = [] - addQuestion(emptyDb, 'test subject', question) - expect(emptyDb.length).toBe(1) + const emptyDb: Subject[] = [] + addQuestion(emptyDb, 'test subject', question) + expect(emptyDb.length).toBe(1) }) test('Adds questions next to existing', () => { - const db: Subject[] = [ - { - Name: 'test subject', - Questions: [question], - }, - ] - addQuestion(db, 'another something', question) - expect(db.length).toBe(2) + const db: Subject[] = [ + { + Name: 'test subject', + Questions: [question], + }, + ] + addQuestion(db, 'another something', question) + expect(db.length).toBe(2) }) test('Does not add new subject, multiple new questions', () => { - const db: Subject[] = [ - { - Name: 'test subject', - Questions: [question], - }, - ] - addQuestion(db, 'test subject', question) - addQuestion(db, 'test subject', question) - addQuestion(db, 'test subject', question) - addQuestion(db, 'test subject', question) - expect(db.length).toBe(1) + const db: Subject[] = [ + { + Name: 'test subject', + Questions: [question], + }, + ] + addQuestion(db, 'test subject', question) + addQuestion(db, 'test subject', question) + addQuestion(db, 'test subject', question) + addQuestion(db, 'test subject', question) + expect(db.length).toBe(1) }) test('Adds new subjects, multiple new questions', () => { - const db: Subject[] = [ - { - Name: 'test subject', - Questions: [question], - }, - ] - addQuestion(db, 'gfjdkglfd', question) - addQuestion(db, ' somrthing test ', question) - addQuestion(db, 'aaaaaaaa', question) - expect(db.length).toBe(4) + const db: Subject[] = [ + { + Name: 'test subject', + Questions: [question], + }, + ] + addQuestion(db, 'gfjdkglfd', question) + addQuestion(db, ' somrthing test ', question) + addQuestion(db, 'aaaaaaaa', question) + expect(db.length).toBe(4) }) diff --git a/src/tests/base64ToText.test.ts b/src/tests/base64ToText.test.ts index 26e7aba..c4597bc 100644 --- a/src/tests/base64ToText.test.ts +++ b/src/tests/base64ToText.test.ts @@ -11,22 +11,22 @@ // const expectedResults = ['Melyik híres zenekar tagja volt Joe Muranyi?'] test('Img text recognition works', async () => { - // TODO: tesseract keeps workers even after terminate(), and jest --detectOpenHandles detects them - expect(true).toBeTruthy() - // await tesseractLoaded - // for (let i = 0; i < imgs.length; i++) { - // const expectedResult = expectedResults[i] - // const img = imgs[i] - // - // const text = await recognizeTextFromBase64(img) - // expect(text.trim() === expectedResult).toBeTruthy() - // } - // - // await terminateWorker() - // - // return new Promise((resolve) => { - // setTimeout(() => { - // resolve() - // }, 1 * 1000) - // }) + // TODO: tesseract keeps workers even after terminate(), and jest --detectOpenHandles detects them + expect(true).toBeTruthy() + // await tesseractLoaded + // for (let i = 0; i < imgs.length; i++) { + // const expectedResult = expectedResults[i] + // const img = imgs[i] + // + // const text = await recognizeTextFromBase64(img) + // expect(text.trim() === expectedResult).toBeTruthy() + // } + // + // await terminateWorker() + // + // return new Promise((resolve) => { + // setTimeout(() => { + // resolve() + // }, 1 * 1000) + // }) }) diff --git a/src/tests/oldQuestionRemoving.test.ts b/src/tests/oldQuestionRemoving.test.ts index 1445aa6..cafc782 100644 --- a/src/tests/oldQuestionRemoving.test.ts +++ b/src/tests/oldQuestionRemoving.test.ts @@ -6,192 +6,192 @@ import { QuestionDb, Subject, Question } from '../types/basicTypes' const date = (x?: number) => new Date().getTime() + (x || 0) const q1 = createQuestion( - 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', - 'piaci áruk eltérhet a névértéktől.', - { - type: 'simple', - date: date(-1000), - } + 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', + 'piaci áruk eltérhet a névértéktől.', + { + type: 'simple', + date: date(-1000), + } ) const q2 = createQuestion( - 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', - 'afjléa gféda gfdjs légf', - { - type: 'simple', - date: date(-1000), - } + 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', + 'afjléa gféda gfdjs légf', + { + type: 'simple', + date: date(-1000), + } ) const q3 = createQuestion( - 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', - 'afjlsd gfds dgfs gf sdgf d', - { - type: 'simple', - date: date(-1000), - } + 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', + 'afjlsd gfds dgfs gf sdgf d', + { + type: 'simple', + date: date(-1000), + } ) const q4 = createQuestion( - 'A kötvény névértéke', - 'A kötvényen feltüntetett meghatározott nagyságú összeg.', - { - type: 'simple', - date: date(-1000), - } + 'A kötvény névértéke', + 'A kötvényen feltüntetett meghatározott nagyságú összeg.', + { + type: 'simple', + date: date(-1000), + } ) const q5 = createQuestion( - 'Mi az osztalék? asd asd', - 'A vállalati profit egy része..', - { - type: 'simple', - date: date(1000), - } + 'Mi az osztalék? asd asd', + 'A vállalati profit egy része..', + { + type: 'simple', + date: date(1000), + } ) const q6 = createQuestion( - 'valaim nagyon értelmes kérdés asd asd', - 'A vállalati profit egy része..', - { - type: 'simple', - date: date(1000), - } + 'valaim nagyon értelmes kérdés asd asd', + 'A vállalati profit egy része..', + { + type: 'simple', + date: date(1000), + } ) function setupTest({ - newQuestions, - data, - subjToClean, + newQuestions, + data, + subjToClean, }: { - newQuestions: Question[] - data: Subject[] - subjToClean?: string + newQuestions: Question[] + data: Subject[] + subjToClean?: string }) { - const recievedQuestions: Question[] = newQuestions.map((x) => { + const recievedQuestions: Question[] = newQuestions.map((x) => { + return { + ...x, + data: { + ...x.data, + date: date(), + }, + } + }) + const subjName = subjToClean || 'subject' + const overwriteFromDate = date(-100) + const qdbIndex = 0 + const qdbs: QuestionDb[] = [ + { + name: 'test', + data: data, + index: 0, + path: '', + shouldSearch: 'asd', + shouldSave: {}, + }, + ] + const subjIndex = qdbs[qdbIndex].data.findIndex((x) => { + return x.Name.toLowerCase().includes(subjName.toLowerCase()) + }) + + const questionIndexesToRemove = cleanDb( + { + questions: recievedQuestions, + subjToClean: subjName, + overwriteFromDate: overwriteFromDate, + qdbIndex: qdbIndex, + }, + qdbs + ) + + const updatedQuestions = updateQuestionsInArray( + questionIndexesToRemove, + qdbs[qdbIndex].data[subjIndex].Questions, + recievedQuestions + ) + return { - ...x, - data: { - ...x.data, - date: date(), - }, + questionIndexesToRemove: questionIndexesToRemove, + updatedQuestions: updatedQuestions, + overwriteFromDate: overwriteFromDate, + subjIndex: subjIndex, } - }) - const subjName = subjToClean || 'subject' - const overwriteFromDate = date(-100) - const qdbIndex = 0 - const qdbs: QuestionDb[] = [ - { - name: 'test', - data: data, - index: 0, - path: '', - shouldSearch: 'asd', - shouldSave: {}, - }, - ] - const subjIndex = qdbs[qdbIndex].data.findIndex((x) => { - return x.Name.toLowerCase().includes(subjName.toLowerCase()) - }) - - const questionIndexesToRemove = cleanDb( - { - questions: recievedQuestions, - subjToClean: subjName, - overwriteFromDate: overwriteFromDate, - qdbIndex: qdbIndex, - }, - qdbs - ) - - const updatedQuestions = updateQuestionsInArray( - questionIndexesToRemove, - qdbs[qdbIndex].data[subjIndex].Questions, - recievedQuestions - ) - - return { - questionIndexesToRemove: questionIndexesToRemove, - updatedQuestions: updatedQuestions, - overwriteFromDate: overwriteFromDate, - subjIndex: subjIndex, - } } const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] } test('Old and duplicate questions should be removed from the database', () => { - const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } = - setupTest({ newQuestions: [q1, q4, q5], data: [s1] }) + const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } = + setupTest({ newQuestions: [q1, q4, q5], data: [s1] }) - expect(questionIndexesToRemove.length).toBe(3) - expect(questionIndexesToRemove[0].length).toBe(2) + expect(questionIndexesToRemove.length).toBe(3) + expect(questionIndexesToRemove[0].length).toBe(2) - expect(updatedQuestions.length).toBe(3) - const toremoveCount = updatedQuestions.filter((question) => { - return question.Q.includes('TOREMOVE') - }).length - expect(toremoveCount).toBe(1) - const newQuestion = updatedQuestions.find((question) => { - return question.Q.includes('TOREMOVE') - }) - expect(newQuestion.data.date > overwriteFromDate).toBeTruthy() + expect(updatedQuestions.length).toBe(3) + const toremoveCount = updatedQuestions.filter((question) => { + return question.Q.includes('TOREMOVE') + }).length + expect(toremoveCount).toBe(1) + const newQuestion = updatedQuestions.find((question) => { + return question.Q.includes('TOREMOVE') + }) + expect(newQuestion.data.date > overwriteFromDate).toBeTruthy() }) const s2: Subject = { - Name: 'test subject', - Questions: [q1, q2, q3, q4, q5, q6], + Name: 'test subject', + Questions: [q1, q2, q3, q4, q5, q6], } test('Old and duplicate questions should be removed from the database round 2', () => { - const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } = - setupTest({ newQuestions: [q1, q4, q5], data: [s2] }) + const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } = + setupTest({ newQuestions: [q1, q4, q5], data: [s2] }) - expect(questionIndexesToRemove.length).toBe(3) - expect(questionIndexesToRemove[0].length).toBe(3) + expect(questionIndexesToRemove.length).toBe(3) + expect(questionIndexesToRemove[0].length).toBe(3) - expect(updatedQuestions.length).toBe(4) - const toremoveCount = updatedQuestions.filter((question) => { - return question.Q.includes('TOREMOVE') - }).length - expect(toremoveCount).toBe(1) - const newQuestion = updatedQuestions.find((question) => { - return question.Q.includes('TOREMOVE') - }) - expect(newQuestion.data.date > overwriteFromDate).toBeTruthy() + expect(updatedQuestions.length).toBe(4) + const toremoveCount = updatedQuestions.filter((question) => { + return question.Q.includes('TOREMOVE') + }).length + expect(toremoveCount).toBe(1) + const newQuestion = updatedQuestions.find((question) => { + return question.Q.includes('TOREMOVE') + }) + expect(newQuestion.data.date > overwriteFromDate).toBeTruthy() }) const s3: Subject = { - Name: 'test subject', - Questions: [q5, q6].map((x) => ({ - ...x, - data: { - ...x.data, - date: date(+50000), - }, - })), + Name: 'test subject', + Questions: [q5, q6].map((x) => ({ + ...x, + data: { + ...x.data, + date: date(+50000), + }, + })), } test('Old and duplicate questions should be removed from the database: questions should be left alone when they are newer', () => { - const { questionIndexesToRemove, updatedQuestions } = setupTest({ - newQuestions: [q5, q6], - data: [s3], - }) + const { questionIndexesToRemove, updatedQuestions } = setupTest({ + newQuestions: [q5, q6], + data: [s3], + }) - expect(questionIndexesToRemove.length).toBe(2) - questionIndexesToRemove.forEach((x) => { - expect(x.length).toBe(0) - }) + expect(questionIndexesToRemove.length).toBe(2) + questionIndexesToRemove.forEach((x) => { + expect(x.length).toBe(0) + }) - expect(updatedQuestions.length).toBe(2) + expect(updatedQuestions.length).toBe(2) }) const s4: Subject = { - Name: 'something else', - Questions: [q5, q6], + Name: 'something else', + Questions: [q5, q6], } test('Old and duplicate questions should be removed from the database:other subjects should be left alone', () => { - const { subjIndex } = setupTest({ - newQuestions: [q5, q6], - data: [s2, s1, s4, s3], - subjToClean: 'else', - }) + const { subjIndex } = setupTest({ + newQuestions: [q5, q6], + data: [s2, s1, s4, s3], + subjToClean: 'else', + }) - expect(subjIndex).toBe(2) + expect(subjIndex).toBe(2) }) diff --git a/src/tests/possibleAnswerPenalty.test.ts b/src/tests/possibleAnswerPenalty.test.ts index bd1defe..55dc815 100644 --- a/src/tests/possibleAnswerPenalty.test.ts +++ b/src/tests/possibleAnswerPenalty.test.ts @@ -1,308 +1,308 @@ import { - setNoPossibleAnswersPenalties, - SearchResultQuestion, - noPossibleAnswerMatchPenalty, + setNoPossibleAnswersPenalties, + SearchResultQuestion, + noPossibleAnswerMatchPenalty, } from '../utils/classes' import { Question } from '../types/basicTypes' const matchPercent = 100 const questionWithNormalPossibleAnswers: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'rubber duck' }, - { type: 'txt', val: 'super laptop' }, - { type: 'txt', val: 'nothing in particular' }, - { type: 'txt', val: 'something giberish' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck' }, + { type: 'txt', val: 'super laptop' }, + { type: 'txt', val: 'nothing in particular' }, + { type: 'txt', val: 'something giberish' }, + ], + }, } const questionWithNormalPossibleAnswersWithLabels: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'a) nothing in particular' }, - { type: 'txt', val: 'b) super laptop' }, - { type: 'txt', val: 'c) something giberish' }, - { type: 'txt', val: 'd) rubber duck' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'a) nothing in particular' }, + { type: 'txt', val: 'b) super laptop' }, + { type: 'txt', val: 'c) something giberish' }, + { type: 'txt', val: 'd) rubber duck' }, + ], + }, } const questionWithNormalPossibleAnswers2: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'rubber duck' }, - { type: 'txt', val: 'cat' }, - { type: 'txt', val: 'nothing in particular' }, - { type: 'txt', val: 'dog' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck' }, + { type: 'txt', val: 'cat' }, + { type: 'txt', val: 'nothing in particular' }, + { type: 'txt', val: 'dog' }, + ], + }, } const questionWithNormalPossibleAnswers3: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'rubber duck 2' }, - { type: 'txt', val: 'whale' }, - { type: 'txt', val: 'nothing in particular 2' }, - { type: 'txt', val: 'sea lion' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck 2' }, + { type: 'txt', val: 'whale' }, + { type: 'txt', val: 'nothing in particular 2' }, + { type: 'txt', val: 'sea lion' }, + ], + }, } const questionWithNormalPossibleAnswers4: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'rubber duck' }, - { type: 'txt', val: 'super laptop' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck' }, + { type: 'txt', val: 'super laptop' }, + ], + }, } const questionWithSimilarPossibleAnswers: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'asd' }, - { type: 'txt', val: 'basd' }, - { type: 'txt', val: 'aaa' }, - { type: 'txt', val: 'bbb' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'asd' }, + { type: 'txt', val: 'basd' }, + { type: 'txt', val: 'aaa' }, + { type: 'txt', val: 'bbb' }, + ], + }, } const questionWithTrueFalsePossibleAnser: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - possibleAnswers: [ - { type: 'txt', val: 'true' }, - { type: 'txt', val: 'false' }, - ], - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'true' }, + { type: 'txt', val: 'false' }, + ], + }, } const questionWithNoPossibleAnswer: Question = { - Q: 'asd', - A: 'asd', - data: { - type: 'simple', - }, + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + }, } const resNormal: SearchResultQuestion = { - q: questionWithNormalPossibleAnswers, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithNormalPossibleAnswers, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const resNormal2: SearchResultQuestion = { - q: questionWithNormalPossibleAnswers2, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithNormalPossibleAnswers2, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const resNormal3: SearchResultQuestion = { - q: questionWithNormalPossibleAnswers3, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithNormalPossibleAnswers3, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const resNormal4: SearchResultQuestion = { - q: questionWithNormalPossibleAnswers4, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithNormalPossibleAnswers4, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const resSimilar: SearchResultQuestion = { - q: questionWithSimilarPossibleAnswers, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithSimilarPossibleAnswers, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const resTrueFalse: SearchResultQuestion = { - q: questionWithTrueFalsePossibleAnser, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithTrueFalsePossibleAnser, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const resNoPossibleAnswer: SearchResultQuestion = { - q: questionWithNoPossibleAnswer, - match: matchPercent, - detailedMatch: { - qMatch: matchPercent, - aMatch: matchPercent, - dMatch: matchPercent, - matchedSubjName: 'testSubj', - avg: matchPercent, - }, + q: questionWithNoPossibleAnswer, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, } const testFunction = ( - question: Question, - searchResult: SearchResultQuestion[], - index: number + question: Question, + searchResult: SearchResultQuestion[], + index: number ) => { - const updated = setNoPossibleAnswersPenalties( - question.data.possibleAnswers, - searchResult - ) + const updated = setNoPossibleAnswersPenalties( + question.data.possibleAnswers, + searchResult + ) - updated.forEach((x, i) => { - if (i !== index) { - expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty) - expect(x.detailedMatch.qMatch).toBe( - matchPercent - noPossibleAnswerMatchPenalty - ) - } else { - expect(x.match).toBe(100) - expect(x.detailedMatch.qMatch).toBe(100) - } - }) + updated.forEach((x, i) => { + if (i !== index) { + expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty) + expect(x.detailedMatch.qMatch).toBe( + matchPercent - noPossibleAnswerMatchPenalty + ) + } else { + expect(x.match).toBe(100) + expect(x.detailedMatch.qMatch).toBe(100) + } + }) - return updated + return updated } test('Possible answer penalty applies correctly (normal possible answers)', () => { - testFunction( - questionWithNormalPossibleAnswers, - [ - resNormal, - resNormal2, - resNormal3, - resNormal4, - resSimilar, - resTrueFalse, - resNoPossibleAnswer, - ], - 0 - ) + testFunction( + questionWithNormalPossibleAnswers, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 0 + ) }) test('Possible answer penalty applies correctly (normal possible answers, with labels)', () => { - testFunction( - questionWithNormalPossibleAnswersWithLabels, - [ - resNormal, - resNormal2, - resNormal3, - resNormal4, - resSimilar, - resTrueFalse, - resNoPossibleAnswer, - ], - 0 - ) + testFunction( + questionWithNormalPossibleAnswersWithLabels, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 0 + ) }) test('Possible answer penalty applies correctly (similar possible answers)', () => { - testFunction( - questionWithSimilarPossibleAnswers, - [ - resNormal, - resNormal2, - resNormal3, - resNormal4, - resSimilar, - resTrueFalse, - resNoPossibleAnswer, - ], - 4 - ) + testFunction( + questionWithSimilarPossibleAnswers, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 4 + ) }) test('Possible answer penalty applies correctly (true false possible answers)', () => { - testFunction( - questionWithTrueFalsePossibleAnser, - [ - resNormal, - resNormal2, - resNormal3, - resNormal4, - resSimilar, - resTrueFalse, - resNoPossibleAnswer, - ], - 5 - ) + testFunction( + questionWithTrueFalsePossibleAnser, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 5 + ) }) test('Possible answer penalty applies correctly (no possible answers)', () => { - const updated = setNoPossibleAnswersPenalties( - questionWithNoPossibleAnswer.data.possibleAnswers, - [ - resNormal, - resNormal2, - resNormal3, - resNormal4, - resSimilar, - resTrueFalse, - resNoPossibleAnswer, - ] - ) + const updated = setNoPossibleAnswersPenalties( + questionWithNoPossibleAnswer.data.possibleAnswers, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ] + ) - updated.forEach((x) => { - expect(x.match).toBe(100) - expect(x.detailedMatch.qMatch).toBe(100) - }) + updated.forEach((x) => { + expect(x.match).toBe(100) + expect(x.detailedMatch.qMatch).toBe(100) + }) }) test('Possible answer penalty applies correctly (empty searchResult)', () => { - const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0) - expect(updated.length).toBe(0) + const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0) + expect(updated.length).toBe(0) }) diff --git a/src/tests/shouldLog.test.ts b/src/tests/shouldLog.test.ts index 33a97da..9f83a45 100644 --- a/src/tests/shouldLog.test.ts +++ b/src/tests/shouldLog.test.ts @@ -6,10 +6,10 @@ const truthy = [1, 2, '10', '40'] const falsey = [5, '55', 47832, 'fhs'] test('ShouldLog works', () => { - truthy.forEach((x) => { - expect(shouldLog(x, noLogIds)).toBeTruthy() - }) - falsey.forEach((x) => { - expect(shouldLog(x, noLogIds)).toBeFalsy() - }) + truthy.forEach((x) => { + expect(shouldLog(x, noLogIds)).toBeTruthy() + }) + falsey.forEach((x) => { + expect(shouldLog(x, noLogIds)).toBeFalsy() + }) }) diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index f05164f..cead294 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -26,136 +26,136 @@ import http from 'http' import https from 'https' export interface QuestionData { - type: string - date?: Date | number - images?: Array - hashedImages?: Array - possibleAnswers?: Array<{ type: string - val: string - selectedByUser?: boolean - }> - base64?: string[] + date?: Date | number + images?: Array + hashedImages?: Array + possibleAnswers?: Array<{ + type: string + val: string + selectedByUser?: boolean + }> + base64?: string[] } export interface Question { - Q: string - A: string - data: QuestionData - cache?: { - Q: Array - A: Array - } + Q: string + A: string + data: QuestionData + cache?: { + Q: Array + A: Array + } } export interface Subject { - Name: string - Questions: Array + Name: string + Questions: Array } export interface DataFile { - path: string - name: string - locked?: Boolean - overwrites?: Array<{ - subjName: string - overwriteFromDate: number - }> - shouldSearch: - | string - | { + path: string + name: string + locked?: Boolean + overwrites?: Array<{ + subjName: string + overwriteFromDate: number + }> + shouldSearch: + | string + | { + location?: { + val: string + } + } + shouldSave: { location?: { - val: string + val: string + } + version?: { + compare: string + val: string } - } - shouldSave: { - location?: { - val: string } - version?: { - compare: string - val: string - } - } } export interface QuestionDb extends DataFile { - data: Array - index: number + data: Array + index: number } export interface User { - id: number - pw: string - created: Date + id: number + pw: string + created: Date } export interface User { - id: number - pw: string - pwRequestCount: number - avaiblePWRequests: number - loginCount: number - createdBy: number + id: number + pw: string + pwRequestCount: number + avaiblePWRequests: number + loginCount: number + createdBy: number } export interface Request extends express.Request { - body: T - cookies: any - session: { - user?: User - sessionID?: string - isException?: boolean - } - files: any - query: { [key: string]: string } + body: T + cookies: any + session: { + user?: User + sessionID?: string + isException?: boolean + } + files: any + query: { [key: string]: string } } export interface SubmoduleData { - app: express.Application - url: string - publicdirs: Array - userDB?: Database - nextdir?: string - moduleSpecificData?: any - httpServer: http.Server - httpsServer: https.Server + app: express.Application + url: string + publicdirs: Array + userDB?: Database + nextdir?: string + moduleSpecificData?: any + httpServer: http.Server + httpsServer: https.Server } export interface QuestionFromScript { - questions: Array - testUrl: string - subj: string + questions: Array + testUrl: string + subj: string } export interface DbSearchResult { - message?: string - recievedData?: string - question: Question - result: SearchResultQuestion[] - success: boolean + message?: string + recievedData?: string + question: Question + result: SearchResultQuestion[] + success: boolean } export interface RegisteredUserEntry { - cid: string - version: string - installSource: string - date: string - userAgent: string - loginDate?: string - uid?: number + cid: string + version: string + installSource: string + date: string + userAgent: string + loginDate?: string + uid?: number } export interface Submodule { - dailyAction?: () => void - load?: () => void + dailyAction?: () => void + load?: () => void } export interface ModuleType { - app: express.Application - dailyAction?: Function + app: express.Application + dailyAction?: Function } export interface Socket extends SocketIoSocket { - user: User + user: User } diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 48cdeca..411a09b 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -23,9 +23,9 @@ const dataLockFile = './data/lockData' import logger from '../utils/logger' import { - createQuestion, - WorkerResult, - SearchResultQuestion, + createQuestion, + WorkerResult, + SearchResultQuestion, } from '../utils/classes' import { doALongTask, msgAllWorker } from './workerPool' import idStats from '../utils/ids' @@ -34,11 +34,11 @@ import { addQuestion, getSubjNameWithoutYear } from './classes' // types import { - QuestionDb, - Subject, - Question, - User, - DataFile, + QuestionDb, + Subject, + Question, + User, + DataFile, } from '../types/basicTypes' // if a recievend question doesnt match at least this % to any other question in the db it gets @@ -49,805 +49,821 @@ const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate let currWrites = 0 export interface RecievedData { - quiz: Array - subj: string - id: string - version: string - location: string + quiz: Array + subj: string + id: string + version: string + location: string } export interface Result { - qdbIndex: number - qdbName: string - subjName: string - newQuestions: Array + qdbIndex: number + qdbName: string + subjName: string + newQuestions: Array } export function logResult( - recievedData: RecievedData, - result: Array, - userId: number, - dryRun?: boolean + recievedData: RecievedData, + result: Array, + userId: number, + dryRun?: boolean ): void { - logger.Log('\t' + recievedData.subj) + logger.Log('\t' + recievedData.subj) - if (dryRun) { - logger.Log('\tDry run') - } + if (dryRun) { + logger.Log('\tDry run') + } - let idRow = '\t' - if (recievedData.version) { - idRow += 'Version: ' + logger.logHashed(recievedData.version) - } - if (recievedData.id) { - idRow += ', CID: ' + logger.logHashed(recievedData.id) - } - idRow += `, User #${logger.logHashed(userId.toString())}` - logger.Log(idRow) + let idRow = '\t' + if (recievedData.version) { + idRow += 'Version: ' + logger.logHashed(recievedData.version) + } + if (recievedData.id) { + idRow += ', CID: ' + logger.logHashed(recievedData.id) + } + idRow += `, User #${logger.logHashed(userId.toString())}` + logger.Log(idRow) - if (result.length > 0) { - result.forEach((res: Result) => { - const allQLength = recievedData.quiz.length - let msg = `${res.qdbName}: ` - let color = logger.GetColor('green') - if (res.newQuestions.length > 0) { - color = logger.GetColor('blue') - msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )` - } else { - msg += `No new data ( ${allQLength} )` - } - logger.Log('\t' + msg, color) - }) - } else { - logger.Log('\tResults length is zero!', logger.GetColor('red')) - } + if (result.length > 0) { + result.forEach((res: Result) => { + const allQLength = recievedData.quiz.length + let msg = `${res.qdbName}: ` + let color = logger.GetColor('green') + if (res.newQuestions.length > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )` + } else { + msg += `No new data ( ${allQLength} )` + } + logger.Log('\t' + msg, color) + }) + } else { + logger.Log('\tResults length is zero!', logger.GetColor('red')) + } } export function processIncomingRequest( - recievedData: RecievedData, - questionDbs: Array, - dryRun: boolean, - user: User + recievedData: RecievedData, + questionDbs: Array, + dryRun: boolean, + user: User ): Promise> { - logger.DebugLog('Processing incoming request', 'isadding', 1) + logger.DebugLog('Processing incoming request', 'isadding', 1) - if (recievedData === undefined) { - logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) - throw new Error('Recieved data is undefined!') - } - - try { - let towrite = utils.GetDateString() + '\n' - towrite += - '------------------------------------------------------------------------------\n' - if (typeof recievedData === 'object') { - towrite += JSON.stringify(recievedData) - } else { - towrite += recievedData + if (recievedData === undefined) { + logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) + throw new Error('Recieved data is undefined!') } - towrite += - '\n------------------------------------------------------------------------------\n' - utils.AppendToFile(towrite, recDataFile) - logger.DebugLog('recDataFile written', 'isadding', 1) - } catch (err) { - logger.Log('Error writing recieved data.') - } - if (utils.FileExists(dataLockFile)) { - logger.Log( - 'Data lock file exists, skipping recieved data processing', - logger.GetColor('red') - ) - return new Promise((resolve) => resolve([])) - } + try { + let towrite = utils.GetDateString() + '\n' + towrite += + '------------------------------------------------------------------------------\n' + if (typeof recievedData === 'object') { + towrite += JSON.stringify(recievedData) + } else { + towrite += recievedData + } + towrite += + '\n------------------------------------------------------------------------------\n' + utils.AppendToFile(towrite, recDataFile) + logger.DebugLog('recDataFile written', 'isadding', 1) + } catch (err) { + logger.Log('Error writing recieved data.') + } - // FIXME: this many promises and stuff might be unnecesarry - const promises: Array> = questionDbs.reduce((acc, qdb) => { - acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) - return acc - }, []) - return Promise.all(promises) + if (utils.FileExists(dataLockFile)) { + logger.Log( + 'Data lock file exists, skipping recieved data processing', + logger.GetColor('red') + ) + return new Promise((resolve) => resolve([])) + } + + // FIXME: this many promises and stuff might be unnecesarry + const promises: Array> = questionDbs.reduce((acc, qdb) => { + acc.push(processIncomingRequestUsingDb(recievedData, qdb, dryRun, user)) + return acc + }, []) + return Promise.all(promises) } function processIncomingRequestUsingDb( - recievedData: RecievedData, - qdb: QuestionDb, - dryRun: boolean, - user: User + recievedData: RecievedData, + qdb: QuestionDb, + dryRun: boolean, + user: User ): Promise { - return new Promise((resolve, reject) => { - try { - const recievedQuestions: Question[] = [] + return new Promise((resolve, reject) => { + try { + const recievedQuestions: Question[] = [] - logger.DebugLog('recievedData JSON parsed', 'isadding', 1) - logger.DebugLog(recievedData, 'isadding', 3) - const allQLength = recievedData.quiz.length - const questionSearchPromises: Promise[] = [] - recievedData.quiz.forEach((question) => { - logger.DebugLog('Question:', 'isadding', 2) - logger.DebugLog(question, 'isadding', 2) - const currentQuestion = createQuestion( - question.Q, - question.A, - question.data - ) - logger.DebugLog( - 'Searching for question in subj ' + recievedData.subj, - 'isadding', - 3 - ) - logger.DebugLog(currentQuestion, 'isadding', 3) - if (isQuestionValid(currentQuestion)) { - recievedQuestions.push(currentQuestion) - // This here searches only in relevant subjects, and not all subjects - questionSearchPromises.push( - doALongTask({ - type: 'work', - data: { - searchIn: [qdb.index], - question: currentQuestion, - subjName: recievedData.subj, - searchTillMatchPercent: minMatchAmmountToAdd, - }, + logger.DebugLog('recievedData JSON parsed', 'isadding', 1) + logger.DebugLog(recievedData, 'isadding', 3) + const allQLength = recievedData.quiz.length + const questionSearchPromises: Promise[] = [] + recievedData.quiz.forEach((question) => { + logger.DebugLog('Question:', 'isadding', 2) + logger.DebugLog(question, 'isadding', 2) + const currentQuestion = createQuestion( + question.Q, + question.A, + question.data + ) + logger.DebugLog( + 'Searching for question in subj ' + recievedData.subj, + 'isadding', + 3 + ) + logger.DebugLog(currentQuestion, 'isadding', 3) + if (isQuestionValid(currentQuestion)) { + recievedQuestions.push(currentQuestion) + // This here searches only in relevant subjects, and not all subjects + questionSearchPromises.push( + doALongTask({ + type: 'work', + data: { + searchIn: [qdb.index], + question: currentQuestion, + subjName: recievedData.subj, + searchTillMatchPercent: minMatchAmmountToAdd, + }, + }) + ) + } else { + logger.DebugLog('Question isnt valid', 'isadding', 1) + logger.DebugLog(currentQuestion, 'isadding', 1) + } }) - ) - } else { - logger.DebugLog('Question isnt valid', 'isadding', 1) - logger.DebugLog(currentQuestion, 'isadding', 1) - } - }) - Promise.all(questionSearchPromises) - .then((results: Array) => { - const allQuestions: Question[] = [] // all new questions here that do not have result - results.forEach((result: WorkerResult, i) => { - const add = (result.result as SearchResultQuestion[]).every( - (res: SearchResultQuestion) => { - return res.match < minMatchAmmountToAdd - } - ) - if (add && !result.error) { - allQuestions.push(recievedQuestions[i]) - } - }) + Promise.all(questionSearchPromises) + .then((results: Array) => { + const allQuestions: Question[] = [] // all new questions here that do not have result + results.forEach((result: WorkerResult, i) => { + const add = ( + result.result as SearchResultQuestion[] + ).every((res: SearchResultQuestion) => { + return res.match < minMatchAmmountToAdd + }) + if (add && !result.error) { + allQuestions.push(recievedQuestions[i]) + } + }) - try { - const subjName = getSubjNameWithoutYear(recievedData.subj) - if (allQuestions.length > 0) { - addQuestionsToDb(allQuestions, subjName, qdb) + try { + const subjName = getSubjNameWithoutYear( + recievedData.subj + ) + if (allQuestions.length > 0) { + addQuestionsToDb(allQuestions, subjName, qdb) - currWrites++ - logger.DebugLog( - 'currWrites for data.json: ' + currWrites, - 'isadding', - 1 - ) - if (currWrites >= writeAfter && !dryRun) { - currWrites = 0 - logger.DebugLog('Writing data.json', 'isadding', 1) - writeData(qdb.data, qdb.path) - } - } + currWrites++ + logger.DebugLog( + 'currWrites for data.json: ' + currWrites, + 'isadding', + 1 + ) + if (currWrites >= writeAfter && !dryRun) { + currWrites = 0 + logger.DebugLog( + 'Writing data.json', + 'isadding', + 1 + ) + writeData(qdb.data, qdb.path) + } + } - idStats.LogId( - user.id, - recievedData.subj, - allQuestions.length, - allQLength - ) + idStats.LogId( + user.id, + recievedData.subj, + allQuestions.length, + allQLength + ) - logger.DebugLog('New Questions:', 'isadding', 2) - logger.DebugLog(allQuestions, 'isadding', 2) + logger.DebugLog('New Questions:', 'isadding', 2) + logger.DebugLog(allQuestions, 'isadding', 2) - logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1) - resolve({ - newQuestions: allQuestions, - subjName: recievedData.subj, - qdbIndex: qdb.index, - qdbName: qdb.name, - }) - runCleanWorker(recievedData.quiz, subjName, qdb) - } catch (error) { - console.error(error) + logger.DebugLog( + 'ProcessIncomingRequest done', + 'isadding', + 1 + ) + resolve({ + newQuestions: allQuestions, + subjName: recievedData.subj, + qdbIndex: qdb.index, + qdbName: qdb.name, + }) + runCleanWorker(recievedData.quiz, subjName, qdb) + } catch (error) { + console.error(error) + logger.Log( + 'Error while processing processData worker result!', + logger.GetColor('redbg') + ) + reject( + new Error( + 'Error while processing processData worker result!' + ) + ) + } + }) + .catch((err) => { + logger.Log( + 'Error while searching for questions in ProcessIncomingRequest!', + logger.GetColor('redbg') + ) + console.error(err) + }) + } catch (err) { + console.error(err) logger.Log( - 'Error while processing processData worker result!', - logger.GetColor('redbg') + 'There was en error handling incoming quiz data, see stderr', + logger.GetColor('redbg') ) - reject( - new Error('Error while processing processData worker result!') - ) - } - }) - .catch((err) => { - logger.Log( - 'Error while searching for questions in ProcessIncomingRequest!', - logger.GetColor('redbg') - ) - console.error(err) - }) - } catch (err) { - console.error(err) - logger.Log( - 'There was en error handling incoming quiz data, see stderr', - logger.GetColor('redbg') - ) - reject(new Error('Couldnt parse JSON data')) - } - }) + reject(new Error('Couldnt parse JSON data')) + } + }) } function addQuestionsToDb( - allQuestions: Question[], - subjName: string, - qdb: QuestionDb + allQuestions: Question[], + subjName: string, + qdb: QuestionDb ) { - allQuestions.forEach((currentQuestion) => { - logger.DebugLog( - 'Adding question with subjName: ' + subjName + ' :', - 'isadding', - 3 - ) - logger.DebugLog(currentQuestion, 'isadding', 3) - addQuestion(qdb.data, subjName, { - ...currentQuestion, - data: { - ...currentQuestion.data, - date: new Date().getTime(), - }, + allQuestions.forEach((currentQuestion) => { + logger.DebugLog( + 'Adding question with subjName: ' + subjName + ' :', + 'isadding', + 3 + ) + logger.DebugLog(currentQuestion, 'isadding', 3) + addQuestion(qdb.data, subjName, { + ...currentQuestion, + data: { + ...currentQuestion.data, + date: new Date().getTime(), + }, + }) }) - }) } function runCleanWorker( - recievedQuesitons: Question[], - subjName: string, - qdb: QuestionDb + recievedQuesitons: Question[], + subjName: string, + qdb: QuestionDb ) { - if (qdb.overwrites && qdb.overwrites.length) { - // check if subject needs to be updated, and qdb has overwriteFromDate - const overwrite = qdb.overwrites.find((x) => { - return subjName.toLowerCase().includes(x.subjName.toLowerCase()) - }) + if (qdb.overwrites && qdb.overwrites.length) { + // check if subject needs to be updated, and qdb has overwriteFromDate + const overwrite = qdb.overwrites.find((x) => { + return subjName.toLowerCase().includes(x.subjName.toLowerCase()) + }) - if (!overwrite) { - return + if (!overwrite) { + return + } + // logger.Log( + // `\tStarting cleaning in subject "${logger.C( + // 'green' + // )}${subjName}${logger.C('')}" (matched: "${logger.C('green')}${ + // overwrite.subjName + // }${logger.C('')}")` + // ) + // pass recieved questions to a worker + doALongTask({ + type: 'dbClean', + data: { + questions: recievedQuesitons, + subjToClean: subjName, + overwriteFromDate: overwrite.overwriteFromDate, + qdbIndex: qdb.index, + }, + }).then(({ result: questionIndexesToRemove }) => { + const subjIndex = qdb.data.findIndex((x) => { + return x.Name.toLowerCase().includes(subjName.toLowerCase()) + }) + // sends msgs to all workers to remove it too + + msgAllWorker({ + type: 'rmQuestions', + data: { + questionIndexesToRemove: + questionIndexesToRemove as number[][], + subjIndex: subjIndex, + qdbIndex: qdb.index, + recievedQuestions: recievedQuesitons, + }, + }) + + // it adds the recieved question WITH DATE! + // recievedQuestions doesnt have date-s + qdb.data[subjIndex].Questions = updateQuestionsInArray( + questionIndexesToRemove as number[][], + qdb.data[subjIndex].Questions, + recievedQuesitons + ) + + // saves the file + writeData(qdb.data, qdb.path) + logger.Log( + `\tRemoved ${logger.C('green')}${ + (questionIndexesToRemove as number[][]).filter( + (x: number[]) => x.length > 1 + ).length + }${logger.C()} old questions from ${logger.C( + 'green' + )}${subjName}${logger.C()}` + ) + }) } - // logger.Log( - // `\tStarting cleaning in subject "${logger.C( - // 'green' - // )}${subjName}${logger.C('')}" (matched: "${logger.C('green')}${ - // overwrite.subjName - // }${logger.C('')}")` - // ) - // pass recieved questions to a worker - doALongTask({ - type: 'dbClean', - data: { - questions: recievedQuesitons, - subjToClean: subjName, - overwriteFromDate: overwrite.overwriteFromDate, - qdbIndex: qdb.index, - }, - }).then(({ result: questionIndexesToRemove }) => { - const subjIndex = qdb.data.findIndex((x) => { - return x.Name.toLowerCase().includes(subjName.toLowerCase()) - }) - // sends msgs to all workers to remove it too - - msgAllWorker({ - type: 'rmQuestions', - data: { - questionIndexesToRemove: questionIndexesToRemove as number[][], - subjIndex: subjIndex, - qdbIndex: qdb.index, - recievedQuestions: recievedQuesitons, - }, - }) - - // it adds the recieved question WITH DATE! - // recievedQuestions doesnt have date-s - qdb.data[subjIndex].Questions = updateQuestionsInArray( - questionIndexesToRemove as number[][], - qdb.data[subjIndex].Questions, - recievedQuesitons - ) - - // saves the file - writeData(qdb.data, qdb.path) - logger.Log( - `\tRemoved ${logger.C('green')}${ - (questionIndexesToRemove as number[][]).filter( - (x: number[]) => x.length > 1 - ).length - }${logger.C()} old questions from ${logger.C( - 'green' - )}${subjName}${logger.C()}` - ) - }) - } } export function updateQuestionsInArray( - questionIndexesToRemove: number[][], - questions: Question[], - newQuestions: Question[] + questionIndexesToRemove: number[][], + questions: Question[], + newQuestions: Question[] ): Question[] { - const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => { - if (x.length > 1) { - return [...acc, ...x] - } - return acc - }, []) + const indexesToRemove = questionIndexesToRemove.reduce((acc, x) => { + if (x.length > 1) { + return [...acc, ...x] + } + return acc + }, []) - const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => { - return questionIndexesToRemove[i].length > 1 - }) + const newQuestionsToAdd: Question[] = newQuestions.filter((_q, i) => { + return questionIndexesToRemove[i].length > 1 + }) - return [ - ...questions.filter((_x, i) => { - return !indexesToRemove.includes(i) - }), - ...newQuestionsToAdd.map((x) => { - x.data.date = new Date() - return x - }), - ] + return [ + ...questions.filter((_x, i) => { + return !indexesToRemove.includes(i) + }), + ...newQuestionsToAdd.map((x) => { + x.data.date = new Date() + return x + }), + ] } export function isQuestionValid(question: Question): boolean { - if (!question.Q) { - return false - } - if (!question.A && question.data.type === 'simple') { - return false - } - return true + if (!question.Q) { + return false + } + if (!question.A && question.data.type === 'simple') { + return false + } + return true } export function shouldSearchDataFile( - df: DataFile, - testUrl: string, - trueIfAlways?: boolean + df: DataFile, + testUrl: string, + trueIfAlways?: boolean ): boolean { - if ( - typeof df.shouldSearch === 'string' && - df.shouldSearch === 'always' && - trueIfAlways - ) { - return true - } - - if (typeof df.shouldSearch === 'object') { - if (df.shouldSearch.location) { - const { val } = df.shouldSearch.location - return testUrl.includes(val) + if ( + typeof df.shouldSearch === 'string' && + df.shouldSearch === 'always' && + trueIfAlways + ) { + return true } - } - logger.DebugLog(`no suitable dbs for ${testUrl}`, 'shouldSearchDataFile', 1) - return false + if (typeof df.shouldSearch === 'object') { + if (df.shouldSearch.location) { + const { val } = df.shouldSearch.location + return testUrl.includes(val) + } + } + + logger.DebugLog(`no suitable dbs for ${testUrl}`, 'shouldSearchDataFile', 1) + return false } export function shouldSaveDataFile( - df: DataFile, - recievedData: RecievedData + df: DataFile, + recievedData: RecievedData ): boolean { - if (df.locked) { + if (df.locked) { + return false + } + + if (df.shouldSave.version) { + const { compare, val } = df.shouldSave.version + if (compare === 'equals') { + return recievedData.version === val + } else if (compare === 'startsWith') { + return recievedData.version.startsWith(val) + } + } + + if (df.shouldSave.version) { + const { compare, val } = df.shouldSave.version + if (compare === 'equals') { + return recievedData.version === val + } else if (compare === 'startsWith') { + return recievedData.version.startsWith(val) + } + } + + if (df.shouldSave.location) { + const { val } = df.shouldSave.location + return recievedData.location.includes(val) + } + return false - } - - if (df.shouldSave.version) { - const { compare, val } = df.shouldSave.version - if (compare === 'equals') { - return recievedData.version === val - } else if (compare === 'startsWith') { - return recievedData.version.startsWith(val) - } - } - - if (df.shouldSave.version) { - const { compare, val } = df.shouldSave.version - if (compare === 'equals') { - return recievedData.version === val - } else if (compare === 'startsWith') { - return recievedData.version.startsWith(val) - } - } - - if (df.shouldSave.location) { - const { val } = df.shouldSave.location - return recievedData.location.includes(val) - } - - return false } export function loadData(path: string): Array { - return JSON.parse(utils.ReadFile(path)).map((subj: Subject) => { - return { - Name: subj.Name, - Questions: subj.Questions.map((question: Question) => { - return createQuestion(question) - }), - } - }) + return JSON.parse(utils.ReadFile(path)).map((subj: Subject) => { + return { + Name: subj.Name, + Questions: subj.Questions.map((question: Question) => { + return createQuestion(question) + }), + } + }) } export function loadJSON( - dataFiles: Array, - dataDir: string + dataFiles: Array, + dataDir: string ): Array { - const res: Array = dataFiles.reduce((acc, dataFile, index) => { - const dataPath = dataDir + dataFile.path + const res: Array = dataFiles.reduce((acc, dataFile, index) => { + const dataPath = dataDir + dataFile.path - if (!utils.FileExists(dataPath)) { - utils.WriteFile(JSON.stringify([]), dataPath) - } + if (!utils.FileExists(dataPath)) { + utils.WriteFile(JSON.stringify([]), dataPath) + } - try { - acc.push({ - ...dataFile, - path: dataPath, - index: index, - data: loadData(dataPath), - }) - } catch (err) { - console.error(err) - logger.Log( - "data is undefined! Couldn't load data!", - logger.GetColor('redbg') - ) - } - return acc - }, []) + try { + acc.push({ + ...dataFile, + path: dataPath, + index: index, + data: loadData(dataPath), + }) + } catch (err) { + console.error(err) + logger.Log( + "data is undefined! Couldn't load data!", + logger.GetColor('redbg') + ) + } + return acc + }, []) - let subjCount = 0 - let questionCount = 0 - res.forEach((qdb) => { - subjCount += qdb.data.length - qdb.data.forEach((subj) => { - questionCount += subj.Questions.length + let subjCount = 0 + let questionCount = 0 + res.forEach((qdb) => { + subjCount += qdb.data.length + qdb.data.forEach((subj) => { + questionCount += subj.Questions.length + }) }) - }) - logger.Log( - `Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`, - logger.GetColor('green') - ) + logger.Log( + `Loaded ${subjCount} subjects with ${questionCount} questions from ${res.length} question db-s`, + logger.GetColor('green') + ) - return res + return res } export function writeData(data: Array, path: string): void { - utils.WriteFile( - JSON.stringify( - data.map((subj) => { - return { - Name: subj.Name, - Questions: subj.Questions.map((question) => { - return { - Q: question.Q, - A: question.A, - data: question.data, - } - }), - } - }) - ), - path - ) + utils.WriteFile( + JSON.stringify( + data.map((subj) => { + return { + Name: subj.Name, + Questions: subj.Questions.map((question) => { + return { + Q: question.Q, + A: question.A, + data: question.data, + } + }), + } + }) + ), + path + ) } export function backupData(questionDbs: Array): void { - questionDbs.forEach((data) => { - const path = './publicDirs/qminingPublic/backs/' - utils.CreatePath(path) - try { - // logger.Log(`Backing up ${data.name}...`) - writeData( - data.data, - `${path}${data.name}_${utils.GetDateString(undefined, true)}.json` - ) - // logger.Log('Done') - } catch (err) { - logger.Log( - `Error backing up data file ${data.name}!`, - logger.GetColor('redbg') - ) - console.error(err) - } - }) + questionDbs.forEach((data) => { + const path = './publicDirs/qminingPublic/backs/' + utils.CreatePath(path) + try { + // logger.Log(`Backing up ${data.name}...`) + writeData( + data.data, + `${path}${data.name}_${utils.GetDateString( + undefined, + true + )}.json` + ) + // logger.Log('Done') + } catch (err) { + logger.Log( + `Error backing up data file ${data.name}!`, + logger.GetColor('redbg') + ) + console.error(err) + } + }) } function deleteFromDb( - questionDb: QuestionDb, - edits: { - index: number - subjName: string - type: string - selectedDb: { path: string; name: string } - } + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + } ): { - success: boolean - msg: string - deletedQuestion?: Question - resultDb?: QuestionDb + success: boolean + msg: string + deletedQuestion?: Question + resultDb?: QuestionDb } { - // { - // "index": 0, - // "subjName": "VHDL programozás", - // "type": "delete", - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } - const { index, subjName } = edits - let deletedQuestion: Question - if (isNaN(index) || !subjName) { + // { + // "index": 0, + // "subjName": "VHDL programozás", + // "type": "delete", + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { index, subjName } = edits + let deletedQuestion: Question + if (isNaN(index) || !subjName) { + return { + success: false, + msg: 'No .index or .subjName !', + } + } + + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.filter((question, i) => { + if (index === i) { + deletedQuestion = question + return false + } else { + return true + } + }), + } + } + }) + return { - success: false, - msg: 'No .index or .subjName !', + success: true, + msg: 'Delete successfull', + deletedQuestion: deletedQuestion, + resultDb: questionDb, } - } - - questionDb.data = questionDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.filter((question, i) => { - if (index === i) { - deletedQuestion = question - return false - } else { - return true - } - }), - } - } - }) - - return { - success: true, - msg: 'Delete successfull', - deletedQuestion: deletedQuestion, - resultDb: questionDb, - } } function editQuestionInDb( - questionDb: QuestionDb, - edits: { - index: number - subjName: string - type: string - selectedDb: { path: string; name: string } - newVal: Question - } + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + newVal: Question + } ): { - success: boolean - msg: string - newVal?: Question - oldVal?: Question - resultDb?: QuestionDb + success: boolean + msg: string + newVal?: Question + oldVal?: Question + resultDb?: QuestionDb } { - // { - // "index": 0, - // "subjName": "Elektronika", - // "type": "edit", - // "newVal": { - // "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.", - // "A": "Igaz", - // "data": { - // "type": "simple", - // "possibleAnswers": [ - // "Igaz" - // ] - // }, - // "possibleAnswers": [ - // "Igaz" - // ] - // }, - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } - const { index, subjName, newVal } = edits + // { + // "index": 0, + // "subjName": "Elektronika", + // "type": "edit", + // "newVal": { + // "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.", + // "A": "Igaz", + // "data": { + // "type": "simple", + // "possibleAnswers": [ + // "Igaz" + // ] + // }, + // "possibleAnswers": [ + // "Igaz" + // ] + // }, + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { index, subjName, newVal } = edits + + let oldVal: Question + if (isNaN(index) || !subjName) { + return { + success: false, + msg: 'No .index or .subjName !', + } + } + if (!isQuestionValid(newVal)) { + return { + success: false, + msg: 'edited question is not valid', + } + } + + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.map((question, i) => { + if (index === i) { + oldVal = question + return createQuestion(newVal) + } else { + return question + } + }), + } + } + }) - let oldVal: Question - if (isNaN(index) || !subjName) { return { - success: false, - msg: 'No .index or .subjName !', + success: true, + msg: 'Edit successfull', + oldVal: oldVal, + newVal: newVal, + resultDb: questionDb, } - } - if (!isQuestionValid(newVal)) { - return { - success: false, - msg: 'edited question is not valid', - } - } - - questionDb.data = questionDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.map((question, i) => { - if (index === i) { - oldVal = question - return createQuestion(newVal) - } else { - return question - } - }), - } - } - }) - - return { - success: true, - msg: 'Edit successfull', - oldVal: oldVal, - newVal: newVal, - resultDb: questionDb, - } } function editSubjInDb( - questionDb: QuestionDb, - edits: { - index: number - subjName: string - type: string - selectedDb: { path: string; name: string } - deletedQuestions?: Array - changedQuestions?: Array<{ - index: number - value: Question - }> - } + questionDb: QuestionDb, + edits: { + index: number + subjName: string + type: string + selectedDb: { path: string; name: string } + deletedQuestions?: Array + changedQuestions?: Array<{ + index: number + value: Question + }> + } ): { - success: boolean - msg: string - deletedQuestions?: Array - changedQuestions?: { oldVal: Question; newVal: Question }[] - resultDb?: QuestionDb + success: boolean + msg: string + deletedQuestions?: Array + changedQuestions?: { oldVal: Question; newVal: Question }[] + resultDb?: QuestionDb } { - // { - // "subjName": "Elektronika", - // "changedQuestions": [ - // { - // "index": 1, - // "value": { - // "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.", - // "A": "Hamis", - // "data": { - // "type": "simple", - // "possibleAnswers": [ - // "Igaz", - // "Hamis" - // ] - // } - // } - // } - // ], - // "deletedQuestions": [ - // 0 - // ], - // "type": "subjEdit", - // "selectedDb": { - // "path": "questionDbs/elearning.uni-obuda.hu.json", - // "name": "elearning.uni-obuda.hu" - // } - // } - const { subjName, changedQuestions, deletedQuestions } = edits - const deletedQuestionsToWrite: Question[] = [] - const changedQuestionsToWrite: { oldVal: Question; newVal: Question }[] = [] - if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) { + // { + // "subjName": "Elektronika", + // "changedQuestions": [ + // { + // "index": 1, + // "value": { + // "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.", + // "A": "Hamis", + // "data": { + // "type": "simple", + // "possibleAnswers": [ + // "Igaz", + // "Hamis" + // ] + // } + // } + // } + // ], + // "deletedQuestions": [ + // 0 + // ], + // "type": "subjEdit", + // "selectedDb": { + // "path": "questionDbs/elearning.uni-obuda.hu.json", + // "name": "elearning.uni-obuda.hu" + // } + // } + const { subjName, changedQuestions, deletedQuestions } = edits + const deletedQuestionsToWrite: Question[] = [] + const changedQuestionsToWrite: { oldVal: Question; newVal: Question }[] = [] + if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) { + return { + success: false, + msg: 'no changedQuestions or deletedQuestions!', + } + } + + // processing changed questions + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.map((question, i) => { + const changedTo = changedQuestions.find((cq) => { + return cq.index === i + }) + if (changedTo) { + changedQuestionsToWrite.push({ + oldVal: question, + newVal: changedTo.value, + }) + return createQuestion(changedTo.value) + } else { + return question + } + }), + } + } + }) + + // processing deletedQuestions + questionDb.data = questionDb.data.map((subj) => { + if (subj.Name !== subjName) { + return subj + } else { + return { + ...subj, + Questions: subj.Questions.filter((question, i) => { + const isDeleted = deletedQuestions.includes(i) + if (isDeleted) { + deletedQuestionsToWrite.push(question) + return false + } else { + return true + } + }), + } + } + }) + return { - success: false, - msg: 'no changedQuestions or deletedQuestions!', + success: true, + msg: 'subj edit successfull', + deletedQuestions: deletedQuestionsToWrite, + changedQuestions: changedQuestionsToWrite, + resultDb: questionDb, } - } - - // processing changed questions - questionDb.data = questionDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.map((question, i) => { - const changedTo = changedQuestions.find((cq) => { - return cq.index === i - }) - if (changedTo) { - changedQuestionsToWrite.push({ - oldVal: question, - newVal: changedTo.value, - }) - return createQuestion(changedTo.value) - } else { - return question - } - }), - } - } - }) - - // processing deletedQuestions - questionDb.data = questionDb.data.map((subj) => { - if (subj.Name !== subjName) { - return subj - } else { - return { - ...subj, - Questions: subj.Questions.filter((question, i) => { - const isDeleted = deletedQuestions.includes(i) - if (isDeleted) { - deletedQuestionsToWrite.push(question) - return false - } else { - return true - } - }), - } - } - }) - - return { - success: true, - msg: 'subj edit successfull', - deletedQuestions: deletedQuestionsToWrite, - changedQuestions: changedQuestionsToWrite, - resultDb: questionDb, - } } // FIXME: newVal is optional in some places but not in others export interface Edits { - index: number - subjName: string - selectedDb: { path: string; name: string } - type: string - newVal: Question - deletedQuestion?: Array - changedQuestions?: Array<{ index: number - value: Question - }> + subjName: string + selectedDb: { path: string; name: string } + type: string + newVal: Question + deletedQuestion?: Array + changedQuestions?: Array<{ + index: number + value: Question + }> } export function editDb( - questionDb: QuestionDb, - edits: Edits + questionDb: QuestionDb, + edits: Edits ): { - success: boolean - msg: string - resultDb?: QuestionDb - deletedQuestion?: Question - newVal?: Question - oldVal?: Question - deletedQuestions?: Array - changedQuestions?: { oldVal: Question; newVal: Question }[] + success: boolean + msg: string + resultDb?: QuestionDb + deletedQuestion?: Question + newVal?: Question + oldVal?: Question + deletedQuestions?: Array + changedQuestions?: { oldVal: Question; newVal: Question }[] } { - if (edits.type === 'delete') { - return deleteFromDb(questionDb, edits) - } - if (edits.type === 'edit') { - return editQuestionInDb(questionDb, edits) - } + if (edits.type === 'delete') { + return deleteFromDb(questionDb, edits) + } + if (edits.type === 'edit') { + return editQuestionInDb(questionDb, edits) + } - if (edits.type === 'subjEdit') { - return editSubjInDb(questionDb, edits) - } + if (edits.type === 'subjEdit') { + return editSubjInDb(questionDb, edits) + } - return { - success: false, - msg: 'DB edit error, no matched type', - } + return { + success: false, + msg: 'DB edit error, no matched type', + } } diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 9e86e21..3888a57 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -24,44 +24,44 @@ import { isMainThread, parentPort, workerData } from 'worker_threads' import { recognizeTextFromBase64, tesseractLoaded } from './tesseract' import logger from './logger' import { - Question, - QuestionData, - QuestionDb, - Subject, + Question, + QuestionData, + QuestionDb, + Subject, } from '../types/basicTypes' import { editDb, Edits, updateQuestionsInArray } from './actions' // import { TaskObject } from './workerPool' export interface WorkerResult { - msg: string - workerIndex: number - result?: SearchResultQuestion[] | number[][] - error?: boolean + msg: string + workerIndex: number + result?: SearchResultQuestion[] | number[][] + error?: boolean } interface DetailedMatch { - qMatch: number - aMatch: number - dMatch: number - matchedSubjName: string - avg: number + qMatch: number + aMatch: number + dMatch: number + matchedSubjName: string + avg: number } export interface SearchResultQuestion { - q: Question - match: number - detailedMatch: DetailedMatch + q: Question + match: number + detailedMatch: DetailedMatch } const commonUselessAnswerParts = [ - 'A helyes válasz az ', - 'A helyes válasz a ', - 'A helyes válaszok: ', - 'A helyes válaszok:', - 'A helyes válasz: ', - 'A helyes válasz:', - 'The correct answer is:', - "'", + 'A helyes válasz az ', + 'A helyes válasz a ', + 'A helyes válaszok: ', + 'A helyes válaszok:', + 'A helyes válasz: ', + 'A helyes válasz:', + 'The correct answer is:', + "'", ] // const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.'] @@ -82,167 +82,169 @@ const minMatchToNotSearchOtherSubjects = 90 // Exported // --------------------------------------------------------------------------------------------------------- function getSubjNameWithoutYear(subjName: string): string { - const t = subjName.split(' - ') - if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { - return t[1] || subjName - } else { - return subjName - } + const t = subjName.split(' - ') + if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t[1] || subjName + } else { + return subjName + } } // Not exported // --------------------------------------------------------------------------------------------------------- function simplifyString(toremove: string): string { - return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase() + return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase() } function removeStuff( - value: string, - removableStrings: Array, - toReplace?: string + value: string, + removableStrings: Array, + toReplace?: string ): string { - removableStrings.forEach((removableString) => { - const regex = new RegExp(removableString, 'g') - value = value.replace(regex, toReplace || '') - }) - return value + removableStrings.forEach((removableString) => { + const regex = new RegExp(removableString, 'g') + value = value.replace(regex, toReplace || '') + }) + return value } // damn nonbreaking space function normalizeSpaces(input: string): string { - return input.replace(/\s/g, ' ') + return input.replace(/\s/g, ' ') } function removeUnnecesarySpaces(toremove: string): string { - return normalizeSpaces(toremove) - .replace(/\s+/g, ' ') - .replace(/(\r\n|\n|\r)/gm, '') - .trim() + return normalizeSpaces(toremove) + .replace(/\s+/g, ' ') + .replace(/(\r\n|\n|\r)/gm, '') + .trim() } function compareString( - s1: string, - s2: string, - s1cache?: Array, - s2cache?: Array + s1: string, + s2: string, + s1cache?: Array, + s2cache?: Array ): number { - const s1a = s1cache || s1.split(' ') - const s2a = s2cache || s2.split(' ') + const s1a = s1cache || s1.split(' ') + const s2a = s2cache || s2.split(' ') - if (s1 === s2) { - return 100 - } - if (!s1a || !s2a) { - if (!s1a && !s2a) { - return 100 - } else { - return 0 + if (s1 === s2) { + return 100 } - } - if (s1a.length < 0 || s2a.length < 0) { - if (s1a.length === 0 && s2a.length === 0) { - return 100 - } else { - return 0 + if (!s1a || !s2a) { + if (!s1a && !s2a) { + return 100 + } else { + return 0 + } } - } - - let match = 0 - let lastMatchIndex = -2 - let i = 0 - - while (i < s1a.length) { - if (match / i < magicNumber) { - break + if (s1a.length < 0 || s2a.length < 0) { + if (s1a.length === 0 && s2a.length === 0) { + return 100 + } else { + return 0 + } } - const currMatchIndex = s2a.indexOf(s1a[i]) - if (currMatchIndex !== -1 && lastMatchIndex < currMatchIndex) { - match++ - lastMatchIndex = currMatchIndex + let match = 0 + let lastMatchIndex = -2 + let i = 0 + + while (i < s1a.length) { + if (match / i < magicNumber) { + break + } + + const currMatchIndex = s2a.indexOf(s1a[i]) + if (currMatchIndex !== -1 && lastMatchIndex < currMatchIndex) { + match++ + lastMatchIndex = currMatchIndex + } + + i++ } - i++ - } - - let percent = Math.round(parseFloat(((match / s1a.length) * 100).toFixed(2))) - const lengthDifference = Math.abs(s2a.length - s1a.length) - percent -= lengthDifference * lengthDiffMultiplier - if (percent < 0) { - percent = 0 - } - return percent + let percent = Math.round( + parseFloat(((match / s1a.length) * 100).toFixed(2)) + ) + const lengthDifference = Math.abs(s2a.length - s1a.length) + percent -= lengthDifference * lengthDiffMultiplier + if (percent < 0) { + percent = 0 + } + return percent } function answerPreProcessor(value: string): string { - if (!value) { - return value - } + if (!value) { + return value + } - return removeStuff(value, commonUselessAnswerParts) + return removeStuff(value, commonUselessAnswerParts) } // 'a. pécsi sör' -> 'pécsi sör' function removeAnswerLetters(value: string): string { - if (!value) { - return value - } + if (!value) { + return value + } - const val = value.split('. ') - if (val[0].length < 2 && val.length > 1) { - val.shift() - return val.join(' ') - } else { - return value - } + const val = value.split('. ') + if (val[0].length < 2 && val.length > 1) { + val.shift() + return val.join(' ') + } else { + return value + } } function simplifyQA(value: string, mods: Array): string { - if (!value) { - return value - } + if (!value) { + return value + } - return mods.reduce((res, fn) => { - return fn(res) - }, value) + return mods.reduce((res, fn) => { + return fn(res) + }, value) } function simplifyAnswer(value: string): string { - if (!value) { - return value - } - return simplifyQA(value, [ - removeUnnecesarySpaces, - answerPreProcessor, - removeAnswerLetters, - ]) + if (!value) { + return value + } + return simplifyQA(value, [ + removeUnnecesarySpaces, + answerPreProcessor, + removeAnswerLetters, + ]) } function simplifyQuestion(question: string): string { - if (!question) { - return question - } - return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters]) + if (!question) { + return question + } + return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters]) } function simplifyQuestionObj(question: Question): Question { - if (!question) { + if (!question) { + return question + } + if (question.Q) { + question.Q = simplifyQA(question.Q, [ + removeUnnecesarySpaces, + removeAnswerLetters, + ]) + } + if (question.A) { + question.A = simplifyQA(question.A, [ + removeUnnecesarySpaces, + removeAnswerLetters, + ]) + } return question - } - if (question.Q) { - question.Q = simplifyQA(question.Q, [ - removeUnnecesarySpaces, - removeAnswerLetters, - ]) - } - if (question.A) { - question.A = simplifyQA(question.A, [ - removeUnnecesarySpaces, - removeAnswerLetters, - ]) - } - return question } // --------------------------------------------------------------------------------------------------------- @@ -250,415 +252,422 @@ function simplifyQuestionObj(question: Question): Question { // --------------------------------------------------------------------------------------------------------- function createQuestion( - question: Question | string, - answer?: string, - data?: QuestionData + question: Question | string, + answer?: string, + data?: QuestionData ): Question { - try { - if (typeof question === 'string') { - return { - Q: simplifyQuestion(question), - A: answer ? simplifyAnswer(answer) : undefined, - data: data, - cache: { - Q: question ? simplifyString(question).split(' ') : [], - A: answer ? simplifyString(answer).split(' ') : [], - }, - } - } else { - return { - ...question, - cache: { - Q: question.Q ? simplifyString(question.Q).split(' ') : [], - A: question.A ? simplifyString(question.A).split(' ') : [], - }, - } + try { + if (typeof question === 'string') { + return { + Q: simplifyQuestion(question), + A: answer ? simplifyAnswer(answer) : undefined, + data: data, + cache: { + Q: question ? simplifyString(question).split(' ') : [], + A: answer ? simplifyString(answer).split(' ') : [], + }, + } + } else { + return { + ...question, + cache: { + Q: question.Q ? simplifyString(question.Q).split(' ') : [], + A: question.A ? simplifyString(question.A).split(' ') : [], + }, + } + } + } catch (err) { + logger.Log('Error creating question', logger.GetColor('redbg')) + console.error(question, answer, data) + console.error(err) + return null } - } catch (err) { - logger.Log('Error creating question', logger.GetColor('redbg')) - console.error(question, answer, data) - console.error(err) - return null - } } async function recognizeQuestionImage(question: Question): Promise { - const base64Data = question.data.base64 - if (Array.isArray(base64Data) && base64Data.length) { - try { - const res: string[] = [] - for (let i = 0; i < base64Data.length; i++) { - const base64 = base64Data[i] - const text = await recognizeTextFromBase64(base64) - res.push(text) - } + const base64Data = question.data.base64 + if (Array.isArray(base64Data) && base64Data.length) { + try { + const res: string[] = [] + for (let i = 0; i < base64Data.length; i++) { + const base64 = base64Data[i] + const text = await recognizeTextFromBase64(base64) + res.push(text) + } - return { - ...question, - Q: res.join(' '), - data: { - ...question.data, - type: 'simple', - }, - } - } catch (e) { - console.error('Error happened in recognizeQuestionImage!') - console.error(e) + return { + ...question, + Q: res.join(' '), + data: { + ...question.data, + type: 'simple', + }, + } + } catch (e) { + console.error('Error happened in recognizeQuestionImage!') + console.error(e) + } } - } - return question + return question } function compareImage(data: QuestionData, data2: QuestionData): number { - if (data.hashedImages && data2.hashedImages) { - return compareString( - data.hashedImages.join(' '), - data2.hashedImages.join(' '), - data.hashedImages, - data2.hashedImages - ) - } else if (data.images && data2.images) { - return ( - compareString( - data.images.join(' '), - data2.images.join(' '), - data.images, - data2.images - ) - 10 - ) - } else { - return 0 - } + if (data.hashedImages && data2.hashedImages) { + return compareString( + data.hashedImages.join(' '), + data2.hashedImages.join(' '), + data.hashedImages, + data2.hashedImages + ) + } else if (data.images && data2.images) { + return ( + compareString( + data.images.join(' '), + data2.images.join(' '), + data.images, + data2.images + ) - 10 + ) + } else { + return 0 + } } function compareData(q1: Question, q2: Question): number { - try { - if (q1.data.type === q2.data.type) { - const dataType = q1.data.type - if (dataType === 'simple') { - return -1 - } else if (dataType === 'image') { - return compareImage(q1.data, q2.data) - } else { - logger.DebugLog( - `Unhandled data type ${dataType}`, - 'Compare question data', - 1 - ) - logger.DebugLog(q1, 'Compare question data', 2) - } - } else { - return 0 + try { + if (q1.data.type === q2.data.type) { + const dataType = q1.data.type + if (dataType === 'simple') { + return -1 + } else if (dataType === 'image') { + return compareImage(q1.data, q2.data) + } else { + logger.DebugLog( + `Unhandled data type ${dataType}`, + 'Compare question data', + 1 + ) + logger.DebugLog(q1, 'Compare question data', 2) + } + } else { + return 0 + } + } catch (error) { + logger.DebugLog('Error comparing data', 'Compare question data', 1) + logger.DebugLog(error.message, 'Compare question data', 1) + logger.DebugLog(error, 'Compare question data', 2) + console.error(error) } - } catch (error) { - logger.DebugLog('Error comparing data', 'Compare question data', 1) - logger.DebugLog(error.message, 'Compare question data', 1) - logger.DebugLog(error, 'Compare question data', 2) - console.error(error) - } - return 0 + return 0 } function compareQuestion(q1: Question, q2: Question): number { - return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q) - // return compareString( - // q1.Q, - // q1.Q ? q1.Q.split(' ') : [], - // q2.Q, - // q2.Q ? q2.Q.split(' ') : [] - // ) + return compareString(q1.Q, q2.Q, q1.cache.Q, q2.cache.Q) + // return compareString( + // q1.Q, + // q1.Q ? q1.Q.split(' ') : [], + // q2.Q, + // q2.Q ? q2.Q.split(' ') : [] + // ) } function compareAnswer(q1: Question, q2: Question): number { - return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A) - // return compareString( - // q1.A, - // q1.A ? q1.A.split(' ') : [], - // q2.A, - // q2.A ? q2.A.split(' ') : [] - // ) + return compareString(q1.A, q2.A, q1.cache.A, q2.cache.A) + // return compareString( + // q1.A, + // q1.A ? q1.A.split(' ') : [], + // q2.A, + // q2.A ? q2.A.split(' ') : [] + // ) } function compareQuestionObj( - q1: Question, - _q1subjName: string, - q2: Question, - q2subjName: string + q1: Question, + _q1subjName: string, + q2: Question, + q2subjName: string ): DetailedMatch { - const qMatch = compareQuestion(q1, q2) - const aMatch = q2.A ? compareAnswer(q1, q2) : 0 - // -1 if botth questions are simple - const dMatch = compareData(q1, q2) + const qMatch = compareQuestion(q1, q2) + const aMatch = q2.A ? compareAnswer(q1, q2) : 0 + // -1 if botth questions are simple + const dMatch = compareData(q1, q2) - let avg = -1 - if (q2.A) { - if (dMatch === -1) { - avg = Math.min(qMatch, aMatch) + let avg = -1 + if (q2.A) { + if (dMatch === -1) { + avg = Math.min(qMatch, aMatch) + } else { + avg = Math.min(qMatch, aMatch, dMatch) + } } else { - avg = Math.min(qMatch, aMatch, dMatch) + if (dMatch === -1) { + avg = qMatch + } else { + avg = Math.min(qMatch, dMatch) + } } - } else { - if (dMatch === -1) { - avg = qMatch - } else { - avg = Math.min(qMatch, dMatch) - } - } - return { - qMatch: qMatch, - aMatch: aMatch, - dMatch: dMatch, - matchedSubjName: q2subjName, - avg: avg, - } + return { + qMatch: qMatch, + aMatch: aMatch, + dMatch: dMatch, + matchedSubjName: q2subjName, + avg: avg, + } } function questionToString(question: Question): string { - const { Q, A, data } = question + const { Q, A, data } = question - if (data.type !== 'simple') { - return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data) - } else { - return '?' + Q + '\n!' + A - } + if (data.type !== 'simple') { + return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data) + } else { + return '?' + Q + '\n!' + A + } } // --------------------------------------------------------------------------------------------------------- // Subject // --------------------------------------------------------------------------------------------------------- function searchSubject( - subj: Subject, - question: Question, - subjName: string, - searchTillMatchPercent?: number + subj: Subject, + question: Question, + subjName: string, + searchTillMatchPercent?: number ): SearchResultQuestion[] { - let result: SearchResultQuestion[] = [] + let result: SearchResultQuestion[] = [] - let stopSearch = false - let i = subj.Questions.length - 1 - while (i >= 0 && !stopSearch) { - const currentQuestion = subj.Questions[i] - const percent = compareQuestionObj( - currentQuestion, - subjName, - question, - subj.Name - ) + let stopSearch = false + let i = subj.Questions.length - 1 + while (i >= 0 && !stopSearch) { + const currentQuestion = subj.Questions[i] + const percent = compareQuestionObj( + currentQuestion, + subjName, + question, + subj.Name + ) - if (percent.avg >= minMatchAmmount) { - result.push({ - q: currentQuestion, - match: percent.avg, - detailedMatch: percent, - }) + if (percent.avg >= minMatchAmmount) { + result.push({ + q: currentQuestion, + match: percent.avg, + detailedMatch: percent, + }) + } + + if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) { + stopSearch = true + } + + i-- } - if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) { - stopSearch = true - } + result = result.sort((q1, q2) => { + if (q1.match < q2.match) { + return 1 + } else if (q1.match > q2.match) { + return -1 + } else { + return 0 + } + }) - i-- - } - - result = result.sort((q1, q2) => { - if (q1.match < q2.match) { - return 1 - } else if (q1.match > q2.match) { - return -1 - } else { - return 0 - } - }) - - return result + return result } function subjectToString(subj: Subject): string { - const { Questions, Name } = subj + const { Questions, Name } = subj - const result: string[] = [] - Questions.forEach((question) => { - result.push(questionToString(question)) - }) + const result: string[] = [] + Questions.forEach((question) => { + result.push(questionToString(question)) + }) - return '+' + Name + '\n' + result.join('\n') + return '+' + Name + '\n' + result.join('\n') } // --------------------------------------------------------------------------------------------------------- // QuestionDB // --------------------------------------------------------------------------------------------------------- function addQuestion( - data: Array, - subj: string, - question: Question + data: Array, + subj: string, + question: Question ): void { - logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1) - logger.DebugLog(question, 'qdb add', 3) + logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1) + logger.DebugLog(question, 'qdb add', 3) - const i = data.findIndex((subject) => { - return subj - .toLowerCase() - .includes(getSubjNameWithoutYear(subject.Name).toLowerCase()) - }) - - if (i !== -1) { - logger.DebugLog('Adding new question to existing subject', 'qdb add', 1) - data[i].Questions.push(question) - } else { - logger.DebugLog('Creating new subject for question', 'qdb add', 1) - data.push({ - Name: subj, - Questions: [question], + const i = data.findIndex((subject) => { + return subj + .toLowerCase() + .includes(getSubjNameWithoutYear(subject.Name).toLowerCase()) }) - } + + if (i !== -1) { + logger.DebugLog('Adding new question to existing subject', 'qdb add', 1) + data[i].Questions.push(question) + } else { + logger.DebugLog('Creating new subject for question', 'qdb add', 1) + data.push({ + Name: subj, + Questions: [question], + }) + } } function prepareQuestion(question: Question): Question { - return simplifyQuestionObj(createQuestion(question)) + return simplifyQuestionObj(createQuestion(question)) } function dataToString(data: Array): string { - const result: string[] = [] - data.forEach((subj) => { - result.push(subjectToString(subj)) - }) - return result.join('\n\n') + const result: string[] = [] + data.forEach((subj) => { + result.push(subjectToString(subj)) + }) + return result.join('\n\n') } function doSearch( - data: Array, - subjName: string, - question: Question, - searchTillMatchPercent?: number, - searchInAllIfNoResult?: Boolean + data: Array, + subjName: string, + question: Question, + searchTillMatchPercent?: number, + searchInAllIfNoResult?: Boolean ): SearchResultQuestion[] { - let result: SearchResultQuestion[] = [] + let result: SearchResultQuestion[] = [] - const questionToSearch = prepareQuestion(question) + const questionToSearch = prepareQuestion(question) - data.every((subj) => { - if ( - subjName - .toLowerCase() - .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) - ) { - logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2) - const subjRes = searchSubject( - subj, - questionToSearch, - subjName, - searchTillMatchPercent - ) - result = result.concat(subjRes) - if (searchTillMatchPercent) { - return !subjRes.some((sr) => { - return sr.match >= searchTillMatchPercent - }) - } - return true - } - return true - }) - - if (searchInAllIfNoResult) { - // FIXME: dont research subject searched above - if ( - result.length === 0 || - result[0].match < minMatchToNotSearchOtherSubjects - ) { - logger.DebugLog( - 'Reqults length is zero when comparing names, trying all subjects', - 'searchworker', - 1 - ) - data.every((subj) => { - const subjRes = searchSubject( - subj, - questionToSearch, - subjName, - searchTillMatchPercent - ) - result = result.concat(subjRes) - - if (searchTillMatchPercent) { - const continueSearching = !subjRes.some((sr) => { - return sr.match >= searchTillMatchPercent - }) - return continueSearching + data.every((subj) => { + if ( + subjName + .toLowerCase() + .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) + ) { + logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2) + const subjRes = searchSubject( + subj, + questionToSearch, + subjName, + searchTillMatchPercent + ) + result = result.concat(subjRes) + if (searchTillMatchPercent) { + return !subjRes.some((sr) => { + return sr.match >= searchTillMatchPercent + }) + } + return true } return true - }) + }) + + if (searchInAllIfNoResult) { + // FIXME: dont research subject searched above + if ( + result.length === 0 || + result[0].match < minMatchToNotSearchOtherSubjects + ) { + logger.DebugLog( + 'Reqults length is zero when comparing names, trying all subjects', + 'searchworker', + 1 + ) + data.every((subj) => { + const subjRes = searchSubject( + subj, + questionToSearch, + subjName, + searchTillMatchPercent + ) + result = result.concat(subjRes) + + if (searchTillMatchPercent) { + const continueSearching = !subjRes.some((sr) => { + return sr.match >= searchTillMatchPercent + }) + return continueSearching + } + return true + }) + } } - } - result = setNoPossibleAnswersPenalties( - questionToSearch.data.possibleAnswers, - result - ) + result = setNoPossibleAnswersPenalties( + questionToSearch.data.possibleAnswers, + result + ) - result = result.sort((q1, q2) => { - if (q1.match < q2.match) { - return 1 - } else if (q1.match > q2.match) { - return -1 - } else { - return 0 - } - }) + result = result.sort((q1, q2) => { + if (q1.match < q2.match) { + return 1 + } else if (q1.match > q2.match) { + return -1 + } else { + return 0 + } + }) - return result + return result } function setNoPossibleAnswersPenalties( - questionPossibleAnswers: QuestionData['possibleAnswers'], - results: SearchResultQuestion[] + questionPossibleAnswers: QuestionData['possibleAnswers'], + results: SearchResultQuestion[] ): SearchResultQuestion[] { - if (!Array.isArray(questionPossibleAnswers)) { - return results - } - const noneHasPossibleAnswers = results.every((x) => { - return !Array.isArray(x.q.data.possibleAnswers) - }) - if (noneHasPossibleAnswers) return results - - let possibleAnswerMatch = false - const updated = results.map((result) => { - const matchCount = Array.isArray(result.q.data.possibleAnswers) - ? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => { - return questionPossibleAnswers.some((questionPossibleAnswer) => { - if (questionPossibleAnswer.val && resultPossibleAnswer.val) { - return questionPossibleAnswer.val.includes( - resultPossibleAnswer.val - ) - } else { - return false - } - }) - }).length - : 0 - - if (matchCount === questionPossibleAnswers.length) { - possibleAnswerMatch = true - return result - } else { - return { - ...result, - match: result.match - noPossibleAnswerMatchPenalty, - detailedMatch: { - ...result.detailedMatch, - qMatch: result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty, - }, - } + if (!Array.isArray(questionPossibleAnswers)) { + return results } - }) + const noneHasPossibleAnswers = results.every((x) => { + return !Array.isArray(x.q.data.possibleAnswers) + }) + if (noneHasPossibleAnswers) return results - if (possibleAnswerMatch) { - return updated - } else { - return results - } + let possibleAnswerMatch = false + const updated = results.map((result) => { + const matchCount = Array.isArray(result.q.data.possibleAnswers) + ? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => { + return questionPossibleAnswers.some( + (questionPossibleAnswer) => { + if ( + questionPossibleAnswer.val && + resultPossibleAnswer.val + ) { + return questionPossibleAnswer.val.includes( + resultPossibleAnswer.val + ) + } else { + return false + } + } + ) + }).length + : 0 + + if (matchCount === questionPossibleAnswers.length) { + possibleAnswerMatch = true + return result + } else { + return { + ...result, + match: result.match - noPossibleAnswerMatchPenalty, + detailedMatch: { + ...result.detailedMatch, + qMatch: + result.detailedMatch.qMatch - + noPossibleAnswerMatchPenalty, + }, + } + } + }) + + if (possibleAnswerMatch) { + return updated + } else { + return results + } } // --------------------------------------------------------------------------------------------------------- @@ -666,293 +675,302 @@ function setNoPossibleAnswersPenalties( // --------------------------------------------------------------------------------------------------------- interface WorkData { - subjName: string - question: Question - searchTillMatchPercent: number - searchInAllIfNoResult: boolean - searchIn: number[] - index: number + subjName: string + question: Question + searchTillMatchPercent: number + searchInAllIfNoResult: boolean + searchIn: number[] + index: number } if (!isMainThread) { - handleWorkerData() + handleWorkerData() } function handleWorkerData() { - const { - workerIndex, - initData, - }: { workerIndex: number; initData: Array } = workerData - let qdbs: Array = initData + const { + workerIndex, + initData, + }: { workerIndex: number; initData: Array } = workerData + let qdbs: Array = initData - logger.Log( - `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty` - ) + logger.Log( + `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty` + ) - parentPort.on('message', async (msg /*: TaskObject */) => { - await tesseractLoaded - if (msg.type === 'work') { - const { - subjName, - question: originalQuestion, - searchTillMatchPercent, - searchInAllIfNoResult, - searchIn, - index, - }: WorkData = msg.data + parentPort.on('message', async (msg /*: TaskObject */) => { + await tesseractLoaded + if (msg.type === 'work') { + const { + subjName, + question: originalQuestion, + searchTillMatchPercent, + searchInAllIfNoResult, + searchIn, + index, + }: WorkData = msg.data - let searchResult: SearchResultQuestion[] = [] - let error = false + let searchResult: SearchResultQuestion[] = [] + let error = false - const question = await recognizeQuestionImage(originalQuestion) + const question = await recognizeQuestionImage(originalQuestion) - try { - qdbs.forEach((qdb) => { - if (searchIn.includes(qdb.index)) { - const res = doSearch( - qdb.data, - subjName, - question, - searchTillMatchPercent, - searchInAllIfNoResult - ) - searchResult = [ - ...searchResult, - ...res.map((x) => { - return { - ...x, - detailedMatch: { - ...x.detailedMatch, - qdb: qdb.name, - }, - } - }), - ] - } - }) - } catch (err) { - logger.Log('Error in worker thread!', logger.GetColor('redbg')) - console.error(err) - console.error( - JSON.stringify( - { - subjName: subjName, - question: question, - searchTillMatchPercent: searchTillMatchPercent, - searchInAllIfNoResult: searchInAllIfNoResult, - searchIn: searchIn, - index: index, - }, - null, - 2 - ) - ) - error = true - } - - // sorting - const sortedResult: SearchResultQuestion[] = searchResult.sort( - (q1, q2) => { - if (q1.match < q2.match) { - return 1 - } else if (q1.match > q2.match) { - return -1 - } else { - return 0 - } - } - ) - - const workerResult: WorkerResult = { - msg: `From thread #${workerIndex}: job ${ - !isNaN(index) ? `#${index}` : '' - }done`, - workerIndex: workerIndex, - result: sortedResult, - error: error, - } - - // ONDONE: - parentPort.postMessage(workerResult) - - // console.log( - // `[THREAD #${workerIndex}]: Work ${ - // !isNaN(index) ? `#${index}` : '' - // }done!` - // ) - } else if (msg.type === 'dbEdit') { - const { dbIndex, edits }: { dbIndex: number; edits: Edits } = msg.data - const { resultDb } = editDb(qdbs[dbIndex], edits) - qdbs[dbIndex] = resultDb - logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1) - - parentPort.postMessage({ - msg: `From thread #${workerIndex}: db edit`, - workerIndex: workerIndex, - }) - } else if (msg.type === 'newQuestions') { - const { - subjName, - qdbIndex, - newQuestions, - }: { - subjName: string - qdbIndex: number - newQuestions: Question[] - } = msg.data - - let added = false - qdbs = qdbs.map((qdb) => { - if (qdb.index === qdbIndex) { - return { - ...qdb, - data: qdb.data.map((subj) => { - if (subj.Name === subjName) { - added = true - return { - Name: subj.Name, - Questions: [...subj.Questions, ...newQuestions], - } - } else { - return subj - } - }), - } - } else { - return qdb - } - }) - - if (!added) { - qdbs = qdbs.map((qdb) => { - if (qdb.index === qdbIndex) { - return { - ...qdb, - data: [ - ...qdb.data, - { - Name: subjName, - Questions: [...newQuestions], - }, - ], + try { + qdbs.forEach((qdb) => { + if (searchIn.includes(qdb.index)) { + const res = doSearch( + qdb.data, + subjName, + question, + searchTillMatchPercent, + searchInAllIfNoResult + ) + searchResult = [ + ...searchResult, + ...res.map((x) => { + return { + ...x, + detailedMatch: { + ...x.detailedMatch, + qdb: qdb.name, + }, + } + }), + ] + } + }) + } catch (err) { + logger.Log('Error in worker thread!', logger.GetColor('redbg')) + console.error(err) + console.error( + JSON.stringify( + { + subjName: subjName, + question: question, + searchTillMatchPercent: searchTillMatchPercent, + searchInAllIfNoResult: searchInAllIfNoResult, + searchIn: searchIn, + index: index, + }, + null, + 2 + ) + ) + error = true } - } else { - return qdb - } - }) - } - logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1) - parentPort.postMessage({ - msg: `From thread #${workerIndex}: update done`, - workerIndex: workerIndex, - }) + // sorting + const sortedResult: SearchResultQuestion[] = searchResult.sort( + (q1, q2) => { + if (q1.match < q2.match) { + return 1 + } else if (q1.match > q2.match) { + return -1 + } else { + return 0 + } + } + ) - // console.log(`[THREAD #${workerIndex}]: update`) - } else if (msg.type === 'newdb') { - const { data }: { data: QuestionDb } = msg - qdbs.push(data) + const workerResult: WorkerResult = { + msg: `From thread #${workerIndex}: job ${ + !isNaN(index) ? `#${index}` : '' + }done`, + workerIndex: workerIndex, + result: sortedResult, + error: error, + } - parentPort.postMessage({ - msg: `From thread #${workerIndex}: new db add done`, - workerIndex: workerIndex, - }) - // console.log(`[THREAD #${workerIndex}]: newdb`) - } else if (msg.type === 'dbClean') { - const removedIndexes = cleanDb(msg.data, qdbs) + // ONDONE: + parentPort.postMessage(workerResult) - const workerResult: WorkerResult = { - msg: `From thread #${workerIndex}: db clean done`, - workerIndex: workerIndex, - result: removedIndexes, - } + // console.log( + // `[THREAD #${workerIndex}]: Work ${ + // !isNaN(index) ? `#${index}` : '' + // }done!` + // ) + } else if (msg.type === 'dbEdit') { + const { dbIndex, edits }: { dbIndex: number; edits: Edits } = + msg.data + const { resultDb } = editDb(qdbs[dbIndex], edits) + qdbs[dbIndex] = resultDb + logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1) - parentPort.postMessage(workerResult) - } else if (msg.type === 'rmQuestions') { - const { - questionIndexesToRemove, - subjIndex, - qdbIndex, - recievedQuestions, - } = msg.data + parentPort.postMessage({ + msg: `From thread #${workerIndex}: db edit`, + workerIndex: workerIndex, + }) + } else if (msg.type === 'newQuestions') { + const { + subjName, + qdbIndex, + newQuestions, + }: { + subjName: string + qdbIndex: number + newQuestions: Question[] + } = msg.data - qdbs[qdbIndex].data[subjIndex].Questions = updateQuestionsInArray( - questionIndexesToRemove, - qdbs[qdbIndex].data[subjIndex].Questions, - recievedQuestions - ) + let added = false + qdbs = qdbs.map((qdb) => { + if (qdb.index === qdbIndex) { + return { + ...qdb, + data: qdb.data.map((subj) => { + if (subj.Name === subjName) { + added = true + return { + Name: subj.Name, + Questions: [ + ...subj.Questions, + ...newQuestions, + ], + } + } else { + return subj + } + }), + } + } else { + return qdb + } + }) - parentPort.postMessage({ - msg: `From thread #${workerIndex}: rm question done`, - workerIndex: workerIndex, - }) - } else { - logger.Log(`Invalid msg type!`, logger.GetColor('redbg')) - console.error(msg) + if (!added) { + qdbs = qdbs.map((qdb) => { + if (qdb.index === qdbIndex) { + return { + ...qdb, + data: [ + ...qdb.data, + { + Name: subjName, + Questions: [...newQuestions], + }, + ], + } + } else { + return qdb + } + }) + } + logger.DebugLog( + `Worker new question ${workerIndex}`, + 'worker update', + 1 + ) - parentPort.postMessage({ - msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`, - workerIndex: workerIndex, - }) - } - }) + parentPort.postMessage({ + msg: `From thread #${workerIndex}: update done`, + workerIndex: workerIndex, + }) + + // console.log(`[THREAD #${workerIndex}]: update`) + } else if (msg.type === 'newdb') { + const { data }: { data: QuestionDb } = msg + qdbs.push(data) + + parentPort.postMessage({ + msg: `From thread #${workerIndex}: new db add done`, + workerIndex: workerIndex, + }) + // console.log(`[THREAD #${workerIndex}]: newdb`) + } else if (msg.type === 'dbClean') { + const removedIndexes = cleanDb(msg.data, qdbs) + + const workerResult: WorkerResult = { + msg: `From thread #${workerIndex}: db clean done`, + workerIndex: workerIndex, + result: removedIndexes, + } + + parentPort.postMessage(workerResult) + } else if (msg.type === 'rmQuestions') { + const { + questionIndexesToRemove, + subjIndex, + qdbIndex, + recievedQuestions, + } = msg.data + + qdbs[qdbIndex].data[subjIndex].Questions = updateQuestionsInArray( + questionIndexesToRemove, + qdbs[qdbIndex].data[subjIndex].Questions, + recievedQuestions + ) + + parentPort.postMessage({ + msg: `From thread #${workerIndex}: rm question done`, + workerIndex: workerIndex, + }) + } else { + logger.Log(`Invalid msg type!`, logger.GetColor('redbg')) + console.error(msg) + + parentPort.postMessage({ + msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`, + workerIndex: workerIndex, + }) + } + }) } export function cleanDb( - { - questions: recievedQuestions, - subjToClean, - overwriteFromDate, - qdbIndex, - }: { - questions: Question[] - subjToClean: string - overwriteFromDate: number - qdbIndex: number - }, - qdbs: QuestionDb[] + { + questions: recievedQuestions, + subjToClean, + overwriteFromDate, + qdbIndex, + }: { + questions: Question[] + subjToClean: string + overwriteFromDate: number + qdbIndex: number + }, + qdbs: QuestionDb[] ): number[][] { - const subjIndex = qdbs[qdbIndex].data.findIndex((x) => { - return x.Name.toLowerCase().includes(subjToClean.toLowerCase()) - }) + const subjIndex = qdbs[qdbIndex].data.findIndex((x) => { + return x.Name.toLowerCase().includes(subjToClean.toLowerCase()) + }) - if (!qdbs[qdbIndex].data[subjIndex]) { - return recievedQuestions.map(() => []) - } + if (!qdbs[qdbIndex].data[subjIndex]) { + return recievedQuestions.map(() => []) + } - const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) => - qdbs[qdbIndex].data[subjIndex].Questions.reduce( - (acc, question, i) => { - const res = compareString( - simplifyQuestion(recievedQuestion.Q), - simplifyQuestion(question.Q) + const questionIndexesToRemove = recievedQuestions.map((recievedQuestion) => + qdbs[qdbIndex].data[subjIndex].Questions.reduce( + (acc, question, i) => { + const res = compareString( + simplifyQuestion(recievedQuestion.Q), + simplifyQuestion(question.Q) + ) + + if ( + res > minMatchToNotSearchOtherSubjects && + (!question.data.date || + question.data.date < overwriteFromDate) + ) { + return [...acc, i] + } + return acc + }, + [] ) - - if ( - res > minMatchToNotSearchOtherSubjects && - (!question.data.date || question.data.date < overwriteFromDate) - ) { - return [...acc, i] - } - return acc - }, - [] ) - ) - return questionIndexesToRemove + return questionIndexesToRemove } // ------------------------------------------------------------------------ export { - compareQuestionObj, - minMatchAmmount, - getSubjNameWithoutYear, - createQuestion, - addQuestion, - dataToString, - doSearch, - setNoPossibleAnswersPenalties, - recognizeQuestionImage, + compareQuestionObj, + minMatchAmmount, + getSubjNameWithoutYear, + createQuestion, + addQuestion, + dataToString, + doSearch, + setNoPossibleAnswersPenalties, + recognizeQuestionImage, } diff --git a/src/utils/dbtools.ts b/src/utils/dbtools.ts index bab6149..388c120 100644 --- a/src/utils/dbtools.ts +++ b/src/utils/dbtools.ts @@ -22,18 +22,18 @@ // https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md export default { - GetDB: GetDB, - AddColumn: AddColumn, - TableInfo: TableInfo, - Update: Update, - Delete: Delete, - CreateTable: CreateTable, - SelectAll: SelectAll, - Select: Select, - Insert: Insert, - CloseDB: CloseDB, - runStatement: runStatement, - sanitizeQuery: sanitizeQuery, + GetDB: GetDB, + AddColumn: AddColumn, + TableInfo: TableInfo, + Update: Update, + Delete: Delete, + CreateTable: CreateTable, + SelectAll: SelectAll, + Select: Select, + Insert: Insert, + CloseDB: CloseDB, + runStatement: runStatement, + sanitizeQuery: sanitizeQuery, } import Sqlite, { Database, RunResult } from 'better-sqlite3' @@ -43,310 +43,310 @@ import utils from '../utils/utils' const debugLog = process.env.NS_SQL_DEBUG_LOG function sanitizeQuery(val: string | number): string | number { - if (typeof val === 'string') { - return val.replace(/'/g, '').replace(/;/g, '') - } - return val + if (typeof val === 'string') { + return val.replace(/'/g, '').replace(/;/g, '') + } + return val } // { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4 function GetSqlQuerry( - conditions: { [key: string]: string | number }, - type?: string, - joiner?: string + conditions: { [key: string]: string | number }, + type?: string, + joiner?: string ) { - const res = Object.keys(conditions).reduce((acc, key) => { - const item = conditions[key] - const conditionKey = sanitizeQuery(key) - const condition = sanitizeQuery(conditions[key]) + const res = Object.keys(conditions).reduce((acc, key) => { + const item = conditions[key] + const conditionKey = sanitizeQuery(key) + const condition = sanitizeQuery(conditions[key]) - if (typeof item === 'string') { - acc.push(`${conditionKey} = '${condition}'`) + if (typeof item === 'string') { + acc.push(`${conditionKey} = '${condition}'`) + } else { + acc.push(`${conditionKey} = ${condition}`) + } + return acc + }, []) + if (type === 'where') { + if (joiner) { + return res.join(` ${joiner} `) + } else { + return res.join(' AND ') + } } else { - acc.push(`${conditionKey} = ${condition}`) + return res.join(', ') } - return acc - }, []) - if (type === 'where') { - if (joiner) { - return res.join(` ${joiner} `) - } else { - return res.join(' AND ') - } - } else { - return res.join(', ') - } } // ------------------------------------------------------------------------- function GetDB(path: string): Database { - utils.CreatePath(path) - const res = new Sqlite(path) - res.pragma('synchronous = OFF') - return res + utils.CreatePath(path) + const res = new Sqlite(path) + res.pragma('synchronous = OFF') + return res } function DebugLog(msg: string) { - if (debugLog) { - logger.DebugLog(msg, 'sql', 0) - } + if (debugLog) { + logger.DebugLog(msg, 'sql', 0) + } } // FIXME: this might not work: what is col exactly, and how we use AddColumn? function AddColumn( - db: Database, - table: string, - col: { [key: string]: string | number } + db: Database, + table: string, + col: { [key: string]: string | number } ): RunResult { - try { - const colName = Object.keys(col)[0] - const colType = col.type + try { + const colName = Object.keys(col)[0] + const colType = col.type - const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}` - const stmt = PrepareStatement(db, command) + const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}` + const stmt = PrepareStatement(db, command) - return stmt.run() - } catch (err) { - console.error(err) - return null - } + return stmt.run() + } catch (err) { + console.error(err) + return null + } } function TableInfo( - db: Database, - table: string + db: Database, + table: string ): { - columns: any[] - dataCount: number + columns: any[] + dataCount: number } { - try { - const command = `PRAGMA table_info(${table})` - const stmt = PrepareStatement(db, command) + try { + const command = `PRAGMA table_info(${table})` + const stmt = PrepareStatement(db, command) - const infoRes = stmt.all() + const infoRes = stmt.all() - const s2 = `SELECT COUNT(*) FROM ${table}` - const stmt2 = PrepareStatement(db, s2) + const s2 = `SELECT COUNT(*) FROM ${table}` + const stmt2 = PrepareStatement(db, s2) - const countRes = stmt2.get() + const countRes = stmt2.get() - return { - columns: infoRes, - dataCount: countRes[Object.keys(countRes)[0]], + return { + columns: infoRes, + dataCount: countRes[Object.keys(countRes)[0]], + } + } catch (err) { + console.error(err) + return null } - } catch (err) { - console.error(err) - return null - } } function Update( - db: Database, - table: string, - newData: { [key: string]: string | number }, - conditions: { [key: string]: string | number } + db: Database, + table: string, + newData: { [key: string]: string | number }, + conditions: { [key: string]: string | number } ): RunResult { - try { - const command = `UPDATE ${table} SET ${GetSqlQuerry( - newData, - 'set' - )} WHERE ${GetSqlQuerry(conditions, 'where')}` - const stmt = PrepareStatement(db, command) + try { + const command = `UPDATE ${table} SET ${GetSqlQuerry( + newData, + 'set' + )} WHERE ${GetSqlQuerry(conditions, 'where')}` + const stmt = PrepareStatement(db, command) - return stmt.run() - } catch (err) { - console.error(err) - return null - } + return stmt.run() + } catch (err) { + console.error(err) + return null + } } function Delete( - db: Database, - table: string, - conditions: { [key: string]: string | number } + db: Database, + table: string, + conditions: { [key: string]: string | number } ): RunResult { - try { - const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry( - conditions, - 'where' - )}` - const stmt = PrepareStatement(db, command) + try { + const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry( + conditions, + 'where' + )}` + const stmt = PrepareStatement(db, command) - return stmt.run() - } catch (err) { - console.error(err) - return null - } + return stmt.run() + } catch (err) { + console.error(err) + return null + } } interface DbColumnDescription { - [key: string]: { - type: string - primary?: boolean - autoIncrement?: boolean - notNull?: boolean - defaultZero?: boolean - [key: string]: any - } + [key: string]: { + type: string + primary?: boolean + autoIncrement?: boolean + notNull?: boolean + defaultZero?: boolean + [key: string]: any + } } function CreateTable( - db: Database, - name: string, - columns: DbColumnDescription, - foreignKeys: { - keysFrom: string[] - table: string - keysTo: string[] - }[] + db: Database, + name: string, + columns: DbColumnDescription, + foreignKeys: { + keysFrom: string[] + table: string + keysTo: string[] + }[] ): RunResult { - // CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount - // number, lastLogin text, lastAccess text - // - // FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) ) + // CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount + // number, lastLogin text, lastAccess text + // + // FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) ) - try { - const cols = Object.keys(columns) - .reduce((acc, key) => { - const item = columns[key] - const flags: string[] = [] - const toCheck = { - primary: 'PRIMARY KEY', - notNull: 'NOT NULL', - unique: 'UNIQUE', - autoIncrement: 'AUTOINCREMENT', - defaultZero: 'DEFAULT 0', + try { + const cols = Object.keys(columns) + .reduce((acc, key) => { + const item = columns[key] + const flags: string[] = [] + const toCheck = { + primary: 'PRIMARY KEY', + notNull: 'NOT NULL', + unique: 'UNIQUE', + autoIncrement: 'AUTOINCREMENT', + defaultZero: 'DEFAULT 0', + } + Object.keys(toCheck).forEach((key) => { + if (item[key]) { + flags.push(toCheck[key]) + } + }) + + acc.push(`${key} ${item.type} ${flags.join(' ')}`) + return acc + }, []) + .join(', ') + + const fKeys: string[] = [] + if (foreignKeys) { + foreignKeys.forEach((foreignKey) => { + const { keysFrom, table, keysTo } = foreignKey + fKeys.push( + `, FOREIGN KEY(${keysFrom.join( + ', ' + )}) REFERENCES ${table}(${keysTo.join(', ')})` + ) + }) } - Object.keys(toCheck).forEach((key) => { - if (item[key]) { - flags.push(toCheck[key]) - } - }) - acc.push(`${key} ${item.type} ${flags.join(' ')}`) - return acc - }, []) - .join(', ') - - const fKeys: string[] = [] - if (foreignKeys) { - foreignKeys.forEach((foreignKey) => { - const { keysFrom, table, keysTo } = foreignKey - fKeys.push( - `, FOREIGN KEY(${keysFrom.join( - ', ' - )}) REFERENCES ${table}(${keysTo.join(', ')})` - ) - }) + // IF NOT EXISTS + const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})` + const stmt = PrepareStatement(db, command) + return stmt.run() + } catch (err) { + console.error(err) + return null } - - // IF NOT EXISTS - const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})` - const stmt = PrepareStatement(db, command) - return stmt.run() - } catch (err) { - console.error(err) - return null - } } function SelectAll(db: Database, from: string): any[] { - try { - const command = `SELECT * from ${from}` + try { + const command = `SELECT * from ${from}` - const stmt = PrepareStatement(db, command) - return stmt.all() - } catch (err) { - console.error(err) - return null - } + const stmt = PrepareStatement(db, command) + return stmt.all() + } catch (err) { + console.error(err) + return null + } } // SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100; function Select( - db: Database, - from: string, - conditions: { [key: string]: string | number }, - options: { joiner?: string; limit?: number } = {} + db: Database, + from: string, + conditions: { [key: string]: string | number }, + options: { joiner?: string; limit?: number } = {} ): any[] { - const { joiner, limit } = options + const { joiner, limit } = options - try { - let command = `SELECT * from ${from} WHERE ${GetSqlQuerry( - conditions, - 'where', - joiner - )}` + try { + let command = `SELECT * from ${from} WHERE ${GetSqlQuerry( + conditions, + 'where', + joiner + )}` - if (!isNaN(limit)) { - command += ` LIMIT ${limit}` + if (!isNaN(limit)) { + command += ` LIMIT ${limit}` + } + + const stmt = PrepareStatement(db, command) + return stmt.all() + } catch (err) { + console.error(err) + return null } - - const stmt = PrepareStatement(db, command) - return stmt.all() - } catch (err) { - console.error(err) - return null - } } function Insert( - db: Database, - table: string, - data: { [key: string]: number | string } + db: Database, + table: string, + data: { [key: string]: number | string } ): RunResult { - try { - const cols = Object.keys(data) - .reduce((acc, key) => { - acc.push(`${key}`) - return acc - }, []) - .join(', ') + try { + const cols = Object.keys(data) + .reduce((acc, key) => { + acc.push(`${key}`) + return acc + }, []) + .join(', ') - const values = Object.keys(data) - .map((key) => { - const item = data[key] - if (typeof item === 'string') { - return `'${item}'` - } else { - return `${item}` - } - }) - .join(', ') + const values = Object.keys(data) + .map((key) => { + const item = data[key] + if (typeof item === 'string') { + return `'${item}'` + } else { + return `${item}` + } + }) + .join(', ') - const command = `INSERT INTO ${table} (${cols}) VALUES (${values})` - const stmt = PrepareStatement(db, command) + const command = `INSERT INTO ${table} (${cols}) VALUES (${values})` + const stmt = PrepareStatement(db, command) - return stmt.run() - } catch (err) { - console.error(err) - return null - } + return stmt.run() + } catch (err) { + console.error(err) + return null + } } function runStatement(db: Database, command: string, runType?: string): any { - const stmt = PrepareStatement(db, command) - if (!runType) { - return stmt.all() - } else if (runType === 'run') { - return stmt.run() - } - return null + const stmt = PrepareStatement(db, command) + if (!runType) { + return stmt.all() + } else if (runType === 'run') { + return stmt.run() + } + return null } function CloseDB(db: Database): void { - db.close() + db.close() } // ------------------------------------------------------------------------- function PrepareStatement(db: Database, command: string) { - if (!db) { - throw new Error( - 'DB is undefined in prepare statement! DB action called with undefined db' - ) - } - DebugLog(command) - return db.prepare(command) + if (!db) { + throw new Error( + 'DB is undefined in prepare statement! DB action called with undefined db' + ) + } + DebugLog(command) + return db.prepare(command) } diff --git a/src/utils/ids.ts b/src/utils/ids.ts index 5bc04c4..0f01a36 100755 --- a/src/utils/ids.ts +++ b/src/utils/ids.ts @@ -19,8 +19,8 @@ ------------------------------------------------------------------------- */ export default { - LogId: LogId, - Load: Load, + LogId: LogId, + Load: Load, } import utils from '../utils/utils' @@ -35,113 +35,113 @@ let idvStatsData = {} let writes = 0 function Load(): void { - try { - idStatsData = utils.ReadJSON(idStatFile) - } catch (err) { - logger.Log( - 'Error at loading id logs! (@ first run its normal)', - logger.GetColor('redbg') - ) - console.error(err) - } + try { + idStatsData = utils.ReadJSON(idStatFile) + } catch (err) { + logger.Log( + 'Error at loading id logs! (@ first run its normal)', + logger.GetColor('redbg') + ) + console.error(err) + } - try { - const prevVData = utils.ReadFile(idVStatFile) - idvStatsData = JSON.parse(prevVData) - } catch (err) { - logger.Log( - 'Error at loading id logs! (@ first run its normal)', - logger.GetColor('redbg') - ) - console.error(err) - } + try { + const prevVData = utils.ReadFile(idVStatFile) + idvStatsData = JSON.parse(prevVData) + } catch (err) { + logger.Log( + 'Error at loading id logs! (@ first run its normal)', + logger.GetColor('redbg') + ) + console.error(err) + } } function LogId( - id: number, - subj: string, - newQuestions: number, - allQuestions: number + id: number, + subj: string, + newQuestions: number, + allQuestions: number ): void { - Inc(id, subj, newQuestions, allQuestions) - AddVisitStat(id, subj, newQuestions, allQuestions) - Save() + Inc(id, subj, newQuestions, allQuestions) + AddVisitStat(id, subj, newQuestions, allQuestions) + Save() } function AddSubjToList(list: { [key: string]: any }, subj: string) { - if (!list[subj]) { - list[subj] = 0 - } - list[subj]++ + if (!list[subj]) { + list[subj] = 0 + } + list[subj]++ } function Inc( - value: number, - subj: string, - newQuestions: number, - allQuestions: number + value: number, + subj: string, + newQuestions: number, + allQuestions: number ) { - if (idStatsData[value] === undefined) { - idStatsData[value] = { - count: 0, - newQuestions: 0, - allQuestions: 0, - subjs: {}, + if (idStatsData[value] === undefined) { + idStatsData[value] = { + count: 0, + newQuestions: 0, + allQuestions: 0, + subjs: {}, + } } - } - idStatsData[value].count++ - idStatsData[value].newQuestions += newQuestions - idStatsData[value].allQuestions += allQuestions - AddSubjToList(idStatsData[value].subjs, subj) + idStatsData[value].count++ + idStatsData[value].newQuestions += newQuestions + idStatsData[value].allQuestions += allQuestions + AddSubjToList(idStatsData[value].subjs, subj) } function AddVisitStat( - name: number, - subj: string, - newQuestions: number, - allQuestions: number + name: number, + subj: string, + newQuestions: number, + allQuestions: number ) { - const date = new Date() - const now = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - if (idvStatsData[now] === undefined) { - idvStatsData[now] = {} - } - if (idvStatsData[now][name] === undefined) { - idvStatsData[now][name] = { - count: 0, - newQuestions: 0, - allQuestions: 0, - subjs: {}, + const date = new Date() + const now = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + if (idvStatsData[now] === undefined) { + idvStatsData[now] = {} } - } - idvStatsData[now][name].count++ - idvStatsData[now][name].newQuestions += newQuestions - idvStatsData[now][name].allQuestions += allQuestions - AddSubjToList(idvStatsData[now][name].subjs, subj) + if (idvStatsData[now][name] === undefined) { + idvStatsData[now][name] = { + count: 0, + newQuestions: 0, + allQuestions: 0, + subjs: {}, + } + } + idvStatsData[now][name].count++ + idvStatsData[now][name].newQuestions += newQuestions + idvStatsData[now][name].allQuestions += allQuestions + AddSubjToList(idvStatsData[now][name].subjs, subj) } function Save() { - writes++ - if (writes === writeInterval) { - try { - utils.WriteFile(JSON.stringify(idStatsData), idStatFile) - // Log("Stats wrote."); - } catch (err) { - logger.Log('Error at writing logs!', logger.GetColor('redbg')) - console.error(err) + writes++ + if (writes === writeInterval) { + try { + utils.WriteFile(JSON.stringify(idStatsData), idStatFile) + // Log("Stats wrote."); + } catch (err) { + logger.Log('Error at writing logs!', logger.GetColor('redbg')) + console.error(err) + } + try { + utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile) + // Log("Stats wrote."); + } catch (err) { + logger.Log('Error at writing visit logs!', logger.GetColor('redbg')) + console.error(err) + } + writes = 0 } - try { - utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile) - // Log("Stats wrote."); - } catch (err) { - logger.Log('Error at writing visit logs!', logger.GetColor('redbg')) - console.error(err) - } - writes = 0 - } } diff --git a/src/utils/logger.ts b/src/utils/logger.ts index d0ae0dc..1620e16 100755 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -19,7 +19,7 @@ ------------------------------------------------------------------------- */ const hr = - '---------------------------------------------------------------------------------' + '---------------------------------------------------------------------------------' const DELIM = C('green') + '|' + C() @@ -50,361 +50,377 @@ let writes = 0 let noLogIds: string[] = [] function getColoredDateString(): string { - const date = new Date() - const dateString = utils.GetDateString() - return GetRandomColor(date.getHours().toString()) + dateString + C() + const date = new Date() + const dateString = utils.GetDateString() + return GetRandomColor(date.getHours().toString()) + dateString + C() } function DebugLog(msg: string | object, name: string, lvl: number): void { - if (lvl <= debugLevel) { - if (msg === 'hr') { - msg = hr + if (lvl <= debugLevel) { + if (msg === 'hr') { + msg = hr + } + let res = msg + const header = `${C('red')}#DEBUG${lvl}#${C( + 'yellow' + )}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}` + if (typeof msg !== 'object') { + res = header + msg + } else { + Log(header + 'OBJECT:', 'yellow') + res = msg + } + Log(res, 'yellow') } - let res = msg - const header = `${C('red')}#DEBUG${lvl}#${C( - 'yellow' - )}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}` - if (typeof msg !== 'object') { - res = header + msg - } else { - Log(header + 'OBJECT:', 'yellow') - res = msg - } - Log(res, 'yellow') - } } function Log(msg: string | object, color?: string): void { - let log = msg - if (typeof msg !== 'object') { - const delimiter = DELIM + C(color) - log = getColoredDateString() + delimiter + C(color) + msg + C() - } + let log = msg + if (typeof msg !== 'object') { + const delimiter = DELIM + C(color) + log = getColoredDateString() + delimiter + C(color) + msg + C() + } - if (!process.env.NS_NOLOG) { - console.log(log) - } - utils.AppendToFile( - typeof log === 'string' ? log : JSON.stringify(log), - logDir + logFileName - ) + if (!process.env.NS_NOLOG) { + console.log(log) + } + utils.AppendToFile( + typeof log === 'string' ? log : JSON.stringify(log), + logDir + logFileName + ) } function expandWithSpaces(text: string, count: number) { - while (text.length < count) { - text += ' ' - } - return text + while (text.length < count) { + text += ' ' + } + return text } function LogReq( - req: Request, - toFile?: boolean, - statusCode?: string | number + req: Request, + toFile?: boolean, + statusCode?: string | number ): void { - try { - let logEntry = '' // logHashed(ip) - let dl = DELIM - if (req.url.includes('lred')) { - dl += C('red') - } - if (req.session && req.session.user && !shouldLog(req.session.user.id, noLogIds)) { - return - } + try { + let logEntry = '' // logHashed(ip) + let dl = DELIM + if (req.url.includes('lred')) { + dl += C('red') + } + if ( + req.session && + req.session.user && + !shouldLog(req.session.user.id, noLogIds) + ) { + return + } - let hostname - if (req.hostname) { - hostname = req.hostname.replace('www.', '').split('.')[0] - } else { - hostname = 'NOHOST' - Log( - 'req.hostname is undefined! req.hostname: ' + req.hostname, - GetColor('redbg') - ) - } - if (!toFile) { - hostname = expandWithSpaces(hostname, 10) - } + let hostname + if (req.hostname) { + hostname = req.hostname.replace('www.', '').split('.')[0] + } else { + hostname = 'NOHOST' + Log( + 'req.hostname is undefined! req.hostname: ' + req.hostname, + GetColor('redbg') + ) + } + if (!toFile) { + hostname = expandWithSpaces(hostname, 10) + } - logEntry += logHashed(hostname) + dl - if (toFile) { - logEntry += req.headers['user-agent'] + dl - logEntry += req.method + dl - } + logEntry += logHashed(hostname) + dl + if (toFile) { + logEntry += req.headers['user-agent'] + dl + logEntry += req.method + dl + } - let uid = '' - if (req.session && req.session.user) { - uid = req.session.user.id.toString() - } else if (req.session && req.session.isException === true) { - uid = 'EX' - } else { - uid = 'NOUSR' - } - if (!toFile) { - uid = expandWithSpaces(uid, 5) - } - logEntry += GetRandomColor(uid.toString()) + uid + C() + dl + let uid = '' + if (req.session && req.session.user) { + uid = req.session.user.id.toString() + } else if (req.session && req.session.isException === true) { + uid = 'EX' + } else { + uid = 'NOUSR' + } + if (!toFile) { + uid = expandWithSpaces(uid, 5) + } + logEntry += GetRandomColor(uid.toString()) + uid + C() + dl - logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C() + logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C() - if (statusCode !== undefined) { - logEntry += dl + statusCode + if (statusCode !== undefined) { + logEntry += dl + statusCode + } + + logEntry += C() + if (!toFile) { + Log(logEntry) + } else { + const defLogs = utils.GetDateString() + dl + logEntry + + utils.AppendToFile(defLogs, vlogDir + logFileName) + } + } catch (err) { + console.error(err) + Log('Error at logging lol', GetColor('redbg')) } - - logEntry += C() - if (!toFile) { - Log(logEntry) - } else { - const defLogs = utils.GetDateString() + dl + logEntry - - utils.AppendToFile(defLogs, vlogDir + logFileName) - } - } catch (err) { - console.error(err) - Log('Error at logging lol', GetColor('redbg')) - } } function parseNoLogFile(newData: string) { - noLogIds = newData.split('\n') - if (noLogIds[noLogIds.length - 1] === '') { - noLogIds.pop() - } - noLogIds = noLogIds.filter((noLogId) => { - return noLogId !== '' - }) - Log('\tNo Log user ID-s changed: ' + noLogIds.join(', ')) + noLogIds = newData.split('\n') + if (noLogIds[noLogIds.length - 1] === '') { + noLogIds.pop() + } + noLogIds = noLogIds.filter((noLogId) => { + return noLogId !== '' + }) + Log('\tNo Log user ID-s changed: ' + noLogIds.join(', ')) } function setNoLogReadInterval() { - utils.WatchFile(nologFile, (newData: string) => { - parseNoLogFile(newData) - }) + utils.WatchFile(nologFile, (newData: string) => { + parseNoLogFile(newData) + }) - parseNoLogFile(utils.ReadFile(nologFile)) + parseNoLogFile(utils.ReadFile(nologFile)) } function Load(): void { - Log('Loading logger...') - try { - uvData = JSON.parse(utils.ReadFile(uStatsFile)) - } catch (err) { - Log('Error at loading logs! (@ first run its normal)', GetColor('redbg')) - console.error(err) - } + Log('Loading logger...') + try { + uvData = JSON.parse(utils.ReadFile(uStatsFile)) + } catch (err) { + Log( + 'Error at loading logs! (@ first run its normal)', + GetColor('redbg') + ) + console.error(err) + } - try { - udvData = JSON.parse(utils.ReadFile(uvStatsFile)) - } catch (err) { - Log('Error at loading logs! (@ first run its normal)', GetColor('redbg')) - console.error(err) - } + try { + udvData = JSON.parse(utils.ReadFile(uvStatsFile)) + } catch (err) { + Log( + 'Error at loading logs! (@ first run its normal)', + GetColor('redbg') + ) + console.error(err) + } - try { - vData = utils.ReadJSON(statFile) - } catch (err) { - Log('Error at loading logs! (@ first run its normal)', GetColor('redbg')) - console.error(err) - } + try { + vData = utils.ReadJSON(statFile) + } catch (err) { + Log( + 'Error at loading logs! (@ first run its normal)', + GetColor('redbg') + ) + console.error(err) + } - try { - dvData = utils.ReadJSON(vStatFile) - } catch (err) { - Log( - 'Error at loading visit logs! (@ first run its normal)', - GetColor('redbg') - ) - console.error(err) - } - setNoLogReadInterval() + try { + dvData = utils.ReadJSON(vStatFile) + } catch (err) { + Log( + 'Error at loading visit logs! (@ first run its normal)', + GetColor('redbg') + ) + console.error(err) + } + setNoLogReadInterval() } export function shouldLog(userId: string | number, nolog: string[]): boolean { - return !nolog.some((noLogId) => { - return noLogId === userId.toString() - }) + return !nolog.some((noLogId) => { + return noLogId === userId.toString() + }) } function LogStat(url: string, hostname: string, userId: number | string): void { - if (!shouldLog(userId, noLogIds)) { - return - } + if (!shouldLog(userId, noLogIds)) { + return + } - url = hostname + url.split('?')[0] - Inc(url) - AddVisitStat(url) - if (shouldAddUserStat(url)) { - AddUserIdStat(userId.toString()) - IncUserStat(userId.toString()) - } - Save() + url = hostname + url.split('?')[0] + Inc(url) + AddVisitStat(url) + if (shouldAddUserStat(url)) { + AddUserIdStat(userId.toString()) + IncUserStat(userId.toString()) + } + Save() } const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg'] function shouldAddUserStat(url: string) { - return !userStatExcludes.some((x) => url.includes(x)) + return !userStatExcludes.some((x) => url.includes(x)) } function IncUserStat(userId: string) { - try { - if (uvData[userId] === undefined) { - uvData[userId] = 0 + try { + if (uvData[userId] === undefined) { + uvData[userId] = 0 + } + uvData[userId]++ + } catch (err) { + Log('Error at making user ID stats!', GetColor('redbg')) + console.error(err) } - uvData[userId]++ - } catch (err) { - Log('Error at making user ID stats!', GetColor('redbg')) - console.error(err) - } } function AddUserIdStat(userId: string) { - try { - const date = new Date() - const now = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - if (udvData[now] === undefined) { - udvData[now] = {} + try { + const date = new Date() + const now = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + if (udvData[now] === undefined) { + udvData[now] = {} + } + if (udvData[now][userId] === undefined) { + udvData[now][userId] = 0 + } + udvData[now][userId]++ + } catch (err) { + Log('Error at making user ID stats!', GetColor('redbg')) + console.error(err) } - if (udvData[now][userId] === undefined) { - udvData[now][userId] = 0 - } - udvData[now][userId]++ - } catch (err) { - Log('Error at making user ID stats!', GetColor('redbg')) - console.error(err) - } } function Inc(value: string) { - if (value.startsWith('/?')) { - value = '/' - } - if (vData[value] === undefined) { - vData[value] = 0 - } - vData[value]++ + if (value.startsWith('/?')) { + value = '/' + } + if (vData[value] === undefined) { + vData[value] = 0 + } + vData[value]++ } function AddVisitStat(name: string) { - const date = new Date() - const now = - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - if (dvData[now] === undefined) { - dvData[now] = {} - } - if (dvData[now][name] === undefined) { - dvData[now][name] = 0 - } - dvData[now][name]++ + const date = new Date() + const now = + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + if (dvData[now] === undefined) { + dvData[now] = {} + } + if (dvData[now][name] === undefined) { + dvData[now][name] = 0 + } + dvData[now][name]++ } function Save() { - writes++ - if (writes === writeInterval) { - try { - utils.WriteFile(JSON.stringify(uvData), uStatsFile) - } catch (err) { - Log('Error at writing logs! (more in stderr)', GetColor('redbg')) - console.error(err) + writes++ + if (writes === writeInterval) { + try { + utils.WriteFile(JSON.stringify(uvData), uStatsFile) + } catch (err) { + Log('Error at writing logs! (more in stderr)', GetColor('redbg')) + console.error(err) + } + try { + utils.WriteFile(JSON.stringify(udvData), uvStatsFile) + } catch (err) { + Log('Error at writing logs! (more in stderr)', GetColor('redbg')) + console.error(err) + } + try { + utils.WriteFile(JSON.stringify(vData), statFile) + // Log("Stats wrote."); + } catch (err) { + Log('Error at writing logs! (more in stderr)', GetColor('redbg')) + console.error(err) + } + try { + utils.WriteFile(JSON.stringify(dvData), vStatFile) + // Log("Stats wrote."); + } catch (err) { + Log( + 'Error at writing visit logs! (more in stderr)', + GetColor('redbg') + ) + console.error(err) + } + writes = 0 } - try { - utils.WriteFile(JSON.stringify(udvData), uvStatsFile) - } catch (err) { - Log('Error at writing logs! (more in stderr)', GetColor('redbg')) - console.error(err) - } - try { - utils.WriteFile(JSON.stringify(vData), statFile) - // Log("Stats wrote."); - } catch (err) { - Log('Error at writing logs! (more in stderr)', GetColor('redbg')) - console.error(err) - } - try { - utils.WriteFile(JSON.stringify(dvData), vStatFile) - // Log("Stats wrote."); - } catch (err) { - Log('Error at writing visit logs! (more in stderr)', GetColor('redbg')) - console.error(err) - } - writes = 0 - } } function logHashed(msg: string): string { - return GetRandomColor(msg.toString()) + msg + C() + return GetRandomColor(msg.toString()) + msg + C() } function GetRandomColor(msg: string): string { - if (!msg) { - return 'red' - } + if (!msg) { + return 'red' + } - const res = msg.split('').reduce((res, character) => { - return res + character.charCodeAt(0) - }, 0) - return C(colors[res % colors.length]) + const res = msg.split('').reduce((res, character) => { + return res + character.charCodeAt(0) + }, 0) + return C(colors[res % colors.length]) } function GetColor(color: string): string { - return color + return color } function C(color?: string): string { - if (color !== undefined) { - color = color.toLowerCase() - } + if (color !== undefined) { + color = color.toLowerCase() + } - if (color === 'redbg') { - return '\x1b[41m' - } - if (color === 'bluebg') { - return '\x1b[44m' - } - if (color === 'cyanbg') { - return '\x1b[46m' - } - if (color === 'green') { - return '\x1b[32m' - } - if (color === 'red') { - return '\x1b[31m' - } - if (color === 'yellow') { - return '\x1b[33m' - } - if (color === 'blue') { - return '\x1b[34m' - } - if (color === 'magenta') { - return '\x1b[35m' - } - if (color === 'cyan') { - return '\x1b[36m' - } - return '\x1b[0m' + if (color === 'redbg') { + return '\x1b[41m' + } + if (color === 'bluebg') { + return '\x1b[44m' + } + if (color === 'cyanbg') { + return '\x1b[46m' + } + if (color === 'green') { + return '\x1b[32m' + } + if (color === 'red') { + return '\x1b[31m' + } + if (color === 'yellow') { + return '\x1b[33m' + } + if (color === 'blue') { + return '\x1b[34m' + } + if (color === 'magenta') { + return '\x1b[35m' + } + if (color === 'cyan') { + return '\x1b[36m' + } + return '\x1b[0m' } export default { - getColoredDateString: getColoredDateString, - Log: Log, - DebugLog: DebugLog, - GetColor: GetColor, - LogReq: LogReq, - LogStat: LogStat, - Load: Load, - logHashed: logHashed, - hr: hr, - C: C, - logFileName: logFileName, - logDir: logDir, - vlogDir: vlogDir, + getColoredDateString: getColoredDateString, + Log: Log, + DebugLog: DebugLog, + GetColor: GetColor, + LogReq: LogReq, + LogStat: LogStat, + Load: Load, + logHashed: logHashed, + hr: hr, + C: C, + logFileName: logFileName, + logDir: logDir, + vlogDir: vlogDir, } diff --git a/src/utils/tesseract.ts b/src/utils/tesseract.ts index fc54d12..25a7290 100644 --- a/src/utils/tesseract.ts +++ b/src/utils/tesseract.ts @@ -1,7 +1,7 @@ import { - createWorker, - Worker as TesseractWorker, - ConfigResult, + createWorker, + Worker as TesseractWorker, + ConfigResult, } from 'tesseract.js' import logger from './logger' @@ -10,44 +10,44 @@ import { isMainThread, workerData } from 'worker_threads' // https://github.com/naptha/tesseract.js/blob/master/docs/api.md let tesseractWorker: TesseractWorker = null export async function initTesseractWorker(): Promise { - const worker = createWorker({ - cacheMethod: 'refresh', - // logger: (m) => console.log(m), - }) - await worker.load() - await worker.loadLanguage('hun+eng') - await worker.initialize('hun+eng') - return worker - // await worker.terminate(); + const worker = createWorker({ + cacheMethod: 'refresh', + // logger: (m) => console.log(m), + }) + await worker.load() + await worker.loadLanguage('hun+eng') + await worker.initialize('hun+eng') + return worker + // await worker.terminate(); } let resolveLoaded: () => void = null export const tesseractLoaded: Promise = new Promise((resolve) => { - resolveLoaded = resolve + resolveLoaded = resolve }) initTesseractWorker().then((worker) => { - tesseractWorker = worker + tesseractWorker = worker - if (isMainThread) { - logger.Log('Tesseract loaded on main thread') - } else { - const { workerIndex }: { workerIndex: number } = workerData - logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`) - } - resolveLoaded() + if (isMainThread) { + logger.Log('Tesseract loaded on main thread') + } else { + const { workerIndex }: { workerIndex: number } = workerData + logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`) + } + resolveLoaded() }) export async function recognizeTextFromBase64(base64: string): Promise { - const { - data: { text }, - } = await tesseractWorker.recognize(base64) - return text + const { + data: { text }, + } = await tesseractWorker.recognize(base64) + return text } export async function terminateWorker(): Promise { - if (tesseractWorker) { - return tesseractWorker.terminate() - } - return + if (tesseractWorker) { + return tesseractWorker.terminate() + } + return } diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 54a89b5..aa47677 100755 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -19,23 +19,23 @@ ------------------------------------------------------------------------- */ export default { - ReadFile: ReadFile, - ReadJSON: ReadJSON, - WriteFile: WriteFile, - writeFileAsync: writeFileAsync, - AppendToFile: AppendToFile, - FileExists: FileExists, - CreatePath: CreatePath, - WatchFile: WatchFile, - ReadDir: ReadDir, - CopyFile: CopyFile, - GetDateString: GetDateString, - formatUrl: formatUrl, - deleteFile: deleteFile, - uploadFile: uploadFile, - statFile: statFile, - renameFile: renameFile, - deleteDir: deleteDir, + ReadFile: ReadFile, + ReadJSON: ReadJSON, + WriteFile: WriteFile, + writeFileAsync: writeFileAsync, + AppendToFile: AppendToFile, + FileExists: FileExists, + CreatePath: CreatePath, + WatchFile: WatchFile, + ReadDir: ReadDir, + CopyFile: CopyFile, + GetDateString: GetDateString, + formatUrl: formatUrl, + deleteFile: deleteFile, + uploadFile: uploadFile, + statFile: statFile, + renameFile: renameFile, + deleteDir: deleteDir, } import fs from 'fs' @@ -45,251 +45,254 @@ import logger from '../utils/logger' import { Request } from '../types/basicTypes' interface URLFormatOptions { - pathname?: string - query?: { [key: string]: string } + pathname?: string + query?: { [key: string]: string } } function formatUrl(options: URLFormatOptions): string { - const path = options.pathname || '' - if (!options.query || Object.keys(options.query).length === 0) { - return path - } - const queryString = - '?' + - Object.keys(options.query) - .map((key) => { - return `${key}=${encodeURIComponent(options.query[key])}` - }) - .join('&') - return path + queryString + const path = options.pathname || '' + if (!options.query || Object.keys(options.query).length === 0) { + return path + } + const queryString = + '?' + + Object.keys(options.query) + .map((key) => { + return `${key}=${encodeURIComponent(options.query[key])}` + }) + .join('&') + return path + queryString } function GetDateString( - referenceDate?: Date | string, - noTime?: boolean + referenceDate?: Date | string, + noTime?: boolean ): string { - const date = referenceDate ? new Date(referenceDate) : new Date() + const date = referenceDate ? new Date(referenceDate) : new Date() - if (noTime) { - return ( - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) - ) - } else { - return ( - date.getFullYear() + - '-' + - ('0' + (date.getMonth() + 1)).slice(-2) + - '-' + - ('0' + date.getDate()).slice(-2) + - ' ' + - ('0' + date.getHours()).slice(-2) + - ':' + - ('0' + date.getMinutes()).slice(-2) + - ':' + - ('0' + date.getSeconds()).slice(-2) - ) - } + if (noTime) { + return ( + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + ) + } else { + return ( + date.getFullYear() + + '-' + + ('0' + (date.getMonth() + 1)).slice(-2) + + '-' + + ('0' + date.getDate()).slice(-2) + + ' ' + + ('0' + date.getHours()).slice(-2) + + ':' + + ('0' + date.getMinutes()).slice(-2) + + ':' + + ('0' + date.getSeconds()).slice(-2) + ) + } } function CopyFile(from: string, to: string): void { - CreatePath(to) - fs.copyFileSync(from, to) + CreatePath(to) + fs.copyFileSync(from, to) } function ReadDir(path: string, listHidden?: boolean): Array { - if (listHidden) { - return fs.readdirSync(path) - } else { - return fs.readdirSync(path).filter((file) => { - return !file.startsWith('.') - }) - } + if (listHidden) { + return fs.readdirSync(path) + } else { + return fs.readdirSync(path).filter((file) => { + return !file.startsWith('.') + }) + } } function ReadJSON(name: string): any { - try { - return JSON.parse(ReadFile(name)) - } catch (err) { - console.error(err) - throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name) - } + try { + return JSON.parse(ReadFile(name)) + } catch (err) { + console.error(err) + throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name) + } } function ReadFile(name: string): string { - if (!FileExists(name)) { - throw new Error('No such file: ' + name) - } - return fs.readFileSync(name, 'utf8') + if (!FileExists(name)) { + throw new Error('No such file: ' + name) + } + return fs.readFileSync(name, 'utf8') } function FileExists(path: string): boolean { - return fs.existsSync(path) + return fs.existsSync(path) } function WatchFile(file: string, callback: Function): void { - if (FileExists(file)) { - fs.watchFile(file, () => { - fs.readFile(file, 'utf8', (err, data) => { - if (err) { - // console.log(err) - } else { - callback(data) - } - }) - }) - } else { - throw new Error(`${file} does not exits to watch`) - } + if (FileExists(file)) { + fs.watchFile(file, () => { + fs.readFile(file, 'utf8', (err, data) => { + if (err) { + // console.log(err) + } else { + callback(data) + } + }) + }) + } else { + throw new Error(`${file} does not exits to watch`) + } } function CreatePath(path: string, onlyPath?: boolean): void { - if (FileExists(path)) { - return - } - - const spath = path.split('/') - let currDir = spath[0] - for (let i = 1; i < spath.length; i++) { - if (currDir !== '' && !fs.existsSync(currDir)) { - try { - fs.mkdirSync(currDir) - } catch (err) { - console.log('Failed to make ' + currDir + ' directory... ') - console.error(err) - } + if (FileExists(path)) { + return + } + + const spath = path.split('/') + let currDir = spath[0] + for (let i = 1; i < spath.length; i++) { + if (currDir !== '' && !fs.existsSync(currDir)) { + try { + fs.mkdirSync(currDir) + } catch (err) { + console.log('Failed to make ' + currDir + ' directory... ') + console.error(err) + } + } + currDir += '/' + spath[i] + } + if (onlyPath) { + fs.mkdirSync(path) } - currDir += '/' + spath[i] - } - if (onlyPath) { - fs.mkdirSync(path) - } } function WriteFile(content: string, path: string): void { - CreatePath(path) - fs.writeFileSync(path, content) + CreatePath(path) + fs.writeFileSync(path, content) } function writeFileAsync(content: string, path: string): void { - CreatePath(path) - fs.writeFile(path, content, function (err) { - if (err) { - logger.Log( - 'Error writing file: ' + path + ' (sync)', - logger.GetColor('redbg') - ) - } - }) + CreatePath(path) + fs.writeFile(path, content, function (err) { + if (err) { + logger.Log( + 'Error writing file: ' + path + ' (sync)', + logger.GetColor('redbg') + ) + } + }) } function AppendToFile(data: string, file: string): void { - CreatePath(file) - try { - fs.appendFileSync(file, '\n' + data) - } catch (err) { - logger.Log( - 'Error appendig to file log file: ' + file + ' (sync)', - logger.GetColor('redbg') - ) - logger.Log(data) - console.error(err) - } + CreatePath(file) + try { + fs.appendFileSync(file, '\n' + data) + } catch (err) { + logger.Log( + 'Error appendig to file log file: ' + file + ' (sync)', + logger.GetColor('redbg') + ) + logger.Log(data) + console.error(err) + } } function deleteFile(fname: string): Boolean { - if (FileExists(fname)) { - fs.unlinkSync(fname) - return true - } - return false + if (FileExists(fname)) { + fs.unlinkSync(fname) + return true + } + return false } function deleteDir(dirName: string): Boolean { - if (FileExists(dirName)) { - fs.rmSync(dirName, { recursive: true }) - return true - } - return false + if (FileExists(dirName)) { + fs.rmSync(dirName, { recursive: true }) + return true + } + return false } function uploadFile( - req: Request, - path: string + req: Request, + path: string ): Promise<{ - body: Request['body'] - fileName: string - filePath: string + body: Request['body'] + fileName: string + filePath: string }> { - return new Promise((resolve, reject) => { - try { - if (!req.files) { - logger.Log( - `Unable to upload file, req.files is undefined`, - logger.GetColor('redbg') - ) - return - } - const file = req.files.file - // FIXME: Object.keys(req.files).forEach((file) => { saveFile() }) + return new Promise((resolve, reject) => { + try { + if (!req.files) { + logger.Log( + `Unable to upload file, req.files is undefined`, + logger.GetColor('redbg') + ) + return + } + const file = req.files.file + // FIXME: Object.keys(req.files).forEach((file) => { saveFile() }) - CreatePath(path, true) + CreatePath(path, true) - let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/') - let fileDestination = path + '/' + fileName - if (FileExists(fileDestination)) { - const id = uuidv4().split('-')[0] + let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/') + let fileDestination = path + '/' + fileName + if (FileExists(fileDestination)) { + const id = uuidv4().split('-')[0] - const temp = file.name.split('.') - const extension = temp.pop() - fileName = temp.join('.') + '_' + id + '.' + extension - fileDestination = path + '/' + fileName - } + const temp = file.name.split('.') + const extension = temp.pop() + fileName = temp.join('.') + '_' + id + '.' + extension + fileDestination = path + '/' + fileName + } - file.mv(fileDestination, (err: Error) => { - if (err) { - logger.Log(`Unable to upload file!`, logger.GetColor('redbg')) - console.error(err) - reject(err) - } else { - logger.Log( - `Uploaded: ${fileName} to ${fileDestination}`, - logger.GetColor('blue') - ) - resolve({ - body: req.body, - fileName: fileName, - filePath: fileDestination, - }) + file.mv(fileDestination, (err: Error) => { + if (err) { + logger.Log( + `Unable to upload file!`, + logger.GetColor('redbg') + ) + console.error(err) + reject(err) + } else { + logger.Log( + `Uploaded: ${fileName} to ${fileDestination}`, + logger.GetColor('blue') + ) + resolve({ + body: req.body, + fileName: fileName, + filePath: fileDestination, + }) + } + }) + } catch (err) { + logger.Log( + `Unable to upload file, error on stderr`, + logger.GetColor('redbg') + ) + console.error(err) + reject(err) } - }) - } catch (err) { - logger.Log( - `Unable to upload file, error on stderr`, - logger.GetColor('redbg') - ) - console.error(err) - reject(err) - } - }) + }) } function statFile(file: string): fs.Stats { - if (FileExists(file)) { - return fs.statSync(file) - } else { - return null - } + if (FileExists(file)) { + return fs.statSync(file) + } else { + return null + } } function renameFile(oldPath: string, newPath: string): string { - if (FileExists(oldPath)) { - fs.renameSync(oldPath, newPath) - return newPath - } else { - return null - } + if (FileExists(oldPath)) { + fs.renameSync(oldPath, newPath) + return newPath + } else { + return null + } } diff --git a/src/utils/workerPool.ts b/src/utils/workerPool.ts index ce1aab7..cdd81f3 100644 --- a/src/utils/workerPool.ts +++ b/src/utils/workerPool.ts @@ -29,268 +29,274 @@ import type { Question, QuestionDb, QuestionData } from '../types/basicTypes' import type { WorkerResult } from './classes' interface WorkerObj { - worker: Worker - index: number - free: Boolean + worker: Worker + index: number + free: Boolean } export interface TaskObject { - type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions' - data: - | { - searchIn: number[] - question: Question - subjName: string - testUrl?: string - questionData?: QuestionData - searchInAllIfNoResult?: boolean - searchTillMatchPercent?: number - [key: string]: any - } - | { dbIndex: number; edits: Edits } - | QuestionDb - | Result - | { - questions: Question[] - subjToClean: string - overwriteFromDate: number - qdbIndex: number - } - | { - questionIndexesToRemove: number[][] - subjIndex: number - qdbIndex: number - recievedQuestions: Question[] - } + type: + | 'work' + | 'dbEdit' + | 'newQuestions' + | 'newdb' + | 'dbClean' + | 'rmQuestions' + data: + | { + searchIn: number[] + question: Question + subjName: string + testUrl?: string + questionData?: QuestionData + searchInAllIfNoResult?: boolean + searchTillMatchPercent?: number + [key: string]: any + } + | { dbIndex: number; edits: Edits } + | QuestionDb + | Result + | { + questions: Question[] + subjToClean: string + overwriteFromDate: number + qdbIndex: number + } + | { + questionIndexesToRemove: number[][] + subjIndex: number + qdbIndex: number + recievedQuestions: Question[] + } } interface PendingJob { - workData: TaskObject - doneEvent: DoneEvent - targetWorkerIndex?: number + workData: TaskObject + doneEvent: DoneEvent + targetWorkerIndex?: number } interface JobEvent extends EventEmitter { - on(event: 'jobDone', listener: () => void): this - on(event: 'newJob', listener: () => void): this - emit(event: 'newJob'): boolean - emit(event: 'jobDone'): boolean + on(event: 'jobDone', listener: () => void): this + on(event: 'newJob', listener: () => void): this + emit(event: 'newJob'): boolean + emit(event: 'jobDone'): boolean } interface DoneEvent extends EventEmitter { - once(event: 'done', listener: (result: WorkerResult) => void): this - emit(event: 'done', res: WorkerResult): boolean + once(event: 'done', listener: (result: WorkerResult) => void): this + emit(event: 'done', res: WorkerResult): boolean } const alertOnPendingCount = 50 const workerFile = './src/utils/classes.ts' let workers: Array const pendingJobs: { - [id: string]: PendingJob + [id: string]: PendingJob } = {} const jobEvents: JobEvent = new EventEmitter() jobEvents.on('jobDone', () => { - processJob() + processJob() }) jobEvents.on('newJob', () => { - processJob() + processJob() }) // --------------------------------------------------------------------------- function handleWorkerError(worker: WorkerObj, err: Error) { - // TODO: restart worker if exited or things like that - logger.Log('resourcePromise error', logger.GetColor('redbg')) - console.error(err, worker) + // TODO: restart worker if exited or things like that + logger.Log('resourcePromise error', logger.GetColor('redbg')) + console.error(err, worker) } // TODO: accuire all workers here, and handle errors so they can be removed if threads exit export function msgAllWorker(data: TaskObject): Promise { - logger.DebugLog('MSGING ALL WORKER', 'job', 1) - return new Promise((resolve) => { - const promises: Promise[] = [] - workers.forEach((worker) => { - promises.push(doALongTask(data, worker.index)) + logger.DebugLog('MSGING ALL WORKER', 'job', 1) + return new Promise((resolve) => { + const promises: Promise[] = [] + workers.forEach((worker) => { + promises.push(doALongTask(data, worker.index)) + }) + Promise.all(promises).then((res) => { + logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1) + resolve(res) + }) }) - Promise.all(promises).then((res) => { - logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1) - resolve(res) - }) - }) } export function doALongTask( - obj: TaskObject, - targetWorkerIndex?: number + obj: TaskObject, + targetWorkerIndex?: number ): Promise { - if (Object.keys(pendingJobs).length > alertOnPendingCount) { - logger.Log( - `More than ${alertOnPendingCount} callers waiting for free resource! (${ - Object.keys(pendingJobs).length - })`, - logger.GetColor('redbg') - ) - } + if (Object.keys(pendingJobs).length > alertOnPendingCount) { + logger.Log( + `More than ${alertOnPendingCount} callers waiting for free resource! (${ + Object.keys(pendingJobs).length + })`, + logger.GetColor('redbg') + ) + } - const jobId = uuidv4() - // FIXME: delete doneEvent? - const doneEvent: DoneEvent = new EventEmitter() - pendingJobs[jobId] = { - workData: obj, - targetWorkerIndex: targetWorkerIndex, - doneEvent: doneEvent, - } - jobEvents.emit('newJob') - return new Promise((resolve) => { - doneEvent.once('done', (result: WorkerResult) => { - jobEvents.emit('jobDone') - resolve(result) + const jobId = uuidv4() + // FIXME: delete doneEvent? + const doneEvent: DoneEvent = new EventEmitter() + pendingJobs[jobId] = { + workData: obj, + targetWorkerIndex: targetWorkerIndex, + doneEvent: doneEvent, + } + jobEvents.emit('newJob') + return new Promise((resolve) => { + doneEvent.once('done', (result: WorkerResult) => { + jobEvents.emit('jobDone') + resolve(result) + }) }) - }) } export function initWorkerPool(initData: Array): Array { - if (workers) { - logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg')) - return null - } - workers = [] + if (workers) { + logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg')) + return null + } + workers = [] - const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length - if (process.env.NS_THREAD_COUNT) { - logger.Log( - `Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`, - logger.GetColor('red') - ) - } + const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length + if (process.env.NS_THREAD_COUNT) { + logger.Log( + `Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`, + logger.GetColor('red') + ) + } - for (let i = 0; i < threadCount; i++) { - workers.push({ - worker: getAWorker(i, initData), - index: i, - free: true, - }) - } + for (let i = 0; i < threadCount; i++) { + workers.push({ + worker: getAWorker(i, initData), + index: i, + free: true, + }) + } - return workers + return workers } // --------------------------------------------------------------------------- function processJob() { - if (Object.keys(pendingJobs).length > 0) { - const keys = Object.keys(pendingJobs) - let jobKey: string, freeWorker: WorkerObj - let i = 0 - while (!freeWorker && i < keys.length) { - jobKey = keys[i] - if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) { - if (workers[pendingJobs[jobKey].targetWorkerIndex].free) { - freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex] - logger.DebugLog( - `RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`, - 'job', - 1 - ) + if (Object.keys(pendingJobs).length > 0) { + const keys = Object.keys(pendingJobs) + let jobKey: string, freeWorker: WorkerObj + let i = 0 + while (!freeWorker && i < keys.length) { + jobKey = keys[i] + if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) { + if (workers[pendingJobs[jobKey].targetWorkerIndex].free) { + freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex] + logger.DebugLog( + `RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`, + 'job', + 1 + ) + } + } else { + freeWorker = workers.find((worker) => { + return worker.free + }) + if (freeWorker) { + logger.DebugLog( + `RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`, + 'job', + 1 + ) + } + } + i++ } - } else { - freeWorker = workers.find((worker) => { - return worker.free - }) - if (freeWorker) { - logger.DebugLog( - `RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`, - 'job', - 1 - ) + + if (!freeWorker) { + logger.DebugLog('NO FREE WORKER', 'job', 1) + return } - } - i++ - } - if (!freeWorker) { - logger.DebugLog('NO FREE WORKER', 'job', 1) - return - } + if (freeWorker.free) { + freeWorker.free = false + } + const job = pendingJobs[jobKey] + delete pendingJobs[jobKey] - if (freeWorker.free) { - freeWorker.free = false + doSomething(freeWorker, job.workData) + .then((res: WorkerResult) => { + freeWorker.free = true + job.doneEvent.emit('done', res) + }) + .catch(function (err) { + handleWorkerError(freeWorker, err) + }) } - const job = pendingJobs[jobKey] - delete pendingJobs[jobKey] - - doSomething(freeWorker, job.workData) - .then((res: WorkerResult) => { - freeWorker.free = true - job.doneEvent.emit('done', res) - }) - .catch(function (err) { - handleWorkerError(freeWorker, err) - }) - } } function getAWorker(i: number, initData: Array) { - const worker = workerTs(workerFile, { - workerData: { - workerIndex: i, - initData: initData, - }, - }) + const worker = workerTs(workerFile, { + workerData: { + workerIndex: i, + initData: initData, + }, + }) - worker.setMaxListeners(50) + worker.setMaxListeners(50) - worker.on('error', (err) => { - logger.Log('Worker error!', logger.GetColor('redbg')) - console.error(err) - }) + worker.on('error', (err) => { + logger.Log('Worker error!', logger.GetColor('redbg')) + console.error(err) + }) - worker.on('exit', (code) => { - // TODO: this is critical, whole server should stop, or child threads should be restarted - logger.Log( - `[MAIN]: worker #${i} exit code: ${code}`, - code === 0 ? logger.GetColor('redbg') : logger.GetColor('green') - ) - }) - return worker + worker.on('exit', (code) => { + // TODO: this is critical, whole server should stop, or child threads should be restarted + logger.Log( + `[MAIN]: worker #${i} exit code: ${code}`, + code === 0 ? logger.GetColor('redbg') : logger.GetColor('green') + ) + }) + return worker } // --------------------------------------------------------------------------- function doSomething(currWorker: WorkerObj, obj: TaskObject) { - const { /* index, */ worker } = currWorker - return new Promise((resolve) => { - worker.postMessage(obj) - worker.once('message', (msg: WorkerResult) => { - resolve(msg) + const { /* index, */ worker } = currWorker + return new Promise((resolve) => { + worker.postMessage(obj) + worker.once('message', (msg: WorkerResult) => { + resolve(msg) + }) }) - }) } const workerTs = ( - file: string, - wkOpts: { - workerData: { - workerIndex: number - initData: QuestionDb[] - __filename?: string + file: string, + wkOpts: { + workerData: { + workerIndex: number + initData: QuestionDb[] + __filename?: string + } + eval?: boolean } - eval?: boolean - } ) => { - wkOpts.eval = true - wkOpts.workerData.__filename = file - return new Worker( - ` + wkOpts.eval = true + wkOpts.workerData.__filename = file + return new Worker( + ` const wk = require('worker_threads'); require('ts-node').register(); let file = wk.workerData.__filename; delete wk.workerData.__filename; require(file); `, - wkOpts - ) + wkOpts + ) } diff --git a/submodules/qmining-page b/submodules/qmining-page index 281d0e0..ed507dc 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 281d0e00ce054d46444f377876786b913b8c1a08 +Subproject commit ed507dc39f5d34703e53585a75a0138e70bcee3a