albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

prettier 4 tabwidth

Browse Source
This commit is contained in:
mrfry
2022-12-10 15:34:54 +01:00
parent 00ec614f1d
commit 96b413a365
42 changed files with 7034 additions and 6905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.prettierrc.js
View File

@@ -1,6 +1,6 @@
module.exports = { module.exports = {
trailingComma: 'es5', trailingComma: 'es5',
tabWidth: 2, tabWidth: 4,
semi: false, semi: false,
singleQuote: true, singleQuote: true,
arrowParens: 'always', arrowParens: 'always',

244
src/middlewares/auth.middleware.ts
View File

@@ -27,148 +27,148 @@ import utils from '../utils/utils'
import dbtools from '../utils/dbtools' import dbtools from '../utils/dbtools'
interface Options { interface Options {
userDB: Database userDB: Database
jsonResponse: boolean jsonResponse: boolean
exceptions: Array<string> exceptions: Array<string>
} }
export const testUser = { export const testUser = {
id: 19, id: 19,
avaiblePWRequests: 645, avaiblePWRequests: 645,
pwRequestCount: 19, pwRequestCount: 19,
created: new Date(), created: new Date(),
pw: 'secret', pw: 'secret',
loginCount: 3, loginCount: 3,
createdBy: 1, createdBy: 1,
} }
function renderLogin(_req: Request, res: Response, jsonResponse: boolean) { function renderLogin(_req: Request, res: Response, jsonResponse: boolean) {
res.status(401) // Unauthorized res.status(401) // Unauthorized
if (jsonResponse) { if (jsonResponse) {
res.json({ res.json({
result: 'nouser', result: 'nouser',
msg: 'You are not logged in', msg: 'You are not logged in',
}) })
} else { } else {
res.render('login', { res.render('login', {
devel: process.env.NS_DEVEL, devel: process.env.NS_DEVEL,
}) })
} }
} }
export default function (options: Options): RequestHandler { export default function (options: Options): RequestHandler {
const { const {
userDB, userDB,
jsonResponse, jsonResponse,
exceptions, exceptions,
}: { }: {
userDB: Database userDB: Database
jsonResponse: boolean jsonResponse: boolean
exceptions: string[] exceptions: string[]
} = options } = options
return function (req: Request, res: Response, next: NextFunction) { return function (req: Request, res: Response, next: NextFunction) {
const sessionID = req.cookies.sessionID const sessionID = req.cookies.sessionID
const isException = exceptions.some((exc) => { const isException = exceptions.some((exc) => {
return req.url.split('?')[0] === exc return req.url.split('?')[0] === exc
}) })
if (process.env.NS_NOUSER) { if (process.env.NS_NOUSER) {
req.session = { req.session = {
user: testUser, user: testUser,
sessionID: sessionID || 11111111111, sessionID: sessionID || 11111111111,
isException: false, isException: false,
} }
next() next()
return return
} }
// FIXME Allowing all urls with _next in it, but not in params // FIXME Allowing all urls with _next in it, but not in params
if ( if (
req.url.split('?')[0].includes('_next') || req.url.split('?')[0].includes('_next') ||
req.url.split('?')[0].includes('well-known/acme-challenge') req.url.split('?')[0].includes('well-known/acme-challenge')
) { ) {
req.session = { isException: true } req.session = { isException: true }
next() next()
return return
} }
if (!sessionID) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
const user = GetUserBySessionID(userDB, sessionID)
if (!user) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No user:${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
req.session = {
user: user,
sessionID: sessionID,
isException: isException,
}
logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1)
dbtools.Update(
userDB,
'sessions',
{
lastAccess: utils.GetDateString(),
},
{
id: sessionID,
}
)
dbtools.Update(
userDB,
'users',
{
lastAccess: utils.GetDateString(),
},
{
id: user.id,
}
)
if (!sessionID) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next() next()
return
}
logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
} }
const user = GetUserBySessionID(userDB, sessionID)
if (!user) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No user:${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
req.session = {
user: user,
sessionID: sessionID,
isException: isException,
}
logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1)
dbtools.Update(
userDB,
'sessions',
{
lastAccess: utils.GetDateString(),
},
{
id: sessionID,
}
)
dbtools.Update(
userDB,
'users',
{
lastAccess: utils.GetDateString(),
},
{
id: user.id,
}
)
next()
}
} }
function GetUserBySessionID(db: Database, sessionID: string) { function GetUserBySessionID(db: Database, sessionID: string) {
logger.DebugLog(`Getting user from db`, 'auth', 2) logger.DebugLog(`Getting user from db`, 'auth', 2)
const session = dbtools.Select(db, 'sessions', { const session = dbtools.Select(db, 'sessions', {
id: sessionID, id: sessionID,
})[0] })[0]
if (!session) { if (!session) {
return return
} }
const user = dbtools.Select(db, 'users', { const user = dbtools.Select(db, 'users', {
id: session.userID, id: session.userID,
})[0] })[0]
if (user) { if (user) {
return user return user
} }
} }

114
src/middlewares/reqlogger.middleware.ts
View File

@@ -23,68 +23,70 @@ import type { Response, NextFunction } from 'express'
import type { Request } from '../types/basicTypes' import type { Request } from '../types/basicTypes'
interface Options { interface Options {
loggableKeywords: Array<string> loggableKeywords: Array<string>
loggableModules: Array<string> loggableModules: Array<string>
exceptions: Array<string> exceptions: Array<string>
excludeFromStats: Array<string> excludeFromStats: Array<string>
} }
export default function (options: Options): any { export default function (options: Options): any {
const loggableKeywords = options ? options.loggableKeywords : undefined const loggableKeywords = options ? options.loggableKeywords : undefined
const loggableModules = options ? options.loggableModules : undefined const loggableModules = options ? options.loggableModules : undefined
const exceptions = options.exceptions || [] const exceptions = options.exceptions || []
const excludeFromStats = options.excludeFromStats || [] const excludeFromStats = options.excludeFromStats || []
return function (req: Request, res: Response, next: NextFunction) { return function (req: Request, res: Response, next: NextFunction) {
res.on('finish', function () { res.on('finish', function () {
// TODO: test this // TODO: test this
const isException = exceptions.some((ex) => { const isException = exceptions.some((ex) => {
return req.url.includes(ex) return req.url.includes(ex)
}) })
if (isException) { if (isException) {
return return
} }
let hostname = 'NOHOST' let hostname = 'NOHOST'
if (req.hostname) { if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0] hostname = req.hostname.replace('www.', '').split('.')[0]
} else { } else {
logger.Log('Hostname is undefined!', logger.GetColor('redbg')) logger.Log('Hostname is undefined!', logger.GetColor('redbg'))
console.log(req.body) console.log(req.body)
console.log(req.query) console.log(req.query)
console.log(req.headers) console.log(req.headers)
} }
const hasLoggableKeyword = const hasLoggableKeyword =
loggableKeywords && loggableKeywords &&
loggableKeywords.some((keyword) => { loggableKeywords.some((keyword) => {
return req.url.includes(keyword) return req.url.includes(keyword)
})
const hasLoggableModule =
loggableModules &&
loggableModules.some((keyword) => {
return hostname.includes(keyword)
})
const toLog = hasLoggableModule || hasLoggableKeyword
logger.LogReq(req, true, res.statusCode)
if (toLog) {
logger.LogReq(req)
}
const shouldLogStat = !excludeFromStats.some((ex) => {
return req.url.includes(ex)
})
if (res.statusCode !== 404 && shouldLogStat) {
logger.LogStat(
req.url,
hostname,
req.session && req.session.user
? req.session.user.id
: 'NOUSER'
)
}
}) })
const hasLoggableModule = next()
loggableModules && }
loggableModules.some((keyword) => {
return hostname.includes(keyword)
})
const toLog = hasLoggableModule || hasLoggableKeyword
logger.LogReq(req, true, res.statusCode)
if (toLog) {
logger.LogReq(req)
}
const shouldLogStat = !excludeFromStats.some((ex) => {
return req.url.includes(ex)
})
if (res.statusCode !== 404 && shouldLogStat) {
logger.LogStat(
req.url,
hostname,
req.session && req.session.user ? req.session.user.id : 'NOUSER'
)
}
})
next()
}
} }

80
src/middlewares/socketAuth.middleware.ts
View File

@@ -27,60 +27,60 @@ import { Socket } from '../types/basicTypes'
import { testUser } from './auth.middleware' import { testUser } from './auth.middleware'
interface Options { interface Options {
userDB: any userDB: any
} }
export default function SocketAuth(options: Options): any { export default function SocketAuth(options: Options): any {
const { userDB } = options const { userDB } = options
return (socket: Socket, next: (arg0?: any) => void) => { return (socket: Socket, next: (arg0?: any) => void) => {
try { try {
const cookies = cookie.parse(socket.handshake.headers.cookie || '') const cookies = cookie.parse(socket.handshake.headers.cookie || '')
const sessionID = cookies.sessionID const sessionID = cookies.sessionID
if (process.env.NS_NOUSER) { if (process.env.NS_NOUSER) {
socket.user = testUser socket.user = testUser
next() next()
return return
} }
if (!sessionID) { if (!sessionID) {
next(new Error('Not authenticated, please log in')) next(new Error('Not authenticated, please log in'))
return return
} }
const user = GetUserBySessionID(userDB, sessionID) const user = GetUserBySessionID(userDB, sessionID)
if (!user) { if (!user) {
next(new Error('Not authenticated, please log in')) next(new Error('Not authenticated, please log in'))
return return
} }
socket.user = user socket.user = user
next() next()
} catch (e) { } catch (e) {
next(new Error('Authentication server error')) next(new Error('Authentication server error'))
console.error('Authentication server error') console.error('Authentication server error')
console.error(e) console.error(e)
}
} }
}
} }
function GetUserBySessionID(db: any, sessionID: string) { function GetUserBySessionID(db: any, sessionID: string) {
logger.DebugLog(`Getting user from db`, 'auth', 2) logger.DebugLog(`Getting user from db`, 'auth', 2)
const session = dbtools.Select(db, 'sessions', { const session = dbtools.Select(db, 'sessions', {
id: sessionID, id: sessionID,
})[0] })[0]
if (!session) { if (!session) {
return return
} }
const user = dbtools.Select(db, 'users', { const user = dbtools.Select(db, 'users', {
id: session.userID, id: session.userID,
})[0] })[0]
if (user) { if (user) {
return user return user
} }
} }

308
src/modules/api/api.ts
View File

@@ -46,188 +46,188 @@ let httpServer: http.Server
let httpsServer: https.Server let httpsServer: https.Server
function GetApp(): ModuleType { function GetApp(): ModuleType {
const app = express() const app = express()
const publicDir = publicdirs[0] const publicDir = publicdirs[0]
if (!publicDir) { if (!publicDir) {
throw new Error(`No public dir! ( API )`) throw new Error(`No public dir! ( API )`)
}
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
// -------------------------------------------------------------------------------------------
app.use(
express.urlencoded({
limit: '10mb',
extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '10mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/api/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: true,
exceptions: [
'/register',
'/favicon.ico',
'/login',
'/postfeedback',
'/fosuploader',
'/badtestsender',
],
})
)
app.use(
fileUpload({
limits: { fileSize: 50 * 1024 * 1024 },
})
)
// -------------------------------------------------------------------------------------------
let rootRedirectURL = ''
function reloadRootRedirectURL() {
if (utils.FileExists(rootRedirectToFile)) {
rootRedirectURL = utils.ReadFile(rootRedirectToFile)
} }
}
const filesToWatch = [ let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
{ domain.shift() // [ "frylabs", "net" ]
fname: rootRedirectToFile, domain = domain.join('.') // "frylabs.net"
logMsg: 'Root redirect URL changed', logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
action: reloadRootRedirectURL,
},
]
function Load() { // -------------------------------------------------------------------------------------------
filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) { app.use(
utils.WatchFile(ftw.fname, () => { express.urlencoded({
logger.Log(ftw.logMsg) limit: '10mb',
ftw.action() extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '10mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/api/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: true,
exceptions: [
'/register',
'/favicon.ico',
'/login',
'/postfeedback',
'/fosuploader',
'/badtestsender',
],
}) })
ftw.action() )
} else { app.use(
logger.Log( fileUpload({
`File ${ftw.fname} does not exists to watch!`, limits: { fileSize: 50 * 1024 * 1024 },
logger.GetColor('redbg') })
) )
} // -------------------------------------------------------------------------------------------
})
}
Load() let rootRedirectURL = ''
// -------------------------------------------------------------- function reloadRootRedirectURL() {
if (utils.FileExists(rootRedirectToFile)) {
app.get('/', function (req: Request, res: any) { rootRedirectURL = utils.ReadFile(rootRedirectToFile)
logger.LogReq(req) }
if (reloadRootRedirectURL) {
res.redirect(rootRedirectURL)
} else {
res.json({ msg: 'hi c:' })
} }
})
// ------------------------------------------------------------------------------------------- const filesToWatch = [
{
fname: rootRedirectToFile,
logMsg: 'Root redirect URL changed',
action: reloadRootRedirectURL,
},
]
const submoduleDatas = setupSubModules(app) function Load() {
filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) {
utils.WatchFile(ftw.fname, () => {
logger.Log(ftw.logMsg)
ftw.action()
})
ftw.action()
} else {
logger.Log(
`File ${ftw.fname} does not exists to watch!`,
logger.GetColor('redbg')
)
}
})
}
// ------------------------------------------------------------------------------------------- Load()
publicdirs.forEach((pdir) => { // --------------------------------------------------------------
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
// ------------------------------------------------------------------------------------------- app.get('/', function (req: Request, res: any) {
logger.LogReq(req)
if (reloadRootRedirectURL) {
res.redirect(rootRedirectURL)
} else {
res.json({ msg: 'hi c:' })
}
})
app.get('*', function (_req: Request, res: any) { // -------------------------------------------------------------------------------------------
res.status(404).render('404')
})
app.post('*', function (_req: Request, res: any) { const submoduleDatas = setupSubModules(app)
res.status(404).render('404')
}) // -------------------------------------------------------------------------------------------
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
// -------------------------------------------------------------------------------------------
app.get('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
app.post('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
function DailyAction() {
submoduleDatas.forEach((data) => {
if (data.dailyAction) {
data.dailyAction()
}
})
}
function DailyAction() {
submoduleDatas.forEach((data) => { submoduleDatas.forEach((data) => {
if (data.dailyAction) { if (data.load) {
data.dailyAction() data.load()
} }
}) })
}
submoduleDatas.forEach((data) => { return {
if (data.load) { dailyAction: DailyAction,
data.load() app: app,
} }
})
return {
dailyAction: DailyAction,
app: app,
}
} }
function setupSubModules( function setupSubModules(
parentApp: express.Application, parentApp: express.Application,
moduleSpecificData?: any moduleSpecificData?: any
): Submodule[] { ): Submodule[] {
const submoduleDir = './submodules/' const submoduleDir = './submodules/'
const absolutePath = __dirname + '/' + submoduleDir const absolutePath = __dirname + '/' + submoduleDir
if (!utils.FileExists(absolutePath)) { if (!utils.FileExists(absolutePath)) {
return null return null
}
const files = utils.ReadDir(absolutePath)
const moduleDatas: Submodule[] = []
files.forEach((file) => {
if (!file.endsWith('.js')) {
return
} }
const submodulePath = submoduleDir + file const files = utils.ReadDir(absolutePath)
const moduleDatas: Submodule[] = []
files.forEach((file) => {
if (!file.endsWith('.js')) {
return
}
const submodulePath = submoduleDir + file
try { try {
logger.Log(`Loading submodule '${file}' for '${moduleName}'...`) logger.Log(`Loading submodule '${file}' for '${moduleName}'...`)
const mod = require(submodulePath).default // eslint-disable-line const mod = require(submodulePath).default // eslint-disable-line
const loadedModData = mod.setup({ const loadedModData = mod.setup({
app: parentApp, app: parentApp,
userDB: userDB, userDB: userDB,
url: url, url: url,
publicdirs: publicdirs, publicdirs: publicdirs,
moduleSpecificData: moduleSpecificData, moduleSpecificData: moduleSpecificData,
httpServer: httpServer, httpServer: httpServer,
httpsServer: httpsServer, httpsServer: httpsServer,
}) })
moduleDatas.push(loadedModData || {}) moduleDatas.push(loadedModData || {})
} catch (e) { } catch (e) {
logger.Log(`Error loading submodule from ${submodulePath}`) logger.Log(`Error loading submodule from ${submodulePath}`)
console.error(e) console.error(e)
} }
}) })
return moduleDatas return moduleDatas
} }
export default { export default {
name: moduleName, name: moduleName,
getApp: GetApp, getApp: GetApp,
setup: (data: SetupData): void => { setup: (data: SetupData): void => {
userDB = data.userDB userDB = data.userDB
url = data.url url = data.url
publicdirs = data.publicdirs publicdirs = data.publicdirs
httpServer = data.httpServer httpServer = data.httpServer
httpsServer = data.httpsServer httpsServer = data.httpsServer
}, },
} }

58
src/modules/api/msgsDbStruct.ts
View File

@@ -19,36 +19,36 @@
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
const DbStruct = { const DbStruct = {
msgs: { msgs: {
tableStruct: { tableStruct: {
id: { id: {
type: 'integer', type: 'integer',
primary: true, primary: true,
autoIncrement: true, autoIncrement: true,
}, },
sender: { sender: {
type: 'integer', type: 'integer',
notNull: true, notNull: true,
}, },
reciever: { reciever: {
type: 'integer', type: 'integer',
notNull: true, notNull: true,
}, },
msg: { msg: {
type: 'text', type: 'text',
}, },
type: { type: {
type: 'text', type: 'text',
}, },
date: { date: {
type: 'integer', type: 'integer',
}, },
unread: { unread: {
type: 'integer', type: 'integer',
defaultZero: true, defaultZero: true,
}, },
},
}, },
},
} }
export default DbStruct export default DbStruct

423
src/modules/api/submodules/chat.ts
View File

@@ -30,65 +30,65 @@ const msgDbPath = './data/dbs/msgs.db'
const msgPaginationLimit = 15 const msgPaginationLimit = 15
interface ExtendedSocket extends Socket { interface ExtendedSocket extends Socket {
user: User user: User
} }
interface Message { interface Message {
id: number id: number
sender: number sender: number
reciever: number reciever: number
msg: string msg: string
type: string type: string
date: number date: number
unread: number unread: number
} }
function setup(data: SubmoduleData): void { function setup(data: SubmoduleData): void {
const { app, httpServer, httpsServer, userDB, publicdirs } = data const { app, httpServer, httpsServer, userDB, publicdirs } = data
const msgDB = dbtools.GetDB(msgDbPath) const msgDB = dbtools.GetDB(msgDbPath)
const publicDir = publicdirs[0] const publicDir = publicdirs[0]
const uloadFiles = publicDir + 'chatFiles' const uloadFiles = publicDir + 'chatFiles'
logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`) logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`)
// https://socket.io/docs/v4/handling-cors/#Configuration // https://socket.io/docs/v4/handling-cors/#Configuration
const io = new socket(httpsServer || httpServer, { const io = new socket(httpsServer || httpServer, {
cors: { cors: {
credentials: true, credentials: true,
origin: true, origin: true,
}, },
}) })
function chatMessageRead({ function chatMessageRead({
sender, sender,
reciever, reciever,
}: { }: {
sender: number sender: number
reciever: number reciever: number
}) { }) {
dbtools.runStatement( dbtools.runStatement(
msgDB, msgDB,
`update msgs `update msgs
set unread = 0 set unread = 0
where sender = ${sender} and reciever = ${reciever}`, where sender = ${sender} and reciever = ${reciever}`,
'run' 'run'
) )
io.sockets.in(sender.toString()).emit('chat message read', { io.sockets.in(sender.toString()).emit('chat message read', {
userReadMsg: reciever, userReadMsg: reciever,
}) })
} }
io.use(socketAuth({ userDB: userDB })) io.use(socketAuth({ userDB: userDB }))
io.on('connection', (socket: ExtendedSocket) => { io.on('connection', (socket: ExtendedSocket) => {
const userid = socket.user.id const userid = socket.user.id
socket.on('join', function (/*data*/) { socket.on('join', function (/*data*/) {
socket.join(userid.toString()) socket.join(userid.toString())
const groups: number[] = dbtools const groups: number[] = dbtools
.runStatement( .runStatement(
msgDB, msgDB,
`select * from `select * from
( (
select sender as a select sender as a
from msgs from msgs
@@ -99,172 +99,177 @@ function setup(data: SubmoduleData): void {
where sender = ${userid} or reciever = ${userid} where sender = ${userid} or reciever = ${userid}
)t )t
order by t.a asc` order by t.a asc`
) )
.reduce((acc: number[], x: { a: number }) => { .reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a) if (x.a !== userid) acc.push(x.a)
return acc return acc
}, []) }, [])
socket.emit('prev messages', { socket.emit('prev messages', {
prevMsgs: groups.map((to) => { prevMsgs: groups.map((to) => {
const first: Message = dbtools.runStatement( const first: Message = dbtools.runStatement(
msgDB, msgDB,
`select * from msgs `select * from msgs
where sender = ${userid} and reciever = ${to} or where sender = ${userid} and reciever = ${to} or
sender = ${to} and reciever = ${userid} sender = ${to} and reciever = ${userid}
order by date desc order by date desc
limit 1` limit 1`
)[0] )[0]
return first return first
}), }),
}) })
socket.on('get chat messages', (data) => { socket.on('get chat messages', (data) => {
const { chatPartner, from } = data const { chatPartner, from } = data
const msgs = dbtools.runStatement( const msgs = dbtools.runStatement(
msgDB, msgDB,
`select * from msgs `select * from msgs
where (sender = ${userid} and reciever = ${chatPartner} or where (sender = ${userid} and reciever = ${chatPartner} or
sender = ${chatPartner} and reciever = ${userid}) sender = ${chatPartner} and reciever = ${userid})
${from ? `and date < ${from}` : ''} ${from ? `and date < ${from}` : ''}
order by date desc order by date desc
limit ${msgPaginationLimit}` limit ${msgPaginationLimit}`
) )
socket.emit('get chat messages', { socket.emit('get chat messages', {
requestsdMsgs: msgs, requestsdMsgs: msgs,
hasMore: msgs.length === msgPaginationLimit, hasMore: msgs.length === msgPaginationLimit,
})
// Read update
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on('chat message read', (data) => {
const { chatPartner } = data
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on(
'chat message',
(message: { reciever: string; msg: string; type: string }) => {
const { reciever, msg, type } = message
if (!reciever || !msg || !msg.trim() || !type) {
return
}
const recieverUser = dbtools.Select(userDB, 'users', {
id: reciever,
})[0]
if (!recieverUser) {
socket.emit('chat message', {
success: false,
date: new Date().getTime(),
sender: reciever,
reciever: userid,
type: 'text',
msg: `A #${reciever} számú felhasználó nem létezik`,
})
return
}
const msgObj = {
sender: userid,
reciever: parseInt(reciever),
msg: dbtools.sanitizeQuery(msg),
type: type || 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
if (userid !== parseInt(reciever)) {
io.sockets
.in(reciever.toString())
.emit('chat message', msgObj)
}
}
)
}) })
// Read update // socket.on('disconnect', () => {})
chatMessageRead({ sender: chatPartner, reciever: userid }) // socket.on('close', () => {})
})
socket.on('chat message read', (data) => {
const { chatPartner } = data
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on(
'chat message',
(message: { reciever: string; msg: string; type: string }) => {
const { reciever, msg, type } = message
if (!reciever || !msg || !msg.trim() || !type) {
return
}
const recieverUser = dbtools.Select(userDB, 'users', {
id: reciever,
})[0]
if (!recieverUser) {
socket.emit('chat message', {
success: false,
date: new Date().getTime(),
sender: reciever,
reciever: userid,
type: 'text',
msg: `A #${reciever} számú felhasználó nem létezik`,
})
return
}
const msgObj = {
sender: userid,
reciever: parseInt(reciever),
msg: dbtools.sanitizeQuery(msg),
type: type || 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
if (userid !== parseInt(reciever)) {
io.sockets.in(reciever.toString()).emit('chat message', msgObj)
}
}
)
}) })
// socket.on('disconnect', () => {}) app.post('/postchatfile', function (req: Request, res) {
// socket.on('close', () => {}) logger.LogReq(req)
}) utils
.uploadFile(req, uloadFiles)
.then((result) => {
res.json({
success: true,
path: result.filePath.replace(publicDir, ''),
})
})
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.post('/postchatfile', function (req: Request, res) { app.post('/postfeedbackfile', function (req: Request, res) {
logger.LogReq(req) logger.LogReq(req)
utils const user: User = req.session.user
.uploadFile(req, uloadFiles)
.then((result) => {
res.json({
success: true,
path: result.filePath.replace(publicDir, ''),
})
})
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.post('/postfeedbackfile', function (req: Request, res) { utils
logger.LogReq(req) .uploadFile(req, uloadFiles)
const user: User = req.session.user .then(({ filePath }) => {
const fileName = filePath.replace(publicDir, '')
const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => {
return fileName.toLowerCase().includes(ext)
})
const msgObj = {
sender: user.id,
reciever: 1,
msg: fileName,
type: isImage ? 'img' : 'file',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
utils res.json({ success: true })
.uploadFile(req, uloadFiles) io.sockets.in('1').emit('chat message', msgObj)
.then(({ filePath }) => { })
const fileName = filePath.replace(publicDir, '') .catch(() => {
const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => { res.json({ success: false, msg: 'error during uploading' })
return fileName.toLowerCase().includes(ext) return
}) })
const msgObj = { })
sender: user.id,
reciever: 1, app.post(
msg: fileName, '/postfeedback',
type: isImage ? 'img' : 'file', function (req: Request<{ content: string }>, res) {
date: new Date().getTime(), logger.LogReq(req)
unread: 1, const user: User = req.session.user
const { content } = req.body
if (!content || !content.trim()) {
res.json({ success: false })
return
}
const msgObj = {
sender: user.id,
reciever: 1,
msg: dbtools.sanitizeQuery(req.body.content),
type: 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
res.json({ success: true })
io.sockets.in('1').emit('chat message', msgObj)
} }
dbtools.Insert(msgDB, 'msgs', msgObj) )
res.json({ success: true }) app.get('/hasNewMsg', (req: Request, res) => {
io.sockets.in('1').emit('chat message', msgObj) const user: User = req.session.user
}) const userid: number = user.id
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.post('/postfeedback', function (req: Request<{ content: string }>, res) { const groups = dbtools
logger.LogReq(req) .runStatement(
const user: User = req.session.user msgDB,
const { content } = req.body `select * from
if (!content || !content.trim()) {
res.json({ success: false })
return
}
const msgObj = {
sender: user.id,
reciever: 1,
msg: dbtools.sanitizeQuery(req.body.content),
type: 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
res.json({ success: true })
io.sockets.in('1').emit('chat message', msgObj)
})
app.get('/hasNewMsg', (req: Request, res) => {
const user: User = req.session.user
const userid: number = user.id
const groups = dbtools
.runStatement(
msgDB,
`select * from
( (
select sender as a select sender as a
from msgs from msgs
@@ -275,35 +280,35 @@ function setup(data: SubmoduleData): void {
where sender = ${userid} or reciever = ${userid} where sender = ${userid} or reciever = ${userid}
)t )t
order by t.a asc` order by t.a asc`
) )
.reduce((acc: number[], x: { a: number }) => { .reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a) if (x.a !== userid) acc.push(x.a)
return acc return acc
}, []) }, [])
const prevMsgs = groups.map((to: number) => { const prevMsgs = groups.map((to: number) => {
const first = dbtools.runStatement( const first = dbtools.runStatement(
msgDB, msgDB,
`select * from msgs `select * from msgs
where sender = ${userid} and reciever = ${to} or where sender = ${userid} and reciever = ${to} or
sender = ${to} and reciever = ${userid} sender = ${to} and reciever = ${userid}
order by date desc order by date desc
limit 1` limit 1`
)[0] )[0]
return first return first
}) })
res.json({ res.json({
unreads: prevMsgs.reduce((acc: number[], msg: Message) => { unreads: prevMsgs.reduce((acc: number[], msg: Message) => {
if (msg && msg.unread === 1 && msg.sender !== userid) { if (msg && msg.unread === 1 && msg.sender !== userid) {
acc.push(msg.sender) acc.push(msg.sender)
} }
return acc return acc
}, []), }, []),
})
}) })
})
} }
export default { export default {
setup: setup, setup: setup,
} }

12
src/modules/api/submodules/feedback.ts
View File

@@ -26,15 +26,15 @@ import { Request, SubmoduleData } from '../../../types/basicTypes'
const uloadFiles = 'data/f' const uloadFiles = 'data/f'
function setup(data: SubmoduleData): void { function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.route('/fosuploader').post(function (req: Request, res: Response) { app.route('/fosuploader').post(function (req: Request, res: Response) {
utils.uploadFile(req, uloadFiles).then(({ fileName }) => { utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
res.redirect('/f/' + fileName) res.redirect('/f/' + fileName)
})
}) })
})
} }
export default { export default {
setup: setup, setup: setup,
} }

1030
src/modules/api/submodules/forum.ts
View File

File diff suppressed because it is too large Load Diff

1877
src/modules/api/submodules/qminingapi.ts
View File

File diff suppressed because it is too large Load Diff

151
src/modules/api/submodules/quickvote.ts
View File

@@ -26,98 +26,101 @@ import type { Response } from 'express'
const quickVoteResultsDir = 'stats/qvote' const quickVoteResultsDir = 'stats/qvote'
const quickVotes = 'stats/qvote/votes.json' const quickVotes = 'stats/qvote/votes.json'
interface QuickVotes { interface QuickVotes {
voteNames?: string[] voteNames?: string[]
} }
interface QuickVote { interface QuickVote {
votes: { votes: {
[key: string]: string [key: string]: string
} }
sum: { sum: {
[key: string]: number [key: string]: number
} }
} }
function setup(data: SubmoduleData): void { function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/quickvote', (req: Request, res: Response) => { app.get('/quickvote', (req: Request, res: Response) => {
const key = req.query.key.toString() const key = req.query.key.toString()
const val: string = req.query.val const val: string = req.query.val
const user: User = req.session.user const user: User = req.session.user
if (!key || !val) { if (!key || !val) {
res.render('votethank', { res.render('votethank', {
results: 'error', results: 'error',
msg: 'no key or val query param!', msg: 'no key or val query param!',
}) })
return return
} }
// FIXME: check vote type in file // FIXME: check vote type in file
let votes: QuickVotes = {} let votes: QuickVotes = {}
if (utils.FileExists(quickVotes)) { if (utils.FileExists(quickVotes)) {
votes = utils.ReadJSON(quickVotes) votes = utils.ReadJSON(quickVotes)
} else { } else {
logger.Log( logger.Log(
`No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`, `No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`,
logger.GetColor('blue') logger.GetColor('blue')
) )
res.render('votethank', { res.render('votethank', {
result: 'no such pool', result: 'no such pool',
}) })
return return
} }
if (!votes.voteNames.includes(key)) { if (!votes.voteNames.includes(key)) {
logger.Log( logger.Log(
`No such vote "${key}" ( #${user.id}: ${key}-${val} )`, `No such vote "${key}" ( #${user.id}: ${key}-${val} )`,
logger.GetColor('blue') logger.GetColor('blue')
) )
res.render('votethank', { res.render('votethank', {
result: 'no such pool', result: 'no such pool',
}) })
return return
} }
const voteFile = quickVoteResultsDir + '/' + key + '.json' const voteFile = quickVoteResultsDir + '/' + key + '.json'
let voteData: QuickVote = { let voteData: QuickVote = {
votes: {}, votes: {},
sum: {}, sum: {},
} }
if (utils.FileExists(voteFile)) { if (utils.FileExists(voteFile)) {
voteData = utils.ReadJSON(voteFile) voteData = utils.ReadJSON(voteFile)
} else { } else {
utils.CreatePath(quickVoteResultsDir) utils.CreatePath(quickVoteResultsDir)
} }
const prevVote = voteData.votes[user.id] const prevVote = voteData.votes[user.id]
voteData.votes[user.id] = val voteData.votes[user.id] = val
if (voteData.sum[val]) { if (voteData.sum[val]) {
voteData.sum[val]++ voteData.sum[val]++
} else { } else {
voteData.sum[val] = 1 voteData.sum[val] = 1
} }
if (prevVote) { if (prevVote) {
if (voteData.sum[prevVote]) { if (voteData.sum[prevVote]) {
voteData.sum[prevVote] -= 1 voteData.sum[prevVote] -= 1
} }
} }
logger.Log(`Vote from #${user.id}: ${key}: ${val}`, logger.GetColor('blue')) logger.Log(
res.render('votethank', { `Vote from #${user.id}: ${key}: ${val}`,
result: prevVote ? 'already voted' : 'success', logger.GetColor('blue')
prevVote: prevVote, )
msg: 'vote added', res.render('votethank', {
result: prevVote ? 'already voted' : 'success',
prevVote: prevVote,
msg: 'vote added',
})
utils.WriteFile(JSON.stringify(voteData), voteFile)
}) })
utils.WriteFile(JSON.stringify(voteData), voteFile)
})
} }
export default { export default {
setup: setup, setup: setup,
} }

181
src/modules/api/submodules/ranklist.ts
View File

@@ -23,121 +23,128 @@ import utils from '../../../utils/utils'
import { Request, SubmoduleData, User } from '../../../types/basicTypes' import { Request, SubmoduleData, User } from '../../../types/basicTypes'
interface Subjects { interface Subjects {
[key: string]: number [key: string]: number
} }
interface IdStat { interface IdStat {
count: number count: number
newQuestions: number newQuestions: number
allQuestions: number allQuestions: number
subjs: Subjects subjs: Subjects
} }
interface IdStats { interface IdStats {
[key: string]: IdStat [key: string]: IdStat
} }
interface IdStatWithUID extends IdStat { interface IdStatWithUID extends IdStat {
userId: number userId: number
} }
const idStatFile = 'stats/idstats' const idStatFile = 'stats/idstats'
const idvStatFile = 'stats/idvstats' const idvStatFile = 'stats/idvstats'
function mergeObjSum(a: Subjects, b: Subjects) { function mergeObjSum(a: Subjects, b: Subjects) {
const res = { ...b } const res = { ...b }
Object.keys(a).forEach((key) => { Object.keys(a).forEach((key) => {
if (res[key]) { if (res[key]) {
res[key] += a[key] res[key] += a[key]
} else { } else {
res[key] = a[key] res[key] = a[key]
} }
}) })
return res return res
} }
function setup(data: SubmoduleData): void { function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/ranklist', (req: Request, res) => { app.get('/ranklist', (req: Request, res) => {
logger.LogReq(req) logger.LogReq(req)
let result: IdStats let result: IdStats
const querySince: string = req.query.since const querySince: string = req.query.since
const user: User = req.session.user const user: User = req.session.user
if (!querySince) { if (!querySince) {
result = utils.ReadJSON(idStatFile) result = utils.ReadJSON(idStatFile)
} else { } else {
try { try {
const since = new Date(querySince) const since = new Date(querySince)
if (!(since instanceof Date) || isNaN(since.getTime())) { if (!(since instanceof Date) || isNaN(since.getTime())) {
throw new Error('Not a date') throw new Error('Not a date')
}
const data = utils.ReadJSON(idvStatFile)
result = {}
Object.keys(data).forEach((key) => {
const dailyStat = data[key]
if (new Date(key) > since) {
Object.keys(dailyStat).forEach((userId) => {
const userStat = dailyStat[userId]
const uidRes = result[userId]
if (!uidRes) {
result[userId] = userStat
} else {
result[userId] = {
count: uidRes.count + userStat.count,
newQuestions: uidRes.newQuestions + userStat.newQuestions,
allQuestions: uidRes.allQuestions + userStat.allQuestions,
subjs: mergeObjSum(uidRes.subjs, userStat.subjs),
} }
} const data = utils.ReadJSON(idvStatFile)
result = {}
Object.keys(data).forEach((key) => {
const dailyStat = data[key]
if (new Date(key) > since) {
Object.keys(dailyStat).forEach((userId) => {
const userStat = dailyStat[userId]
const uidRes = result[userId]
if (!uidRes) {
result[userId] = userStat
} else {
result[userId] = {
count: uidRes.count + userStat.count,
newQuestions:
uidRes.newQuestions +
userStat.newQuestions,
allQuestions:
uidRes.allQuestions +
userStat.allQuestions,
subjs: mergeObjSum(
uidRes.subjs,
userStat.subjs
),
}
}
})
}
})
} catch (err) {
res.json({
msg: 'invalid date format, or other error occured',
})
}
}
const list: Array<IdStatWithUID> = []
const sum = {
count: 0,
newQuestions: 0,
allQuestions: 0,
}
Object.keys(result).forEach((key) => {
list.push({
userId: parseInt(key),
...result[key],
}) })
}
sum.count = sum.count + result[key].count
sum.newQuestions = sum.newQuestions + result[key].newQuestions
sum.allQuestions = sum.allQuestions + result[key].allQuestions
}) })
} catch (err) {
if (list.length === 0) {
res.json({
msg: 'There are no users in the stats db :c',
})
return
}
res.json({ res.json({
msg: 'invalid date format, or other error occured', since: querySince,
sum: sum,
list: list,
selfuserId: user.id,
}) })
}
}
const list: Array<IdStatWithUID> = []
const sum = {
count: 0,
newQuestions: 0,
allQuestions: 0,
}
Object.keys(result).forEach((key) => {
list.push({
userId: parseInt(key),
...result[key],
})
sum.count = sum.count + result[key].count
sum.newQuestions = sum.newQuestions + result[key].newQuestions
sum.allQuestions = sum.allQuestions + result[key].allQuestions
}) })
if (list.length === 0) {
res.json({
msg: 'There are no users in the stats db :c',
})
return
}
res.json({
since: querySince,
sum: sum,
list: list,
selfuserId: user.id,
})
})
} }
export default { export default {
setup: setup, setup: setup,
} }

152
src/modules/api/submodules/todos.ts
View File

@@ -25,110 +25,110 @@ import utils from '../../../utils/utils'
import { Request, SubmoduleData } from '../../../types/basicTypes' import { Request, SubmoduleData } from '../../../types/basicTypes'
interface Categories { interface Categories {
[key: string]: { [key: string]: {
name: string name: string
color: string color: string
} }
} }
enum CardState { enum CardState {
TODO = 'todo', TODO = 'todo',
INPROGRESS = 'inprogress', INPROGRESS = 'inprogress',
TESTING = 'testing', TESTING = 'testing',
DONE = 'done', DONE = 'done',
INPROD = 'inprod', INPROD = 'inprod',
NOTPOSSIBLE = 'notpossible', NOTPOSSIBLE = 'notpossible',
} }
interface Card { interface Card {
id: number id: number
name: string name: string
description: string description: string
category: string category: string
points: number points: number
state: CardState state: CardState
votes: number[] votes: number[]
} }
type Columns = { type Columns = {
[key in CardState]: { [key in CardState]: {
name: string name: string
clickable: boolean clickable: boolean
} }
} }
interface Groups { interface Groups {
[key: string]: { [key: string]: {
name: string name: string
description: string description: string
} }
} }
interface Todos { interface Todos {
categories: Categories categories: Categories
cards: Card[] cards: Card[]
columns: Columns columns: Columns
groups: Groups groups: Groups
} }
const todosFile = 'data/todos.json' const todosFile = 'data/todos.json'
function setup(data: SubmoduleData): void { function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/voteTodo', (req: Request, res: Response) => { app.get('/voteTodo', (req: Request, res: Response) => {
logger.LogReq(req) logger.LogReq(req)
const userId = req.session.user.id const userId = req.session.user.id
const id: string = req.query.id const id: string = req.query.id
const todos: Todos = utils.ReadJSON(todosFile) const todos: Todos = utils.ReadJSON(todosFile)
if (!id) { if (!id) {
res.json({ res.json({
msg: 'id query undefined', msg: 'id query undefined',
result: 'not ok', result: 'not ok',
}) })
} }
const cardIndex = todos.cards.findIndex((currcard) => { const cardIndex = todos.cards.findIndex((currcard) => {
return currcard.id === parseInt(id) return currcard.id === parseInt(id)
})
if (cardIndex === -1) {
res.json({
msg: 'card not found',
result: 'not ok',
})
return
}
const ind = todos.cards[cardIndex].votes.indexOf(userId)
if (ind === -1) {
todos.cards[cardIndex].votes.push(userId)
} else {
todos.cards[cardIndex].votes.splice(ind, 1)
}
utils.WriteFile(JSON.stringify(todos, null, 2), todosFile)
res.json({
todos: todos,
userId: userId,
msg: 'updated',
result: 'ok',
})
}) })
if (cardIndex === -1) {
res.json({
msg: 'card not found',
result: 'not ok',
})
return
}
const ind = todos.cards[cardIndex].votes.indexOf(userId) app.get('/todos', (req: Request, res: Response) => {
if (ind === -1) { logger.LogReq(req)
todos.cards[cardIndex].votes.push(userId) const userId = req.session.user.id
} else { const todos = utils.ReadJSON(todosFile)
todos.cards[cardIndex].votes.splice(ind, 1)
}
utils.WriteFile(JSON.stringify(todos, null, 2), todosFile) res.json({
res.json({ todos: todos,
todos: todos, userId: userId,
userId: userId, result: 'ok',
msg: 'updated', })
result: 'ok',
}) })
})
app.get('/todos', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const todos = utils.ReadJSON(todosFile)
res.json({
todos: todos,
userId: userId,
result: 'ok',
})
})
} }
export default { export default {
setup: setup, setup: setup,
} }

576
src/modules/api/submodules/userFiles.ts
View File

@@ -27,320 +27,324 @@ import { Request, SubmoduleData, User } from '../../../types/basicTypes'
const dataFileName = '.data.json' const dataFileName = '.data.json'
function listDir(publicDir: string, subdir: string, userFilesDir: string) { function listDir(publicDir: string, subdir: string, userFilesDir: string) {
const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '') const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '')
const dir = userFilesDir + '/' + safeSubdir const dir = userFilesDir + '/' + safeSubdir
const usersFile = dir + '/' + dataFileName const usersFile = dir + '/' + dataFileName
if (!utils.FileExists(dir)) { if (!utils.FileExists(dir)) {
return { return {
success: false, success: false,
msg: `Directory ${subdir} does not exists`, msg: `Directory ${subdir} does not exists`,
}
} }
} if (!utils.FileExists(usersFile)) {
if (!utils.FileExists(usersFile)) { utils.WriteFile('{}', usersFile)
utils.WriteFile('{}', usersFile)
}
const users = utils.ReadJSON(usersFile)
if (!utils.FileExists(dir)) {
return {
success: false,
msg: `Path '${safeSubdir}' does not exists`,
} }
} const users = utils.ReadJSON(usersFile)
return { if (!utils.FileExists(dir)) {
success: true, return {
files: utils.ReadDir(dir).reduce((acc, file) => { success: false,
const stat = fs.lstatSync(dir + '/' + file) msg: `Path '${safeSubdir}' does not exists`,
}
}
if (stat.isDirectory()) { return {
return acc success: true,
} files: utils.ReadDir(dir).reduce((acc, file) => {
const stat = fs.lstatSync(dir + '/' + file)
acc.push({ if (stat.isDirectory()) {
name: file, return acc
path: dir.replace(publicDir, '') + '/' + file, }
size: stat.size,
date: stat.mtime.getTime(), acc.push({
user: users && users[file] ? users[file].uid : -1, name: file,
views: path: dir.replace(publicDir, '') + '/' + file,
users && users[file] && users[file].views ? users[file].views : 0, size: stat.size,
upvotes: date: stat.mtime.getTime(),
users && users[file] && users[file].upvotes user: users && users[file] ? users[file].uid : -1,
? users[file].upvotes views:
: [], users && users[file] && users[file].views
downvotes: ? users[file].views
users && users[file] && users[file].downvotes : 0,
? users[file].downvotes upvotes:
: [], users && users[file] && users[file].upvotes
}) ? users[file].upvotes
return acc : [],
}, []), downvotes:
} users && users[file] && users[file].downvotes
? users[file].downvotes
: [],
})
return acc
}, []),
}
} }
function setup(data: SubmoduleData): void { function setup(data: SubmoduleData): void {
const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data
app.use((req: Request, _res, next) => { app.use((req: Request, _res, next) => {
// /userFiles/test/2021-04-28_10-59.png // /userFiles/test/2021-04-28_10-59.png
try { try {
if (req.url.includes('/userFiles/')) { if (req.url.includes('/userFiles/')) {
logger.LogReq(req)
const safePath = decodeURIComponent(req.url)
.split('?')[0]
.replace(/\.+/g, '.')
.replace(/\/+/g, '/')
const x = safePath.split('/')
const dir = x[2]
const fname = x.pop()
const dataFilePath =
userFilesDir + '/' + dir + '/' + dataFileName
const data = utils.ReadJSON(dataFilePath)
if (data[fname]) {
if (!data[fname].views) {
data[fname].views = 0
}
data[fname].views = data[fname].views + 1
utils.WriteFile(JSON.stringify(data), dataFilePath)
}
}
} catch (e) {
console.error(e)
logger.Log(
`Error trying to update view count on ${req.url}`,
logger.GetColor('redbg')
)
}
next()
})
const publicDir = publicdirs[0]
const userFilesDir = publicDir + 'userFiles'
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
app.get('/listUserDir', (req: Request, res) => {
logger.LogReq(req) logger.LogReq(req)
const safePath = decodeURIComponent(req.url)
.split('?')[0] if (!utils.FileExists(userFilesDir)) {
.replace(/\.+/g, '.') utils.CreatePath(userFilesDir, true)
.replace(/\/+/g, '/') }
const subdir: string = req.query.subdir
if (subdir) {
const result = listDir(publicDir, subdir, userFilesDir)
res.json(result)
} else {
res.json({
success: true,
dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => {
const stat = fs.lstatSync(userFilesDir + '/' + file)
if (!stat.isDirectory()) {
return acc
}
acc.push({
name: file,
date: stat.mtime.getTime(),
size: utils.ReadDir(userFilesDir + '/' + file).length,
})
return acc
}, []),
})
}
})
app.post(
'/deleteUserFile',
(req: Request<{ dir: string; fname: string }>, res) => {
logger.LogReq(req)
const dir: string = req.body.dir
const fname: string = req.body.fname
if (!dir || !fname) {
res.json({
success: false,
msg: `'dir' or 'fname' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '')
const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '')
const filePath = userFilesDir + '/' + safeDir + '/' + safeFname
if (!utils.FileExists(filePath)) {
res.json({
success: false,
msg: `path does not exists!`,
})
return
}
utils.deleteFile(filePath)
const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
delete users[safeFname]
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
}
)
app.post('/newUserDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const name: string = req.body.name
if (!name) {
res.json({
success: false,
msg: `name is undefined!`,
})
return
}
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
if (utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} already exists`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
res.json({
success: true,
})
})
app.post('/uploadUserFile', (req: Request<{ dir: string }>, res) => {
logger.LogReq(req)
const user: User = req.session.user
const dir = req.body.dir
if (!dir) {
res.json({
success: false,
msg: `dir '${dir}' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/')
if (!utils.FileExists(userFilesDir + '/' + safeDir)) {
res.json({
success: false,
msg: `dir '${dir}' does not exists!`,
})
return
}
utils
.uploadFile(req, userFilesDir + '/' + safeDir)
.then((body) => {
logger.Log(
`Successfull upload ${body.filePath}`,
logger.GetColor('blue')
)
const usersFile =
userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
users[body.fileName] = { uid: user.id }
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
})
.catch(() => {
res.json({ success: false, msg: 'something bad happened :s' })
})
})
app.post('/voteFile', (req: Request<{ path: string; to: string }>, res) => {
logger.LogReq(req)
const user: User = req.session.user
// { path: 'userFiles/test/2021-04-28_10-59.png', to: 'up' } 19
const { path, to } = req.body
const safePath = path.replace(/\.+/g, '.').replace(/\/+/g, '/')
const x = safePath.split('/') const x = safePath.split('/')
const dir = x[2] const dir = x[1]
const fname = x.pop() const fname = x.pop()
const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName
const data = utils.ReadJSON(dataFilePath) const data = utils.ReadJSON(dataFilePath)
if (data[fname]) { if (data[fname]) {
if (!data[fname].views) { if (!data[fname].upvotes) {
data[fname].views = 0 data[fname].upvotes = []
} }
data[fname].views = data[fname].views + 1 if (!data[fname].downvotes) {
data[fname].downvotes = []
}
utils.WriteFile(JSON.stringify(data), dataFilePath) const removeVote = (from: number[], uid: number) => {
if (!from.includes(uid)) {
return from
}
return from.reduce((acc, id) => {
if (id !== uid) {
acc = [...acc, id]
}
return acc
}, [])
}
data[fname].downvotes = removeVote(data[fname].downvotes, user.id)
data[fname].upvotes = removeVote(data[fname].upvotes, user.id)
if (to === 'up') {
data[fname].upvotes = [...data[fname].upvotes, user.id]
} else if (to === 'down') {
data[fname].downvotes = [...data[fname].downvotes, user.id]
} else if (to === 'clear') {
// ... already cleared
}
utils.WriteFile(JSON.stringify(data), dataFilePath)
} }
}
} catch (e) {
console.error(e)
logger.Log(
`Error trying to update view count on ${req.url}`,
logger.GetColor('redbg')
)
}
next()
})
const publicDir = publicdirs[0] const result = listDir(publicDir, dir, userFilesDir)
res.json(result)
const userFilesDir = publicDir + 'userFiles'
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
app.get('/listUserDir', (req: Request, res) => {
logger.LogReq(req)
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
const subdir: string = req.query.subdir
if (subdir) {
const result = listDir(publicDir, subdir, userFilesDir)
res.json(result)
} else {
res.json({
success: true,
dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => {
const stat = fs.lstatSync(userFilesDir + '/' + file)
if (!stat.isDirectory()) {
return acc
}
acc.push({
name: file,
date: stat.mtime.getTime(),
size: utils.ReadDir(userFilesDir + '/' + file).length,
})
return acc
}, []),
})
}
})
app.post(
'/deleteUserFile',
(req: Request<{ dir: string; fname: string }>, res) => {
logger.LogReq(req)
const dir: string = req.body.dir
const fname: string = req.body.fname
if (!dir || !fname) {
res.json({
success: false,
msg: `'dir' or 'fname' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '')
const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '')
const filePath = userFilesDir + '/' + safeDir + '/' + safeFname
if (!utils.FileExists(filePath)) {
res.json({
success: false,
msg: `path does not exists!`,
})
return
}
utils.deleteFile(filePath)
const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
delete users[safeFname]
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
}
)
app.post('/newUserDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const name: string = req.body.name
if (!name) {
res.json({
success: false,
msg: `name is undefined!`,
})
return
}
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
if (utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} already exists`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
res.json({
success: true,
}) })
})
app.post('/uploadUserFile', (req: Request<{ dir: string }>, res) => { app.post('/deleteDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req) logger.LogReq(req)
const { name } = req.body
const user: User = req.session.user const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
const dir = req.body.dir
if (!dir) {
res.json({
success: false,
msg: `dir '${dir}' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/')
if (!utils.FileExists(userFilesDir + '/' + safeDir)) {
res.json({
success: false,
msg: `dir '${dir}' does not exists!`,
})
return
}
utils if (!utils.FileExists(userFilesDir + '/' + safeName)) {
.uploadFile(req, userFilesDir + '/' + safeDir) res.json({
.then((body) => { success: false,
logger.Log( msg: `Dir ${name} does not exist!`,
`Successfull upload ${body.filePath}`, })
logger.GetColor('blue') return
) }
utils.CreatePath(userFilesDir + '/' + safeName, true)
const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName const result = listDir(publicDir, name, userFilesDir)
const users = utils.ReadJSON(usersFile) if (result.files.length === 0) {
users[body.fileName] = { uid: user.id } utils.deleteDir(userFilesDir + '/' + safeName)
utils.WriteFile(JSON.stringify(users), usersFile) } else {
res.json({ succes: false, msg: `Dir ${name} is not empty!` })
res.json({ return
success: true,
})
})
.catch(() => {
res.json({ success: false, msg: 'something bad happened :s' })
})
})
app.post('/voteFile', (req: Request<{ path: string; to: string }>, res) => {
logger.LogReq(req)
const user: User = req.session.user
// { path: 'userFiles/test/2021-04-28_10-59.png', to: 'up' } 19
const { path, to } = req.body
const safePath = path.replace(/\.+/g, '.').replace(/\/+/g, '/')
const x = safePath.split('/')
const dir = x[1]
const fname = x.pop()
const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName
const data = utils.ReadJSON(dataFilePath)
if (data[fname]) {
if (!data[fname].upvotes) {
data[fname].upvotes = []
}
if (!data[fname].downvotes) {
data[fname].downvotes = []
}
const removeVote = (from: number[], uid: number) => {
if (!from.includes(uid)) {
return from
} }
return from.reduce((acc, id) => {
if (id !== uid) {
acc = [...acc, id]
}
return acc
}, [])
}
data[fname].downvotes = removeVote(data[fname].downvotes, user.id) res.json({ succes: true })
data[fname].upvotes = removeVote(data[fname].upvotes, user.id) })
if (to === 'up') {
data[fname].upvotes = [...data[fname].upvotes, user.id]
} else if (to === 'down') {
data[fname].downvotes = [...data[fname].downvotes, user.id]
} else if (to === 'clear') {
// ... already cleared
}
utils.WriteFile(JSON.stringify(data), dataFilePath)
}
const result = listDir(publicDir, dir, userFilesDir)
res.json(result)
})
app.post('/deleteDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const { name } = req.body
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
if (!utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} does not exist!`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
const result = listDir(publicDir, name, userFilesDir)
if (result.files.length === 0) {
utils.deleteDir(userFilesDir + '/' + safeName)
} else {
res.json({ succes: false, msg: `Dir ${name} is not empty!` })
return
}
res.json({ succes: true })
})
} }
export default { export default {
setup: setup, setup: setup,
} }

587
src/modules/api/submodules/userManagement.ts
View File

@@ -24,10 +24,10 @@ import type { Database } from 'better-sqlite3'
import logger from '../../../utils/logger' import logger from '../../../utils/logger'
import utils from '../../../utils/utils' import utils from '../../../utils/utils'
import { import {
Request, Request,
SubmoduleData, SubmoduleData,
User, User,
Submodule, Submodule,
} from '../../../types/basicTypes' } from '../../../types/basicTypes'
import dbtools from '../../../utils/dbtools' import dbtools from '../../../utils/dbtools'
@@ -38,321 +38,322 @@ const maxPWCount = 3
const daysAfterUserGetsPWs = 7 // days after user gets pw-s const daysAfterUserGetsPWs = 7 // days after user gets pw-s
interface Session { interface Session {
id: string id: string
userId: number userId: number
createDate: string createDate: string
lastAccess: string lastAccess: string
isScript: number isScript: number
} }
function BackupDB(usersDbBackupPath: string, userDB: Database) { function BackupDB(usersDbBackupPath: string, userDB: Database) {
logger.Log('Backing up auth DB ...') logger.Log('Backing up auth DB ...')
utils.CreatePath(usersDbBackupPath, true) utils.CreatePath(usersDbBackupPath, true)
userDB userDB
.backup( .backup(
`${usersDbBackupPath}/users.${utils `${usersDbBackupPath}/users.${utils
.GetDateString() .GetDateString()
.replace(/ /g, '_')}.db` .replace(/ /g, '_')}.db`
) )
.then(() => { .then(() => {
logger.Log('Auth DB backup complete!') logger.Log('Auth DB backup complete!')
}) })
.catch((err: Error) => { .catch((err: Error) => {
logger.Log('Auth DB backup failed!', logger.GetColor('redbg')) logger.Log('Auth DB backup failed!', logger.GetColor('redbg'))
console.error(err) console.error(err)
}) })
} }
function setup(data: SubmoduleData): Submodule { function setup(data: SubmoduleData): Submodule {
const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ] let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ] domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net" domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1) logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
app.get('/avaiblePWS', (req: Request, res: any) => { app.get('/avaiblePWS', (req: Request, res: any) => {
logger.LogReq(req) logger.LogReq(req)
const user: User = req.session.user const user: User = req.session.user
res.json({ res.json({
success: true, success: true,
userCreated: user.created, userCreated: user.created,
availablePWS: user.avaiblePWRequests, availablePWS: user.avaiblePWRequests,
requestedPWS: user.pwRequestCount, requestedPWS: user.pwRequestCount,
maxPWCount: maxPWCount, maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs, daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(user.created), dayDiff: getDayDiff(user.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount, userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
app.post('/getpw', function (req: Request, res: any) {
logger.LogReq(req)
const requestingUser = req.session.user
if (requestingUser.avaiblePWRequests <= 0) {
res.json({
result: 'error',
success: false,
msg: 'Too many passwords requested or cant request password yet, try later',
})
logger.Log(
`User #${requestingUser.id} requested too much passwords`,
logger.GetColor('cyan')
)
return
}
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: requestingUser.avaiblePWRequests - 1,
pwRequestCount: requestingUser.pwRequestCount + 1,
},
{
id: requestingUser.id,
}
)
const pw = uuidv4()
const insertRes = dbtools.Insert(userDB, 'users', {
pw: pw,
avaiblePWRequests: 0,
created: utils.GetDateString(),
createdBy: requestingUser.id,
})
logger.Log(
`User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`,
logger.GetColor('cyan')
)
res.json({
pw: pw,
success: true,
userCreated: requestingUser.created,
availablePWS: requestingUser.avaiblePWRequests,
requestedPWS: requestingUser.pwRequestCount,
maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(requestingUser.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
app.post('/login', (req: Request, res: any) => {
logger.LogReq(req)
const pw = req.body.pw
? req.body.pw.replace(/'/g, '').replace(/"/g, '').replace(/;/g, '')
: false
const isScript = req.body.script
const user: User = dbtools.Select(userDB, 'users', {
pw: pw,
})[0]
if (user) {
const sessionID = uuidv4()
const existingSessions = dbtools
.Select(userDB, 'sessions', {
userID: user.id,
isScript: isScript ? 1 : 0,
})
.sort((a: Session, b: Session) => {
return (
new Date(a.lastAccess).getTime() - new Date(b.lastAccess).getTime()
)
}) })
})
const diff = existingSessions.length - minimumAlowwedSessions app.post('/getpw', function (req: Request, res: any) {
if (diff > 0) { logger.LogReq(req)
logger.Log(
`Multiple ${isScript ? 'script' : 'website'} sessions ( ${ const requestingUser = req.session.user
existingSessions.length
} ) for #${user.id}, deleting olds`, if (requestingUser.avaiblePWRequests <= 0) {
logger.GetColor('cyan') res.json({
result: 'error',
success: false,
msg: 'Too many passwords requested or cant request password yet, try later',
})
logger.Log(
`User #${requestingUser.id} requested too much passwords`,
logger.GetColor('cyan')
)
return
}
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: requestingUser.avaiblePWRequests - 1,
pwRequestCount: requestingUser.pwRequestCount + 1,
},
{
id: requestingUser.id,
}
) )
for (let i = 0; i < diff; i++) {
const id = existingSessions[i].id
dbtools.Delete(userDB, 'sessions', {
id: id,
isScript: isScript ? 1 : 0,
})
}
}
dbtools.Update( const pw = uuidv4()
userDB, const insertRes = dbtools.Insert(userDB, 'users', {
'users', pw: pw,
{ avaiblePWRequests: 0,
loginCount: user.loginCount + 1, created: utils.GetDateString(),
lastLogin: utils.GetDateString(), createdBy: requestingUser.id,
}, })
{
id: user.id,
}
)
dbtools.Insert(userDB, 'sessions', { logger.Log(
id: sessionID, `User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`,
userID: user.id,
isScript: isScript ? 1 : 0,
createDate: utils.GetDateString(),
})
// https://www.npmjs.com/package/cookie
res.cookie('sessionID', sessionID, {
domain: domain,
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.cookie('sessionID', sessionID, {
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.json({
result: 'success',
msg: 'you are now logged in',
})
logger.Log(
`Successfull login to ${
isScript ? 'script' : 'website'
} with user ID: #${user.id}`,
logger.GetColor('cyan')
)
} else {
logger.Log(
`Login attempt with invalid pw: ${pw} to ${
isScript ? 'script' : 'website'
}`,
logger.GetColor('cyan')
)
res.json({
result: 'error',
msg: 'Invalid password',
})
}
})
app.get('/logout', (req: Request, res: any) => {
logger.LogReq(req)
const sessionID = req.cookies.sessionID
const user: User = req.session.user
const { all } = req.query
if (!user) {
res.json({
msg: 'You are not logged in',
success: false,
})
return
}
logger.Log(
`Successfull logout with user ID: #${user.id}`,
logger.GetColor('cyan')
)
if (all) {
dbtools.Delete(userDB, 'sessions', {
userID: user.id,
})
} else {
dbtools.Delete(userDB, 'sessions', {
id: sessionID,
})
}
res.clearCookie('sessionID').json({
msg: 'Successfull logout',
result: 'success',
})
})
function getDayDiff(dateString: string | Date) {
const msdiff = new Date().getTime() - new Date(dateString).getTime()
return Math.floor(msdiff / (1000 * 3600 * 24))
}
function IncrementAvaiblePWs() {
// FIXME: check this if this is legit and works
logger.Log('Incrementing avaible PW-s ...')
const users: Array<User> = dbtools.SelectAll(userDB, 'users')
const day = new Date().getDay()
if (day === 1) {
users.forEach((user) => {
const dayDiff = getDayDiff(user.created)
if (dayDiff < daysAfterUserGetsPWs) {
logger.Log(
`User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`,
logger.GetColor('cyan') logger.GetColor('cyan')
) )
return
}
if (user.avaiblePWRequests >= maxPWCount) { res.json({
return pw: pw,
success: true,
userCreated: requestingUser.created,
availablePWS: requestingUser.avaiblePWRequests,
requestedPWS: requestingUser.pwRequestCount,
maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(requestingUser.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
app.post('/login', (req: Request, res: any) => {
logger.LogReq(req)
const pw = req.body.pw
? req.body.pw.replace(/'/g, '').replace(/"/g, '').replace(/;/g, '')
: false
const isScript = req.body.script
const user: User = dbtools.Select(userDB, 'users', {
pw: pw,
})[0]
if (user) {
const sessionID = uuidv4()
const existingSessions = dbtools
.Select(userDB, 'sessions', {
userID: user.id,
isScript: isScript ? 1 : 0,
})
.sort((a: Session, b: Session) => {
return (
new Date(a.lastAccess).getTime() -
new Date(b.lastAccess).getTime()
)
})
const diff = existingSessions.length - minimumAlowwedSessions
if (diff > 0) {
logger.Log(
`Multiple ${isScript ? 'script' : 'website'} sessions ( ${
existingSessions.length
} ) for #${user.id}, deleting olds`,
logger.GetColor('cyan')
)
for (let i = 0; i < diff; i++) {
const id = existingSessions[i].id
dbtools.Delete(userDB, 'sessions', {
id: id,
isScript: isScript ? 1 : 0,
})
}
}
dbtools.Update(
userDB,
'users',
{
loginCount: user.loginCount + 1,
lastLogin: utils.GetDateString(),
},
{
id: user.id,
}
)
dbtools.Insert(userDB, 'sessions', {
id: sessionID,
userID: user.id,
isScript: isScript ? 1 : 0,
createDate: utils.GetDateString(),
})
// https://www.npmjs.com/package/cookie
res.cookie('sessionID', sessionID, {
domain: domain,
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.cookie('sessionID', sessionID, {
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.json({
result: 'success',
msg: 'you are now logged in',
})
logger.Log(
`Successfull login to ${
isScript ? 'script' : 'website'
} with user ID: #${user.id}`,
logger.GetColor('cyan')
)
} else {
logger.Log(
`Login attempt with invalid pw: ${pw} to ${
isScript ? 'script' : 'website'
}`,
logger.GetColor('cyan')
)
res.json({
result: 'error',
msg: 'Invalid password',
})
}
})
app.get('/logout', (req: Request, res: any) => {
logger.LogReq(req)
const sessionID = req.cookies.sessionID
const user: User = req.session.user
const { all } = req.query
if (!user) {
res.json({
msg: 'You are not logged in',
success: false,
})
return
} }
logger.Log( logger.Log(
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`, `Successfull logout with user ID: #${user.id}`,
logger.GetColor('cyan') logger.GetColor('cyan')
) )
dbtools.Update( if (all) {
userDB, dbtools.Delete(userDB, 'sessions', {
'users', userID: user.id,
{ })
avaiblePWRequests: maxPWCount, } else {
}, dbtools.Delete(userDB, 'sessions', {
{ id: sessionID,
id: user.id, })
} }
)
}) res.clearCookie('sessionID').json({
msg: 'Successfull logout',
result: 'success',
})
})
function getDayDiff(dateString: string | Date) {
const msdiff = new Date().getTime() - new Date(dateString).getTime()
return Math.floor(msdiff / (1000 * 3600 * 24))
} }
users.forEach((user) => { function IncrementAvaiblePWs() {
const dayDiff = getDayDiff(user.created) // FIXME: check this if this is legit and works
if (dayDiff === daysAfterUserGetsPWs) { logger.Log('Incrementing avaible PW-s ...')
logger.Log( const users: Array<User> = dbtools.SelectAll(userDB, 'users')
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`, const day = new Date().getDay()
logger.GetColor('cyan')
)
dbtools.Update( if (day === 1) {
userDB, users.forEach((user) => {
'users', const dayDiff = getDayDiff(user.created)
{ if (dayDiff < daysAfterUserGetsPWs) {
avaiblePWRequests: maxPWCount, logger.Log(
}, `User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`,
{ logger.GetColor('cyan')
id: user.id, )
} return
) }
}
})
}
return { if (user.avaiblePWRequests >= maxPWCount) {
dailyAction: () => { return
BackupDB(usersDbBackupPath, userDB) }
IncrementAvaiblePWs()
}, logger.Log(
} `Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`,
logger.GetColor('cyan')
)
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: maxPWCount,
},
{
id: user.id,
}
)
})
}
users.forEach((user) => {
const dayDiff = getDayDiff(user.created)
if (dayDiff === daysAfterUserGetsPWs) {
logger.Log(
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`,
logger.GetColor('cyan')
)
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: maxPWCount,
},
{
id: user.id,
}
)
}
})
}
return {
dailyAction: () => {
BackupDB(usersDbBackupPath, userDB)
IncrementAvaiblePWs()
},
}
} }
export default { export default {
setup: setup, setup: setup,
} }

142
src/modules/api/usersDBStruct.ts
View File

@@ -19,78 +19,78 @@
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
const DBStruct = { const DBStruct = {
users: { users: {
tableStruct: { tableStruct: {
id: { id: {
type: 'integer', type: 'integer',
primary: true, primary: true,
autoIncrement: true, autoIncrement: true,
}, },
pw: { pw: {
type: 'text', type: 'text',
notNull: true, notNull: true,
unique: true, unique: true,
}, },
notes: { notes: {
type: 'text', type: 'text',
}, },
loginCount: { loginCount: {
type: 'number', type: 'number',
defaultZero: true, defaultZero: true,
}, },
created: { created: {
type: 'text', type: 'text',
notNull: true, notNull: true,
}, },
lastLogin: { lastLogin: {
type: 'text', type: 'text',
}, },
lastAccess: { lastAccess: {
type: 'text', type: 'text',
}, },
avaiblePWRequests: { avaiblePWRequests: {
type: 'number', type: 'number',
defaultZero: true, defaultZero: true,
}, },
pwRequestCount: { pwRequestCount: {
type: 'number', type: 'number',
defaultZero: true, defaultZero: true,
}, },
createdBy: { createdBy: {
type: 'number', type: 'number',
}, },
},
}, },
}, sessions: {
sessions: { foreignKey: [
foreignKey: [ {
{ keysFrom: ['userID'],
keysFrom: ['userID'], table: 'users',
table: 'users', keysTo: ['id'],
keysTo: ['id'], },
}, ],
], tableStruct: {
tableStruct: { id: {
id: { type: 'text',
type: 'text', primary: true,
primary: true, notNull: true,
notNull: true, },
}, userID: {
userID: { type: 'number',
type: 'number', notNull: true,
notNull: true, },
}, createDate: {
createDate: { type: 'text',
type: 'text', notNull: true,
notNull: true, },
}, lastAccess: {
lastAccess: { type: 'text',
type: 'text', },
}, isScript: {
isScript: { type: 'number',
type: 'number', notNull: true,
notNull: true, },
}, },
}, },
},
} }
export default DBStruct export default DBStruct

140
src/modules/dataEditor/dataEditor.ts
View File

@@ -36,85 +36,85 @@ let publicdirs: string[] = []
let nextdir = '' let nextdir = ''
function GetApp(): ModuleType { function GetApp(): ModuleType {
app.use( app.use(
express.urlencoded({ express.urlencoded({
limit: '5mb', limit: '5mb',
extended: true, extended: true,
}) as RequestHandler }) as RequestHandler
) )
app.use( app.use(
express.json({ express.json({
limit: '5mb', limit: '5mb',
}) as RequestHandler }) as RequestHandler
) )
app.set('view engine', 'ejs') app.set('view engine', 'ejs')
app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews']) app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews'])
app.use( app.use(
auth({ auth({
userDB: userDB, userDB: userDB,
jsonResponse: false, jsonResponse: false,
exceptions: ['/favicon.ico'], exceptions: ['/favicon.ico'],
})
)
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
}
next()
}) })
) publicdirs.forEach((pdir) => {
app.use((req: Request, _res, next) => { logger.Log(`Using public dir: ${pdir}`)
const url = req.url.split('?')[0] app.use(express.static(pdir))
if (url.includes('.html') || url === '/') { })
logger.LogReq(req) app.use(express.static(nextdir))
// --------------------------------------------------------------
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1)
app.get(`/${route}`, function (_req: Request, res) {
res.redirect(`${route}.html`)
})
})
} }
next() AddHtmlRoutes(utils.ReadDir(nextdir))
})
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
// -------------------------------------------------------------- // --------------------------------------------------------------
function AddHtmlRoutes(files: string[]) { app.get('/', function (req: Request, res) {
const routes = files.reduce((acc, file) => { res.end('hai')
if (file.includes('html')) { logger.LogReq(req)
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1)
app.get(`/${route}`, function (_req: Request, res) {
res.redirect(`${route}.html`)
})
}) })
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// -------------------------------------------------------------- app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.get('/', function (req: Request, res) { app.post('*', function (_req: Request, res) {
res.end('hai') res.status(404).render('404')
logger.LogReq(req) })
})
app.get('*', function (_req: Request, res) { return {
res.status(404).render('404') app: app,
}) }
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})
return {
app: app,
}
} }
export default { export default {
name: 'Data editor', name: 'Data editor',
getApp: GetApp, getApp: GetApp,
setup: (data: SetupData): void => { setup: (data: SetupData): void => {
userDB = data.userDB userDB = data.userDB
publicdirs = data.publicdirs publicdirs = data.publicdirs
nextdir = data.nextdir nextdir = data.nextdir
}, },
} }

72
src/modules/main/main.ts
View File

@@ -32,47 +32,47 @@ let publicdirs: string[] = []
let url = '' // http(s)//asd.basd let url = '' // http(s)//asd.basd
function GetApp(): ModuleType { function GetApp(): ModuleType {
app.set('view engine', 'ejs') app.set('view engine', 'ejs')
app.set('views', ['./src/modules/main/views', './src/sharedViews']) app.set('views', ['./src/modules/main/views', './src/sharedViews'])
publicdirs.forEach((pdir) => { publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`) logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir)) app.use(express.static(pdir))
})
app.use(express.json() as RequestHandler)
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
// --------------------------------------------------------------
app.get('/', function (_req, res) {
res.render('main', {
siteurl: url,
}) })
})
app.get('*', function (_req, res) { app.use(express.json() as RequestHandler)
res.status(404).render('404') app.use(
}) express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
app.post('*', function (_req, res) { // --------------------------------------------------------------
res.status(404).render('404')
})
return { app.get('/', function (_req, res) {
app: app, res.render('main', {
} siteurl: url,
})
})
app.get('*', function (_req, res) {
res.status(404).render('404')
})
app.post('*', function (_req, res) {
res.status(404).render('404')
})
return {
app: app,
}
} }
export default { export default {
name: 'Main', name: 'Main',
getApp: GetApp, getApp: GetApp,
setup: (data: SetupData): void => { setup: (data: SetupData): void => {
url = data.url url = data.url
publicdirs = data.publicdirs publicdirs = data.publicdirs
}, },
} }

391
src/modules/qmining/qmining.ts
View File

@@ -36,212 +36,219 @@ let userDB: Database
let nextdir = '' let nextdir = ''
function GetApp(): ModuleType { function GetApp(): ModuleType {
app.use( app.use(
express.urlencoded({ express.urlencoded({
limit: '5mb', limit: '5mb',
extended: true, extended: true,
}) as RequestHandler }) as RequestHandler
)
app.use(
express.json({
limit: '5mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/qmining/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: false,
exceptions: ['/favicon.ico', '/img/frylabs-logo_large_transparent.png'],
})
)
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
}
next()
})
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
const linksFile = 'data/links.json'
let links: { [key: string]: string } = {}
function loadDonateURL() {
try {
links = utils.ReadJSON(linksFile)
} catch (err) {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
console.error(err)
}
}
loadDonateURL()
if (utils.FileExists(linksFile)) {
utils.WatchFile(linksFile, (newData: string) => {
logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`)
loadDonateURL()
})
} else {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
}
// --------------------------------------------------------------
// REDIRECTS
// --------------------------------------------------------------
// to be backwards compatible
app.get('/ask', function (req: Request, res) {
logger.DebugLog(`Qmining module ask redirect`, 'ask', 1)
res.redirect(
`http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}`
) )
}) app.use(
express.json({
const simpleRedirects = [ limit: '5mb',
{ }) as RequestHandler
from: '/dataeditor', )
to: 'https://dataeditor.frylabs.net', app.set('view engine', 'ejs')
}, app.set('views', ['./src/modules/qmining/views', './src/sharedViews'])
{ app.use(
from: '/install', auth({
to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js', userDB: userDB,
}, jsonResponse: false,
{ exceptions: [
from: '/servergit', '/favicon.ico',
to: 'https://gitlab.com/MrFry/mrfrys-node-server', '/img/frylabs-logo_large_transparent.png',
}, ],
{ })
from: '/scriptgit', )
to: 'https://gitlab.com/MrFry/moodle-test-userscript', app.use((req: Request, _res, next) => {
}, const url = req.url.split('?')[0]
{ if (url.includes('.html') || url === '/') {
from: '/qminingSite', logger.LogReq(req)
to: 'https://gitlab.com/MrFry/qmining-page', }
}, next()
{
from: '/classesgit',
to: 'https://gitlab.com/MrFry/question-classes',
},
{
from: '/addQuestion',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/donate',
to: links.donate,
},
{
from: '/menuClick',
to: '/',
},
{
from: '/legacy',
to: '/allQuestions.html',
},
{
from: '/subjectBrowser',
to: '/allQuestions.html',
},
{
from: '/lred',
to: '/allQuestions',
},
{
from: '/allqr',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/allqr.txt',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/infos',
to: 'https://api.frylabs.net/infos?version=true&motd=true&subjinfo=true',
nolog: true,
},
{
from: '/irc',
to: '/chat',
},
{
from: '/patreon',
to: links.patreon,
},
]
simpleRedirects.forEach((redirect) => {
app.get(redirect.from, function (req: Request, res) {
if (!redirect.nolog) {
logger.LogReq(req)
}
logger.DebugLog(`Qmining module ${redirect.from} redirect`, 'infos', 1)
let target = redirect.to
if (!redirect.to.includes('https://')) {
target += utils.formatUrl({ query: req.query })
}
res.redirect(target)
}) })
}) publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
const linksFile = 'data/links.json'
let links: { [key: string]: string } = {}
// -------------------------------------------------------------- function loadDonateURL() {
try {
links = utils.ReadJSON(linksFile)
} catch (err) {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
console.error(err)
}
}
function AddHtmlRoutes(files: string[]) { loadDonateURL()
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route: string) => { if (utils.FileExists(linksFile)) {
logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1) utils.WatchFile(linksFile, (newData: string) => {
app.get(`/${route}`, function (req: Request, res) { logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`)
loadDonateURL()
})
} else {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
}
// --------------------------------------------------------------
// REDIRECTS
// --------------------------------------------------------------
// to be backwards compatible
app.get('/ask', function (req: Request, res) {
logger.DebugLog(`Qmining module ask redirect`, 'ask', 1)
res.redirect( res.redirect(
utils.formatUrl({ `http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}`
pathname: `${route}.html`,
query: req.query,
})
) )
})
}) })
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// -------------------------------------------------------------- const simpleRedirects = [
{
from: '/dataeditor',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/install',
to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js',
},
{
from: '/servergit',
to: 'https://gitlab.com/MrFry/mrfrys-node-server',
},
{
from: '/scriptgit',
to: 'https://gitlab.com/MrFry/moodle-test-userscript',
},
{
from: '/qminingSite',
to: 'https://gitlab.com/MrFry/qmining-page',
},
{
from: '/classesgit',
to: 'https://gitlab.com/MrFry/question-classes',
},
{
from: '/addQuestion',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/donate',
to: links.donate,
},
{
from: '/menuClick',
to: '/',
},
{
from: '/legacy',
to: '/allQuestions.html',
},
{
from: '/subjectBrowser',
to: '/allQuestions.html',
},
{
from: '/lred',
to: '/allQuestions',
},
{
from: '/allqr',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/allqr.txt',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/infos',
to: 'https://api.frylabs.net/infos?version=true&motd=true&subjinfo=true',
nolog: true,
},
{
from: '/irc',
to: '/chat',
},
{
from: '/patreon',
to: links.patreon,
},
]
app.get('/', function (req: Request, res) { simpleRedirects.forEach((redirect) => {
res.end('hai') app.get(redirect.from, function (req: Request, res) {
logger.LogReq(req) if (!redirect.nolog) {
}) logger.LogReq(req)
}
logger.DebugLog(
`Qmining module ${redirect.from} redirect`,
'infos',
1
)
app.get('*', function (_req: Request, res) { let target = redirect.to
res.status(404).render('404') if (!redirect.to.includes('https://')) {
}) target += utils.formatUrl({ query: req.query })
}
app.post('*', function (_req: Request, res) { res.redirect(target)
res.status(404).render('404') })
}) })
return { // --------------------------------------------------------------
app: app,
} function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route: string) => {
logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1)
app.get(`/${route}`, function (req: Request, res) {
res.redirect(
utils.formatUrl({
pathname: `${route}.html`,
query: req.query,
})
)
})
})
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
app.get('/', function (req: Request, res) {
res.end('hai')
logger.LogReq(req)
})
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})
return {
app: app,
}
} }
export default { export default {
name: 'Qmining', name: 'Qmining',
getApp: GetApp, getApp: GetApp,
setup: (data: SetupData): void => { setup: (data: SetupData): void => {
userDB = data.userDB userDB = data.userDB
publicdirs = data.publicdirs publicdirs = data.publicdirs
nextdir = data.nextdir nextdir = data.nextdir
}, },
} }

390
src/server.ts
View File

@@ -53,17 +53,17 @@ const logFile = logger.logDir + logger.logFileName
const vlogFile = logger.vlogDir + logger.logFileName const vlogFile = logger.vlogDir + logger.logFileName
function moveLogIfNotFromToday(path: string, to: string) { function moveLogIfNotFromToday(path: string, to: string) {
if (utils.FileExists(path)) { if (utils.FileExists(path)) {
const today = new Date() const today = new Date()
const stat = utils.statFile(path) const stat = utils.statFile(path)
if ( if (
today.getFullYear() !== stat.mtime.getFullYear() || today.getFullYear() !== stat.mtime.getFullYear() ||
today.getMonth() !== stat.mtime.getMonth() || today.getMonth() !== stat.mtime.getMonth() ||
today.getDate() !== stat.mtime.getDate() today.getDate() !== stat.mtime.getDate()
) { ) {
utils.renameFile(path, to + utils.GetDateString(stat.mtime)) utils.renameFile(path, to + utils.GetDateString(stat.mtime))
}
} }
}
} }
moveLogIfNotFromToday(logFile, logger.logDir) moveLogIfNotFromToday(logFile, logger.logDir)
moveLogIfNotFromToday(vlogFile, logger.vlogDir) moveLogIfNotFromToday(vlogFile, logger.vlogDir)
@@ -72,32 +72,32 @@ idStats.Load()
logger.Load() logger.Load()
interface Modules { interface Modules {
[name: string]: Module [name: string]: Module
} }
interface Module { interface Module {
path: string path: string
publicdirs: Array<string> publicdirs: Array<string>
name: string name: string
urls: Array<string> urls: Array<string>
nextdir?: string nextdir?: string
isNextJs?: boolean isNextJs?: boolean
app: express.Application app: express.Application
dailyAction: Function dailyAction: Function
cleanup: Function cleanup: Function
} }
export interface SetupData { export interface SetupData {
url: string url: string
publicdirs: Array<string> publicdirs: Array<string>
userDB?: Database userDB?: Database
nextdir?: string nextdir?: string
httpServer: http.Server httpServer: http.Server
httpsServer: https.Server httpsServer: https.Server
} }
if (!utils.FileExists(usersDBPath)) { if (!utils.FileExists(usersDBPath)) {
throw new Error('No user DB exists yet! please run utils/dbSetup.js first!') throw new Error('No user DB exists yet! please run utils/dbSetup.js first!')
} }
const userDB = dbtools.GetDB(usersDBPath) const userDB = dbtools.GetDB(usersDBPath)
let modules: Modules = utils.ReadJSON(modulesFile) let modules: Modules = utils.ReadJSON(modulesFile)
@@ -108,43 +108,43 @@ logger.Log(`Log path: ${logFile}`)
logger.Log(`vLog path: ${vlogFile}`) logger.Log(`vLog path: ${vlogFile}`)
try { try {
if (utils.FileExists(extraModulesFile)) { if (utils.FileExists(extraModulesFile)) {
const extraModules = JSON.parse(utils.ReadFile(extraModulesFile)) const extraModules = JSON.parse(utils.ReadFile(extraModulesFile))
modules = { modules = {
...extraModules, ...extraModules,
...modules, ...modules,
}
} }
}
} catch (err) { } catch (err) {
logger.Log('Failed to read extra modules file') logger.Log('Failed to read extra modules file')
console.error(err) console.error(err)
} }
process.on('SIGINT', () => exit('SIGINT')) process.on('SIGINT', () => exit('SIGINT'))
process.on('SIGTERM', () => exit('SIGTERM')) process.on('SIGTERM', () => exit('SIGTERM'))
function exit(reason: string) { function exit(reason: string) {
console.log() console.log()
logger.Log(`Exiting, reason: ${reason}`) logger.Log(`Exiting, reason: ${reason}`)
Object.keys(modules).forEach((key) => { Object.keys(modules).forEach((key) => {
const module = modules[key] const module = modules[key]
if (module.cleanup) { if (module.cleanup) {
try { try {
module.cleanup() module.cleanup()
} catch (err) { } catch (err) {
logger.Log( logger.Log(
`Error in ${key} cleanup! Details in STDERR`, `Error in ${key} cleanup! Details in STDERR`,
logger.GetColor('redbg') logger.GetColor('redbg')
) )
console.error(err) console.error(err)
} }
} }
}) })
logger.Log('Closing Auth DB') logger.Log('Closing Auth DB')
userDB.close() userDB.close()
process.exit() process.exit()
} }
// https://certbot.eff.org/ // https://certbot.eff.org/
@@ -156,201 +156,201 @@ let certsLoaded = false
let certs: { key: string; cert: string; ca: string } let certs: { key: string; cert: string; ca: string }
if ( if (
startHTTPS && startHTTPS &&
utils.FileExists(privkeyFile) && utils.FileExists(privkeyFile) &&
utils.FileExists(fullchainFile) && utils.FileExists(fullchainFile) &&
utils.FileExists(chainFile) utils.FileExists(chainFile)
) { ) {
try { try {
const key = utils.ReadFile(privkeyFile) const key = utils.ReadFile(privkeyFile)
const cert = utils.ReadFile(fullchainFile) const cert = utils.ReadFile(fullchainFile)
const ca = utils.ReadFile(chainFile) const ca = utils.ReadFile(chainFile)
certs = { certs = {
key: key, key: key,
cert: cert, cert: cert,
ca: ca, ca: ca,
}
certsLoaded = true
} catch (err) {
logger.Log('Error loading cert files!', logger.GetColor('redbg'))
console.error(err)
} }
certsLoaded = true
} catch (err) {
logger.Log('Error loading cert files!', logger.GetColor('redbg'))
console.error(err)
}
} }
const app = express() const app = express()
const httpServer = http.createServer(app) const httpServer = http.createServer(app)
let httpsServer: https.Server let httpsServer: https.Server
if (certsLoaded) { if (certsLoaded) {
httpsServer = https.createServer(certs, app) httpsServer = https.createServer(certs, app)
logger.Log('Listening on port: ' + httpsport + ' (https)') logger.Log('Listening on port: ' + httpsport + ' (https)')
} else { } else {
logger.Log('Https not avaible') logger.Log('Https not avaible')
} }
if (!process.env.NS_DEVEL) { if (!process.env.NS_DEVEL) {
app.use(function (req, res, next) { app.use(function (req, res, next) {
if (req.secure) { if (req.secure) {
next() next()
} else { } else {
logger.DebugLog( logger.DebugLog(
`HTTPS ${req.method} redirect to: ${ `HTTPS ${req.method} redirect to: ${
'https://' + req.headers.host + req.url 'https://' + req.headers.host + req.url
}`, }`,
'https', 'https',
1 1
) )
if (req.method === 'POST') { if (req.method === 'POST') {
res.redirect(307, 'https://' + req.headers.host + req.url) res.redirect(307, 'https://' + req.headers.host + req.url)
} else { } else {
res.redirect('https://' + req.headers.host + req.url) res.redirect('https://' + req.headers.host + req.url)
} }
} }
}) })
} }
// https://github.com/expressjs/cors#configuration-options // https://github.com/expressjs/cors#configuration-options
app.use( app.use(
cors({ cors({
credentials: true, credentials: true,
origin: true, origin: true,
// origin: [ /\.frylabs\.net$/ ] // origin: [ /\.frylabs\.net$/ ]
}) })
) )
const cookieSecret = uuidv4() const cookieSecret = uuidv4()
app.use(cookieParser(cookieSecret)) app.use(cookieParser(cookieSecret))
if (!utils.FileExists(statExcludeFile)) { if (!utils.FileExists(statExcludeFile)) {
utils.WriteFile('[]', statExcludeFile) utils.WriteFile('[]', statExcludeFile)
} }
const excludeFromStats = utils.ReadJSON(statExcludeFile) const excludeFromStats = utils.ReadJSON(statExcludeFile)
app.use( app.use(
reqlogger({ reqlogger({
loggableKeywords: ['news.json'], loggableKeywords: ['news.json'],
loggableModules: [], loggableModules: [],
exceptions: ['_next/static'], exceptions: ['_next/static'],
excludeFromStats: excludeFromStats, excludeFromStats: excludeFromStats,
}) })
) )
Object.keys(modules).forEach(function (key) { Object.keys(modules).forEach(function (key) {
const module = modules[key] const module = modules[key]
try { try {
const mod = require(module.path).default // eslint-disable-line const mod = require(module.path).default // eslint-disable-line
// const mod = require(module.path) // const mod = require(module.path)
logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow')) logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow'))
module.publicdirs.forEach((pdir) => { module.publicdirs.forEach((pdir) => {
utils.CreatePath(pdir) utils.CreatePath(pdir)
}) })
if (mod.setup) { if (mod.setup) {
mod.setup({ mod.setup({
url: 'https://' + module.urls[0], url: 'https://' + module.urls[0],
userDB: userDB, userDB: userDB,
publicdirs: module.publicdirs, publicdirs: module.publicdirs,
nextdir: module.nextdir, nextdir: module.nextdir,
httpServer: httpServer, httpServer: httpServer,
httpsServer: httpsServer, httpsServer: httpsServer,
}) })
}
const modApp = mod.getApp()
module.app = modApp.app
module.dailyAction = modApp.dailyAction
module.cleanup = modApp.cleanup
module.urls.forEach((url) => {
app.use(vhost(url, module.app))
})
} catch (err) {
console.error(err)
} }
const modApp = mod.getApp()
module.app = modApp.app
module.dailyAction = modApp.dailyAction
module.cleanup = modApp.cleanup
module.urls.forEach((url) => {
app.use(vhost(url, module.app))
})
} catch (err) {
console.error(err)
}
}) })
setLogTimer() setLogTimer()
function setLogTimer() { function setLogTimer() {
const now = new Date() const now = new Date()
const night = new Date( const night = new Date(
now.getFullYear(), now.getFullYear(),
now.getMonth(), now.getMonth(),
now.getDate() + 1, now.getDate() + 1,
0, 0,
0, 0,
1 1
)
logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
const msToMidnight = night.getTime() - now.getTime() + 10000
logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)
if (msToMidnight < 0) {
logger.Log(
`Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`,
logger.GetColor('redbg')
) )
return logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
} const msToMidnight = night.getTime() - now.getTime() + 10000
logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)
setTimeout(function () { if (msToMidnight < 0) {
LogTimerAction() logger.Log(
rotateLog() `Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`,
setLogTimer() logger.GetColor('redbg')
}, msToMidnight) )
return
}
setTimeout(function () {
LogTimerAction()
rotateLog()
setLogTimer()
}, msToMidnight)
} }
function rotateLog() { function rotateLog() {
const date = new Date() const date = new Date()
date.setDate(date.getDate() - 1) date.setDate(date.getDate() - 1)
const fname = const fname =
date.getFullYear() + date.getFullYear() +
'-' + '-' +
('0' + (date.getMonth() + 1)).slice(-2) + ('0' + (date.getMonth() + 1)).slice(-2) +
'-' + '-' +
('0' + date.getDate()).slice(-2) ('0' + date.getDate()).slice(-2)
if (utils.FileExists(logFile)) { if (utils.FileExists(logFile)) {
utils.CopyFile(logFile, logger.logDir + fname) utils.CopyFile(logFile, logger.logDir + fname)
} }
if (utils.FileExists(vlogFile)) { if (utils.FileExists(vlogFile)) {
utils.CopyFile(vlogFile, logger.vlogDir + fname) utils.CopyFile(vlogFile, logger.vlogDir + fname)
} }
utils.WriteFile(fname, logFile) utils.WriteFile(fname, logFile)
utils.WriteFile(fname, vlogFile) utils.WriteFile(fname, vlogFile)
} }
function LogTimerAction() { function LogTimerAction() {
logger.DebugLog(`Running Log Timer Action`, 'daily', 1) logger.DebugLog(`Running Log Timer Action`, 'daily', 1)
Object.keys(modules).forEach((key) => { Object.keys(modules).forEach((key) => {
const module = modules[key] const module = modules[key]
logger.DebugLog(`Ckecking ${key}`, 'daily', 1) logger.DebugLog(`Ckecking ${key}`, 'daily', 1)
if (module.dailyAction) { if (module.dailyAction) {
try { try {
logger.Log(`Running daily action of ${key}`) logger.Log(`Running daily action of ${key}`)
module.dailyAction() module.dailyAction()
} catch (err) { } catch (err) {
logger.Log( logger.Log(
`Error in ${key} daily action! Details in STDERR`, `Error in ${key} daily action! Details in STDERR`,
logger.GetColor('redbg') logger.GetColor('redbg')
) )
console.error(err) console.error(err)
} }
} }
}) })
const line = const line =
'===================================================================================================================================================' '==================================================================================================================================================='
logger.Log(line) logger.Log(line)
} }
logger.Log('Node version: ' + process.version) logger.Log('Node version: ' + process.version)
logger.Log('Current working directory: ' + process.cwd()) logger.Log('Current working directory: ' + process.cwd())
logger.Log('Listening on port: ' + port) logger.Log('Listening on port: ' + port)
if (isRoot) { if (isRoot) {
logger.Log('Running as root', logger.GetColor('red')) logger.Log('Running as root', logger.GetColor('red'))
} }
httpServer.listen(port) httpServer.listen(port)
if (httpsServer) { if (httpsServer) {
httpsServer.listen(httpsport) httpsServer.listen(httpsport)
} }

42
src/standaloneUtils/createJSON.js
View File

@@ -3,36 +3,36 @@ const fs = require('fs')
const params = process.argv const params = process.argv
const file = params[2] const file = params[2]
const data = fs.readFileSync(file,'utf8').split('\n') const data = fs.readFileSync(file, 'utf8').split('\n')
console.log(data) console.log(data)
console.log("TODO: remove 'Q: ' and 'A: '") console.log("TODO: remove 'Q: ' and 'A: '")
let currVal = {} let currVal = {}
const res = data.reduce((acc, val) => { const res = data.reduce((acc, val) => {
const formattedVal = val.replace(/\r/g, '').trim() const formattedVal = val.replace(/\r/g, '').trim()
if (formattedVal.startsWith('#')) return acc if (formattedVal.startsWith('#')) return acc
if (formattedVal.startsWith('Q')) { if (formattedVal.startsWith('Q')) {
currVal = { currVal = {
Q: formattedVal Q: formattedVal,
}
return acc
}
if (formattedVal.startsWith('A')) {
currVal.A = formattedVal
return [
...acc,
{
...currVal,
data: {
type: 'simple'
} }
} return acc
] }
} if (formattedVal.startsWith('A')) {
currVal.A = formattedVal
return [
...acc,
{
...currVal,
data: {
type: 'simple',
},
},
]
}
return acc return acc
}, []) }, [])
console.log(res) console.log(res)

32
src/standaloneUtils/dataSimplifyer.js
View File

@@ -7,23 +7,23 @@ const data = JSON.parse(fs.readFileSync(file, 'utf8'))
const res = [] const res = []
data.forEach((subj) => { data.forEach((subj) => {
const questions = [] const questions = []
subj.Questions.forEach((question) => { subj.Questions.forEach((question) => {
const res = {} const res = {}
if (question.Q) { if (question.Q) {
res.Q = simplifyString(question.Q) res.Q = simplifyString(question.Q)
} }
if (question.A) { if (question.A) {
res.A = simplifyString(question.A) res.A = simplifyString(question.A)
} }
res.data = question.data res.data = question.data
questions.push(res) questions.push(res)
}) })
res.push({ res.push({
Name: subj.Name, Name: subj.Name,
Questions: questions, Questions: questions,
}) })
}) })
fs.writeFileSync(file + '.res', JSON.stringify(res)) fs.writeFileSync(file + '.res', JSON.stringify(res))

46
src/standaloneUtils/dbSetup.js
View File

@@ -4,38 +4,38 @@ const dbtools = require('../../dist/utils/dbtools.js').default // eslint-disable
const { v4: uuidv4 } = require('uuid') // eslint-disable-line const { v4: uuidv4 } = require('uuid') // eslint-disable-line
const dbStructPaths = [ const dbStructPaths = [
{ structPath: '../modules/api/usersDBStruct.json', name: 'users.db' }, { structPath: '../modules/api/usersDBStruct.json', name: 'users.db' },
{ structPath: '../modules/api/msgsDbStruct.json', name: 'msgs.db' }, { structPath: '../modules/api/msgsDbStruct.json', name: 'msgs.db' },
] ]
dbStructPaths.forEach((data) => { dbStructPaths.forEach((data) => {
const { structPath, name } = data const { structPath, name } = data
createDB(structPath, name) createDB(structPath, name)
}) })
function createDB(path, name) { function createDB(path, name) {
const dbStruct = utils.ReadJSON(path) const dbStruct = utils.ReadJSON(path)
const db = dbtools.GetDB(`./${name}`) const db = dbtools.GetDB(`./${name}`)
db.pragma('synchronous = OFF') db.pragma('synchronous = OFF')
Object.keys(dbStruct).forEach((tableName) => { Object.keys(dbStruct).forEach((tableName) => {
const tableData = dbStruct[tableName] const tableData = dbStruct[tableName]
dbtools.CreateTable( dbtools.CreateTable(
db, db,
tableName, tableName,
tableData.tableStruct, tableData.tableStruct,
tableData.foreignKey tableData.foreignKey
) )
}) })
printDb(db, dbStruct) printDb(db, dbStruct)
db.close() db.close()
logger.Log('Done') logger.Log('Done')
} }
function printDb(db, dbStruct) { function printDb(db, dbStruct) {
Object.keys(dbStruct).forEach((key) => { Object.keys(dbStruct).forEach((key) => {
console.log(dbtools.TableInfo(db, key)) console.log(dbtools.TableInfo(db, key))
console.log(dbtools.SelectAll(db, key)) console.log(dbtools.SelectAll(db, key))
}) })
} }

54
src/standaloneUtils/invalidQuestionRemover.js
View File

@@ -1,13 +1,13 @@
const fs = require('fs') const fs = require('fs')
function GetParams() { function GetParams() {
return process.argv.splice(2) return process.argv.splice(2)
} }
const params = GetParams() const params = GetParams()
console.log(params) console.log(params)
if (params.length === 0) { if (params.length === 0) {
console.error('No params! Need a path to a question database!') console.error('No params! Need a path to a question database!')
process.exit() process.exit()
} }
const file = params[0] const file = params[0]
@@ -16,36 +16,36 @@ const res = []
let invalidQuestionCount = 0 let invalidQuestionCount = 0
data.forEach((subj) => { data.forEach((subj) => {
const questions = [] const questions = []
subj.Questions.forEach((question) => { subj.Questions.forEach((question) => {
if (isInvalidQuestion(question)) { if (isInvalidQuestion(question)) {
console.log(`invalid question in ${subj.Name}:`) console.log(`invalid question in ${subj.Name}:`)
console.log(question) console.log(question)
invalidQuestionCount++ invalidQuestionCount++
} else { } else {
questions.push(question) questions.push(question)
} }
}) })
res.push({ res.push({
Name: subj.Name, Name: subj.Name,
Questions: questions, Questions: questions,
}) })
}) })
function isInvalidQuestion(q) { function isInvalidQuestion(q) {
if (q.Q === 'Ugrás...' || q.A === 'Ugrás...') { if (q.Q === 'Ugrás...' || q.A === 'Ugrás...') {
return true return true
} }
if (!q.Q && !q.A) { if (!q.Q && !q.A) {
return true return true
} }
if (!q.Q && q.data.type === 'simple') { if (!q.Q && q.data.type === 'simple') {
return true return true
} }
return false return false
} }
console.log(`${invalidQuestionCount} invalid questions, writing results...`) console.log(`${invalidQuestionCount} invalid questions, writing results...`)

78
src/standaloneUtils/manualSearch.js
View File

@@ -21,16 +21,16 @@ const answer = params[2]
console.time('SEARCH') console.time('SEARCH')
const searchRes = search({ const searchRes = search({
qdb: loadData(path), qdb: loadData(path),
subjName: 'Elektronika', subjName: 'Elektronika',
question: { question: {
Q: question, Q: question,
A: answer, A: answer,
data: { data: {
type: 'simple', type: 'simple',
},
}, },
}, searchTillMatchPercent: 80,
searchTillMatchPercent: 80,
}) })
hr() hr()
console.log('Search result') console.log('Search result')
@@ -39,9 +39,9 @@ showSearchResult(searchRes)
hr() hr()
console.timeEnd('SEARCH') console.timeEnd('SEARCH')
log( log(
`Searched for question: "${C('green')}${question}${C()}" answer: "${C( `Searched for question: "${C('green')}${question}${C()}" answer: "${C(
'green' 'green'
)}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"` )}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"`
) )
hr() hr()
@@ -50,44 +50,44 @@ hr()
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
function showSearchResult(res) { function showSearchResult(res) {
res.forEach((x) => { res.forEach((x) => {
console.log(`${C('green')}Q:${C()}`, x.q.Q) console.log(`${C('green')}Q:${C()}`, x.q.Q)
console.log(`${C('green')}A:${C()}`, x.q.A) console.log(`${C('green')}A:${C()}`, x.q.A)
console.log(`${C('green')}match:${C()}`, x.match) console.log(`${C('green')}match:${C()}`, x.match)
console.log() console.log()
}) })
console.log(`Result length: ${C('green')}${res.length}${C()}`) console.log(`Result length: ${C('green')}${res.length}${C()}`)
} }
function search({ qdb, subjName, question, searchInAllIfNoResult }) { function search({ qdb, subjName, question, searchInAllIfNoResult }) {
return doSearch( return doSearch(
qdb, qdb,
subjName, subjName,
question, question,
null, null,
minpercent, minpercent,
searchInAllIfNoResult searchInAllIfNoResult
) )
} }
function hr() { function hr() {
let res = '' let res = ''
for (let i = 0; i < process.stdout.columns; i++) { for (let i = 0; i < process.stdout.columns; i++) {
res += '=' res += '='
} }
log(`${C('cyan')}${res}${C()}`) log(`${C('cyan')}${res}${C()}`)
} }
function log(text) { function log(text) {
utils.AppendToFile(text, globalLog) utils.AppendToFile(text, globalLog)
if (process.stdout.isTTY) { if (process.stdout.isTTY) {
process.stdout.clearLine() process.stdout.clearLine()
process.stdout.cursorTo(0) process.stdout.cursorTo(0)
} }
console.log(text) console.log(text)
} }
function C(color) { function C(color) {
return logger.C(color) return logger.C(color)
} }

684
src/standaloneUtils/rmDuplicates.js
View File

@@ -1,10 +1,10 @@
const utils = require('../../dist/utils/utils.js').default // eslint-disable-line const utils = require('../../dist/utils/utils.js').default // eslint-disable-line
const logger = require('../../dist/utils/logger.js').default // eslint-disable-line const logger = require('../../dist/utils/logger.js').default // eslint-disable-line
const { const {
addQuestion, addQuestion,
doSearch, doSearch,
compareQuestionObj, compareQuestionObj,
createQuestion, createQuestion,
} = require('../../dist/utils/classes.js') // eslint-disable-line } = require('../../dist/utils/classes.js') // eslint-disable-line
const { loadData, writeData } = require('../../dist/utils/actions.js') // eslint-disable-line const { loadData, writeData } = require('../../dist/utils/actions.js') // eslint-disable-line
const fs = require('fs') // eslint-disable-line const fs = require('fs') // eslint-disable-line
@@ -35,7 +35,7 @@ const fs = require('fs') // eslint-disable-line
const minpercent = 95 const minpercent = 95
const line = const line =
'====================================================================' '===================================================================='
const logPath = './duplicateRemovingLog/' const logPath = './duplicateRemovingLog/'
const globalLog = './duplicateRemovingLog/log' const globalLog = './duplicateRemovingLog/log'
utils.CreatePath(logPath) utils.CreatePath(logPath)
@@ -45,25 +45,25 @@ utils.WriteFile('', globalLog)
let currentMaxIndex = -1 let currentMaxIndex = -1
let currentIndex = -1 let currentIndex = -1
process.on('message', function () { process.on('message', function () {
process.send({ process.send({
currentMaxIndex: currentMaxIndex, currentMaxIndex: currentMaxIndex,
currentIndex: currentIndex, currentIndex: currentIndex,
}) })
}) })
// ---------------------------------------------- // ----------------------------------------------
let params = process.argv.splice(2) let params = process.argv.splice(2)
let silenced = false let silenced = false
if (params.includes('-s')) { if (params.includes('-s')) {
silenced = true silenced = true
} }
params = params.filter((x) => { params = params.filter((x) => {
return !x.startsWith('-') return !x.startsWith('-')
}) })
console.log(params) console.log(params)
if (params.length === 0) { if (params.length === 0) {
console.log('At least 1 parameter required (path to DB)') console.log('At least 1 parameter required (path to DB)')
process.exit(1) process.exit(1)
} }
const pathA = params[0] const pathA = params[0]
@@ -71,61 +71,67 @@ const pathB = params[1]
const stat = fs.lstatSync(pathA) const stat = fs.lstatSync(pathA)
if (stat.isDirectory()) { if (stat.isDirectory()) {
if (pathB) { if (pathB) {
log( log(
`Clearing possible questions from ${C( `Clearing possible questions from ${C(
'green' 'green'
)}${pathA}${C()} based on ${C('green')}${pathB}${C()} db` )}${pathA}${C()} based on ${C('green')}${pathB}${C()} db`
) )
const db = pathB ? loadData(pathB) : null const db = pathB ? loadData(pathB) : null
clearPossibleAnswers(pathA, db) clearPossibleAnswers(pathA, db)
log( log(
`Cleared possible questions from ${C('green')}${pathA}${C()} based on ${C( `Cleared possible questions from ${C(
'green' 'green'
)}${pathB}${C()} db` )}${pathA}${C()} based on ${C('green')}${pathB}${C()} db`
) )
} else { } else {
log( log(
`Removing possible question duplicates from ${C('green')}${pathA}${C()}` `Removing possible question duplicates from ${C(
) 'green'
removePossibleAnswersDuplicates(pathA) )}${pathA}${C()}`
log(`Removed possible question duplicates from ${C('green')}${pathA}${C()}`) )
} removePossibleAnswersDuplicates(pathA)
log(
`Removed possible question duplicates from ${C(
'green'
)}${pathA}${C()}`
)
}
} else { } else {
console.time('load') console.time('load')
const dbA = loadData(pathA) const dbA = loadData(pathA)
const dbB = pathB ? loadData(pathB) : null const dbB = pathB ? loadData(pathB) : null
console.timeEnd('load') console.timeEnd('load')
console.time('rmduplicates') console.time('rmduplicates')
if (!dbB) { if (!dbB) {
log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`) log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`)
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1] const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
const res = rmDuplicates(dbA) const res = rmDuplicates(dbA)
console.timeEnd('rmduplicates') console.timeEnd('rmduplicates')
writeData(res, resultDbFileName + '.res') writeData(res, resultDbFileName + '.res')
log('File written') log('File written')
log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`) log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`)
} else { } else {
log( log(
`Removing questions found in ${C('green')}${pathB}${C()} from ${C( `Removing questions found in ${C('green')}${pathB}${C()} from ${C(
'green' 'green'
)}${pathA}${C()}` )}${pathA}${C()}`
) )
const res = difference({ dbA: dbA, dbB: dbB }) const res = difference({ dbA: dbA, dbB: dbB })
console.timeEnd('rmduplicates') console.timeEnd('rmduplicates')
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1] const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
writeData(res, resultDbFileName + '.res') writeData(res, resultDbFileName + '.res')
log('File written') log('File written')
log( log(
`Removed questions found in ${C('green')}${pathB}${C()} from ${C( `Removed questions found in ${C('green')}${pathB}${C()} from ${C(
'green' 'green'
)}${pathA}${C()}` )}${pathA}${C()}`
) )
} }
} }
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
@@ -135,141 +141,141 @@ if (stat.isDirectory()) {
// TODO: dont check every file, only check per directorires // TODO: dont check every file, only check per directorires
// only compare questions of same subjects // only compare questions of same subjects
function removePossibleAnswersDuplicates(path) { function removePossibleAnswersDuplicates(path) {
const dirs = fs.readdirSync(path) const dirs = fs.readdirSync(path)
let count = 0 let count = 0
let currIndex = 1 let currIndex = 1
let delets = 0 let delets = 0
iterateDir(path, () => { iterateDir(path, () => {
count++ count++
})
dirs.forEach((currDir) => {
const contents = fs.readdirSync(path + '/' + currDir)
contents.forEach((currFile) => {
const currPath = path + '/' + currDir + '/' + currFile
if (currPath.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath)) {
return
}
const dataA = utils.ReadJSON(currPath)
currIndex++
printProgressBar(currIndex, count - 1)
contents.forEach((currFile2) => {
const currPath2 = path + '/' + currDir + '/' + currFile2
if (currPath2.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath2)) {
return
}
if (currPath === currPath2) {
return
}
const dataB = utils.ReadJSON(currPath2)
dataA.questions.forEach((q1) => {
dataB.questions.some((q2) => {
const percent = compareQuestionObj(
createQuestion(q1),
'',
createQuestion(q2),
''
)
if (percent.avg === 100) {
utils.deleteFile(currPath2)
count--
delets++
return true
}
})
})
})
}) })
})
log(`${C('green')}Deleting empty directories ...${C()}`) dirs.forEach((currDir) => {
count = dirs.length const contents = fs.readdirSync(path + '/' + currDir)
currIndex = 0
let deletedDirCount = 0
dirs.forEach((dir) => {
currIndex++
const currDirContent = fs.readdirSync(path + '/' + dir)
if (currDirContent.length === 0) {
fs.rmdirSync(path + '/' + dir)
deletedDirCount++
}
printProgressBar(currIndex, count)
})
log(`${C('green')}Updating savedQuestions.json ...${C()}`) contents.forEach((currFile) => {
count = dirs.length const currPath = path + '/' + currDir + '/' + currFile
currIndex = 0 if (currPath.includes('savedQuestions.json')) {
dirs.forEach((dir) => { return
currIndex++ }
updateSavedQuestionsFile(path + '/' + dir) if (!utils.FileExists(currPath)) {
printProgressBar(currIndex, count) return
}) }
const dataA = utils.ReadJSON(currPath)
log( currIndex++
`Deleted ${C('green')}${delets}${C()} files, and ${C( printProgressBar(currIndex, count - 1)
'green'
)}${deletedDirCount}${C()} directories` contents.forEach((currFile2) => {
) const currPath2 = path + '/' + currDir + '/' + currFile2
if (currPath2.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath2)) {
return
}
if (currPath === currPath2) {
return
}
const dataB = utils.ReadJSON(currPath2)
dataA.questions.forEach((q1) => {
dataB.questions.some((q2) => {
const percent = compareQuestionObj(
createQuestion(q1),
'',
createQuestion(q2),
''
)
if (percent.avg === 100) {
utils.deleteFile(currPath2)
count--
delets++
return true
}
})
})
})
})
})
log(`${C('green')}Deleting empty directories ...${C()}`)
count = dirs.length
currIndex = 0
let deletedDirCount = 0
dirs.forEach((dir) => {
currIndex++
const currDirContent = fs.readdirSync(path + '/' + dir)
if (currDirContent.length === 0) {
fs.rmdirSync(path + '/' + dir)
deletedDirCount++
}
printProgressBar(currIndex, count)
})
log(`${C('green')}Updating savedQuestions.json ...${C()}`)
count = dirs.length
currIndex = 0
dirs.forEach((dir) => {
currIndex++
updateSavedQuestionsFile(path + '/' + dir)
printProgressBar(currIndex, count)
})
log(
`Deleted ${C('green')}${delets}${C()} files, and ${C(
'green'
)}${deletedDirCount}${C()} directories`
)
} }
function clearPossibleAnswers(path, db) { function clearPossibleAnswers(path, db) {
let count = 0 let count = 0
let currIndex = 1 let currIndex = 1
let delets = 0 let delets = 0
iterateDir(path, () => { iterateDir(path, () => {
count++ count++
})
iterateDir(path, (currPath) => {
currIndex++
if (currPath.includes('savedQuestions.json')) {
return
}
const { subj, questions } = utils.ReadJSON(currPath)
questions.forEach((question) => {
const searchRes = search({
qdb: db,
subjName: subj,
question: question,
searchTillMatchPercent: 80,
})
if (searchRes.length > 0) {
utils.deleteFile(currPath)
delets++
}
}) })
printProgressBar(currIndex, count)
}) iterateDir(path, (currPath) => {
log(`Deleted ${C('green')}${delets}${C()} files`) currIndex++
if (currPath.includes('savedQuestions.json')) {
return
}
const { subj, questions } = utils.ReadJSON(currPath)
questions.forEach((question) => {
const searchRes = search({
qdb: db,
subjName: subj,
question: question,
searchTillMatchPercent: 80,
})
if (searchRes.length > 0) {
utils.deleteFile(currPath)
delets++
}
})
printProgressBar(currIndex, count)
})
log(`Deleted ${C('green')}${delets}${C()} files`)
} }
function updateSavedQuestionsFile(path) { function updateSavedQuestionsFile(path) {
const filePath = path + '/' + 'savedQuestions.json' const filePath = path + '/' + 'savedQuestions.json'
if (!utils.FileExists(filePath)) { if (!utils.FileExists(filePath)) {
log(`${filePath} does not exists!`) log(`${filePath} does not exists!`)
return return
} }
const savedQuestions = utils.ReadJSON(filePath) const savedQuestions = utils.ReadJSON(filePath)
const filtered = savedQuestions.filter((sq) => { const filtered = savedQuestions.filter((sq) => {
return utils.FileExists(path + '/' + sq.fname) return utils.FileExists(path + '/' + sq.fname)
}) })
if (savedQuestions.length !== filtered.length) { if (savedQuestions.length !== filtered.length) {
utils.WriteFile(JSON.stringify(filtered), filePath) utils.WriteFile(JSON.stringify(filtered), filePath)
} }
} }
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
@@ -277,103 +283,105 @@ function updateSavedQuestionsFile(path) {
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
function rmDuplicates(db) { function rmDuplicates(db) {
return difference({ dbA: db }) return difference({ dbA: db })
} }
function difference({ dbA, dbB }) { function difference({ dbA, dbB }) {
const doingDifference = !!dbB const doingDifference = !!dbB
// Stuff only from A // Stuff only from A
const resultDb = [] const resultDb = []
let dbLength = 0 let dbLength = 0
let removedTotal = 0 let removedTotal = 0
let processedQuestions = 0 let processedQuestions = 0
iterateSubjects(dbA, () => { iterateSubjects(dbA, () => {
dbLength++ dbLength++
})
currentMaxIndex = dbLength
const getResultDbLength = () => {
let resultDbLength = 0
iterateSubjects(resultDb, () => {
resultDbLength++
}) })
return resultDbLength currentMaxIndex = dbLength
}
for (let i = 0; i < dbA.length; i++) { const getResultDbLength = () => {
const subj = dbA[i] let resultDbLength = 0
const subjLogPath = logPath + subj.Name iterateSubjects(resultDb, () => {
utils.WriteFile('', subjLogPath) resultDbLength++
let removedCount = 0 })
return resultDbLength
}
for (let i = 0; i < dbA.length; i++) {
const subj = dbA[i]
const subjLogPath = logPath + subj.Name
utils.WriteFile('', subjLogPath)
let removedCount = 0
hr()
log(
`${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${
subj.Name
}, ${C('blue')}${subj.Questions.length}${C(
'green'
)} questions${C()}`
)
printProgressBar(i + 1, dbA.length)
for (let j = 0; j < subj.Questions.length; j++) {
const question = subj.Questions[j]
const searchRes = search({
qdb: doingDifference ? dbB : resultDb,
subjName: subj.Name,
question: question,
searchInAllIfNoResult: doingDifference,
searchTillMatchPercent: minpercent,
})
printProgressBar(processedQuestions, dbLength)
processedQuestions++
currentIndex = processedQuestions
const res = hasRequiredPercent(searchRes, minpercent)
// no result: adding to difference
if (res.length === 0) {
// no result: adding to difference
addQuestion(resultDb, subj.Name, question)
} else {
// has result, not adding to difference
utils.AppendToFile(
line +
'\n' +
line +
'\n' +
JSON.stringify(question, null, 2) +
'\n' +
line +
JSON.stringify(res, null, 2) +
'\n',
subjLogPath
)
removedCount++
removedTotal++
}
}
log(
`${C('yellow')}Removed ${C('red')}${removedCount}${C(
'yellow'
)} questions${C()}`
)
}
hr() hr()
log( log(
`${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${subj.Name}, ${C( `Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions`
'blue'
)}${subj.Questions.length}${C('green')} questions${C()}`
) )
return resultDb
printProgressBar(i + 1, dbA.length)
for (let j = 0; j < subj.Questions.length; j++) {
const question = subj.Questions[j]
const searchRes = search({
qdb: doingDifference ? dbB : resultDb,
subjName: subj.Name,
question: question,
searchInAllIfNoResult: doingDifference,
searchTillMatchPercent: minpercent,
})
printProgressBar(processedQuestions, dbLength)
processedQuestions++
currentIndex = processedQuestions
const res = hasRequiredPercent(searchRes, minpercent)
// no result: adding to difference
if (res.length === 0) {
// no result: adding to difference
addQuestion(resultDb, subj.Name, question)
} else {
// has result, not adding to difference
utils.AppendToFile(
line +
'\n' +
line +
'\n' +
JSON.stringify(question, null, 2) +
'\n' +
line +
JSON.stringify(res, null, 2) +
'\n',
subjLogPath
)
removedCount++
removedTotal++
}
}
log(
`${C('yellow')}Removed ${C('red')}${removedCount}${C(
'yellow'
)} questions${C()}`
)
}
hr()
log(
`Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions`
)
return resultDb
} }
function hasRequiredPercent(result, minpercent) { function hasRequiredPercent(result, minpercent) {
return result.reduce((acc, res) => { return result.reduce((acc, res) => {
if (res.match >= minpercent) { if (res.match >= minpercent) {
acc.push(res) acc.push(res)
} }
return acc return acc
}, []) }, [])
} }
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
@@ -381,22 +389,22 @@ function hasRequiredPercent(result, minpercent) {
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
function search({ qdb, subjName, question, searchInAllIfNoResult }) { function search({ qdb, subjName, question, searchInAllIfNoResult }) {
return doSearch( return doSearch(
qdb, qdb,
subjName, subjName,
question, question,
null, null,
minpercent, minpercent,
searchInAllIfNoResult searchInAllIfNoResult
) )
} }
function iterateSubjects(db, fn) { function iterateSubjects(db, fn) {
db.forEach((subj) => { db.forEach((subj) => {
subj.Questions.forEach((question) => { subj.Questions.forEach((question) => {
fn(subj, question) fn(subj, question)
})
}) })
})
} }
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
@@ -404,19 +412,19 @@ function iterateSubjects(db, fn) {
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
function iterateDir(path, action) { function iterateDir(path, action) {
if (!utils.FileExists(path)) { if (!utils.FileExists(path)) {
return return
} }
const stat = fs.lstatSync(path) const stat = fs.lstatSync(path)
if (stat.isDirectory()) { if (stat.isDirectory()) {
const content = fs.readdirSync(path) const content = fs.readdirSync(path)
content.forEach((currContent) => { content.forEach((currContent) => {
iterateDir(`${path}/${currContent}`, action) iterateDir(`${path}/${currContent}`, action)
}) })
} else { } else {
action(path) action(path)
} }
} }
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
@@ -424,69 +432,69 @@ function iterateDir(path, action) {
// --------------------------------------------------------------------------------- // ---------------------------------------------------------------------------------
function hr() { function hr() {
let res = '' let res = ''
for (let i = 0; i < process.stdout.columns; i++) { for (let i = 0; i < process.stdout.columns; i++) {
res += '=' res += '='
} }
log(`${C('cyan')}${res}${C()}`) log(`${C('cyan')}${res}${C()}`)
} }
function log(text) { function log(text) {
utils.AppendToFile(text, globalLog) utils.AppendToFile(text, globalLog)
if (silenced) return if (silenced) return
if (process.stdout.isTTY) { if (process.stdout.isTTY) {
process.stdout.clearLine() process.stdout.clearLine()
process.stdout.cursorTo(0) process.stdout.cursorTo(0)
} }
console.log(text) console.log(text)
} }
function writeInSameLine(text, returnToLineStart) { function writeInSameLine(text, returnToLineStart) {
if (!process.stdout.isTTY) { if (!process.stdout.isTTY) {
return return
} }
process.stdout.clearLine() process.stdout.clearLine()
process.stdout.cursorTo(0) process.stdout.cursorTo(0)
process.stdout.write(text) process.stdout.write(text)
if (returnToLineStart) { if (returnToLineStart) {
process.stdout.write('\r') process.stdout.write('\r')
} else { } else {
process.stdout.write('\n') process.stdout.write('\n')
} }
} }
function printProgressBar(current, total) { function printProgressBar(current, total) {
if (!process.stdout.isTTY || silenced) { if (!process.stdout.isTTY || silenced) {
return return
} }
const width = process.stdout.columns - 30 const width = process.stdout.columns - 30
if (width <= 0) { if (width <= 0) {
return return
} }
const x = width / total const x = width / total
const xCurrent = Math.floor(current * x) const xCurrent = Math.floor(current * x)
const xTotal = Math.floor(total * x) const xTotal = Math.floor(total * x)
let line = '' let line = ''
for (let i = 0; i < xCurrent; i++) { for (let i = 0; i < xCurrent; i++) {
line += '=' line += '='
} }
for (let i = 0; i < xTotal - xCurrent; i++) { for (let i = 0; i < xTotal - xCurrent; i++) {
line += ' ' line += ' '
} }
const numbers = `${current} / ${total}` const numbers = `${current} / ${total}`
writeInSameLine( writeInSameLine(
`${C('magenta')} [${line}]${C('green')} ${numbers}${C()}`, `${C('magenta')} [${line}]${C('green')} ${numbers}${C()}`,
current !== total current !== total
) )
} }
function C(color) { function C(color) {
return logger.C(color) return logger.C(color)
} }
process.exit() process.exit()

130
src/standaloneUtils/sqliteTest.js
View File

@@ -6,73 +6,73 @@ const dbtools = require('../utils/dbtools.js')
Main() Main()
function Main() { function Main() {
const cols = { const cols = {
uname: { uname: {
type: 'text', type: 'text',
}, },
pw: { pw: {
type: 'text', type: 'text',
}, },
notes: { notes: {
type: 'text', type: 'text',
}, },
} }
const dbName = 'test' const dbName = 'test'
const db = dbtools.GetDB('./testdb.db') const db = dbtools.GetDB('./testdb.db')
// Creating table // Creating table
dbtools.CreateTable(db, dbName, cols) dbtools.CreateTable(db, dbName, cols)
console.log(dbtools.TableInfo(db, dbName)) console.log(dbtools.TableInfo(db, dbName))
dbtools.SelectAll(db, dbName) dbtools.SelectAll(db, dbName)
// inserting test val to table // inserting test val to table
dbtools.Insert(db, dbName, { dbtools.Insert(db, dbName, {
uname: 'mrfry', uname: 'mrfry',
pw: 'dsads', pw: 'dsads',
})
// Selecting a record
console.log(
dbtools.Select(db, dbName, {
uname: 'mrfry',
}) })
) // Selecting a record
console.log(dbtools.TableInfo(db, dbName)) console.log(
console.log(dbtools.SelectAll(db, dbName)) dbtools.Select(db, dbName, {
// Updating record uname: 'mrfry',
dbtools.Update( })
db, )
dbName, console.log(dbtools.TableInfo(db, dbName))
{ console.log(dbtools.SelectAll(db, dbName))
pw: 'sspw', // Updating record
}, dbtools.Update(
{ db,
uname: 'mrfry', dbName,
} {
) pw: 'sspw',
console.log(dbtools.SelectAll(db, dbName)) },
// Updating record again {
dbtools.Update( uname: 'mrfry',
db, }
dbName, )
{ console.log(dbtools.SelectAll(db, dbName))
notes: 'new note!', // Updating record again
}, dbtools.Update(
{ db,
uname: 'mrfry', dbName,
} {
) notes: 'new note!',
console.log(dbtools.SelectAll(db, dbName)) },
// Adding new column and {
dbtools.AddColumn(db, dbName, { uname: 'mrfry',
test: 'text', }
}) )
console.log(dbtools.TableInfo(db, dbName)) console.log(dbtools.SelectAll(db, dbName))
console.log(dbtools.SelectAll(db, dbName)) // Adding new column and
// Deleting stuff dbtools.AddColumn(db, dbName, {
dbtools.Delete(db, dbName, { test: 'text',
uname: 'mrfry', })
}) console.log(dbtools.TableInfo(db, dbName))
console.log(dbtools.SelectAll(db, dbName)) console.log(dbtools.SelectAll(db, dbName))
// Deleting stuff
dbtools.Delete(db, dbName, {
uname: 'mrfry',
})
console.log(dbtools.SelectAll(db, dbName))
dbtools.CloseDB(db) dbtools.CloseDB(db)
} }

64
src/tests/addQuestion.test.ts
View File

@@ -4,45 +4,45 @@ import { Subject, Question } from '../types/basicTypes'
const question: Question = createQuestion('asd', 'asd', { type: 'simple' }) const question: Question = createQuestion('asd', 'asd', { type: 'simple' })
test('Adds questions to empty db', () => { test('Adds questions to empty db', () => {
const emptyDb: Subject[] = [] const emptyDb: Subject[] = []
addQuestion(emptyDb, 'test subject', question) addQuestion(emptyDb, 'test subject', question)
expect(emptyDb.length).toBe(1) expect(emptyDb.length).toBe(1)
}) })
test('Adds questions next to existing', () => { test('Adds questions next to existing', () => {
const db: Subject[] = [ const db: Subject[] = [
{ {
Name: 'test subject', Name: 'test subject',
Questions: [question], Questions: [question],
}, },
] ]
addQuestion(db, 'another something', question) addQuestion(db, 'another something', question)
expect(db.length).toBe(2) expect(db.length).toBe(2)
}) })
test('Does not add new subject, multiple new questions', () => { test('Does not add new subject, multiple new questions', () => {
const db: Subject[] = [ const db: Subject[] = [
{ {
Name: 'test subject', Name: 'test subject',
Questions: [question], Questions: [question],
}, },
] ]
addQuestion(db, 'test subject', question) addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question) addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question) addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question) addQuestion(db, 'test subject', question)
expect(db.length).toBe(1) expect(db.length).toBe(1)
}) })
test('Adds new subjects, multiple new questions', () => { test('Adds new subjects, multiple new questions', () => {
const db: Subject[] = [ const db: Subject[] = [
{ {
Name: 'test subject', Name: 'test subject',
Questions: [question], Questions: [question],
}, },
] ]
addQuestion(db, 'gfjdkglfd', question) addQuestion(db, 'gfjdkglfd', question)
addQuestion(db, ' somrthing test ', question) addQuestion(db, ' somrthing test ', question)
addQuestion(db, 'aaaaaaaa', question) addQuestion(db, 'aaaaaaaa', question)
expect(db.length).toBe(4) expect(db.length).toBe(4)
}) })

36
src/tests/base64ToText.test.ts
View File

@@ -11,22 +11,22 @@
// const expectedResults = ['Melyik híres zenekar tagja volt Joe Muranyi?'] // const expectedResults = ['Melyik híres zenekar tagja volt Joe Muranyi?']
test('Img text recognition works', async () => { test('Img text recognition works', async () => {
// TODO: tesseract keeps workers even after terminate(), and jest --detectOpenHandles detects them // TODO: tesseract keeps workers even after terminate(), and jest --detectOpenHandles detects them
expect(true).toBeTruthy() expect(true).toBeTruthy()
// await tesseractLoaded // await tesseractLoaded
// for (let i = 0; i < imgs.length; i++) { // for (let i = 0; i < imgs.length; i++) {
// const expectedResult = expectedResults[i] // const expectedResult = expectedResults[i]
// const img = imgs[i] // const img = imgs[i]
// //
// const text = await recognizeTextFromBase64(img) // const text = await recognizeTextFromBase64(img)
// expect(text.trim() === expectedResult).toBeTruthy() // expect(text.trim() === expectedResult).toBeTruthy()
// } // }
// //
// await terminateWorker() // await terminateWorker()
// //
// return new Promise<void>((resolve) => { // return new Promise<void>((resolve) => {
// setTimeout(() => { // setTimeout(() => {
// resolve() // resolve()
// }, 1 * 1000) // }, 1 * 1000)
// }) // })
}) })

282
src/tests/oldQuestionRemoving.test.ts
View File

@@ -6,192 +6,192 @@ import { QuestionDb, Subject, Question } from '../types/basicTypes'
const date = (x?: number) => new Date().getTime() + (x || 0) const date = (x?: number) => new Date().getTime() + (x || 0)
const q1 = createQuestion( const q1 = createQuestion(
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'piaci áruk eltérhet a névértéktől.', 'piaci áruk eltérhet a névértéktől.',
{ {
type: 'simple', type: 'simple',
date: date(-1000), date: date(-1000),
} }
) )
const q2 = createQuestion( const q2 = createQuestion(
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'afjléa gféda gfdjs légf', 'afjléa gféda gfdjs légf',
{ {
type: 'simple', type: 'simple',
date: date(-1000), date: date(-1000),
} }
) )
const q3 = createQuestion( const q3 = createQuestion(
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE', 'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'afjlsd gfds dgfs gf sdgf d', 'afjlsd gfds dgfs gf sdgf d',
{ {
type: 'simple', type: 'simple',
date: date(-1000), date: date(-1000),
} }
) )
const q4 = createQuestion( const q4 = createQuestion(
'A kötvény névértéke', 'A kötvény névértéke',
'A kötvényen feltüntetett meghatározott nagyságú összeg.', 'A kötvényen feltüntetett meghatározott nagyságú összeg.',
{ {
type: 'simple', type: 'simple',
date: date(-1000), date: date(-1000),
} }
) )
const q5 = createQuestion( const q5 = createQuestion(
'Mi az osztalék? asd asd', 'Mi az osztalék? asd asd',
'A vállalati profit egy része..', 'A vállalati profit egy része..',
{ {
type: 'simple', type: 'simple',
date: date(1000), date: date(1000),
} }
) )
const q6 = createQuestion( const q6 = createQuestion(
'valaim nagyon értelmes kérdés asd asd', 'valaim nagyon értelmes kérdés asd asd',
'A vállalati profit egy része..', 'A vállalati profit egy része..',
{ {
type: 'simple', type: 'simple',
date: date(1000), date: date(1000),
} }
) )
function setupTest({ function setupTest({
newQuestions, newQuestions,
data, data,
subjToClean, subjToClean,
}: { }: {
newQuestions: Question[] newQuestions: Question[]
data: Subject[] data: Subject[]
subjToClean?: string subjToClean?: string
}) { }) {
const recievedQuestions: Question[] = newQuestions.map((x) => { const recievedQuestions: Question[] = newQuestions.map((x) => {
return {
...x,
data: {
...x.data,
date: date(),
},
}
})
const subjName = subjToClean || 'subject'
const overwriteFromDate = date(-100)
const qdbIndex = 0
const qdbs: QuestionDb[] = [
{
name: 'test',
data: data,
index: 0,
path: '',
shouldSearch: 'asd',
shouldSave: {},
},
]
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
return x.Name.toLowerCase().includes(subjName.toLowerCase())
})
const questionIndexesToRemove = cleanDb(
{
questions: recievedQuestions,
subjToClean: subjName,
overwriteFromDate: overwriteFromDate,
qdbIndex: qdbIndex,
},
qdbs
)
const updatedQuestions = updateQuestionsInArray(
questionIndexesToRemove,
qdbs[qdbIndex].data[subjIndex].Questions,
recievedQuestions
)
return { return {
...x, questionIndexesToRemove: questionIndexesToRemove,
data: { updatedQuestions: updatedQuestions,
...x.data, overwriteFromDate: overwriteFromDate,
date: date(), subjIndex: subjIndex,
},
} }
})
const subjName = subjToClean || 'subject'
const overwriteFromDate = date(-100)
const qdbIndex = 0
const qdbs: QuestionDb[] = [
{
name: 'test',
data: data,
index: 0,
path: '',
shouldSearch: 'asd',
shouldSave: {},
},
]
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
return x.Name.toLowerCase().includes(subjName.toLowerCase())
})
const questionIndexesToRemove = cleanDb(
{
questions: recievedQuestions,
subjToClean: subjName,
overwriteFromDate: overwriteFromDate,
qdbIndex: qdbIndex,
},
qdbs
)
const updatedQuestions = updateQuestionsInArray(
questionIndexesToRemove,
qdbs[qdbIndex].data[subjIndex].Questions,
recievedQuestions
)
return {
questionIndexesToRemove: questionIndexesToRemove,
updatedQuestions: updatedQuestions,
overwriteFromDate: overwriteFromDate,
subjIndex: subjIndex,
}
} }
const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] } const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] }
test('Old and duplicate questions should be removed from the database', () => { test('Old and duplicate questions should be removed from the database', () => {
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } = const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
setupTest({ newQuestions: [q1, q4, q5], data: [s1] }) setupTest({ newQuestions: [q1, q4, q5], data: [s1] })
expect(questionIndexesToRemove.length).toBe(3) expect(questionIndexesToRemove.length).toBe(3)
expect(questionIndexesToRemove[0].length).toBe(2) expect(questionIndexesToRemove[0].length).toBe(2)
expect(updatedQuestions.length).toBe(3) expect(updatedQuestions.length).toBe(3)
const toremoveCount = updatedQuestions.filter((question) => { const toremoveCount = updatedQuestions.filter((question) => {
return question.Q.includes('TOREMOVE') return question.Q.includes('TOREMOVE')
}).length }).length
expect(toremoveCount).toBe(1) expect(toremoveCount).toBe(1)
const newQuestion = updatedQuestions.find((question) => { const newQuestion = updatedQuestions.find((question) => {
return question.Q.includes('TOREMOVE') return question.Q.includes('TOREMOVE')
}) })
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy() expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
}) })
const s2: Subject = { const s2: Subject = {
Name: 'test subject', Name: 'test subject',
Questions: [q1, q2, q3, q4, q5, q6], Questions: [q1, q2, q3, q4, q5, q6],
} }
test('Old and duplicate questions should be removed from the database round 2', () => { test('Old and duplicate questions should be removed from the database round 2', () => {
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } = const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
setupTest({ newQuestions: [q1, q4, q5], data: [s2] }) setupTest({ newQuestions: [q1, q4, q5], data: [s2] })
expect(questionIndexesToRemove.length).toBe(3) expect(questionIndexesToRemove.length).toBe(3)
expect(questionIndexesToRemove[0].length).toBe(3) expect(questionIndexesToRemove[0].length).toBe(3)
expect(updatedQuestions.length).toBe(4) expect(updatedQuestions.length).toBe(4)
const toremoveCount = updatedQuestions.filter((question) => { const toremoveCount = updatedQuestions.filter((question) => {
return question.Q.includes('TOREMOVE') return question.Q.includes('TOREMOVE')
}).length }).length
expect(toremoveCount).toBe(1) expect(toremoveCount).toBe(1)
const newQuestion = updatedQuestions.find((question) => { const newQuestion = updatedQuestions.find((question) => {
return question.Q.includes('TOREMOVE') return question.Q.includes('TOREMOVE')
}) })
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy() expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
}) })
const s3: Subject = { const s3: Subject = {
Name: 'test subject', Name: 'test subject',
Questions: [q5, q6].map((x) => ({ Questions: [q5, q6].map((x) => ({
...x, ...x,
data: { data: {
...x.data, ...x.data,
date: date(+50000), date: date(+50000),
}, },
})), })),
} }
test('Old and duplicate questions should be removed from the database: questions should be left alone when they are newer', () => { test('Old and duplicate questions should be removed from the database: questions should be left alone when they are newer', () => {
const { questionIndexesToRemove, updatedQuestions } = setupTest({ const { questionIndexesToRemove, updatedQuestions } = setupTest({
newQuestions: [q5, q6], newQuestions: [q5, q6],
data: [s3], data: [s3],
}) })
expect(questionIndexesToRemove.length).toBe(2) expect(questionIndexesToRemove.length).toBe(2)
questionIndexesToRemove.forEach((x) => { questionIndexesToRemove.forEach((x) => {
expect(x.length).toBe(0) expect(x.length).toBe(0)
}) })
expect(updatedQuestions.length).toBe(2) expect(updatedQuestions.length).toBe(2)
}) })
const s4: Subject = { const s4: Subject = {
Name: 'something else', Name: 'something else',
Questions: [q5, q6], Questions: [q5, q6],
} }
test('Old and duplicate questions should be removed from the database:other subjects should be left alone', () => { test('Old and duplicate questions should be removed from the database:other subjects should be left alone', () => {
const { subjIndex } = setupTest({ const { subjIndex } = setupTest({
newQuestions: [q5, q6], newQuestions: [q5, q6],
data: [s2, s1, s4, s3], data: [s2, s1, s4, s3],
subjToClean: 'else', subjToClean: 'else',
}) })
expect(subjIndex).toBe(2) expect(subjIndex).toBe(2)
}) })

466
src/tests/possibleAnswerPenalty.test.ts
View File

@@ -1,308 +1,308 @@
import { import {
setNoPossibleAnswersPenalties, setNoPossibleAnswersPenalties,
SearchResultQuestion, SearchResultQuestion,
noPossibleAnswerMatchPenalty, noPossibleAnswerMatchPenalty,
} from '../utils/classes' } from '../utils/classes'
import { Question } from '../types/basicTypes' import { Question } from '../types/basicTypes'
const matchPercent = 100 const matchPercent = 100
const questionWithNormalPossibleAnswers: Question = { const questionWithNormalPossibleAnswers: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'rubber duck' }, { type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'super laptop' }, { type: 'txt', val: 'super laptop' },
{ type: 'txt', val: 'nothing in particular' }, { type: 'txt', val: 'nothing in particular' },
{ type: 'txt', val: 'something giberish' }, { type: 'txt', val: 'something giberish' },
], ],
}, },
} }
const questionWithNormalPossibleAnswersWithLabels: Question = { const questionWithNormalPossibleAnswersWithLabels: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'a) nothing in particular' }, { type: 'txt', val: 'a) nothing in particular' },
{ type: 'txt', val: 'b) super laptop' }, { type: 'txt', val: 'b) super laptop' },
{ type: 'txt', val: 'c) something giberish' }, { type: 'txt', val: 'c) something giberish' },
{ type: 'txt', val: 'd) rubber duck' }, { type: 'txt', val: 'd) rubber duck' },
], ],
}, },
} }
const questionWithNormalPossibleAnswers2: Question = { const questionWithNormalPossibleAnswers2: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'rubber duck' }, { type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'cat' }, { type: 'txt', val: 'cat' },
{ type: 'txt', val: 'nothing in particular' }, { type: 'txt', val: 'nothing in particular' },
{ type: 'txt', val: 'dog' }, { type: 'txt', val: 'dog' },
], ],
}, },
} }
const questionWithNormalPossibleAnswers3: Question = { const questionWithNormalPossibleAnswers3: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'rubber duck 2' }, { type: 'txt', val: 'rubber duck 2' },
{ type: 'txt', val: 'whale' }, { type: 'txt', val: 'whale' },
{ type: 'txt', val: 'nothing in particular 2' }, { type: 'txt', val: 'nothing in particular 2' },
{ type: 'txt', val: 'sea lion' }, { type: 'txt', val: 'sea lion' },
], ],
}, },
} }
const questionWithNormalPossibleAnswers4: Question = { const questionWithNormalPossibleAnswers4: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'rubber duck' }, { type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'super laptop' }, { type: 'txt', val: 'super laptop' },
], ],
}, },
} }
const questionWithSimilarPossibleAnswers: Question = { const questionWithSimilarPossibleAnswers: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'asd' }, { type: 'txt', val: 'asd' },
{ type: 'txt', val: 'basd' }, { type: 'txt', val: 'basd' },
{ type: 'txt', val: 'aaa' }, { type: 'txt', val: 'aaa' },
{ type: 'txt', val: 'bbb' }, { type: 'txt', val: 'bbb' },
], ],
}, },
} }
const questionWithTrueFalsePossibleAnser: Question = { const questionWithTrueFalsePossibleAnser: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
possibleAnswers: [ possibleAnswers: [
{ type: 'txt', val: 'true' }, { type: 'txt', val: 'true' },
{ type: 'txt', val: 'false' }, { type: 'txt', val: 'false' },
], ],
}, },
} }
const questionWithNoPossibleAnswer: Question = { const questionWithNoPossibleAnswer: Question = {
Q: 'asd', Q: 'asd',
A: 'asd', A: 'asd',
data: { data: {
type: 'simple', type: 'simple',
}, },
} }
const resNormal: SearchResultQuestion = { const resNormal: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers, q: questionWithNormalPossibleAnswers,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const resNormal2: SearchResultQuestion = { const resNormal2: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers2, q: questionWithNormalPossibleAnswers2,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const resNormal3: SearchResultQuestion = { const resNormal3: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers3, q: questionWithNormalPossibleAnswers3,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const resNormal4: SearchResultQuestion = { const resNormal4: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers4, q: questionWithNormalPossibleAnswers4,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const resSimilar: SearchResultQuestion = { const resSimilar: SearchResultQuestion = {
q: questionWithSimilarPossibleAnswers, q: questionWithSimilarPossibleAnswers,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const resTrueFalse: SearchResultQuestion = { const resTrueFalse: SearchResultQuestion = {
q: questionWithTrueFalsePossibleAnser, q: questionWithTrueFalsePossibleAnser,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const resNoPossibleAnswer: SearchResultQuestion = { const resNoPossibleAnswer: SearchResultQuestion = {
q: questionWithNoPossibleAnswer, q: questionWithNoPossibleAnswer,
match: matchPercent, match: matchPercent,
detailedMatch: { detailedMatch: {
qMatch: matchPercent, qMatch: matchPercent,
aMatch: matchPercent, aMatch: matchPercent,
dMatch: matchPercent, dMatch: matchPercent,
matchedSubjName: 'testSubj', matchedSubjName: 'testSubj',
avg: matchPercent, avg: matchPercent,
}, },
} }
const testFunction = ( const testFunction = (
question: Question, question: Question,
searchResult: SearchResultQuestion[], searchResult: SearchResultQuestion[],
index: number index: number
) => { ) => {
const updated = setNoPossibleAnswersPenalties( const updated = setNoPossibleAnswersPenalties(
question.data.possibleAnswers, question.data.possibleAnswers,
searchResult searchResult
) )
updated.forEach((x, i) => { updated.forEach((x, i) => {
if (i !== index) { if (i !== index) {
expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty) expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty)
expect(x.detailedMatch.qMatch).toBe( expect(x.detailedMatch.qMatch).toBe(
matchPercent - noPossibleAnswerMatchPenalty matchPercent - noPossibleAnswerMatchPenalty
) )
} else { } else {
expect(x.match).toBe(100) expect(x.match).toBe(100)
expect(x.detailedMatch.qMatch).toBe(100) expect(x.detailedMatch.qMatch).toBe(100)
} }
}) })
return updated return updated
} }
test('Possible answer penalty applies correctly (normal possible answers)', () => { test('Possible answer penalty applies correctly (normal possible answers)', () => {
testFunction( testFunction(
questionWithNormalPossibleAnswers, questionWithNormalPossibleAnswers,
[ [
resNormal, resNormal,
resNormal2, resNormal2,
resNormal3, resNormal3,
resNormal4, resNormal4,
resSimilar, resSimilar,
resTrueFalse, resTrueFalse,
resNoPossibleAnswer, resNoPossibleAnswer,
], ],
0 0
) )
}) })
test('Possible answer penalty applies correctly (normal possible answers, with labels)', () => { test('Possible answer penalty applies correctly (normal possible answers, with labels)', () => {
testFunction( testFunction(
questionWithNormalPossibleAnswersWithLabels, questionWithNormalPossibleAnswersWithLabels,
[ [
resNormal, resNormal,
resNormal2, resNormal2,
resNormal3, resNormal3,
resNormal4, resNormal4,
resSimilar, resSimilar,
resTrueFalse, resTrueFalse,
resNoPossibleAnswer, resNoPossibleAnswer,
], ],
0 0
) )
}) })
test('Possible answer penalty applies correctly (similar possible answers)', () => { test('Possible answer penalty applies correctly (similar possible answers)', () => {
testFunction( testFunction(
questionWithSimilarPossibleAnswers, questionWithSimilarPossibleAnswers,
[ [
resNormal, resNormal,
resNormal2, resNormal2,
resNormal3, resNormal3,
resNormal4, resNormal4,
resSimilar, resSimilar,
resTrueFalse, resTrueFalse,
resNoPossibleAnswer, resNoPossibleAnswer,
], ],
4 4
) )
}) })
test('Possible answer penalty applies correctly (true false possible answers)', () => { test('Possible answer penalty applies correctly (true false possible answers)', () => {
testFunction( testFunction(
questionWithTrueFalsePossibleAnser, questionWithTrueFalsePossibleAnser,
[ [
resNormal, resNormal,
resNormal2, resNormal2,
resNormal3, resNormal3,
resNormal4, resNormal4,
resSimilar, resSimilar,
resTrueFalse, resTrueFalse,
resNoPossibleAnswer, resNoPossibleAnswer,
], ],
5 5
) )
}) })
test('Possible answer penalty applies correctly (no possible answers)', () => { test('Possible answer penalty applies correctly (no possible answers)', () => {
const updated = setNoPossibleAnswersPenalties( const updated = setNoPossibleAnswersPenalties(
questionWithNoPossibleAnswer.data.possibleAnswers, questionWithNoPossibleAnswer.data.possibleAnswers,
[ [
resNormal, resNormal,
resNormal2, resNormal2,
resNormal3, resNormal3,
resNormal4, resNormal4,
resSimilar, resSimilar,
resTrueFalse, resTrueFalse,
resNoPossibleAnswer, resNoPossibleAnswer,
] ]
) )
updated.forEach((x) => { updated.forEach((x) => {
expect(x.match).toBe(100) expect(x.match).toBe(100)
expect(x.detailedMatch.qMatch).toBe(100) expect(x.detailedMatch.qMatch).toBe(100)
}) })
}) })
test('Possible answer penalty applies correctly (empty searchResult)', () => { test('Possible answer penalty applies correctly (empty searchResult)', () => {
const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0) const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0)
expect(updated.length).toBe(0) expect(updated.length).toBe(0)
}) })

12
src/tests/shouldLog.test.ts
View File

@@ -6,10 +6,10 @@ const truthy = [1, 2, '10', '40']
const falsey = [5, '55', 47832, 'fhs'] const falsey = [5, '55', 47832, 'fhs']
test('ShouldLog works', () => { test('ShouldLog works', () => {
truthy.forEach((x) => { truthy.forEach((x) => {
expect(shouldLog(x, noLogIds)).toBeTruthy() expect(shouldLog(x, noLogIds)).toBeTruthy()
}) })
falsey.forEach((x) => { falsey.forEach((x) => {
expect(shouldLog(x, noLogIds)).toBeFalsy() expect(shouldLog(x, noLogIds)).toBeFalsy()
}) })
}) })

172
src/types/basicTypes.ts
View File

@@ -26,136 +26,136 @@ import http from 'http'
import https from 'https' import https from 'https'
export interface QuestionData { export interface QuestionData {
type: string
date?: Date | number
images?: Array<string>
hashedImages?: Array<string>
possibleAnswers?: Array<{
type: string type: string
val: string date?: Date | number
selectedByUser?: boolean images?: Array<string>
}> hashedImages?: Array<string>
base64?: string[] possibleAnswers?: Array<{
type: string
val: string
selectedByUser?: boolean
}>
base64?: string[]
} }
export interface Question { export interface Question {
Q: string Q: string
A: string A: string
data: QuestionData data: QuestionData
cache?: { cache?: {
Q: Array<string> Q: Array<string>
A: Array<string> A: Array<string>
} }
} }
export interface Subject { export interface Subject {
Name: string Name: string
Questions: Array<Question> Questions: Array<Question>
} }
export interface DataFile { export interface DataFile {
path: string path: string
name: string name: string
locked?: Boolean locked?: Boolean
overwrites?: Array<{ overwrites?: Array<{
subjName: string subjName: string
overwriteFromDate: number overwriteFromDate: number
}> }>
shouldSearch: shouldSearch:
| string | string
| { | {
location?: {
val: string
}
}
shouldSave: {
location?: { location?: {
val: string val: string
}
version?: {
compare: string
val: string
} }
}
shouldSave: {
location?: {
val: string
} }
version?: {
compare: string
val: string
}
}
} }
export interface QuestionDb extends DataFile { export interface QuestionDb extends DataFile {
data: Array<Subject> data: Array<Subject>
index: number index: number
} }
export interface User { export interface User {
id: number id: number
pw: string pw: string
created: Date created: Date
} }
export interface User { export interface User {
id: number id: number
pw: string pw: string
pwRequestCount: number pwRequestCount: number
avaiblePWRequests: number avaiblePWRequests: number
loginCount: number loginCount: number
createdBy: number createdBy: number
} }
export interface Request<T = any> extends express.Request { export interface Request<T = any> extends express.Request {
body: T body: T
cookies: any cookies: any
session: { session: {
user?: User user?: User
sessionID?: string sessionID?: string
isException?: boolean isException?: boolean
} }
files: any files: any
query: { [key: string]: string } query: { [key: string]: string }
} }
export interface SubmoduleData { export interface SubmoduleData {
app: express.Application app: express.Application
url: string url: string
publicdirs: Array<string> publicdirs: Array<string>
userDB?: Database userDB?: Database
nextdir?: string nextdir?: string
moduleSpecificData?: any moduleSpecificData?: any
httpServer: http.Server httpServer: http.Server
httpsServer: https.Server httpsServer: https.Server
} }
export interface QuestionFromScript { export interface QuestionFromScript {
questions: Array<Question> questions: Array<Question>
testUrl: string testUrl: string
subj: string subj: string
} }
export interface DbSearchResult { export interface DbSearchResult {
message?: string message?: string
recievedData?: string recievedData?: string
question: Question question: Question
result: SearchResultQuestion[] result: SearchResultQuestion[]
success: boolean success: boolean
} }
export interface RegisteredUserEntry { export interface RegisteredUserEntry {
cid: string cid: string
version: string version: string
installSource: string installSource: string
date: string date: string
userAgent: string userAgent: string
loginDate?: string loginDate?: string
uid?: number uid?: number
} }
export interface Submodule { export interface Submodule {
dailyAction?: () => void dailyAction?: () => void
load?: () => void load?: () => void
} }
export interface ModuleType { export interface ModuleType {
app: express.Application app: express.Application
dailyAction?: Function dailyAction?: Function
} }
export interface Socket extends SocketIoSocket { export interface Socket extends SocketIoSocket {
user: User user: User
} }

1404
src/utils/actions.ts
View File

File diff suppressed because it is too large Load Diff

1448
src/utils/classes.ts
View File

File diff suppressed because it is too large Load Diff

468
src/utils/dbtools.ts
View File

@@ -22,18 +22,18 @@
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md // https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
export default { export default {
GetDB: GetDB, GetDB: GetDB,
AddColumn: AddColumn, AddColumn: AddColumn,
TableInfo: TableInfo, TableInfo: TableInfo,
Update: Update, Update: Update,
Delete: Delete, Delete: Delete,
CreateTable: CreateTable, CreateTable: CreateTable,
SelectAll: SelectAll, SelectAll: SelectAll,
Select: Select, Select: Select,
Insert: Insert, Insert: Insert,
CloseDB: CloseDB, CloseDB: CloseDB,
runStatement: runStatement, runStatement: runStatement,
sanitizeQuery: sanitizeQuery, sanitizeQuery: sanitizeQuery,
} }
import Sqlite, { Database, RunResult } from 'better-sqlite3' import Sqlite, { Database, RunResult } from 'better-sqlite3'
@@ -43,310 +43,310 @@ import utils from '../utils/utils'
const debugLog = process.env.NS_SQL_DEBUG_LOG const debugLog = process.env.NS_SQL_DEBUG_LOG
function sanitizeQuery(val: string | number): string | number { function sanitizeQuery(val: string | number): string | number {
if (typeof val === 'string') { if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '') return val.replace(/'/g, '').replace(/;/g, '')
} }
return val return val
} }
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4 // { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
function GetSqlQuerry( function GetSqlQuerry(
conditions: { [key: string]: string | number }, conditions: { [key: string]: string | number },
type?: string, type?: string,
joiner?: string joiner?: string
) { ) {
const res = Object.keys(conditions).reduce((acc, key) => { const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key] const item = conditions[key]
const conditionKey = sanitizeQuery(key) const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key]) const condition = sanitizeQuery(conditions[key])
if (typeof item === 'string') { if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`) acc.push(`${conditionKey} = '${condition}'`)
} else {
acc.push(`${conditionKey} = ${condition}`)
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else { } else {
acc.push(`${conditionKey} = ${condition}`) return res.join(', ')
} }
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
return res.join(', ')
}
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
function GetDB(path: string): Database { function GetDB(path: string): Database {
utils.CreatePath(path) utils.CreatePath(path)
const res = new Sqlite(path) const res = new Sqlite(path)
res.pragma('synchronous = OFF') res.pragma('synchronous = OFF')
return res return res
} }
function DebugLog(msg: string) { function DebugLog(msg: string) {
if (debugLog) { if (debugLog) {
logger.DebugLog(msg, 'sql', 0) logger.DebugLog(msg, 'sql', 0)
} }
} }
// FIXME: this might not work: what is col exactly, and how we use AddColumn? // FIXME: this might not work: what is col exactly, and how we use AddColumn?
function AddColumn( function AddColumn(
db: Database, db: Database,
table: string, table: string,
col: { [key: string]: string | number } col: { [key: string]: string | number }
): RunResult { ): RunResult {
try { try {
const colName = Object.keys(col)[0] const colName = Object.keys(col)[0]
const colType = col.type const colType = col.type
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}` const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
return stmt.run() return stmt.run()
} catch (err) { } catch (err) {
console.error(err) console.error(err)
return null return null
} }
} }
function TableInfo( function TableInfo(
db: Database, db: Database,
table: string table: string
): { ): {
columns: any[] columns: any[]
dataCount: number dataCount: number
} { } {
try { try {
const command = `PRAGMA table_info(${table})` const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
const infoRes = stmt.all() const infoRes = stmt.all()
const s2 = `SELECT COUNT(*) FROM ${table}` const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2) const stmt2 = PrepareStatement(db, s2)
const countRes = stmt2.get() const countRes = stmt2.get()
return { return {
columns: infoRes, columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]], dataCount: countRes[Object.keys(countRes)[0]],
}
} catch (err) {
console.error(err)
return null
} }
} catch (err) {
console.error(err)
return null
}
} }
function Update( function Update(
db: Database, db: Database,
table: string, table: string,
newData: { [key: string]: string | number }, newData: { [key: string]: string | number },
conditions: { [key: string]: string | number } conditions: { [key: string]: string | number }
): RunResult { ): RunResult {
try { try {
const command = `UPDATE ${table} SET ${GetSqlQuerry( const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData, newData,
'set' 'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}` )} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
return stmt.run() return stmt.run()
} catch (err) { } catch (err) {
console.error(err) console.error(err)
return null return null
} }
} }
function Delete( function Delete(
db: Database, db: Database,
table: string, table: string,
conditions: { [key: string]: string | number } conditions: { [key: string]: string | number }
): RunResult { ): RunResult {
try { try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry( const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions, conditions,
'where' 'where'
)}` )}`
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
return stmt.run() return stmt.run()
} catch (err) { } catch (err) {
console.error(err) console.error(err)
return null return null
} }
} }
interface DbColumnDescription { interface DbColumnDescription {
[key: string]: { [key: string]: {
type: string type: string
primary?: boolean primary?: boolean
autoIncrement?: boolean autoIncrement?: boolean
notNull?: boolean notNull?: boolean
defaultZero?: boolean defaultZero?: boolean
[key: string]: any [key: string]: any
} }
} }
function CreateTable( function CreateTable(
db: Database, db: Database,
name: string, name: string,
columns: DbColumnDescription, columns: DbColumnDescription,
foreignKeys: { foreignKeys: {
keysFrom: string[] keysFrom: string[]
table: string table: string
keysTo: string[] keysTo: string[]
}[] }[]
): RunResult { ): RunResult {
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount // CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text // number, lastLogin text, lastAccess text
// //
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) ) // FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
try { try {
const cols = Object.keys(columns) const cols = Object.keys(columns)
.reduce((acc, key) => { .reduce((acc, key) => {
const item = columns[key] const item = columns[key]
const flags: string[] = [] const flags: string[] = []
const toCheck = { const toCheck = {
primary: 'PRIMARY KEY', primary: 'PRIMARY KEY',
notNull: 'NOT NULL', notNull: 'NOT NULL',
unique: 'UNIQUE', unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT', autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0', defaultZero: 'DEFAULT 0',
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
} }
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`) // IF NOT EXISTS
return acc const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
}, []) const stmt = PrepareStatement(db, command)
.join(', ') return stmt.run()
} catch (err) {
const fKeys: string[] = [] console.error(err)
if (foreignKeys) { return null
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
} }
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
} }
function SelectAll(db: Database, from: string): any[] { function SelectAll(db: Database, from: string): any[] {
try { try {
const command = `SELECT * from ${from}` const command = `SELECT * from ${from}`
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
return stmt.all() return stmt.all()
} catch (err) { } catch (err) {
console.error(err) console.error(err)
return null return null
} }
} }
// SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100; // SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100;
function Select( function Select(
db: Database, db: Database,
from: string, from: string,
conditions: { [key: string]: string | number }, conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {} options: { joiner?: string; limit?: number } = {}
): any[] { ): any[] {
const { joiner, limit } = options const { joiner, limit } = options
try { try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry( let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions, conditions,
'where', 'where',
joiner joiner
)}` )}`
if (!isNaN(limit)) { if (!isNaN(limit)) {
command += ` LIMIT ${limit}` command += ` LIMIT ${limit}`
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
} }
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
} }
function Insert( function Insert(
db: Database, db: Database,
table: string, table: string,
data: { [key: string]: number | string } data: { [key: string]: number | string }
): RunResult { ): RunResult {
try { try {
const cols = Object.keys(data) const cols = Object.keys(data)
.reduce((acc, key) => { .reduce((acc, key) => {
acc.push(`${key}`) acc.push(`${key}`)
return acc return acc
}, []) }, [])
.join(', ') .join(', ')
const values = Object.keys(data) const values = Object.keys(data)
.map((key) => { .map((key) => {
const item = data[key] const item = data[key]
if (typeof item === 'string') { if (typeof item === 'string') {
return `'${item}'` return `'${item}'`
} else { } else {
return `${item}` return `${item}`
} }
}) })
.join(', ') .join(', ')
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})` const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
return stmt.run() return stmt.run()
} catch (err) { } catch (err) {
console.error(err) console.error(err)
return null return null
} }
} }
function runStatement(db: Database, command: string, runType?: string): any { function runStatement(db: Database, command: string, runType?: string): any {
const stmt = PrepareStatement(db, command) const stmt = PrepareStatement(db, command)
if (!runType) { if (!runType) {
return stmt.all() return stmt.all()
} else if (runType === 'run') { } else if (runType === 'run') {
return stmt.run() return stmt.run()
} }
return null return null
} }
function CloseDB(db: Database): void { function CloseDB(db: Database): void {
db.close() db.close()
} }
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
function PrepareStatement(db: Database, command: string) { function PrepareStatement(db: Database, command: string) {
if (!db) { if (!db) {
throw new Error( throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db' 'DB is undefined in prepare statement! DB action called with undefined db'
) )
} }
DebugLog(command) DebugLog(command)
return db.prepare(command) return db.prepare(command)
} }

178
src/utils/ids.ts
View File

@@ -19,8 +19,8 @@
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
export default { export default {
LogId: LogId, LogId: LogId,
Load: Load, Load: Load,
} }
import utils from '../utils/utils' import utils from '../utils/utils'
@@ -35,113 +35,113 @@ let idvStatsData = {}
let writes = 0 let writes = 0
function Load(): void { function Load(): void {
try { try {
idStatsData = utils.ReadJSON(idStatFile) idStatsData = utils.ReadJSON(idStatFile)
} catch (err) { } catch (err) {
logger.Log( logger.Log(
'Error at loading id logs! (@ first run its normal)', 'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg') logger.GetColor('redbg')
) )
console.error(err) console.error(err)
} }
try { try {
const prevVData = utils.ReadFile(idVStatFile) const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData) idvStatsData = JSON.parse(prevVData)
} catch (err) { } catch (err) {
logger.Log( logger.Log(
'Error at loading id logs! (@ first run its normal)', 'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg') logger.GetColor('redbg')
) )
console.error(err) console.error(err)
} }
} }
function LogId( function LogId(
id: number, id: number,
subj: string, subj: string,
newQuestions: number, newQuestions: number,
allQuestions: number allQuestions: number
): void { ): void {
Inc(id, subj, newQuestions, allQuestions) Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions) AddVisitStat(id, subj, newQuestions, allQuestions)
Save() Save()
} }
function AddSubjToList(list: { [key: string]: any }, subj: string) { function AddSubjToList(list: { [key: string]: any }, subj: string) {
if (!list[subj]) { if (!list[subj]) {
list[subj] = 0 list[subj] = 0
} }
list[subj]++ list[subj]++
} }
function Inc( function Inc(
value: number, value: number,
subj: string, subj: string,
newQuestions: number, newQuestions: number,
allQuestions: number allQuestions: number
) { ) {
if (idStatsData[value] === undefined) { if (idStatsData[value] === undefined) {
idStatsData[value] = { idStatsData[value] = {
count: 0, count: 0,
newQuestions: 0, newQuestions: 0,
allQuestions: 0, allQuestions: 0,
subjs: {}, subjs: {},
}
} }
} idStatsData[value].count++
idStatsData[value].count++ idStatsData[value].newQuestions += newQuestions
idStatsData[value].newQuestions += newQuestions idStatsData[value].allQuestions += allQuestions
idStatsData[value].allQuestions += allQuestions AddSubjToList(idStatsData[value].subjs, subj)
AddSubjToList(idStatsData[value].subjs, subj)
} }
function AddVisitStat( function AddVisitStat(
name: number, name: number,
subj: string, subj: string,
newQuestions: number, newQuestions: number,
allQuestions: number allQuestions: number
) { ) {
const date = new Date() const date = new Date()
const now = const now =
date.getFullYear() + date.getFullYear() +
'-' + '-' +
('0' + (date.getMonth() + 1)).slice(-2) + ('0' + (date.getMonth() + 1)).slice(-2) +
'-' + '-' +
('0' + date.getDate()).slice(-2) ('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) { if (idvStatsData[now] === undefined) {
idvStatsData[now] = {} idvStatsData[now] = {}
}
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
} }
} if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name].count++ idvStatsData[now][name] = {
idvStatsData[now][name].newQuestions += newQuestions count: 0,
idvStatsData[now][name].allQuestions += allQuestions newQuestions: 0,
AddSubjToList(idvStatsData[now][name].subjs, subj) allQuestions: 0,
subjs: {},
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
} }
function Save() { function Save() {
writes++ writes++
if (writes === writeInterval) { if (writes === writeInterval) {
try { try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile) utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote."); // Log("Stats wrote.");
} catch (err) { } catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg')) logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err) console.error(err)
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
} }
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
} }

572
src/utils/logger.ts
View File

@@ -19,7 +19,7 @@
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
const hr = const hr =
'---------------------------------------------------------------------------------' '---------------------------------------------------------------------------------'
const DELIM = C('green') + '|' + C() const DELIM = C('green') + '|' + C()
@@ -50,361 +50,377 @@ let writes = 0
let noLogIds: string[] = [] let noLogIds: string[] = []
function getColoredDateString(): string { function getColoredDateString(): string {
const date = new Date() const date = new Date()
const dateString = utils.GetDateString() const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C() return GetRandomColor(date.getHours().toString()) + dateString + C()
} }
function DebugLog(msg: string | object, name: string, lvl: number): void { function DebugLog(msg: string | object, name: string, lvl: number): void {
if (lvl <= debugLevel) { if (lvl <= debugLevel) {
if (msg === 'hr') { if (msg === 'hr') {
msg = hr msg = hr
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
} }
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
} }
function Log(msg: string | object, color?: string): void { function Log(msg: string | object, color?: string): void {
let log = msg let log = msg
if (typeof msg !== 'object') { if (typeof msg !== 'object') {
const delimiter = DELIM + C(color) const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C() log = getColoredDateString() + delimiter + C(color) + msg + C()
} }
if (!process.env.NS_NOLOG) { if (!process.env.NS_NOLOG) {
console.log(log) console.log(log)
} }
utils.AppendToFile( utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log), typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName logDir + logFileName
) )
} }
function expandWithSpaces(text: string, count: number) { function expandWithSpaces(text: string, count: number) {
while (text.length < count) { while (text.length < count) {
text += ' ' text += ' '
} }
return text return text
} }
function LogReq( function LogReq(
req: Request, req: Request,
toFile?: boolean, toFile?: boolean,
statusCode?: string | number statusCode?: string | number
): void { ): void {
try { try {
let logEntry = '' // logHashed(ip) let logEntry = '' // logHashed(ip)
let dl = DELIM let dl = DELIM
if (req.url.includes('lred')) { if (req.url.includes('lred')) {
dl += C('red') dl += C('red')
} }
if (req.session && req.session.user && !shouldLog(req.session.user.id, noLogIds)) { if (
return req.session &&
} req.session.user &&
!shouldLog(req.session.user.id, noLogIds)
) {
return
}
let hostname let hostname
if (req.hostname) { if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0] hostname = req.hostname.replace('www.', '').split('.')[0]
} else { } else {
hostname = 'NOHOST' hostname = 'NOHOST'
Log( Log(
'req.hostname is undefined! req.hostname: ' + req.hostname, 'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg') GetColor('redbg')
) )
} }
if (!toFile) { if (!toFile) {
hostname = expandWithSpaces(hostname, 10) hostname = expandWithSpaces(hostname, 10)
} }
logEntry += logHashed(hostname) + dl logEntry += logHashed(hostname) + dl
if (toFile) { if (toFile) {
logEntry += req.headers['user-agent'] + dl logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl logEntry += req.method + dl
} }
let uid = '' let uid = ''
if (req.session && req.session.user) { if (req.session && req.session.user) {
uid = req.session.user.id.toString() uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) { } else if (req.session && req.session.isException === true) {
uid = 'EX' uid = 'EX'
} else { } else {
uid = 'NOUSR' uid = 'NOUSR'
} }
if (!toFile) { if (!toFile) {
uid = expandWithSpaces(uid, 5) uid = expandWithSpaces(uid, 5)
} }
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C() logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
if (statusCode !== undefined) { if (statusCode !== undefined) {
logEntry += dl + statusCode logEntry += dl + statusCode
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
} }
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
} }
function parseNoLogFile(newData: string) { function parseNoLogFile(newData: string) {
noLogIds = newData.split('\n') noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') { if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop() noLogIds.pop()
} }
noLogIds = noLogIds.filter((noLogId) => { noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== '' return noLogId !== ''
}) })
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', ')) Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
} }
function setNoLogReadInterval() { function setNoLogReadInterval() {
utils.WatchFile(nologFile, (newData: string) => { utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData) parseNoLogFile(newData)
}) })
parseNoLogFile(utils.ReadFile(nologFile)) parseNoLogFile(utils.ReadFile(nologFile))
} }
function Load(): void { function Load(): void {
Log('Loading logger...') Log('Loading logger...')
try { try {
uvData = JSON.parse(utils.ReadFile(uStatsFile)) uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) { } catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg')) Log(
console.error(err) 'Error at loading logs! (@ first run its normal)',
} GetColor('redbg')
)
console.error(err)
}
try { try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile)) udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) { } catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg')) Log(
console.error(err) 'Error at loading logs! (@ first run its normal)',
} GetColor('redbg')
)
console.error(err)
}
try { try {
vData = utils.ReadJSON(statFile) vData = utils.ReadJSON(statFile)
} catch (err) { } catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg')) Log(
console.error(err) 'Error at loading logs! (@ first run its normal)',
} GetColor('redbg')
)
console.error(err)
}
try { try {
dvData = utils.ReadJSON(vStatFile) dvData = utils.ReadJSON(vStatFile)
} catch (err) { } catch (err) {
Log( Log(
'Error at loading visit logs! (@ first run its normal)', 'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg') GetColor('redbg')
) )
console.error(err) console.error(err)
} }
setNoLogReadInterval() setNoLogReadInterval()
} }
export function shouldLog(userId: string | number, nolog: string[]): boolean { export function shouldLog(userId: string | number, nolog: string[]): boolean {
return !nolog.some((noLogId) => { return !nolog.some((noLogId) => {
return noLogId === userId.toString() return noLogId === userId.toString()
}) })
} }
function LogStat(url: string, hostname: string, userId: number | string): void { function LogStat(url: string, hostname: string, userId: number | string): void {
if (!shouldLog(userId, noLogIds)) { if (!shouldLog(userId, noLogIds)) {
return return
} }
url = hostname + url.split('?')[0] url = hostname + url.split('?')[0]
Inc(url) Inc(url)
AddVisitStat(url) AddVisitStat(url)
if (shouldAddUserStat(url)) { if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString()) AddUserIdStat(userId.toString())
IncUserStat(userId.toString()) IncUserStat(userId.toString())
} }
Save() Save()
} }
const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg'] const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
function shouldAddUserStat(url: string) { function shouldAddUserStat(url: string) {
return !userStatExcludes.some((x) => url.includes(x)) return !userStatExcludes.some((x) => url.includes(x))
} }
function IncUserStat(userId: string) { function IncUserStat(userId: string) {
try { try {
if (uvData[userId] === undefined) { if (uvData[userId] === undefined) {
uvData[userId] = 0 uvData[userId] = 0
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
} }
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
} }
function AddUserIdStat(userId: string) { function AddUserIdStat(userId: string) {
try { try {
const date = new Date() const date = new Date()
const now = const now =
date.getFullYear() + date.getFullYear() +
'-' + '-' +
('0' + (date.getMonth() + 1)).slice(-2) + ('0' + (date.getMonth() + 1)).slice(-2) +
'-' + '-' +
('0' + date.getDate()).slice(-2) ('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) { if (udvData[now] === undefined) {
udvData[now] = {} udvData[now] = {}
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
} }
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
} }
function Inc(value: string) { function Inc(value: string) {
if (value.startsWith('/?')) { if (value.startsWith('/?')) {
value = '/' value = '/'
} }
if (vData[value] === undefined) { if (vData[value] === undefined) {
vData[value] = 0 vData[value] = 0
} }
vData[value]++ vData[value]++
} }
function AddVisitStat(name: string) { function AddVisitStat(name: string) {
const date = new Date() const date = new Date()
const now = const now =
date.getFullYear() + date.getFullYear() +
'-' + '-' +
('0' + (date.getMonth() + 1)).slice(-2) + ('0' + (date.getMonth() + 1)).slice(-2) +
'-' + '-' +
('0' + date.getDate()).slice(-2) ('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) { if (dvData[now] === undefined) {
dvData[now] = {} dvData[now] = {}
} }
if (dvData[now][name] === undefined) { if (dvData[now][name] === undefined) {
dvData[now][name] = 0 dvData[now][name] = 0
} }
dvData[now][name]++ dvData[now][name]++
} }
function Save() { function Save() {
writes++ writes++
if (writes === writeInterval) { if (writes === writeInterval) {
try { try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile) utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) { } catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg')) Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err) console.error(err)
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log(
'Error at writing visit logs! (more in stderr)',
GetColor('redbg')
)
console.error(err)
}
writes = 0
} }
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
writes = 0
}
} }
function logHashed(msg: string): string { function logHashed(msg: string): string {
return GetRandomColor(msg.toString()) + msg + C() return GetRandomColor(msg.toString()) + msg + C()
} }
function GetRandomColor(msg: string): string { function GetRandomColor(msg: string): string {
if (!msg) { if (!msg) {
return 'red' return 'red'
} }
const res = msg.split('').reduce((res, character) => { const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0) return res + character.charCodeAt(0)
}, 0) }, 0)
return C(colors[res % colors.length]) return C(colors[res % colors.length])
} }
function GetColor(color: string): string { function GetColor(color: string): string {
return color return color
} }
function C(color?: string): string { function C(color?: string): string {
if (color !== undefined) { if (color !== undefined) {
color = color.toLowerCase() color = color.toLowerCase()
} }
if (color === 'redbg') { if (color === 'redbg') {
return '\x1b[41m' return '\x1b[41m'
} }
if (color === 'bluebg') { if (color === 'bluebg') {
return '\x1b[44m' return '\x1b[44m'
} }
if (color === 'cyanbg') { if (color === 'cyanbg') {
return '\x1b[46m' return '\x1b[46m'
} }
if (color === 'green') { if (color === 'green') {
return '\x1b[32m' return '\x1b[32m'
} }
if (color === 'red') { if (color === 'red') {
return '\x1b[31m' return '\x1b[31m'
} }
if (color === 'yellow') { if (color === 'yellow') {
return '\x1b[33m' return '\x1b[33m'
} }
if (color === 'blue') { if (color === 'blue') {
return '\x1b[34m' return '\x1b[34m'
} }
if (color === 'magenta') { if (color === 'magenta') {
return '\x1b[35m' return '\x1b[35m'
} }
if (color === 'cyan') { if (color === 'cyan') {
return '\x1b[36m' return '\x1b[36m'
} }
return '\x1b[0m' return '\x1b[0m'
} }
export default { export default {
getColoredDateString: getColoredDateString, getColoredDateString: getColoredDateString,
Log: Log, Log: Log,
DebugLog: DebugLog, DebugLog: DebugLog,
GetColor: GetColor, GetColor: GetColor,
LogReq: LogReq, LogReq: LogReq,
LogStat: LogStat, LogStat: LogStat,
Load: Load, Load: Load,
logHashed: logHashed, logHashed: logHashed,
hr: hr, hr: hr,
C: C, C: C,
logFileName: logFileName, logFileName: logFileName,
logDir: logDir, logDir: logDir,
vlogDir: vlogDir, vlogDir: vlogDir,
} }

58
src/utils/tesseract.ts
View File

@@ -1,7 +1,7 @@
import { import {
createWorker, createWorker,
Worker as TesseractWorker, Worker as TesseractWorker,
ConfigResult, ConfigResult,
} from 'tesseract.js' } from 'tesseract.js'
import logger from './logger' import logger from './logger'
@@ -10,44 +10,44 @@ import { isMainThread, workerData } from 'worker_threads'
// https://github.com/naptha/tesseract.js/blob/master/docs/api.md // https://github.com/naptha/tesseract.js/blob/master/docs/api.md
let tesseractWorker: TesseractWorker = null let tesseractWorker: TesseractWorker = null
export async function initTesseractWorker(): Promise<TesseractWorker> { export async function initTesseractWorker(): Promise<TesseractWorker> {
const worker = createWorker({ const worker = createWorker({
cacheMethod: 'refresh', cacheMethod: 'refresh',
// logger: (m) => console.log(m), // logger: (m) => console.log(m),
}) })
await worker.load() await worker.load()
await worker.loadLanguage('hun+eng') await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng') await worker.initialize('hun+eng')
return worker return worker
// await worker.terminate(); // await worker.terminate();
} }
let resolveLoaded: () => void = null let resolveLoaded: () => void = null
export const tesseractLoaded: Promise<void> = new Promise((resolve) => { export const tesseractLoaded: Promise<void> = new Promise((resolve) => {
resolveLoaded = resolve resolveLoaded = resolve
}) })
initTesseractWorker().then((worker) => { initTesseractWorker().then((worker) => {
tesseractWorker = worker tesseractWorker = worker
if (isMainThread) { if (isMainThread) {
logger.Log('Tesseract loaded on main thread') logger.Log('Tesseract loaded on main thread')
} else { } else {
const { workerIndex }: { workerIndex: number } = workerData const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`) logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
} }
resolveLoaded() resolveLoaded()
}) })
export async function recognizeTextFromBase64(base64: string): Promise<string> { export async function recognizeTextFromBase64(base64: string): Promise<string> {
const { const {
data: { text }, data: { text },
} = await tesseractWorker.recognize(base64) } = await tesseractWorker.recognize(base64)
return text return text
} }
export async function terminateWorker(): Promise<void | ConfigResult> { export async function terminateWorker(): Promise<void | ConfigResult> {
if (tesseractWorker) { if (tesseractWorker) {
return tesseractWorker.terminate() return tesseractWorker.terminate()
} }
return return
} }

409
src/utils/utils.ts
View File

@@ -19,23 +19,23 @@
------------------------------------------------------------------------- */ ------------------------------------------------------------------------- */
export default { export default {
ReadFile: ReadFile, ReadFile: ReadFile,
ReadJSON: ReadJSON, ReadJSON: ReadJSON,
WriteFile: WriteFile, WriteFile: WriteFile,
writeFileAsync: writeFileAsync, writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile, AppendToFile: AppendToFile,
FileExists: FileExists, FileExists: FileExists,
CreatePath: CreatePath, CreatePath: CreatePath,
WatchFile: WatchFile, WatchFile: WatchFile,
ReadDir: ReadDir, ReadDir: ReadDir,
CopyFile: CopyFile, CopyFile: CopyFile,
GetDateString: GetDateString, GetDateString: GetDateString,
formatUrl: formatUrl, formatUrl: formatUrl,
deleteFile: deleteFile, deleteFile: deleteFile,
uploadFile: uploadFile, uploadFile: uploadFile,
statFile: statFile, statFile: statFile,
renameFile: renameFile, renameFile: renameFile,
deleteDir: deleteDir, deleteDir: deleteDir,
} }
import fs from 'fs' import fs from 'fs'
@@ -45,251 +45,254 @@ import logger from '../utils/logger'
import { Request } from '../types/basicTypes' import { Request } from '../types/basicTypes'
interface URLFormatOptions { interface URLFormatOptions {
pathname?: string pathname?: string
query?: { [key: string]: string } query?: { [key: string]: string }
} }
function formatUrl(options: URLFormatOptions): string { function formatUrl(options: URLFormatOptions): string {
const path = options.pathname || '' const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) { if (!options.query || Object.keys(options.query).length === 0) {
return path return path
} }
const queryString = const queryString =
'?' + '?' +
Object.keys(options.query) Object.keys(options.query)
.map((key) => { .map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}` return `${key}=${encodeURIComponent(options.query[key])}`
}) })
.join('&') .join('&')
return path + queryString return path + queryString
} }
function GetDateString( function GetDateString(
referenceDate?: Date | string, referenceDate?: Date | string,
noTime?: boolean noTime?: boolean
): string { ): string {
const date = referenceDate ? new Date(referenceDate) : new Date() const date = referenceDate ? new Date(referenceDate) : new Date()
if (noTime) { if (noTime) {
return ( return (
date.getFullYear() + date.getFullYear() +
'-' + '-' +
('0' + (date.getMonth() + 1)).slice(-2) + ('0' + (date.getMonth() + 1)).slice(-2) +
'-' + '-' +
('0' + date.getDate()).slice(-2) ('0' + date.getDate()).slice(-2)
) )
} else { } else {
return ( return (
date.getFullYear() + date.getFullYear() +
'-' + '-' +
('0' + (date.getMonth() + 1)).slice(-2) + ('0' + (date.getMonth() + 1)).slice(-2) +
'-' + '-' +
('0' + date.getDate()).slice(-2) + ('0' + date.getDate()).slice(-2) +
' ' + ' ' +
('0' + date.getHours()).slice(-2) + ('0' + date.getHours()).slice(-2) +
':' + ':' +
('0' + date.getMinutes()).slice(-2) + ('0' + date.getMinutes()).slice(-2) +
':' + ':' +
('0' + date.getSeconds()).slice(-2) ('0' + date.getSeconds()).slice(-2)
) )
} }
} }
function CopyFile(from: string, to: string): void { function CopyFile(from: string, to: string): void {
CreatePath(to) CreatePath(to)
fs.copyFileSync(from, to) fs.copyFileSync(from, to)
} }
function ReadDir(path: string, listHidden?: boolean): Array<string> { function ReadDir(path: string, listHidden?: boolean): Array<string> {
if (listHidden) { if (listHidden) {
return fs.readdirSync(path) return fs.readdirSync(path)
} else { } else {
return fs.readdirSync(path).filter((file) => { return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.') return !file.startsWith('.')
}) })
} }
} }
function ReadJSON(name: string): any { function ReadJSON(name: string): any {
try { try {
return JSON.parse(ReadFile(name)) return JSON.parse(ReadFile(name))
} catch (err) { } catch (err) {
console.error(err) console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name) throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
} }
} }
function ReadFile(name: string): string { function ReadFile(name: string): string {
if (!FileExists(name)) { if (!FileExists(name)) {
throw new Error('No such file: ' + name) throw new Error('No such file: ' + name)
} }
return fs.readFileSync(name, 'utf8') return fs.readFileSync(name, 'utf8')
} }
function FileExists(path: string): boolean { function FileExists(path: string): boolean {
return fs.existsSync(path) return fs.existsSync(path)
} }
function WatchFile(file: string, callback: Function): void { function WatchFile(file: string, callback: Function): void {
if (FileExists(file)) { if (FileExists(file)) {
fs.watchFile(file, () => { fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => { fs.readFile(file, 'utf8', (err, data) => {
if (err) { if (err) {
// console.log(err) // console.log(err)
} else { } else {
callback(data) callback(data)
} }
}) })
}) })
} else { } else {
throw new Error(`${file} does not exits to watch`) throw new Error(`${file} does not exits to watch`)
} }
} }
function CreatePath(path: string, onlyPath?: boolean): void { function CreatePath(path: string, onlyPath?: boolean): void {
if (FileExists(path)) { if (FileExists(path)) {
return return
} }
const spath = path.split('/') const spath = path.split('/')
let currDir = spath[0] let currDir = spath[0]
for (let i = 1; i < spath.length; i++) { for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) { if (currDir !== '' && !fs.existsSync(currDir)) {
try { try {
fs.mkdirSync(currDir) fs.mkdirSync(currDir)
} catch (err) { } catch (err) {
console.log('Failed to make ' + currDir + ' directory... ') console.log('Failed to make ' + currDir + ' directory... ')
console.error(err) console.error(err)
} }
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
} }
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
} }
function WriteFile(content: string, path: string): void { function WriteFile(content: string, path: string): void {
CreatePath(path) CreatePath(path)
fs.writeFileSync(path, content) fs.writeFileSync(path, content)
} }
function writeFileAsync(content: string, path: string): void { function writeFileAsync(content: string, path: string): void {
CreatePath(path) CreatePath(path)
fs.writeFile(path, content, function (err) { fs.writeFile(path, content, function (err) {
if (err) { if (err) {
logger.Log( logger.Log(
'Error writing file: ' + path + ' (sync)', 'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg') logger.GetColor('redbg')
) )
} }
}) })
} }
function AppendToFile(data: string, file: string): void { function AppendToFile(data: string, file: string): void {
CreatePath(file) CreatePath(file)
try { try {
fs.appendFileSync(file, '\n' + data) fs.appendFileSync(file, '\n' + data)
} catch (err) { } catch (err) {
logger.Log( logger.Log(
'Error appendig to file log file: ' + file + ' (sync)', 'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg') logger.GetColor('redbg')
) )
logger.Log(data) logger.Log(data)
console.error(err) console.error(err)
} }
} }
function deleteFile(fname: string): Boolean { function deleteFile(fname: string): Boolean {
if (FileExists(fname)) { if (FileExists(fname)) {
fs.unlinkSync(fname) fs.unlinkSync(fname)
return true return true
} }
return false return false
} }
function deleteDir(dirName: string): Boolean { function deleteDir(dirName: string): Boolean {
if (FileExists(dirName)) { if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true }) fs.rmSync(dirName, { recursive: true })
return true return true
} }
return false return false
} }
function uploadFile( function uploadFile(
req: Request, req: Request,
path: string path: string
): Promise<{ ): Promise<{
body: Request['body'] body: Request['body']
fileName: string fileName: string
filePath: string filePath: string
}> { }> {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
try { try {
if (!req.files) { if (!req.files) {
logger.Log( logger.Log(
`Unable to upload file, req.files is undefined`, `Unable to upload file, req.files is undefined`,
logger.GetColor('redbg') logger.GetColor('redbg')
) )
return return
} }
const file = req.files.file const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() }) // FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
CreatePath(path, true) CreatePath(path, true)
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/') let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) { if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0] const id = uuidv4().split('-')[0]
const temp = file.name.split('.') const temp = file.name.split('.')
const extension = temp.pop() const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName fileDestination = path + '/' + fileName
} }
file.mv(fileDestination, (err: Error) => { file.mv(fileDestination, (err: Error) => {
if (err) { if (err) {
logger.Log(`Unable to upload file!`, logger.GetColor('redbg')) logger.Log(
console.error(err) `Unable to upload file!`,
reject(err) logger.GetColor('redbg')
} else { )
logger.Log( console.error(err)
`Uploaded: ${fileName} to ${fileDestination}`, reject(err)
logger.GetColor('blue') } else {
) logger.Log(
resolve({ `Uploaded: ${fileName} to ${fileDestination}`,
body: req.body, logger.GetColor('blue')
fileName: fileName, )
filePath: fileDestination, resolve({
}) body: req.body,
fileName: fileName,
filePath: fileDestination,
})
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
} }
}) })
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
} }
function statFile(file: string): fs.Stats { function statFile(file: string): fs.Stats {
if (FileExists(file)) { if (FileExists(file)) {
return fs.statSync(file) return fs.statSync(file)
} else { } else {
return null return null
} }
} }
function renameFile(oldPath: string, newPath: string): string { function renameFile(oldPath: string, newPath: string): string {
if (FileExists(oldPath)) { if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath) fs.renameSync(oldPath, newPath)
return newPath return newPath
} else { } else {
return null return null
} }
} }

372
src/utils/workerPool.ts
View File

@@ -29,268 +29,274 @@ import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
import type { WorkerResult } from './classes' import type { WorkerResult } from './classes'
interface WorkerObj { interface WorkerObj {
worker: Worker worker: Worker
index: number index: number
free: Boolean free: Boolean
} }
export interface TaskObject { export interface TaskObject {
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions' type:
data: | 'work'
| { | 'dbEdit'
searchIn: number[] | 'newQuestions'
question: Question | 'newdb'
subjName: string | 'dbClean'
testUrl?: string | 'rmQuestions'
questionData?: QuestionData data:
searchInAllIfNoResult?: boolean | {
searchTillMatchPercent?: number searchIn: number[]
[key: string]: any question: Question
} subjName: string
| { dbIndex: number; edits: Edits } testUrl?: string
| QuestionDb questionData?: QuestionData
| Result searchInAllIfNoResult?: boolean
| { searchTillMatchPercent?: number
questions: Question[] [key: string]: any
subjToClean: string }
overwriteFromDate: number | { dbIndex: number; edits: Edits }
qdbIndex: number | QuestionDb
} | Result
| { | {
questionIndexesToRemove: number[][] questions: Question[]
subjIndex: number subjToClean: string
qdbIndex: number overwriteFromDate: number
recievedQuestions: Question[] qdbIndex: number
} }
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
} }
interface PendingJob { interface PendingJob {
workData: TaskObject workData: TaskObject
doneEvent: DoneEvent doneEvent: DoneEvent
targetWorkerIndex?: number targetWorkerIndex?: number
} }
interface JobEvent extends EventEmitter { interface JobEvent extends EventEmitter {
on(event: 'jobDone', listener: () => void): this on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean emit(event: 'jobDone'): boolean
} }
interface DoneEvent extends EventEmitter { interface DoneEvent extends EventEmitter {
once(event: 'done', listener: (result: WorkerResult) => void): this once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean emit(event: 'done', res: WorkerResult): boolean
} }
const alertOnPendingCount = 50 const alertOnPendingCount = 50
const workerFile = './src/utils/classes.ts' const workerFile = './src/utils/classes.ts'
let workers: Array<WorkerObj> let workers: Array<WorkerObj>
const pendingJobs: { const pendingJobs: {
[id: string]: PendingJob [id: string]: PendingJob
} = {} } = {}
const jobEvents: JobEvent = new EventEmitter() const jobEvents: JobEvent = new EventEmitter()
jobEvents.on('jobDone', () => { jobEvents.on('jobDone', () => {
processJob() processJob()
}) })
jobEvents.on('newJob', () => { jobEvents.on('newJob', () => {
processJob() processJob()
}) })
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function handleWorkerError(worker: WorkerObj, err: Error) { function handleWorkerError(worker: WorkerObj, err: Error) {
// TODO: restart worker if exited or things like that // TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg')) logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker) console.error(err, worker)
} }
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit // TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> { export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
logger.DebugLog('MSGING ALL WORKER', 'job', 1) logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => { return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = [] const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => { workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index)) promises.push(doALongTask(data, worker.index))
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
}) })
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
} }
export function doALongTask( export function doALongTask(
obj: TaskObject, obj: TaskObject,
targetWorkerIndex?: number targetWorkerIndex?: number
): Promise<WorkerResult> { ): Promise<WorkerResult> {
if (Object.keys(pendingJobs).length > alertOnPendingCount) { if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log( logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${ `More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length Object.keys(pendingJobs).length
})`, })`,
logger.GetColor('redbg') logger.GetColor('redbg')
) )
} }
const jobId = uuidv4() const jobId = uuidv4()
// FIXME: delete doneEvent? // FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter() const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = { pendingJobs[jobId] = {
workData: obj, workData: obj,
targetWorkerIndex: targetWorkerIndex, targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent, doneEvent: doneEvent,
} }
jobEvents.emit('newJob') jobEvents.emit('newJob')
return new Promise((resolve) => { return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => { doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone') jobEvents.emit('jobDone')
resolve(result) resolve(result)
})
}) })
})
} }
export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> { export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> {
if (workers) { if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg')) logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null return null
} }
workers = [] workers = []
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) { if (process.env.NS_THREAD_COUNT) {
logger.Log( logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`, `Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red') logger.GetColor('red')
) )
} }
for (let i = 0; i < threadCount; i++) { for (let i = 0; i < threadCount; i++) {
workers.push({ workers.push({
worker: getAWorker(i, initData), worker: getAWorker(i, initData),
index: i, index: i,
free: true, free: true,
}) })
} }
return workers return workers
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function processJob() { function processJob() {
if (Object.keys(pendingJobs).length > 0) { if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs) const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj let jobKey: string, freeWorker: WorkerObj
let i = 0 let i = 0
while (!freeWorker && i < keys.length) { while (!freeWorker && i < keys.length) {
jobKey = keys[i] jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) { if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) { if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex] freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog( logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`, `RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job', 'job',
1 1
) )
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
}
}
i++
} }
} else {
freeWorker = workers.find((worker) => { if (!freeWorker) {
return worker.free logger.DebugLog('NO FREE WORKER', 'job', 1)
}) return
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
} }
}
i++
}
if (!freeWorker) { if (freeWorker.free) {
logger.DebugLog('NO FREE WORKER', 'job', 1) freeWorker.free = false
return }
} const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
if (freeWorker.free) { doSomething(freeWorker, job.workData)
freeWorker.free = false .then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
} }
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
} }
function getAWorker(i: number, initData: Array<QuestionDb>) { function getAWorker(i: number, initData: Array<QuestionDb>) {
const worker = workerTs(workerFile, { const worker = workerTs(workerFile, {
workerData: { workerData: {
workerIndex: i, workerIndex: i,
initData: initData, initData: initData,
}, },
}) })
worker.setMaxListeners(50) worker.setMaxListeners(50)
worker.on('error', (err) => { worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg')) logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err) console.error(err)
}) })
worker.on('exit', (code) => { worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted // TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log( logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`, `[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green') code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
) )
}) })
return worker return worker
} }
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
function doSomething(currWorker: WorkerObj, obj: TaskObject) { function doSomething(currWorker: WorkerObj, obj: TaskObject) {
const { /* index, */ worker } = currWorker const { /* index, */ worker } = currWorker
return new Promise((resolve) => { return new Promise((resolve) => {
worker.postMessage(obj) worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => { worker.once('message', (msg: WorkerResult) => {
resolve(msg) resolve(msg)
})
}) })
})
} }
const workerTs = ( const workerTs = (
file: string, file: string,
wkOpts: { wkOpts: {
workerData: { workerData: {
workerIndex: number workerIndex: number
initData: QuestionDb[] initData: QuestionDb[]
__filename?: string __filename?: string
}
eval?: boolean
} }
eval?: boolean
}
) => { ) => {
wkOpts.eval = true wkOpts.eval = true
wkOpts.workerData.__filename = file wkOpts.workerData.__filename = file
return new Worker( return new Worker(
` `
const wk = require('worker_threads'); const wk = require('worker_threads');
require('ts-node').register(); require('ts-node').register();
let file = wk.workerData.__filename; let file = wk.workerData.__filename;
delete wk.workerData.__filename; delete wk.workerData.__filename;
require(file); require(file);
`, `,
wkOpts wkOpts
) )
} }

2
submodules/qmining-page

Submodule submodules/qmining-page updated: 281d0e00ce...ed507dc39f