albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

prettier 4 tabwidth

Browse source
This commit is contained in:
mrfry 2022-12-10 15:34:54 +01:00
parent 00ec614f1d
commit 96b413a365
42 changed files with 7034 additions and 6905 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.prettierrc.js
View file

@ -1,6 +1,6 @@
module.exports = {
trailingComma: 'es5',
tabWidth: 2,
tabWidth: 4,
semi: false,
singleQuote: true,
arrowParens: 'always',

244
src/middlewares/auth.middleware.ts
View file

@ -27,148 +27,148 @@ import utils from '../utils/utils'
import dbtools from '../utils/dbtools'
interface Options {
userDB: Database
jsonResponse: boolean
exceptions: Array<string>
userDB: Database
jsonResponse: boolean
exceptions: Array<string>
}
export const testUser = {
id: 19,
avaiblePWRequests: 645,
pwRequestCount: 19,
created: new Date(),
pw: 'secret',
loginCount: 3,
createdBy: 1,
id: 19,
avaiblePWRequests: 645,
pwRequestCount: 19,
created: new Date(),
pw: 'secret',
loginCount: 3,
createdBy: 1,
}
function renderLogin(_req: Request, res: Response, jsonResponse: boolean) {
res.status(401) // Unauthorized
if (jsonResponse) {
res.json({
result: 'nouser',
msg: 'You are not logged in',
})
} else {
res.render('login', {
devel: process.env.NS_DEVEL,
})
}
res.status(401) // Unauthorized
if (jsonResponse) {
res.json({
result: 'nouser',
msg: 'You are not logged in',
})
} else {
res.render('login', {
devel: process.env.NS_DEVEL,
})
}
}
export default function (options: Options): RequestHandler {
const {
userDB,
jsonResponse,
exceptions,
}: {
userDB: Database
jsonResponse: boolean
exceptions: string[]
} = options
const {
userDB,
jsonResponse,
exceptions,
}: {
userDB: Database
jsonResponse: boolean
exceptions: string[]
} = options
return function (req: Request, res: Response, next: NextFunction) {
const sessionID = req.cookies.sessionID
const isException = exceptions.some((exc) => {
return req.url.split('?')[0] === exc
})
return function (req: Request, res: Response, next: NextFunction) {
const sessionID = req.cookies.sessionID
const isException = exceptions.some((exc) => {
return req.url.split('?')[0] === exc
})
if (process.env.NS_NOUSER) {
req.session = {
user: testUser,
sessionID: sessionID || 11111111111,
isException: false,
}
next()
return
}
if (process.env.NS_NOUSER) {
req.session = {
user: testUser,
sessionID: sessionID || 11111111111,
isException: false,
}
next()
return
}
// FIXME Allowing all urls with _next in it, but not in params
if (
req.url.split('?')[0].includes('_next') ||
req.url.split('?')[0].includes('well-known/acme-challenge')
) {
req.session = { isException: true }
next()
return
}
// FIXME Allowing all urls with _next in it, but not in params
if (
req.url.split('?')[0].includes('_next') ||
req.url.split('?')[0].includes('well-known/acme-challenge')
) {
req.session = { isException: true }
next()
return
}
if (!sessionID) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
const user = GetUserBySessionID(userDB, sessionID)
if (!user) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No user:${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
req.session = {
user: user,
sessionID: sessionID,
isException: isException,
}
logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1)
dbtools.Update(
userDB,
'sessions',
{
lastAccess: utils.GetDateString(),
},
{
id: sessionID,
}
)
dbtools.Update(
userDB,
'users',
{
lastAccess: utils.GetDateString(),
},
{
id: user.id,
}
)
if (!sessionID) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
const user = GetUserBySessionID(userDB, sessionID)
if (!user) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No user:${req.url}`, 'auth', 1)
renderLogin(req, res, jsonResponse)
return
}
req.session = {
user: user,
sessionID: sessionID,
isException: isException,
}
logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1)
dbtools.Update(
userDB,
'sessions',
{
lastAccess: utils.GetDateString(),
},
{
id: sessionID,
}
)
dbtools.Update(
userDB,
'users',
{
lastAccess: utils.GetDateString(),
},
{
id: user.id,
}
)
next()
}
}
function GetUserBySessionID(db: Database, sessionID: string) {
logger.DebugLog(`Getting user from db`, 'auth', 2)
logger.DebugLog(`Getting user from db`, 'auth', 2)
const session = dbtools.Select(db, 'sessions', {
id: sessionID,
})[0]
const session = dbtools.Select(db, 'sessions', {
id: sessionID,
})[0]
if (!session) {
return
}
if (!session) {
return
}
const user = dbtools.Select(db, 'users', {
id: session.userID,
})[0]
const user = dbtools.Select(db, 'users', {
id: session.userID,
})[0]
if (user) {
return user
}
if (user) {
return user
}
}

114
src/middlewares/reqlogger.middleware.ts
View file

@ -23,68 +23,70 @@ import type { Response, NextFunction } from 'express'
import type { Request } from '../types/basicTypes'
interface Options {
loggableKeywords: Array<string>
loggableModules: Array<string>
exceptions: Array<string>
excludeFromStats: Array<string>
loggableKeywords: Array<string>
loggableModules: Array<string>
exceptions: Array<string>
excludeFromStats: Array<string>
}
export default function (options: Options): any {
const loggableKeywords = options ? options.loggableKeywords : undefined
const loggableModules = options ? options.loggableModules : undefined
const exceptions = options.exceptions || []
const excludeFromStats = options.excludeFromStats || []
const loggableKeywords = options ? options.loggableKeywords : undefined
const loggableModules = options ? options.loggableModules : undefined
const exceptions = options.exceptions || []
const excludeFromStats = options.excludeFromStats || []
return function (req: Request, res: Response, next: NextFunction) {
res.on('finish', function () {
// TODO: test this
const isException = exceptions.some((ex) => {
return req.url.includes(ex)
})
return function (req: Request, res: Response, next: NextFunction) {
res.on('finish', function () {
// TODO: test this
const isException = exceptions.some((ex) => {
return req.url.includes(ex)
})
if (isException) {
return
}
if (isException) {
return
}
let hostname = 'NOHOST'
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
logger.Log('Hostname is undefined!', logger.GetColor('redbg'))
console.log(req.body)
console.log(req.query)
console.log(req.headers)
}
let hostname = 'NOHOST'
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
logger.Log('Hostname is undefined!', logger.GetColor('redbg'))
console.log(req.body)
console.log(req.query)
console.log(req.headers)
}
const hasLoggableKeyword =
loggableKeywords &&
loggableKeywords.some((keyword) => {
return req.url.includes(keyword)
const hasLoggableKeyword =
loggableKeywords &&
loggableKeywords.some((keyword) => {
return req.url.includes(keyword)
})
const hasLoggableModule =
loggableModules &&
loggableModules.some((keyword) => {
return hostname.includes(keyword)
})
const toLog = hasLoggableModule || hasLoggableKeyword
logger.LogReq(req, true, res.statusCode)
if (toLog) {
logger.LogReq(req)
}
const shouldLogStat = !excludeFromStats.some((ex) => {
return req.url.includes(ex)
})
if (res.statusCode !== 404 && shouldLogStat) {
logger.LogStat(
req.url,
hostname,
req.session && req.session.user
? req.session.user.id
: 'NOUSER'
)
}
})
const hasLoggableModule =
loggableModules &&
loggableModules.some((keyword) => {
return hostname.includes(keyword)
})
const toLog = hasLoggableModule || hasLoggableKeyword
logger.LogReq(req, true, res.statusCode)
if (toLog) {
logger.LogReq(req)
}
const shouldLogStat = !excludeFromStats.some((ex) => {
return req.url.includes(ex)
})
if (res.statusCode !== 404 && shouldLogStat) {
logger.LogStat(
req.url,
hostname,
req.session && req.session.user ? req.session.user.id : 'NOUSER'
)
}
})
next()
}
next()
}
}

80
src/middlewares/socketAuth.middleware.ts
View file

@ -27,60 +27,60 @@ import { Socket } from '../types/basicTypes'
import { testUser } from './auth.middleware'
interface Options {
userDB: any
userDB: any
}
export default function SocketAuth(options: Options): any {
const { userDB } = options
const { userDB } = options
return (socket: Socket, next: (arg0?: any) => void) => {
try {
const cookies = cookie.parse(socket.handshake.headers.cookie || '')
const sessionID = cookies.sessionID
return (socket: Socket, next: (arg0?: any) => void) => {
try {
const cookies = cookie.parse(socket.handshake.headers.cookie || '')
const sessionID = cookies.sessionID
if (process.env.NS_NOUSER) {
socket.user = testUser
next()
return
}
if (process.env.NS_NOUSER) {
socket.user = testUser
next()
return
}
if (!sessionID) {
next(new Error('Not authenticated, please log in'))
return
}
if (!sessionID) {
next(new Error('Not authenticated, please log in'))
return
}
const user = GetUserBySessionID(userDB, sessionID)
const user = GetUserBySessionID(userDB, sessionID)
if (!user) {
next(new Error('Not authenticated, please log in'))
return
}
socket.user = user
next()
} catch (e) {
next(new Error('Authentication server error'))
console.error('Authentication server error')
console.error(e)
if (!user) {
next(new Error('Not authenticated, please log in'))
return
}
socket.user = user
next()
} catch (e) {
next(new Error('Authentication server error'))
console.error('Authentication server error')
console.error(e)
}
}
}
}
function GetUserBySessionID(db: any, sessionID: string) {
logger.DebugLog(`Getting user from db`, 'auth', 2)
logger.DebugLog(`Getting user from db`, 'auth', 2)
const session = dbtools.Select(db, 'sessions', {
id: sessionID,
})[0]
const session = dbtools.Select(db, 'sessions', {
id: sessionID,
})[0]
if (!session) {
return
}
if (!session) {
return
}
const user = dbtools.Select(db, 'users', {
id: session.userID,
})[0]
const user = dbtools.Select(db, 'users', {
id: session.userID,
})[0]
if (user) {
return user
}
if (user) {
return user
}
}

308
src/modules/api/api.ts
View file

@ -46,188 +46,188 @@ let httpServer: http.Server
let httpsServer: https.Server
function GetApp(): ModuleType {
const app = express()
const app = express()
const publicDir = publicdirs[0]
if (!publicDir) {
throw new Error(`No public dir! ( API )`)
}
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
// -------------------------------------------------------------------------------------------
app.use(
express.urlencoded({
limit: '10mb',
extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '10mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/api/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: true,
exceptions: [
'/register',
'/favicon.ico',
'/login',
'/postfeedback',
'/fosuploader',
'/badtestsender',
],
})
)
app.use(
fileUpload({
limits: { fileSize: 50 * 1024 * 1024 },
})
)
// -------------------------------------------------------------------------------------------
let rootRedirectURL = ''
function reloadRootRedirectURL() {
if (utils.FileExists(rootRedirectToFile)) {
rootRedirectURL = utils.ReadFile(rootRedirectToFile)
const publicDir = publicdirs[0]
if (!publicDir) {
throw new Error(`No public dir! ( API )`)
}
}
const filesToWatch = [
{
fname: rootRedirectToFile,
logMsg: 'Root redirect URL changed',
action: reloadRootRedirectURL,
},
]
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
function Load() {
filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) {
utils.WatchFile(ftw.fname, () => {
logger.Log(ftw.logMsg)
ftw.action()
// -------------------------------------------------------------------------------------------
app.use(
express.urlencoded({
limit: '10mb',
extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '10mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/api/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: true,
exceptions: [
'/register',
'/favicon.ico',
'/login',
'/postfeedback',
'/fosuploader',
'/badtestsender',
],
})
ftw.action()
} else {
logger.Log(
`File ${ftw.fname} does not exists to watch!`,
logger.GetColor('redbg')
)
}
})
}
)
app.use(
fileUpload({
limits: { fileSize: 50 * 1024 * 1024 },
})
)
// -------------------------------------------------------------------------------------------
Load()
let rootRedirectURL = ''
// --------------------------------------------------------------
app.get('/', function (req: Request, res: any) {
logger.LogReq(req)
if (reloadRootRedirectURL) {
res.redirect(rootRedirectURL)
} else {
res.json({ msg: 'hi c:' })
function reloadRootRedirectURL() {
if (utils.FileExists(rootRedirectToFile)) {
rootRedirectURL = utils.ReadFile(rootRedirectToFile)
}
}
})
// -------------------------------------------------------------------------------------------
const filesToWatch = [
{
fname: rootRedirectToFile,
logMsg: 'Root redirect URL changed',
action: reloadRootRedirectURL,
},
]
const submoduleDatas = setupSubModules(app)
function Load() {
filesToWatch.forEach((ftw) => {
if (utils.FileExists(ftw.fname)) {
utils.WatchFile(ftw.fname, () => {
logger.Log(ftw.logMsg)
ftw.action()
})
ftw.action()
} else {
logger.Log(
`File ${ftw.fname} does not exists to watch!`,
logger.GetColor('redbg')
)
}
})
}
// -------------------------------------------------------------------------------------------
Load()
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
// --------------------------------------------------------------
// -------------------------------------------------------------------------------------------
app.get('/', function (req: Request, res: any) {
logger.LogReq(req)
if (reloadRootRedirectURL) {
res.redirect(rootRedirectURL)
} else {
res.json({ msg: 'hi c:' })
}
})
app.get('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
// -------------------------------------------------------------------------------------------
app.post('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
const submoduleDatas = setupSubModules(app)
// -------------------------------------------------------------------------------------------
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
// -------------------------------------------------------------------------------------------
app.get('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
app.post('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
function DailyAction() {
submoduleDatas.forEach((data) => {
if (data.dailyAction) {
data.dailyAction()
}
})
}
function DailyAction() {
submoduleDatas.forEach((data) => {
if (data.dailyAction) {
data.dailyAction()
}
if (data.load) {
data.load()
}
})
}
submoduleDatas.forEach((data) => {
if (data.load) {
data.load()
return {
dailyAction: DailyAction,
app: app,
}
})
return {
dailyAction: DailyAction,
app: app,
}
}
function setupSubModules(
parentApp: express.Application,
moduleSpecificData?: any
parentApp: express.Application,
moduleSpecificData?: any
): Submodule[] {
const submoduleDir = './submodules/'
const absolutePath = __dirname + '/' + submoduleDir
if (!utils.FileExists(absolutePath)) {
return null
}
const files = utils.ReadDir(absolutePath)
const moduleDatas: Submodule[] = []
files.forEach((file) => {
if (!file.endsWith('.js')) {
return
const submoduleDir = './submodules/'
const absolutePath = __dirname + '/' + submoduleDir
if (!utils.FileExists(absolutePath)) {
return null
}
const submodulePath = submoduleDir + file
const files = utils.ReadDir(absolutePath)
const moduleDatas: Submodule[] = []
files.forEach((file) => {
if (!file.endsWith('.js')) {
return
}
const submodulePath = submoduleDir + file
try {
logger.Log(`Loading submodule '${file}' for '${moduleName}'...`)
const mod = require(submodulePath).default // eslint-disable-line
const loadedModData = mod.setup({
app: parentApp,
userDB: userDB,
url: url,
publicdirs: publicdirs,
moduleSpecificData: moduleSpecificData,
httpServer: httpServer,
httpsServer: httpsServer,
})
moduleDatas.push(loadedModData || {})
} catch (e) {
logger.Log(`Error loading submodule from ${submodulePath}`)
console.error(e)
}
})
try {
logger.Log(`Loading submodule '${file}' for '${moduleName}'...`)
const mod = require(submodulePath).default // eslint-disable-line
const loadedModData = mod.setup({
app: parentApp,
userDB: userDB,
url: url,
publicdirs: publicdirs,
moduleSpecificData: moduleSpecificData,
httpServer: httpServer,
httpsServer: httpsServer,
})
moduleDatas.push(loadedModData || {})
} catch (e) {
logger.Log(`Error loading submodule from ${submodulePath}`)
console.error(e)
}
})
return moduleDatas
return moduleDatas
}
export default {
name: moduleName,
getApp: GetApp,
setup: (data: SetupData): void => {
userDB = data.userDB
url = data.url
publicdirs = data.publicdirs
httpServer = data.httpServer
httpsServer = data.httpsServer
},
name: moduleName,
getApp: GetApp,
setup: (data: SetupData): void => {
userDB = data.userDB
url = data.url
publicdirs = data.publicdirs
httpServer = data.httpServer
httpsServer = data.httpsServer
},
}

58
src/modules/api/msgsDbStruct.ts
View file

@ -19,36 +19,36 @@
------------------------------------------------------------------------- */
const DbStruct = {
msgs: {
tableStruct: {
id: {
type: 'integer',
primary: true,
autoIncrement: true,
},
sender: {
type: 'integer',
notNull: true,
},
reciever: {
type: 'integer',
notNull: true,
},
msg: {
type: 'text',
},
type: {
type: 'text',
},
date: {
type: 'integer',
},
unread: {
type: 'integer',
defaultZero: true,
},
msgs: {
tableStruct: {
id: {
type: 'integer',
primary: true,
autoIncrement: true,
},
sender: {
type: 'integer',
notNull: true,
},
reciever: {
type: 'integer',
notNull: true,
},
msg: {
type: 'text',
},
type: {
type: 'text',
},
date: {
type: 'integer',
},
unread: {
type: 'integer',
defaultZero: true,
},
},
},
},
}
export default DbStruct

423
src/modules/api/submodules/chat.ts
View file

@ -30,65 +30,65 @@ const msgDbPath = './data/dbs/msgs.db'
const msgPaginationLimit = 15
interface ExtendedSocket extends Socket {
user: User
user: User
}
interface Message {
id: number
sender: number
reciever: number
msg: string
type: string
date: number
unread: number
id: number
sender: number
reciever: number
msg: string
type: string
date: number
unread: number
}
function setup(data: SubmoduleData): void {
const { app, httpServer, httpsServer, userDB, publicdirs } = data
const msgDB = dbtools.GetDB(msgDbPath)
const { app, httpServer, httpsServer, userDB, publicdirs } = data
const msgDB = dbtools.GetDB(msgDbPath)
const publicDir = publicdirs[0]
const uloadFiles = publicDir + 'chatFiles'
logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`)
// https://socket.io/docs/v4/handling-cors/#Configuration
const io = new socket(httpsServer || httpServer, {
cors: {
credentials: true,
origin: true,
},
})
const publicDir = publicdirs[0]
const uloadFiles = publicDir + 'chatFiles'
logger.Log(`Starting Socket.io Server on ${httpsServer ? 'https' : 'http'}`)
// https://socket.io/docs/v4/handling-cors/#Configuration
const io = new socket(httpsServer || httpServer, {
cors: {
credentials: true,
origin: true,
},
})
function chatMessageRead({
sender,
reciever,
}: {
sender: number
reciever: number
}) {
dbtools.runStatement(
msgDB,
`update msgs
function chatMessageRead({
sender,
reciever,
}: {
sender: number
reciever: number
}) {
dbtools.runStatement(
msgDB,
`update msgs
set unread = 0
where sender = ${sender} and reciever = ${reciever}`,
'run'
)
io.sockets.in(sender.toString()).emit('chat message read', {
userReadMsg: reciever,
})
}
'run'
)
io.sockets.in(sender.toString()).emit('chat message read', {
userReadMsg: reciever,
})
}
io.use(socketAuth({ userDB: userDB }))
io.use(socketAuth({ userDB: userDB }))
io.on('connection', (socket: ExtendedSocket) => {
const userid = socket.user.id
io.on('connection', (socket: ExtendedSocket) => {
const userid = socket.user.id
socket.on('join', function (/*data*/) {
socket.join(userid.toString())
socket.on('join', function (/*data*/) {
socket.join(userid.toString())
const groups: number[] = dbtools
.runStatement(
msgDB,
`select * from
const groups: number[] = dbtools
.runStatement(
msgDB,
`select * from
(
select sender as a
from msgs
@ -99,172 +99,177 @@ function setup(data: SubmoduleData): void {
where sender = ${userid} or reciever = ${userid}
)t
order by t.a asc`
)
.reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a)
return acc
}, [])
)
.reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a)
return acc
}, [])
socket.emit('prev messages', {
prevMsgs: groups.map((to) => {
const first: Message = dbtools.runStatement(
msgDB,
`select * from msgs
socket.emit('prev messages', {
prevMsgs: groups.map((to) => {
const first: Message = dbtools.runStatement(
msgDB,
`select * from msgs
where sender = ${userid} and reciever = ${to} or
sender = ${to} and reciever = ${userid}
order by date desc
limit 1`
)[0]
return first
}),
})
)[0]
return first
}),
})
socket.on('get chat messages', (data) => {
const { chatPartner, from } = data
socket.on('get chat messages', (data) => {
const { chatPartner, from } = data
const msgs = dbtools.runStatement(
msgDB,
`select * from msgs
const msgs = dbtools.runStatement(
msgDB,
`select * from msgs
where (sender = ${userid} and reciever = ${chatPartner} or
sender = ${chatPartner} and reciever = ${userid})
${from ? `and date < ${from}` : ''}
order by date desc
limit ${msgPaginationLimit}`
)
)
socket.emit('get chat messages', {
requestsdMsgs: msgs,
hasMore: msgs.length === msgPaginationLimit,
socket.emit('get chat messages', {
requestsdMsgs: msgs,
hasMore: msgs.length === msgPaginationLimit,
})
// Read update
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on('chat message read', (data) => {
const { chatPartner } = data
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on(
'chat message',
(message: { reciever: string; msg: string; type: string }) => {
const { reciever, msg, type } = message
if (!reciever || !msg || !msg.trim() || !type) {
return
}
const recieverUser = dbtools.Select(userDB, 'users', {
id: reciever,
})[0]
if (!recieverUser) {
socket.emit('chat message', {
success: false,
date: new Date().getTime(),
sender: reciever,
reciever: userid,
type: 'text',
msg: `A #${reciever} számú felhasználó nem létezik`,
})
return
}
const msgObj = {
sender: userid,
reciever: parseInt(reciever),
msg: dbtools.sanitizeQuery(msg),
type: type || 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
if (userid !== parseInt(reciever)) {
io.sockets
.in(reciever.toString())
.emit('chat message', msgObj)
}
}
)
})
// Read update
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on('chat message read', (data) => {
const { chatPartner } = data
chatMessageRead({ sender: chatPartner, reciever: userid })
})
socket.on(
'chat message',
(message: { reciever: string; msg: string; type: string }) => {
const { reciever, msg, type } = message
if (!reciever || !msg || !msg.trim() || !type) {
return
}
const recieverUser = dbtools.Select(userDB, 'users', {
id: reciever,
})[0]
if (!recieverUser) {
socket.emit('chat message', {
success: false,
date: new Date().getTime(),
sender: reciever,
reciever: userid,
type: 'text',
msg: `A #${reciever} számú felhasználó nem létezik`,
})
return
}
const msgObj = {
sender: userid,
reciever: parseInt(reciever),
msg: dbtools.sanitizeQuery(msg),
type: type || 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
if (userid !== parseInt(reciever)) {
io.sockets.in(reciever.toString()).emit('chat message', msgObj)
}
}
)
// socket.on('disconnect', () => {})
// socket.on('close', () => {})
})
// socket.on('disconnect', () => {})
// socket.on('close', () => {})
})
app.post('/postchatfile', function (req: Request, res) {
logger.LogReq(req)
utils
.uploadFile(req, uloadFiles)
.then((result) => {
res.json({
success: true,
path: result.filePath.replace(publicDir, ''),
})
})
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.post('/postchatfile', function (req: Request, res) {
logger.LogReq(req)
utils
.uploadFile(req, uloadFiles)
.then((result) => {
res.json({
success: true,
path: result.filePath.replace(publicDir, ''),
})
})
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.post('/postfeedbackfile', function (req: Request, res) {
logger.LogReq(req)
const user: User = req.session.user
app.post('/postfeedbackfile', function (req: Request, res) {
logger.LogReq(req)
const user: User = req.session.user
utils
.uploadFile(req, uloadFiles)
.then(({ filePath }) => {
const fileName = filePath.replace(publicDir, '')
const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => {
return fileName.toLowerCase().includes(ext)
})
const msgObj = {
sender: user.id,
reciever: 1,
msg: fileName,
type: isImage ? 'img' : 'file',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
utils
.uploadFile(req, uloadFiles)
.then(({ filePath }) => {
const fileName = filePath.replace(publicDir, '')
const isImage = ['png', 'jpg', 'jpeg', 'gif'].some((ext) => {
return fileName.toLowerCase().includes(ext)
})
const msgObj = {
sender: user.id,
reciever: 1,
msg: fileName,
type: isImage ? 'img' : 'file',
date: new Date().getTime(),
unread: 1,
res.json({ success: true })
io.sockets.in('1').emit('chat message', msgObj)
})
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.post(
'/postfeedback',
function (req: Request<{ content: string }>, res) {
logger.LogReq(req)
const user: User = req.session.user
const { content } = req.body
if (!content || !content.trim()) {
res.json({ success: false })
return
}
const msgObj = {
sender: user.id,
reciever: 1,
msg: dbtools.sanitizeQuery(req.body.content),
type: 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
res.json({ success: true })
io.sockets.in('1').emit('chat message', msgObj)
}
dbtools.Insert(msgDB, 'msgs', msgObj)
)
res.json({ success: true })
io.sockets.in('1').emit('chat message', msgObj)
})
.catch(() => {
res.json({ success: false, msg: 'error during uploading' })
return
})
})
app.get('/hasNewMsg', (req: Request, res) => {
const user: User = req.session.user
const userid: number = user.id
app.post('/postfeedback', function (req: Request<{ content: string }>, res) {
logger.LogReq(req)
const user: User = req.session.user
const { content } = req.body
if (!content || !content.trim()) {
res.json({ success: false })
return
}
const msgObj = {
sender: user.id,
reciever: 1,
msg: dbtools.sanitizeQuery(req.body.content),
type: 'text',
date: new Date().getTime(),
unread: 1,
}
dbtools.Insert(msgDB, 'msgs', msgObj)
res.json({ success: true })
io.sockets.in('1').emit('chat message', msgObj)
})
app.get('/hasNewMsg', (req: Request, res) => {
const user: User = req.session.user
const userid: number = user.id
const groups = dbtools
.runStatement(
msgDB,
`select * from
const groups = dbtools
.runStatement(
msgDB,
`select * from
(
select sender as a
from msgs
@ -275,35 +280,35 @@ function setup(data: SubmoduleData): void {
where sender = ${userid} or reciever = ${userid}
)t
order by t.a asc`
)
.reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a)
return acc
}, [])
)
.reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a)
return acc
}, [])
const prevMsgs = groups.map((to: number) => {
const first = dbtools.runStatement(
msgDB,
`select * from msgs
const prevMsgs = groups.map((to: number) => {
const first = dbtools.runStatement(
msgDB,
`select * from msgs
where sender = ${userid} and reciever = ${to} or
sender = ${to} and reciever = ${userid}
order by date desc
limit 1`
)[0]
return first
})
)[0]
return first
})
res.json({
unreads: prevMsgs.reduce((acc: number[], msg: Message) => {
if (msg && msg.unread === 1 && msg.sender !== userid) {
acc.push(msg.sender)
}
return acc
}, []),
res.json({
unreads: prevMsgs.reduce((acc: number[], msg: Message) => {
if (msg && msg.unread === 1 && msg.sender !== userid) {
acc.push(msg.sender)
}
return acc
}, []),
})
})
})
}
export default {
setup: setup,
setup: setup,
}

12
src/modules/api/submodules/feedback.ts
View file

@ -26,15 +26,15 @@ import { Request, SubmoduleData } from '../../../types/basicTypes'
const uloadFiles = 'data/f'
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.route('/fosuploader').post(function (req: Request, res: Response) {
utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
res.redirect('/f/' + fileName)
app.route('/fosuploader').post(function (req: Request, res: Response) {
utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
res.redirect('/f/' + fileName)
})
})
})
}
export default {
setup: setup,
setup: setup,
}

1030
src/modules/api/submodules/forum.ts
View file

File diff suppressed because it is too large Load diff

1877
src/modules/api/submodules/qminingapi.ts
View file

File diff suppressed because it is too large Load diff

151
src/modules/api/submodules/quickvote.ts
View file

@ -26,98 +26,101 @@ import type { Response } from 'express'
const quickVoteResultsDir = 'stats/qvote'
const quickVotes = 'stats/qvote/votes.json'
interface QuickVotes {
voteNames?: string[]
voteNames?: string[]
}
interface QuickVote {
votes: {
[key: string]: string
}
sum: {
[key: string]: number
}
votes: {
[key: string]: string
}
sum: {
[key: string]: number
}
}
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/quickvote', (req: Request, res: Response) => {
const key = req.query.key.toString()
const val: string = req.query.val
const user: User = req.session.user
app.get('/quickvote', (req: Request, res: Response) => {
const key = req.query.key.toString()
const val: string = req.query.val
const user: User = req.session.user
if (!key || !val) {
res.render('votethank', {
results: 'error',
msg: 'no key or val query param!',
})
return
}
if (!key || !val) {
res.render('votethank', {
results: 'error',
msg: 'no key or val query param!',
})
return
}
// FIXME: check vote type in file
let votes: QuickVotes = {}
if (utils.FileExists(quickVotes)) {
votes = utils.ReadJSON(quickVotes)
} else {
logger.Log(
`No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`,
logger.GetColor('blue')
)
res.render('votethank', {
result: 'no such pool',
})
return
}
// FIXME: check vote type in file
let votes: QuickVotes = {}
if (utils.FileExists(quickVotes)) {
votes = utils.ReadJSON(quickVotes)
} else {
logger.Log(
`No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`,
logger.GetColor('blue')
)
res.render('votethank', {
result: 'no such pool',
})
return
}
if (!votes.voteNames.includes(key)) {
logger.Log(
`No such vote "${key}" ( #${user.id}: ${key}-${val} )`,
logger.GetColor('blue')
)
res.render('votethank', {
result: 'no such pool',
})
return
}
if (!votes.voteNames.includes(key)) {
logger.Log(
`No such vote "${key}" ( #${user.id}: ${key}-${val} )`,
logger.GetColor('blue')
)
res.render('votethank', {
result: 'no such pool',
})
return
}
const voteFile = quickVoteResultsDir + '/' + key + '.json'
const voteFile = quickVoteResultsDir + '/' + key + '.json'
let voteData: QuickVote = {
votes: {},
sum: {},
}
let voteData: QuickVote = {
votes: {},
sum: {},
}
if (utils.FileExists(voteFile)) {
voteData = utils.ReadJSON(voteFile)
} else {
utils.CreatePath(quickVoteResultsDir)
}
if (utils.FileExists(voteFile)) {
voteData = utils.ReadJSON(voteFile)
} else {
utils.CreatePath(quickVoteResultsDir)
}
const prevVote = voteData.votes[user.id]
const prevVote = voteData.votes[user.id]
voteData.votes[user.id] = val
if (voteData.sum[val]) {
voteData.sum[val]++
} else {
voteData.sum[val] = 1
}
if (prevVote) {
if (voteData.sum[prevVote]) {
voteData.sum[prevVote] -= 1
}
}
voteData.votes[user.id] = val
if (voteData.sum[val]) {
voteData.sum[val]++
} else {
voteData.sum[val] = 1
}
if (prevVote) {
if (voteData.sum[prevVote]) {
voteData.sum[prevVote] -= 1
}
}
logger.Log(`Vote from #${user.id}: ${key}: ${val}`, logger.GetColor('blue'))
res.render('votethank', {
result: prevVote ? 'already voted' : 'success',
prevVote: prevVote,
msg: 'vote added',
logger.Log(
`Vote from #${user.id}: ${key}: ${val}`,
logger.GetColor('blue')
)
res.render('votethank', {
result: prevVote ? 'already voted' : 'success',
prevVote: prevVote,
msg: 'vote added',
})
utils.WriteFile(JSON.stringify(voteData), voteFile)
})
utils.WriteFile(JSON.stringify(voteData), voteFile)
})
}
export default {
setup: setup,
setup: setup,
}

181
src/modules/api/submodules/ranklist.ts
View file

@ -23,121 +23,128 @@ import utils from '../../../utils/utils'
import { Request, SubmoduleData, User } from '../../../types/basicTypes'
interface Subjects {
[key: string]: number
[key: string]: number
}
interface IdStat {
count: number
newQuestions: number
allQuestions: number
subjs: Subjects
count: number
newQuestions: number
allQuestions: number
subjs: Subjects
}
interface IdStats {
[key: string]: IdStat
[key: string]: IdStat
}
interface IdStatWithUID extends IdStat {
userId: number
userId: number
}
const idStatFile = 'stats/idstats'
const idvStatFile = 'stats/idvstats'
function mergeObjSum(a: Subjects, b: Subjects) {
const res = { ...b }
Object.keys(a).forEach((key) => {
if (res[key]) {
res[key] += a[key]
} else {
res[key] = a[key]
}
})
const res = { ...b }
Object.keys(a).forEach((key) => {
if (res[key]) {
res[key] += a[key]
} else {
res[key] = a[key]
}
})
return res
return res
}
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/ranklist', (req: Request, res) => {
logger.LogReq(req)
let result: IdStats
const querySince: string = req.query.since
const user: User = req.session.user
app.get('/ranklist', (req: Request, res) => {
logger.LogReq(req)
let result: IdStats
const querySince: string = req.query.since
const user: User = req.session.user
if (!querySince) {
result = utils.ReadJSON(idStatFile)
} else {
try {
const since = new Date(querySince)
if (!(since instanceof Date) || isNaN(since.getTime())) {
throw new Error('Not a date')
}
const data = utils.ReadJSON(idvStatFile)
result = {}
Object.keys(data).forEach((key) => {
const dailyStat = data[key]
if (new Date(key) > since) {
Object.keys(dailyStat).forEach((userId) => {
const userStat = dailyStat[userId]
const uidRes = result[userId]
if (!uidRes) {
result[userId] = userStat
} else {
result[userId] = {
count: uidRes.count + userStat.count,
newQuestions: uidRes.newQuestions + userStat.newQuestions,
allQuestions: uidRes.allQuestions + userStat.allQuestions,
subjs: mergeObjSum(uidRes.subjs, userStat.subjs),
if (!querySince) {
result = utils.ReadJSON(idStatFile)
} else {
try {
const since = new Date(querySince)
if (!(since instanceof Date) || isNaN(since.getTime())) {
throw new Error('Not a date')
}
}
const data = utils.ReadJSON(idvStatFile)
result = {}
Object.keys(data).forEach((key) => {
const dailyStat = data[key]
if (new Date(key) > since) {
Object.keys(dailyStat).forEach((userId) => {
const userStat = dailyStat[userId]
const uidRes = result[userId]
if (!uidRes) {
result[userId] = userStat
} else {
result[userId] = {
count: uidRes.count + userStat.count,
newQuestions:
uidRes.newQuestions +
userStat.newQuestions,
allQuestions:
uidRes.allQuestions +
userStat.allQuestions,
subjs: mergeObjSum(
uidRes.subjs,
userStat.subjs
),
}
}
})
}
})
} catch (err) {
res.json({
msg: 'invalid date format, or other error occured',
})
}
}
const list: Array<IdStatWithUID> = []
const sum = {
count: 0,
newQuestions: 0,
allQuestions: 0,
}
Object.keys(result).forEach((key) => {
list.push({
userId: parseInt(key),
...result[key],
})
}
sum.count = sum.count + result[key].count
sum.newQuestions = sum.newQuestions + result[key].newQuestions
sum.allQuestions = sum.allQuestions + result[key].allQuestions
})
} catch (err) {
if (list.length === 0) {
res.json({
msg: 'There are no users in the stats db :c',
})
return
}
res.json({
msg: 'invalid date format, or other error occured',
since: querySince,
sum: sum,
list: list,
selfuserId: user.id,
})
}
}
const list: Array<IdStatWithUID> = []
const sum = {
count: 0,
newQuestions: 0,
allQuestions: 0,
}
Object.keys(result).forEach((key) => {
list.push({
userId: parseInt(key),
...result[key],
})
sum.count = sum.count + result[key].count
sum.newQuestions = sum.newQuestions + result[key].newQuestions
sum.allQuestions = sum.allQuestions + result[key].allQuestions
})
if (list.length === 0) {
res.json({
msg: 'There are no users in the stats db :c',
})
return
}
res.json({
since: querySince,
sum: sum,
list: list,
selfuserId: user.id,
})
})
}
export default {
setup: setup,
setup: setup,
}

152
src/modules/api/submodules/todos.ts
View file

@ -25,110 +25,110 @@ import utils from '../../../utils/utils'
import { Request, SubmoduleData } from '../../../types/basicTypes'
interface Categories {
[key: string]: {
name: string
color: string
}
[key: string]: {
name: string
color: string
}
}
enum CardState {
TODO = 'todo',
INPROGRESS = 'inprogress',
TESTING = 'testing',
DONE = 'done',
INPROD = 'inprod',
NOTPOSSIBLE = 'notpossible',
TODO = 'todo',
INPROGRESS = 'inprogress',
TESTING = 'testing',
DONE = 'done',
INPROD = 'inprod',
NOTPOSSIBLE = 'notpossible',
}
interface Card {
id: number
name: string
description: string
category: string
points: number
state: CardState
votes: number[]
id: number
name: string
description: string
category: string
points: number
state: CardState
votes: number[]
}
type Columns = {
[key in CardState]: {
name: string
clickable: boolean
}
[key in CardState]: {
name: string
clickable: boolean
}
}
interface Groups {
[key: string]: {
name: string
description: string
}
[key: string]: {
name: string
description: string
}
}
interface Todos {
categories: Categories
cards: Card[]
columns: Columns
groups: Groups
categories: Categories
cards: Card[]
columns: Columns
groups: Groups
}
const todosFile = 'data/todos.json'
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/voteTodo', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const id: string = req.query.id
const todos: Todos = utils.ReadJSON(todosFile)
app.get('/voteTodo', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const id: string = req.query.id
const todos: Todos = utils.ReadJSON(todosFile)
if (!id) {
res.json({
msg: 'id query undefined',
result: 'not ok',
})
}
if (!id) {
res.json({
msg: 'id query undefined',
result: 'not ok',
})
}
const cardIndex = todos.cards.findIndex((currcard) => {
return currcard.id === parseInt(id)
const cardIndex = todos.cards.findIndex((currcard) => {
return currcard.id === parseInt(id)
})
if (cardIndex === -1) {
res.json({
msg: 'card not found',
result: 'not ok',
})
return
}
const ind = todos.cards[cardIndex].votes.indexOf(userId)
if (ind === -1) {
todos.cards[cardIndex].votes.push(userId)
} else {
todos.cards[cardIndex].votes.splice(ind, 1)
}
utils.WriteFile(JSON.stringify(todos, null, 2), todosFile)
res.json({
todos: todos,
userId: userId,
msg: 'updated',
result: 'ok',
})
})
if (cardIndex === -1) {
res.json({
msg: 'card not found',
result: 'not ok',
})
return
}
const ind = todos.cards[cardIndex].votes.indexOf(userId)
if (ind === -1) {
todos.cards[cardIndex].votes.push(userId)
} else {
todos.cards[cardIndex].votes.splice(ind, 1)
}
app.get('/todos', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const todos = utils.ReadJSON(todosFile)
utils.WriteFile(JSON.stringify(todos, null, 2), todosFile)
res.json({
todos: todos,
userId: userId,
msg: 'updated',
result: 'ok',
res.json({
todos: todos,
userId: userId,
result: 'ok',
})
})
})
app.get('/todos', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const todos = utils.ReadJSON(todosFile)
res.json({
todos: todos,
userId: userId,
result: 'ok',
})
})
}
export default {
setup: setup,
setup: setup,
}

576
src/modules/api/submodules/userFiles.ts
View file

@ -27,320 +27,324 @@ import { Request, SubmoduleData, User } from '../../../types/basicTypes'
const dataFileName = '.data.json'
function listDir(publicDir: string, subdir: string, userFilesDir: string) {
const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '')
const dir = userFilesDir + '/' + safeSubdir
const usersFile = dir + '/' + dataFileName
const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '')
const dir = userFilesDir + '/' + safeSubdir
const usersFile = dir + '/' + dataFileName
if (!utils.FileExists(dir)) {
return {
success: false,
msg: `Directory ${subdir} does not exists`,
if (!utils.FileExists(dir)) {
return {
success: false,
msg: `Directory ${subdir} does not exists`,
}
}
}
if (!utils.FileExists(usersFile)) {
utils.WriteFile('{}', usersFile)
}
const users = utils.ReadJSON(usersFile)
if (!utils.FileExists(dir)) {
return {
success: false,
msg: `Path '${safeSubdir}' does not exists`,
if (!utils.FileExists(usersFile)) {
utils.WriteFile('{}', usersFile)
}
}
const users = utils.ReadJSON(usersFile)
return {
success: true,
files: utils.ReadDir(dir).reduce((acc, file) => {
const stat = fs.lstatSync(dir + '/' + file)
if (!utils.FileExists(dir)) {
return {
success: false,
msg: `Path '${safeSubdir}' does not exists`,
}
}
if (stat.isDirectory()) {
return acc
}
return {
success: true,
files: utils.ReadDir(dir).reduce((acc, file) => {
const stat = fs.lstatSync(dir + '/' + file)
acc.push({
name: file,
path: dir.replace(publicDir, '') + '/' + file,
size: stat.size,
date: stat.mtime.getTime(),
user: users && users[file] ? users[file].uid : -1,
views:
users && users[file] && users[file].views ? users[file].views : 0,
upvotes:
users && users[file] && users[file].upvotes
? users[file].upvotes
: [],
downvotes:
users && users[file] && users[file].downvotes
? users[file].downvotes
: [],
})
return acc
}, []),
}
if (stat.isDirectory()) {
return acc
}
acc.push({
name: file,
path: dir.replace(publicDir, '') + '/' + file,
size: stat.size,
date: stat.mtime.getTime(),
user: users && users[file] ? users[file].uid : -1,
views:
users && users[file] && users[file].views
? users[file].views
: 0,
upvotes:
users && users[file] && users[file].upvotes
? users[file].upvotes
: [],
downvotes:
users && users[file] && users[file].downvotes
? users[file].downvotes
: [],
})
return acc
}, []),
}
}
function setup(data: SubmoduleData): void {
const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data
const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data
app.use((req: Request, _res, next) => {
// /userFiles/test/2021-04-28_10-59.png
try {
if (req.url.includes('/userFiles/')) {
app.use((req: Request, _res, next) => {
// /userFiles/test/2021-04-28_10-59.png
try {
if (req.url.includes('/userFiles/')) {
logger.LogReq(req)
const safePath = decodeURIComponent(req.url)
.split('?')[0]
.replace(/\.+/g, '.')
.replace(/\/+/g, '/')
const x = safePath.split('/')
const dir = x[2]
const fname = x.pop()
const dataFilePath =
userFilesDir + '/' + dir + '/' + dataFileName
const data = utils.ReadJSON(dataFilePath)
if (data[fname]) {
if (!data[fname].views) {
data[fname].views = 0
}
data[fname].views = data[fname].views + 1
utils.WriteFile(JSON.stringify(data), dataFilePath)
}
}
} catch (e) {
console.error(e)
logger.Log(
`Error trying to update view count on ${req.url}`,
logger.GetColor('redbg')
)
}
next()
})
const publicDir = publicdirs[0]
const userFilesDir = publicDir + 'userFiles'
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
app.get('/listUserDir', (req: Request, res) => {
logger.LogReq(req)
const safePath = decodeURIComponent(req.url)
.split('?')[0]
.replace(/\.+/g, '.')
.replace(/\/+/g, '/')
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
const subdir: string = req.query.subdir
if (subdir) {
const result = listDir(publicDir, subdir, userFilesDir)
res.json(result)
} else {
res.json({
success: true,
dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => {
const stat = fs.lstatSync(userFilesDir + '/' + file)
if (!stat.isDirectory()) {
return acc
}
acc.push({
name: file,
date: stat.mtime.getTime(),
size: utils.ReadDir(userFilesDir + '/' + file).length,
})
return acc
}, []),
})
}
})
app.post(
'/deleteUserFile',
(req: Request<{ dir: string; fname: string }>, res) => {
logger.LogReq(req)
const dir: string = req.body.dir
const fname: string = req.body.fname
if (!dir || !fname) {
res.json({
success: false,
msg: `'dir' or 'fname' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '')
const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '')
const filePath = userFilesDir + '/' + safeDir + '/' + safeFname
if (!utils.FileExists(filePath)) {
res.json({
success: false,
msg: `path does not exists!`,
})
return
}
utils.deleteFile(filePath)
const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
delete users[safeFname]
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
}
)
app.post('/newUserDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const name: string = req.body.name
if (!name) {
res.json({
success: false,
msg: `name is undefined!`,
})
return
}
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
if (utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} already exists`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
res.json({
success: true,
})
})
app.post('/uploadUserFile', (req: Request<{ dir: string }>, res) => {
logger.LogReq(req)
const user: User = req.session.user
const dir = req.body.dir
if (!dir) {
res.json({
success: false,
msg: `dir '${dir}' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/')
if (!utils.FileExists(userFilesDir + '/' + safeDir)) {
res.json({
success: false,
msg: `dir '${dir}' does not exists!`,
})
return
}
utils
.uploadFile(req, userFilesDir + '/' + safeDir)
.then((body) => {
logger.Log(
`Successfull upload ${body.filePath}`,
logger.GetColor('blue')
)
const usersFile =
userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
users[body.fileName] = { uid: user.id }
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
})
.catch(() => {
res.json({ success: false, msg: 'something bad happened :s' })
})
})
app.post('/voteFile', (req: Request<{ path: string; to: string }>, res) => {
logger.LogReq(req)
const user: User = req.session.user
// { path: 'userFiles/test/2021-04-28_10-59.png', to: 'up' } 19
const { path, to } = req.body
const safePath = path.replace(/\.+/g, '.').replace(/\/+/g, '/')
const x = safePath.split('/')
const dir = x[2]
const dir = x[1]
const fname = x.pop()
const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName
const data = utils.ReadJSON(dataFilePath)
if (data[fname]) {
if (!data[fname].views) {
data[fname].views = 0
}
data[fname].views = data[fname].views + 1
if (!data[fname].upvotes) {
data[fname].upvotes = []
}
if (!data[fname].downvotes) {
data[fname].downvotes = []
}
utils.WriteFile(JSON.stringify(data), dataFilePath)
const removeVote = (from: number[], uid: number) => {
if (!from.includes(uid)) {
return from
}
return from.reduce((acc, id) => {
if (id !== uid) {
acc = [...acc, id]
}
return acc
}, [])
}
data[fname].downvotes = removeVote(data[fname].downvotes, user.id)
data[fname].upvotes = removeVote(data[fname].upvotes, user.id)
if (to === 'up') {
data[fname].upvotes = [...data[fname].upvotes, user.id]
} else if (to === 'down') {
data[fname].downvotes = [...data[fname].downvotes, user.id]
} else if (to === 'clear') {
// ... already cleared
}
utils.WriteFile(JSON.stringify(data), dataFilePath)
}
}
} catch (e) {
console.error(e)
logger.Log(
`Error trying to update view count on ${req.url}`,
logger.GetColor('redbg')
)
}
next()
})
const publicDir = publicdirs[0]
const userFilesDir = publicDir + 'userFiles'
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
app.get('/listUserDir', (req: Request, res) => {
logger.LogReq(req)
if (!utils.FileExists(userFilesDir)) {
utils.CreatePath(userFilesDir, true)
}
const subdir: string = req.query.subdir
if (subdir) {
const result = listDir(publicDir, subdir, userFilesDir)
res.json(result)
} else {
res.json({
success: true,
dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => {
const stat = fs.lstatSync(userFilesDir + '/' + file)
if (!stat.isDirectory()) {
return acc
}
acc.push({
name: file,
date: stat.mtime.getTime(),
size: utils.ReadDir(userFilesDir + '/' + file).length,
})
return acc
}, []),
})
}
})
app.post(
'/deleteUserFile',
(req: Request<{ dir: string; fname: string }>, res) => {
logger.LogReq(req)
const dir: string = req.body.dir
const fname: string = req.body.fname
if (!dir || !fname) {
res.json({
success: false,
msg: `'dir' or 'fname' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '')
const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '')
const filePath = userFilesDir + '/' + safeDir + '/' + safeFname
if (!utils.FileExists(filePath)) {
res.json({
success: false,
msg: `path does not exists!`,
})
return
}
utils.deleteFile(filePath)
const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
delete users[safeFname]
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
}
)
app.post('/newUserDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const name: string = req.body.name
if (!name) {
res.json({
success: false,
msg: `name is undefined!`,
})
return
}
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
if (utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} already exists`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
res.json({
success: true,
const result = listDir(publicDir, dir, userFilesDir)
res.json(result)
})
})
app.post('/uploadUserFile', (req: Request<{ dir: string }>, res) => {
logger.LogReq(req)
app.post('/deleteDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const { name } = req.body
const user: User = req.session.user
const dir = req.body.dir
if (!dir) {
res.json({
success: false,
msg: `dir '${dir}' is undefined!`,
})
return
}
const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/')
if (!utils.FileExists(userFilesDir + '/' + safeDir)) {
res.json({
success: false,
msg: `dir '${dir}' does not exists!`,
})
return
}
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
utils
.uploadFile(req, userFilesDir + '/' + safeDir)
.then((body) => {
logger.Log(
`Successfull upload ${body.filePath}`,
logger.GetColor('blue')
)
const usersFile = userFilesDir + '/' + safeDir + '/' + dataFileName
const users = utils.ReadJSON(usersFile)
users[body.fileName] = { uid: user.id }
utils.WriteFile(JSON.stringify(users), usersFile)
res.json({
success: true,
})
})
.catch(() => {
res.json({ success: false, msg: 'something bad happened :s' })
})
})
app.post('/voteFile', (req: Request<{ path: string; to: string }>, res) => {
logger.LogReq(req)
const user: User = req.session.user
// { path: 'userFiles/test/2021-04-28_10-59.png', to: 'up' } 19
const { path, to } = req.body
const safePath = path.replace(/\.+/g, '.').replace(/\/+/g, '/')
const x = safePath.split('/')
const dir = x[1]
const fname = x.pop()
const dataFilePath = userFilesDir + '/' + dir + '/' + dataFileName
const data = utils.ReadJSON(dataFilePath)
if (data[fname]) {
if (!data[fname].upvotes) {
data[fname].upvotes = []
}
if (!data[fname].downvotes) {
data[fname].downvotes = []
}
const removeVote = (from: number[], uid: number) => {
if (!from.includes(uid)) {
return from
if (!utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} does not exist!`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
const result = listDir(publicDir, name, userFilesDir)
if (result.files.length === 0) {
utils.deleteDir(userFilesDir + '/' + safeName)
} else {
res.json({ succes: false, msg: `Dir ${name} is not empty!` })
return
}
return from.reduce((acc, id) => {
if (id !== uid) {
acc = [...acc, id]
}
return acc
}, [])
}
data[fname].downvotes = removeVote(data[fname].downvotes, user.id)
data[fname].upvotes = removeVote(data[fname].upvotes, user.id)
if (to === 'up') {
data[fname].upvotes = [...data[fname].upvotes, user.id]
} else if (to === 'down') {
data[fname].downvotes = [...data[fname].downvotes, user.id]
} else if (to === 'clear') {
// ... already cleared
}
utils.WriteFile(JSON.stringify(data), dataFilePath)
}
const result = listDir(publicDir, dir, userFilesDir)
res.json(result)
})
app.post('/deleteDir', (req: Request<{ name: string }>, res) => {
logger.LogReq(req)
const { name } = req.body
const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
if (!utils.FileExists(userFilesDir + '/' + safeName)) {
res.json({
success: false,
msg: `Dir ${name} does not exist!`,
})
return
}
utils.CreatePath(userFilesDir + '/' + safeName, true)
const result = listDir(publicDir, name, userFilesDir)
if (result.files.length === 0) {
utils.deleteDir(userFilesDir + '/' + safeName)
} else {
res.json({ succes: false, msg: `Dir ${name} is not empty!` })
return
}
res.json({ succes: true })
})
res.json({ succes: true })
})
}
export default {
setup: setup,
setup: setup,
}

587
src/modules/api/submodules/userManagement.ts
View file

@ -24,10 +24,10 @@ import type { Database } from 'better-sqlite3'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import {
Request,
SubmoduleData,
User,
Submodule,
Request,
SubmoduleData,
User,
Submodule,
} from '../../../types/basicTypes'
import dbtools from '../../../utils/dbtools'
@ -38,321 +38,322 @@ const maxPWCount = 3
const daysAfterUserGetsPWs = 7 // days after user gets pw-s
interface Session {
id: string
userId: number
createDate: string
lastAccess: string
isScript: number
id: string
userId: number
createDate: string
lastAccess: string
isScript: number
}
function BackupDB(usersDbBackupPath: string, userDB: Database) {
logger.Log('Backing up auth DB ...')
utils.CreatePath(usersDbBackupPath, true)
userDB
.backup(
`${usersDbBackupPath}/users.${utils
.GetDateString()
.replace(/ /g, '_')}.db`
)
.then(() => {
logger.Log('Auth DB backup complete!')
})
.catch((err: Error) => {
logger.Log('Auth DB backup failed!', logger.GetColor('redbg'))
console.error(err)
})
logger.Log('Backing up auth DB ...')
utils.CreatePath(usersDbBackupPath, true)
userDB
.backup(
`${usersDbBackupPath}/users.${utils
.GetDateString()
.replace(/ /g, '_')}.db`
)
.then(() => {
logger.Log('Auth DB backup complete!')
})
.catch((err: Error) => {
logger.Log('Auth DB backup failed!', logger.GetColor('redbg'))
console.error(err)
})
}
function setup(data: SubmoduleData): Submodule {
const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
app.get('/avaiblePWS', (req: Request, res: any) => {
logger.LogReq(req)
app.get('/avaiblePWS', (req: Request, res: any) => {
logger.LogReq(req)
const user: User = req.session.user
const user: User = req.session.user
res.json({
success: true,
userCreated: user.created,
availablePWS: user.avaiblePWRequests,
requestedPWS: user.pwRequestCount,
maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(user.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
app.post('/getpw', function (req: Request, res: any) {
logger.LogReq(req)
const requestingUser = req.session.user
if (requestingUser.avaiblePWRequests <= 0) {
res.json({
result: 'error',
success: false,
msg: 'Too many passwords requested or cant request password yet, try later',
})
logger.Log(
`User #${requestingUser.id} requested too much passwords`,
logger.GetColor('cyan')
)
return
}
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: requestingUser.avaiblePWRequests - 1,
pwRequestCount: requestingUser.pwRequestCount + 1,
},
{
id: requestingUser.id,
}
)
const pw = uuidv4()
const insertRes = dbtools.Insert(userDB, 'users', {
pw: pw,
avaiblePWRequests: 0,
created: utils.GetDateString(),
createdBy: requestingUser.id,
})
logger.Log(
`User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`,
logger.GetColor('cyan')
)
res.json({
pw: pw,
success: true,
userCreated: requestingUser.created,
availablePWS: requestingUser.avaiblePWRequests,
requestedPWS: requestingUser.pwRequestCount,
maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(requestingUser.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
app.post('/login', (req: Request, res: any) => {
logger.LogReq(req)
const pw = req.body.pw
? req.body.pw.replace(/'/g, '').replace(/"/g, '').replace(/;/g, '')
: false
const isScript = req.body.script
const user: User = dbtools.Select(userDB, 'users', {
pw: pw,
})[0]
if (user) {
const sessionID = uuidv4()
const existingSessions = dbtools
.Select(userDB, 'sessions', {
userID: user.id,
isScript: isScript ? 1 : 0,
})
.sort((a: Session, b: Session) => {
return (
new Date(a.lastAccess).getTime() - new Date(b.lastAccess).getTime()
)
res.json({
success: true,
userCreated: user.created,
availablePWS: user.avaiblePWRequests,
requestedPWS: user.pwRequestCount,
maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(user.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
const diff = existingSessions.length - minimumAlowwedSessions
if (diff > 0) {
logger.Log(
`Multiple ${isScript ? 'script' : 'website'} sessions ( ${
existingSessions.length
} ) for #${user.id}, deleting olds`,
logger.GetColor('cyan')
app.post('/getpw', function (req: Request, res: any) {
logger.LogReq(req)
const requestingUser = req.session.user
if (requestingUser.avaiblePWRequests <= 0) {
res.json({
result: 'error',
success: false,
msg: 'Too many passwords requested or cant request password yet, try later',
})
logger.Log(
`User #${requestingUser.id} requested too much passwords`,
logger.GetColor('cyan')
)
return
}
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: requestingUser.avaiblePWRequests - 1,
pwRequestCount: requestingUser.pwRequestCount + 1,
},
{
id: requestingUser.id,
}
)
for (let i = 0; i < diff; i++) {
const id = existingSessions[i].id
dbtools.Delete(userDB, 'sessions', {
id: id,
isScript: isScript ? 1 : 0,
})
}
}
dbtools.Update(
userDB,
'users',
{
loginCount: user.loginCount + 1,
lastLogin: utils.GetDateString(),
},
{
id: user.id,
}
)
const pw = uuidv4()
const insertRes = dbtools.Insert(userDB, 'users', {
pw: pw,
avaiblePWRequests: 0,
created: utils.GetDateString(),
createdBy: requestingUser.id,
})
dbtools.Insert(userDB, 'sessions', {
id: sessionID,
userID: user.id,
isScript: isScript ? 1 : 0,
createDate: utils.GetDateString(),
})
// https://www.npmjs.com/package/cookie
res.cookie('sessionID', sessionID, {
domain: domain,
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.cookie('sessionID', sessionID, {
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.json({
result: 'success',
msg: 'you are now logged in',
})
logger.Log(
`Successfull login to ${
isScript ? 'script' : 'website'
} with user ID: #${user.id}`,
logger.GetColor('cyan')
)
} else {
logger.Log(
`Login attempt with invalid pw: ${pw} to ${
isScript ? 'script' : 'website'
}`,
logger.GetColor('cyan')
)
res.json({
result: 'error',
msg: 'Invalid password',
})
}
})
app.get('/logout', (req: Request, res: any) => {
logger.LogReq(req)
const sessionID = req.cookies.sessionID
const user: User = req.session.user
const { all } = req.query
if (!user) {
res.json({
msg: 'You are not logged in',
success: false,
})
return
}
logger.Log(
`Successfull logout with user ID: #${user.id}`,
logger.GetColor('cyan')
)
if (all) {
dbtools.Delete(userDB, 'sessions', {
userID: user.id,
})
} else {
dbtools.Delete(userDB, 'sessions', {
id: sessionID,
})
}
res.clearCookie('sessionID').json({
msg: 'Successfull logout',
result: 'success',
})
})
function getDayDiff(dateString: string | Date) {
const msdiff = new Date().getTime() - new Date(dateString).getTime()
return Math.floor(msdiff / (1000 * 3600 * 24))
}
function IncrementAvaiblePWs() {
// FIXME: check this if this is legit and works
logger.Log('Incrementing avaible PW-s ...')
const users: Array<User> = dbtools.SelectAll(userDB, 'users')
const day = new Date().getDay()
if (day === 1) {
users.forEach((user) => {
const dayDiff = getDayDiff(user.created)
if (dayDiff < daysAfterUserGetsPWs) {
logger.Log(
`User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`,
logger.Log(
`User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`,
logger.GetColor('cyan')
)
return
}
)
if (user.avaiblePWRequests >= maxPWCount) {
return
res.json({
pw: pw,
success: true,
userCreated: requestingUser.created,
availablePWS: requestingUser.avaiblePWRequests,
requestedPWS: requestingUser.pwRequestCount,
maxPWCount: maxPWCount,
daysAfterUserGetsPWs: daysAfterUserGetsPWs,
dayDiff: getDayDiff(requestingUser.created),
userCount: dbtools.TableInfo(userDB, 'users').dataCount,
})
})
app.post('/login', (req: Request, res: any) => {
logger.LogReq(req)
const pw = req.body.pw
? req.body.pw.replace(/'/g, '').replace(/"/g, '').replace(/;/g, '')
: false
const isScript = req.body.script
const user: User = dbtools.Select(userDB, 'users', {
pw: pw,
})[0]
if (user) {
const sessionID = uuidv4()
const existingSessions = dbtools
.Select(userDB, 'sessions', {
userID: user.id,
isScript: isScript ? 1 : 0,
})
.sort((a: Session, b: Session) => {
return (
new Date(a.lastAccess).getTime() -
new Date(b.lastAccess).getTime()
)
})
const diff = existingSessions.length - minimumAlowwedSessions
if (diff > 0) {
logger.Log(
`Multiple ${isScript ? 'script' : 'website'} sessions ( ${
existingSessions.length
} ) for #${user.id}, deleting olds`,
logger.GetColor('cyan')
)
for (let i = 0; i < diff; i++) {
const id = existingSessions[i].id
dbtools.Delete(userDB, 'sessions', {
id: id,
isScript: isScript ? 1 : 0,
})
}
}
dbtools.Update(
userDB,
'users',
{
loginCount: user.loginCount + 1,
lastLogin: utils.GetDateString(),
},
{
id: user.id,
}
)
dbtools.Insert(userDB, 'sessions', {
id: sessionID,
userID: user.id,
isScript: isScript ? 1 : 0,
createDate: utils.GetDateString(),
})
// https://www.npmjs.com/package/cookie
res.cookie('sessionID', sessionID, {
domain: domain,
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.cookie('sessionID', sessionID, {
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
sameSite: 'none',
secure: true,
})
res.json({
result: 'success',
msg: 'you are now logged in',
})
logger.Log(
`Successfull login to ${
isScript ? 'script' : 'website'
} with user ID: #${user.id}`,
logger.GetColor('cyan')
)
} else {
logger.Log(
`Login attempt with invalid pw: ${pw} to ${
isScript ? 'script' : 'website'
}`,
logger.GetColor('cyan')
)
res.json({
result: 'error',
msg: 'Invalid password',
})
}
})
app.get('/logout', (req: Request, res: any) => {
logger.LogReq(req)
const sessionID = req.cookies.sessionID
const user: User = req.session.user
const { all } = req.query
if (!user) {
res.json({
msg: 'You are not logged in',
success: false,
})
return
}
logger.Log(
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`,
logger.GetColor('cyan')
`Successfull logout with user ID: #${user.id}`,
logger.GetColor('cyan')
)
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: maxPWCount,
},
{
id: user.id,
}
)
})
if (all) {
dbtools.Delete(userDB, 'sessions', {
userID: user.id,
})
} else {
dbtools.Delete(userDB, 'sessions', {
id: sessionID,
})
}
res.clearCookie('sessionID').json({
msg: 'Successfull logout',
result: 'success',
})
})
function getDayDiff(dateString: string | Date) {
const msdiff = new Date().getTime() - new Date(dateString).getTime()
return Math.floor(msdiff / (1000 * 3600 * 24))
}
users.forEach((user) => {
const dayDiff = getDayDiff(user.created)
if (dayDiff === daysAfterUserGetsPWs) {
logger.Log(
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`,
logger.GetColor('cyan')
)
function IncrementAvaiblePWs() {
// FIXME: check this if this is legit and works
logger.Log('Incrementing avaible PW-s ...')
const users: Array<User> = dbtools.SelectAll(userDB, 'users')
const day = new Date().getDay()
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: maxPWCount,
},
{
id: user.id,
}
)
}
})
}
if (day === 1) {
users.forEach((user) => {
const dayDiff = getDayDiff(user.created)
if (dayDiff < daysAfterUserGetsPWs) {
logger.Log(
`User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`,
logger.GetColor('cyan')
)
return
}
return {
dailyAction: () => {
BackupDB(usersDbBackupPath, userDB)
IncrementAvaiblePWs()
},
}
if (user.avaiblePWRequests >= maxPWCount) {
return
}
logger.Log(
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`,
logger.GetColor('cyan')
)
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: maxPWCount,
},
{
id: user.id,
}
)
})
}
users.forEach((user) => {
const dayDiff = getDayDiff(user.created)
if (dayDiff === daysAfterUserGetsPWs) {
logger.Log(
`Setting avaible PW-s for user #${user.id}: ${user.avaiblePWRequests} -> ${maxPWCount}`,
logger.GetColor('cyan')
)
dbtools.Update(
userDB,
'users',
{
avaiblePWRequests: maxPWCount,
},
{
id: user.id,
}
)
}
})
}
return {
dailyAction: () => {
BackupDB(usersDbBackupPath, userDB)
IncrementAvaiblePWs()
},
}
}
export default {
setup: setup,
setup: setup,
}

142
src/modules/api/usersDBStruct.ts
View file

@ -19,78 +19,78 @@
------------------------------------------------------------------------- */
const DBStruct = {
users: {
tableStruct: {
id: {
type: 'integer',
primary: true,
autoIncrement: true,
},
pw: {
type: 'text',
notNull: true,
unique: true,
},
notes: {
type: 'text',
},
loginCount: {
type: 'number',
defaultZero: true,
},
created: {
type: 'text',
notNull: true,
},
lastLogin: {
type: 'text',
},
lastAccess: {
type: 'text',
},
avaiblePWRequests: {
type: 'number',
defaultZero: true,
},
pwRequestCount: {
type: 'number',
defaultZero: true,
},
createdBy: {
type: 'number',
},
users: {
tableStruct: {
id: {
type: 'integer',
primary: true,
autoIncrement: true,
},
pw: {
type: 'text',
notNull: true,
unique: true,
},
notes: {
type: 'text',
},
loginCount: {
type: 'number',
defaultZero: true,
},
created: {
type: 'text',
notNull: true,
},
lastLogin: {
type: 'text',
},
lastAccess: {
type: 'text',
},
avaiblePWRequests: {
type: 'number',
defaultZero: true,
},
pwRequestCount: {
type: 'number',
defaultZero: true,
},
createdBy: {
type: 'number',
},
},
},
},
sessions: {
foreignKey: [
{
keysFrom: ['userID'],
table: 'users',
keysTo: ['id'],
},
],
tableStruct: {
id: {
type: 'text',
primary: true,
notNull: true,
},
userID: {
type: 'number',
notNull: true,
},
createDate: {
type: 'text',
notNull: true,
},
lastAccess: {
type: 'text',
},
isScript: {
type: 'number',
notNull: true,
},
sessions: {
foreignKey: [
{
keysFrom: ['userID'],
table: 'users',
keysTo: ['id'],
},
],
tableStruct: {
id: {
type: 'text',
primary: true,
notNull: true,
},
userID: {
type: 'number',
notNull: true,
},
createDate: {
type: 'text',
notNull: true,
},
lastAccess: {
type: 'text',
},
isScript: {
type: 'number',
notNull: true,
},
},
},
},
}
export default DBStruct

140
src/modules/dataEditor/dataEditor.ts
View file

@ -36,85 +36,85 @@ let publicdirs: string[] = []
let nextdir = ''
function GetApp(): ModuleType {
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '5mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: false,
exceptions: ['/favicon.ico'],
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '5mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: false,
exceptions: ['/favicon.ico'],
})
)
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
}
next()
})
)
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
// --------------------------------------------------------------
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1)
app.get(`/${route}`, function (_req: Request, res) {
res.redirect(`${route}.html`)
})
})
}
next()
})
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
// --------------------------------------------------------------
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1)
app.get(`/${route}`, function (_req: Request, res) {
res.redirect(`${route}.html`)
})
app.get('/', function (req: Request, res) {
res.end('hai')
logger.LogReq(req)
})
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.get('/', function (req: Request, res) {
res.end('hai')
logger.LogReq(req)
})
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})
return {
app: app,
}
return {
app: app,
}
}
export default {
name: 'Data editor',
getApp: GetApp,
setup: (data: SetupData): void => {
userDB = data.userDB
publicdirs = data.publicdirs
nextdir = data.nextdir
},
name: 'Data editor',
getApp: GetApp,
setup: (data: SetupData): void => {
userDB = data.userDB
publicdirs = data.publicdirs
nextdir = data.nextdir
},
}

72
src/modules/main/main.ts
View file

@ -32,47 +32,47 @@ let publicdirs: string[] = []
let url = '' // http(s)//asd.basd
function GetApp(): ModuleType {
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/main/views', './src/sharedViews'])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.json() as RequestHandler)
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
// --------------------------------------------------------------
app.get('/', function (_req, res) {
res.render('main', {
siteurl: url,
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/main/views', './src/sharedViews'])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
})
app.get('*', function (_req, res) {
res.status(404).render('404')
})
app.use(express.json() as RequestHandler)
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
app.post('*', function (_req, res) {
res.status(404).render('404')
})
// --------------------------------------------------------------
return {
app: app,
}
app.get('/', function (_req, res) {
res.render('main', {
siteurl: url,
})
})
app.get('*', function (_req, res) {
res.status(404).render('404')
})
app.post('*', function (_req, res) {
res.status(404).render('404')
})
return {
app: app,
}
}
export default {
name: 'Main',
getApp: GetApp,
setup: (data: SetupData): void => {
url = data.url
publicdirs = data.publicdirs
},
name: 'Main',
getApp: GetApp,
setup: (data: SetupData): void => {
url = data.url
publicdirs = data.publicdirs
},
}

391
src/modules/qmining/qmining.ts
View file

@ -36,212 +36,219 @@ let userDB: Database
let nextdir = ''
function GetApp(): ModuleType {
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
app.use(
express.json({
limit: '5mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/qmining/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: false,
exceptions: ['/favicon.ico', '/img/frylabs-logo_large_transparent.png'],
})
)
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
}
next()
})
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
const linksFile = 'data/links.json'
let links: { [key: string]: string } = {}
function loadDonateURL() {
try {
links = utils.ReadJSON(linksFile)
} catch (err) {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
console.error(err)
}
}
loadDonateURL()
if (utils.FileExists(linksFile)) {
utils.WatchFile(linksFile, (newData: string) => {
logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`)
loadDonateURL()
})
} else {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
}
// --------------------------------------------------------------
// REDIRECTS
// --------------------------------------------------------------
// to be backwards compatible
app.get('/ask', function (req: Request, res) {
logger.DebugLog(`Qmining module ask redirect`, 'ask', 1)
res.redirect(
`http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}`
app.use(
express.urlencoded({
limit: '5mb',
extended: true,
}) as RequestHandler
)
})
const simpleRedirects = [
{
from: '/dataeditor',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/install',
to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js',
},
{
from: '/servergit',
to: 'https://gitlab.com/MrFry/mrfrys-node-server',
},
{
from: '/scriptgit',
to: 'https://gitlab.com/MrFry/moodle-test-userscript',
},
{
from: '/qminingSite',
to: 'https://gitlab.com/MrFry/qmining-page',
},
{
from: '/classesgit',
to: 'https://gitlab.com/MrFry/question-classes',
},
{
from: '/addQuestion',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/donate',
to: links.donate,
},
{
from: '/menuClick',
to: '/',
},
{
from: '/legacy',
to: '/allQuestions.html',
},
{
from: '/subjectBrowser',
to: '/allQuestions.html',
},
{
from: '/lred',
to: '/allQuestions',
},
{
from: '/allqr',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/allqr.txt',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/infos',
to: 'https://api.frylabs.net/infos?version=true&motd=true&subjinfo=true',
nolog: true,
},
{
from: '/irc',
to: '/chat',
},
{
from: '/patreon',
to: links.patreon,
},
]
simpleRedirects.forEach((redirect) => {
app.get(redirect.from, function (req: Request, res) {
if (!redirect.nolog) {
logger.LogReq(req)
}
logger.DebugLog(`Qmining module ${redirect.from} redirect`, 'infos', 1)
let target = redirect.to
if (!redirect.to.includes('https://')) {
target += utils.formatUrl({ query: req.query })
}
res.redirect(target)
app.use(
express.json({
limit: '5mb',
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/qmining/views', './src/sharedViews'])
app.use(
auth({
userDB: userDB,
jsonResponse: false,
exceptions: [
'/favicon.ico',
'/img/frylabs-logo_large_transparent.png',
],
})
)
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
}
next()
})
})
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
const linksFile = 'data/links.json'
let links: { [key: string]: string } = {}
// --------------------------------------------------------------
function loadDonateURL() {
try {
links = utils.ReadJSON(linksFile)
} catch (err) {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
console.error(err)
}
}
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
loadDonateURL()
routes.forEach((route: string) => {
logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1)
app.get(`/${route}`, function (req: Request, res) {
if (utils.FileExists(linksFile)) {
utils.WatchFile(linksFile, (newData: string) => {
logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`)
loadDonateURL()
})
} else {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
}
// --------------------------------------------------------------
// REDIRECTS
// --------------------------------------------------------------
// to be backwards compatible
app.get('/ask', function (req: Request, res) {
logger.DebugLog(`Qmining module ask redirect`, 'ask', 1)
res.redirect(
utils.formatUrl({
pathname: `${route}.html`,
query: req.query,
})
`http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}`
)
})
})
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
const simpleRedirects = [
{
from: '/dataeditor',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/install',
to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js',
},
{
from: '/servergit',
to: 'https://gitlab.com/MrFry/mrfrys-node-server',
},
{
from: '/scriptgit',
to: 'https://gitlab.com/MrFry/moodle-test-userscript',
},
{
from: '/qminingSite',
to: 'https://gitlab.com/MrFry/qmining-page',
},
{
from: '/classesgit',
to: 'https://gitlab.com/MrFry/question-classes',
},
{
from: '/addQuestion',
to: 'https://dataeditor.frylabs.net',
},
{
from: '/donate',
to: links.donate,
},
{
from: '/menuClick',
to: '/',
},
{
from: '/legacy',
to: '/allQuestions.html',
},
{
from: '/subjectBrowser',
to: '/allQuestions.html',
},
{
from: '/lred',
to: '/allQuestions',
},
{
from: '/allqr',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/allqr.txt',
to: 'https://api.frylabs.net/allqr.txt',
},
{
from: '/infos',
to: 'https://api.frylabs.net/infos?version=true&motd=true&subjinfo=true',
nolog: true,
},
{
from: '/irc',
to: '/chat',
},
{
from: '/patreon',
to: links.patreon,
},
]
app.get('/', function (req: Request, res) {
res.end('hai')
logger.LogReq(req)
})
simpleRedirects.forEach((redirect) => {
app.get(redirect.from, function (req: Request, res) {
if (!redirect.nolog) {
logger.LogReq(req)
}
logger.DebugLog(
`Qmining module ${redirect.from} redirect`,
'infos',
1
)
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
let target = redirect.to
if (!redirect.to.includes('https://')) {
target += utils.formatUrl({ query: req.query })
}
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})
res.redirect(target)
})
})
return {
app: app,
}
// --------------------------------------------------------------
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route: string) => {
logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1)
app.get(`/${route}`, function (req: Request, res) {
res.redirect(
utils.formatUrl({
pathname: `${route}.html`,
query: req.query,
})
)
})
})
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
app.get('/', function (req: Request, res) {
res.end('hai')
logger.LogReq(req)
})
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})
return {
app: app,
}
}
export default {
name: 'Qmining',
getApp: GetApp,
setup: (data: SetupData): void => {
userDB = data.userDB
publicdirs = data.publicdirs
nextdir = data.nextdir
},
name: 'Qmining',
getApp: GetApp,
setup: (data: SetupData): void => {
userDB = data.userDB
publicdirs = data.publicdirs
nextdir = data.nextdir
},
}

390
src/server.ts
View file

@ -53,17 +53,17 @@ const logFile = logger.logDir + logger.logFileName
const vlogFile = logger.vlogDir + logger.logFileName
function moveLogIfNotFromToday(path: string, to: string) {
if (utils.FileExists(path)) {
const today = new Date()
const stat = utils.statFile(path)
if (
today.getFullYear() !== stat.mtime.getFullYear() ||
today.getMonth() !== stat.mtime.getMonth() ||
today.getDate() !== stat.mtime.getDate()
) {
utils.renameFile(path, to + utils.GetDateString(stat.mtime))
if (utils.FileExists(path)) {
const today = new Date()
const stat = utils.statFile(path)
if (
today.getFullYear() !== stat.mtime.getFullYear() ||
today.getMonth() !== stat.mtime.getMonth() ||
today.getDate() !== stat.mtime.getDate()
) {
utils.renameFile(path, to + utils.GetDateString(stat.mtime))
}
}
}
}
moveLogIfNotFromToday(logFile, logger.logDir)
moveLogIfNotFromToday(vlogFile, logger.vlogDir)
@ -72,32 +72,32 @@ idStats.Load()
logger.Load()
interface Modules {
[name: string]: Module
[name: string]: Module
}
interface Module {
path: string
publicdirs: Array<string>
name: string
urls: Array<string>
nextdir?: string
isNextJs?: boolean
app: express.Application
dailyAction: Function
cleanup: Function
path: string
publicdirs: Array<string>
name: string
urls: Array<string>
nextdir?: string
isNextJs?: boolean
app: express.Application
dailyAction: Function
cleanup: Function
}
export interface SetupData {
url: string
publicdirs: Array<string>
userDB?: Database
nextdir?: string
httpServer: http.Server
httpsServer: https.Server
url: string
publicdirs: Array<string>
userDB?: Database
nextdir?: string
httpServer: http.Server
httpsServer: https.Server
}
if (!utils.FileExists(usersDBPath)) {
throw new Error('No user DB exists yet! please run utils/dbSetup.js first!')
throw new Error('No user DB exists yet! please run utils/dbSetup.js first!')
}
const userDB = dbtools.GetDB(usersDBPath)
let modules: Modules = utils.ReadJSON(modulesFile)
@ -108,43 +108,43 @@ logger.Log(`Log path: ${logFile}`)
logger.Log(`vLog path: ${vlogFile}`)
try {
if (utils.FileExists(extraModulesFile)) {
const extraModules = JSON.parse(utils.ReadFile(extraModulesFile))
modules = {
...extraModules,
...modules,
if (utils.FileExists(extraModulesFile)) {
const extraModules = JSON.parse(utils.ReadFile(extraModulesFile))
modules = {
...extraModules,
...modules,
}
}
}
} catch (err) {
logger.Log('Failed to read extra modules file')
console.error(err)
logger.Log('Failed to read extra modules file')
console.error(err)
}
process.on('SIGINT', () => exit('SIGINT'))
process.on('SIGTERM', () => exit('SIGTERM'))
function exit(reason: string) {
console.log()
logger.Log(`Exiting, reason: ${reason}`)
Object.keys(modules).forEach((key) => {
const module = modules[key]
if (module.cleanup) {
try {
module.cleanup()
} catch (err) {
logger.Log(
`Error in ${key} cleanup! Details in STDERR`,
logger.GetColor('redbg')
)
console.error(err)
}
}
})
console.log()
logger.Log(`Exiting, reason: ${reason}`)
Object.keys(modules).forEach((key) => {
const module = modules[key]
if (module.cleanup) {
try {
module.cleanup()
} catch (err) {
logger.Log(
`Error in ${key} cleanup! Details in STDERR`,
logger.GetColor('redbg')
)
console.error(err)
}
}
})
logger.Log('Closing Auth DB')
userDB.close()
logger.Log('Closing Auth DB')
userDB.close()
process.exit()
process.exit()
}
// https://certbot.eff.org/
@ -156,201 +156,201 @@ let certsLoaded = false
let certs: { key: string; cert: string; ca: string }
if (
startHTTPS &&
utils.FileExists(privkeyFile) &&
utils.FileExists(fullchainFile) &&
utils.FileExists(chainFile)
startHTTPS &&
utils.FileExists(privkeyFile) &&
utils.FileExists(fullchainFile) &&
utils.FileExists(chainFile)
) {
try {
const key = utils.ReadFile(privkeyFile)
const cert = utils.ReadFile(fullchainFile)
const ca = utils.ReadFile(chainFile)
certs = {
key: key,
cert: cert,
ca: ca,
try {
const key = utils.ReadFile(privkeyFile)
const cert = utils.ReadFile(fullchainFile)
const ca = utils.ReadFile(chainFile)
certs = {
key: key,
cert: cert,
ca: ca,
}
certsLoaded = true
} catch (err) {
logger.Log('Error loading cert files!', logger.GetColor('redbg'))
console.error(err)
}
certsLoaded = true
} catch (err) {
logger.Log('Error loading cert files!', logger.GetColor('redbg'))
console.error(err)
}
}
const app = express()
const httpServer = http.createServer(app)
let httpsServer: https.Server
if (certsLoaded) {
httpsServer = https.createServer(certs, app)
logger.Log('Listening on port: ' + httpsport + ' (https)')
httpsServer = https.createServer(certs, app)
logger.Log('Listening on port: ' + httpsport + ' (https)')
} else {
logger.Log('Https not avaible')
logger.Log('Https not avaible')
}
if (!process.env.NS_DEVEL) {
app.use(function (req, res, next) {
if (req.secure) {
next()
} else {
logger.DebugLog(
`HTTPS ${req.method} redirect to: ${
'https://' + req.headers.host + req.url
}`,
'https',
1
)
if (req.method === 'POST') {
res.redirect(307, 'https://' + req.headers.host + req.url)
} else {
res.redirect('https://' + req.headers.host + req.url)
}
}
})
app.use(function (req, res, next) {
if (req.secure) {
next()
} else {
logger.DebugLog(
`HTTPS ${req.method} redirect to: ${
'https://' + req.headers.host + req.url
}`,
'https',
1
)
if (req.method === 'POST') {
res.redirect(307, 'https://' + req.headers.host + req.url)
} else {
res.redirect('https://' + req.headers.host + req.url)
}
}
})
}
// https://github.com/expressjs/cors#configuration-options
app.use(
cors({
credentials: true,
origin: true,
// origin: [ /\.frylabs\.net$/ ]
})
cors({
credentials: true,
origin: true,
// origin: [ /\.frylabs\.net$/ ]
})
)
const cookieSecret = uuidv4()
app.use(cookieParser(cookieSecret))
if (!utils.FileExists(statExcludeFile)) {
utils.WriteFile('[]', statExcludeFile)
utils.WriteFile('[]', statExcludeFile)
}
const excludeFromStats = utils.ReadJSON(statExcludeFile)
app.use(
reqlogger({
loggableKeywords: ['news.json'],
loggableModules: [],
exceptions: ['_next/static'],
excludeFromStats: excludeFromStats,
})
reqlogger({
loggableKeywords: ['news.json'],
loggableModules: [],
exceptions: ['_next/static'],
excludeFromStats: excludeFromStats,
})
)
Object.keys(modules).forEach(function (key) {
const module = modules[key]
try {
const mod = require(module.path).default // eslint-disable-line
// const mod = require(module.path)
logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow'))
const module = modules[key]
try {
const mod = require(module.path).default // eslint-disable-line
// const mod = require(module.path)
logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow'))
module.publicdirs.forEach((pdir) => {
utils.CreatePath(pdir)
})
module.publicdirs.forEach((pdir) => {
utils.CreatePath(pdir)
})
if (mod.setup) {
mod.setup({
url: 'https://' + module.urls[0],
userDB: userDB,
publicdirs: module.publicdirs,
nextdir: module.nextdir,
httpServer: httpServer,
httpsServer: httpsServer,
})
if (mod.setup) {
mod.setup({
url: 'https://' + module.urls[0],
userDB: userDB,
publicdirs: module.publicdirs,
nextdir: module.nextdir,
httpServer: httpServer,
httpsServer: httpsServer,
})
}
const modApp = mod.getApp()
module.app = modApp.app
module.dailyAction = modApp.dailyAction
module.cleanup = modApp.cleanup
module.urls.forEach((url) => {
app.use(vhost(url, module.app))
})
} catch (err) {
console.error(err)
}
const modApp = mod.getApp()
module.app = modApp.app
module.dailyAction = modApp.dailyAction
module.cleanup = modApp.cleanup
module.urls.forEach((url) => {
app.use(vhost(url, module.app))
})
} catch (err) {
console.error(err)
}
})
setLogTimer()
function setLogTimer() {
const now = new Date()
const night = new Date(
now.getFullYear(),
now.getMonth(),
now.getDate() + 1,
0,
0,
1
)
logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
const msToMidnight = night.getTime() - now.getTime() + 10000
logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)
if (msToMidnight < 0) {
logger.Log(
`Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`,
logger.GetColor('redbg')
const now = new Date()
const night = new Date(
now.getFullYear(),
now.getMonth(),
now.getDate() + 1,
0,
0,
1
)
return
}
logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
const msToMidnight = night.getTime() - now.getTime() + 10000
logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)
setTimeout(function () {
LogTimerAction()
rotateLog()
setLogTimer()
}, msToMidnight)
if (msToMidnight < 0) {
logger.Log(
`Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`,
logger.GetColor('redbg')
)
return
}
setTimeout(function () {
LogTimerAction()
rotateLog()
setLogTimer()
}, msToMidnight)
}
function rotateLog() {
const date = new Date()
date.setDate(date.getDate() - 1)
const fname =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
const date = new Date()
date.setDate(date.getDate() - 1)
const fname =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (utils.FileExists(logFile)) {
utils.CopyFile(logFile, logger.logDir + fname)
}
if (utils.FileExists(vlogFile)) {
utils.CopyFile(vlogFile, logger.vlogDir + fname)
}
if (utils.FileExists(logFile)) {
utils.CopyFile(logFile, logger.logDir + fname)
}
if (utils.FileExists(vlogFile)) {
utils.CopyFile(vlogFile, logger.vlogDir + fname)
}
utils.WriteFile(fname, logFile)
utils.WriteFile(fname, vlogFile)
utils.WriteFile(fname, logFile)
utils.WriteFile(fname, vlogFile)
}
function LogTimerAction() {
logger.DebugLog(`Running Log Timer Action`, 'daily', 1)
Object.keys(modules).forEach((key) => {
const module = modules[key]
logger.DebugLog(`Ckecking ${key}`, 'daily', 1)
if (module.dailyAction) {
try {
logger.Log(`Running daily action of ${key}`)
module.dailyAction()
} catch (err) {
logger.Log(
`Error in ${key} daily action! Details in STDERR`,
logger.GetColor('redbg')
)
console.error(err)
}
}
})
logger.DebugLog(`Running Log Timer Action`, 'daily', 1)
Object.keys(modules).forEach((key) => {
const module = modules[key]
logger.DebugLog(`Ckecking ${key}`, 'daily', 1)
if (module.dailyAction) {
try {
logger.Log(`Running daily action of ${key}`)
module.dailyAction()
} catch (err) {
logger.Log(
`Error in ${key} daily action! Details in STDERR`,
logger.GetColor('redbg')
)
console.error(err)
}
}
})
const line =
'==================================================================================================================================================='
logger.Log(line)
const line =
'==================================================================================================================================================='
logger.Log(line)
}
logger.Log('Node version: ' + process.version)
logger.Log('Current working directory: ' + process.cwd())
logger.Log('Listening on port: ' + port)
if (isRoot) {
logger.Log('Running as root', logger.GetColor('red'))
logger.Log('Running as root', logger.GetColor('red'))
}
httpServer.listen(port)
if (httpsServer) {
httpsServer.listen(httpsport)
httpsServer.listen(httpsport)
}

42
src/standaloneUtils/createJSON.js
View file

@ -3,36 +3,36 @@ const fs = require('fs')
const params = process.argv
const file = params[2]
const data = fs.readFileSync(file,'utf8').split('\n')
const data = fs.readFileSync(file, 'utf8').split('\n')
console.log(data)
console.log("TODO: remove 'Q: ' and 'A: '")
let currVal = {}
const res = data.reduce((acc, val) => {
const formattedVal = val.replace(/\r/g, '').trim()
const formattedVal = val.replace(/\r/g, '').trim()
if (formattedVal.startsWith('#')) return acc
if (formattedVal.startsWith('Q')) {
currVal = {
Q: formattedVal
}
return acc
}
if (formattedVal.startsWith('A')) {
currVal.A = formattedVal
return [
...acc,
{
...currVal,
data: {
type: 'simple'
if (formattedVal.startsWith('#')) return acc
if (formattedVal.startsWith('Q')) {
currVal = {
Q: formattedVal,
}
}
]
}
return acc
}
if (formattedVal.startsWith('A')) {
currVal.A = formattedVal
return [
...acc,
{
...currVal,
data: {
type: 'simple',
},
},
]
}
return acc
return acc
}, [])
console.log(res)

32
src/standaloneUtils/dataSimplifyer.js
View file

@ -7,23 +7,23 @@ const data = JSON.parse(fs.readFileSync(file, 'utf8'))
const res = []
data.forEach((subj) => {
const questions = []
subj.Questions.forEach((question) => {
const res = {}
if (question.Q) {
res.Q = simplifyString(question.Q)
}
if (question.A) {
res.A = simplifyString(question.A)
}
res.data = question.data
const questions = []
subj.Questions.forEach((question) => {
const res = {}
if (question.Q) {
res.Q = simplifyString(question.Q)
}
if (question.A) {
res.A = simplifyString(question.A)
}
res.data = question.data
questions.push(res)
})
res.push({
Name: subj.Name,
Questions: questions,
})
questions.push(res)
})
res.push({
Name: subj.Name,
Questions: questions,
})
})
fs.writeFileSync(file + '.res', JSON.stringify(res))

46
src/standaloneUtils/dbSetup.js
View file

@ -4,38 +4,38 @@ const dbtools = require('../../dist/utils/dbtools.js').default // eslint-disable
const { v4: uuidv4 } = require('uuid') // eslint-disable-line
const dbStructPaths = [
{ structPath: '../modules/api/usersDBStruct.json', name: 'users.db' },
{ structPath: '../modules/api/msgsDbStruct.json', name: 'msgs.db' },
{ structPath: '../modules/api/usersDBStruct.json', name: 'users.db' },
{ structPath: '../modules/api/msgsDbStruct.json', name: 'msgs.db' },
]
dbStructPaths.forEach((data) => {
const { structPath, name } = data
createDB(structPath, name)
const { structPath, name } = data
createDB(structPath, name)
})
function createDB(path, name) {
const dbStruct = utils.ReadJSON(path)
const db = dbtools.GetDB(`./${name}`)
db.pragma('synchronous = OFF')
const dbStruct = utils.ReadJSON(path)
const db = dbtools.GetDB(`./${name}`)
db.pragma('synchronous = OFF')
Object.keys(dbStruct).forEach((tableName) => {
const tableData = dbStruct[tableName]
dbtools.CreateTable(
db,
tableName,
tableData.tableStruct,
tableData.foreignKey
)
})
printDb(db, dbStruct)
db.close()
Object.keys(dbStruct).forEach((tableName) => {
const tableData = dbStruct[tableName]
dbtools.CreateTable(
db,
tableName,
tableData.tableStruct,
tableData.foreignKey
)
})
printDb(db, dbStruct)
db.close()
logger.Log('Done')
logger.Log('Done')
}
function printDb(db, dbStruct) {
Object.keys(dbStruct).forEach((key) => {
console.log(dbtools.TableInfo(db, key))
console.log(dbtools.SelectAll(db, key))
})
Object.keys(dbStruct).forEach((key) => {
console.log(dbtools.TableInfo(db, key))
console.log(dbtools.SelectAll(db, key))
})
}

54
src/standaloneUtils/invalidQuestionRemover.js
View file

@ -1,13 +1,13 @@
const fs = require('fs')
function GetParams() {
return process.argv.splice(2)
return process.argv.splice(2)
}
const params = GetParams()
console.log(params)
if (params.length === 0) {
console.error('No params! Need a path to a question database!')
process.exit()
console.error('No params! Need a path to a question database!')
process.exit()
}
const file = params[0]
@ -16,36 +16,36 @@ const res = []
let invalidQuestionCount = 0
data.forEach((subj) => {
const questions = []
subj.Questions.forEach((question) => {
if (isInvalidQuestion(question)) {
console.log(`invalid question in ${subj.Name}:`)
console.log(question)
invalidQuestionCount++
} else {
questions.push(question)
}
})
res.push({
Name: subj.Name,
Questions: questions,
})
const questions = []
subj.Questions.forEach((question) => {
if (isInvalidQuestion(question)) {
console.log(`invalid question in ${subj.Name}:`)
console.log(question)
invalidQuestionCount++
} else {
questions.push(question)
}
})
res.push({
Name: subj.Name,
Questions: questions,
})
})
function isInvalidQuestion(q) {
if (q.Q === 'Ugrás...' || q.A === 'Ugrás...') {
return true
}
if (q.Q === 'Ugrás...' || q.A === 'Ugrás...') {
return true
}
if (!q.Q && !q.A) {
return true
}
if (!q.Q && !q.A) {
return true
}
if (!q.Q && q.data.type === 'simple') {
return true
}
if (!q.Q && q.data.type === 'simple') {
return true
}
return false
return false
}
console.log(`${invalidQuestionCount} invalid questions, writing results...`)

78
src/standaloneUtils/manualSearch.js
View file

@ -21,16 +21,16 @@ const answer = params[2]
console.time('SEARCH')
const searchRes = search({
qdb: loadData(path),
subjName: 'Elektronika',
question: {
Q: question,
A: answer,
data: {
type: 'simple',
qdb: loadData(path),
subjName: 'Elektronika',
question: {
Q: question,
A: answer,
data: {
type: 'simple',
},
},
},
searchTillMatchPercent: 80,
searchTillMatchPercent: 80,
})
hr()
console.log('Search result')
@ -39,9 +39,9 @@ showSearchResult(searchRes)
hr()
console.timeEnd('SEARCH')
log(
`Searched for question: "${C('green')}${question}${C()}" answer: "${C(
'green'
)}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"`
`Searched for question: "${C('green')}${question}${C()}" answer: "${C(
'green'
)}${answer || ''}${C()}" in "${C('cyan')}${path}${C()}"`
)
hr()
@ -50,44 +50,44 @@ hr()
// ---------------------------------------------------------------------------------
function showSearchResult(res) {
res.forEach((x) => {
console.log(`${C('green')}Q:${C()}`, x.q.Q)
console.log(`${C('green')}A:${C()}`, x.q.A)
console.log(`${C('green')}match:${C()}`, x.match)
console.log()
})
console.log(`Result length: ${C('green')}${res.length}${C()}`)
res.forEach((x) => {
console.log(`${C('green')}Q:${C()}`, x.q.Q)
console.log(`${C('green')}A:${C()}`, x.q.A)
console.log(`${C('green')}match:${C()}`, x.match)
console.log()
})
console.log(`Result length: ${C('green')}${res.length}${C()}`)
}
function search({ qdb, subjName, question, searchInAllIfNoResult }) {
return doSearch(
qdb,
subjName,
question,
null,
minpercent,
searchInAllIfNoResult
)
return doSearch(
qdb,
subjName,
question,
null,
minpercent,
searchInAllIfNoResult
)
}
function hr() {
let res = ''
for (let i = 0; i < process.stdout.columns; i++) {
res += '='
}
log(`${C('cyan')}${res}${C()}`)
let res = ''
for (let i = 0; i < process.stdout.columns; i++) {
res += '='
}
log(`${C('cyan')}${res}${C()}`)
}
function log(text) {
utils.AppendToFile(text, globalLog)
if (process.stdout.isTTY) {
process.stdout.clearLine()
process.stdout.cursorTo(0)
}
utils.AppendToFile(text, globalLog)
if (process.stdout.isTTY) {
process.stdout.clearLine()
process.stdout.cursorTo(0)
}
console.log(text)
console.log(text)
}
function C(color) {
return logger.C(color)
return logger.C(color)
}

684
src/standaloneUtils/rmDuplicates.js
View file

@ -1,10 +1,10 @@
const utils = require('../../dist/utils/utils.js').default // eslint-disable-line
const logger = require('../../dist/utils/logger.js').default // eslint-disable-line
const {
addQuestion,
doSearch,
compareQuestionObj,
createQuestion,
addQuestion,
doSearch,
compareQuestionObj,
createQuestion,
} = require('../../dist/utils/classes.js') // eslint-disable-line
const { loadData, writeData } = require('../../dist/utils/actions.js') // eslint-disable-line
const fs = require('fs') // eslint-disable-line
@ -35,7 +35,7 @@ const fs = require('fs') // eslint-disable-line
const minpercent = 95
const line =
'===================================================================='
'===================================================================='
const logPath = './duplicateRemovingLog/'
const globalLog = './duplicateRemovingLog/log'
utils.CreatePath(logPath)
@ -45,25 +45,25 @@ utils.WriteFile('', globalLog)
let currentMaxIndex = -1
let currentIndex = -1
process.on('message', function () {
process.send({
currentMaxIndex: currentMaxIndex,
currentIndex: currentIndex,
})
process.send({
currentMaxIndex: currentMaxIndex,
currentIndex: currentIndex,
})
})
// ----------------------------------------------
let params = process.argv.splice(2)
let silenced = false
if (params.includes('-s')) {
silenced = true
silenced = true
}
params = params.filter((x) => {
return !x.startsWith('-')
return !x.startsWith('-')
})
console.log(params)
if (params.length === 0) {
console.log('At least 1 parameter required (path to DB)')
process.exit(1)
console.log('At least 1 parameter required (path to DB)')
process.exit(1)
}
const pathA = params[0]
@ -71,61 +71,67 @@ const pathB = params[1]
const stat = fs.lstatSync(pathA)
if (stat.isDirectory()) {
if (pathB) {
log(
`Clearing possible questions from ${C(
'green'
)}${pathA}${C()} based on ${C('green')}${pathB}${C()} db`
)
const db = pathB ? loadData(pathB) : null
if (pathB) {
log(
`Clearing possible questions from ${C(
'green'
)}${pathA}${C()} based on ${C('green')}${pathB}${C()} db`
)
const db = pathB ? loadData(pathB) : null
clearPossibleAnswers(pathA, db)
clearPossibleAnswers(pathA, db)
log(
`Cleared possible questions from ${C('green')}${pathA}${C()} based on ${C(
'green'
)}${pathB}${C()} db`
)
} else {
log(
`Removing possible question duplicates from ${C('green')}${pathA}${C()}`
)
removePossibleAnswersDuplicates(pathA)
log(`Removed possible question duplicates from ${C('green')}${pathA}${C()}`)
}
log(
`Cleared possible questions from ${C(
'green'
)}${pathA}${C()} based on ${C('green')}${pathB}${C()} db`
)
} else {
log(
`Removing possible question duplicates from ${C(
'green'
)}${pathA}${C()}`
)
removePossibleAnswersDuplicates(pathA)
log(
`Removed possible question duplicates from ${C(
'green'
)}${pathA}${C()}`
)
}
} else {
console.time('load')
const dbA = loadData(pathA)
const dbB = pathB ? loadData(pathB) : null
console.timeEnd('load')
console.time('load')
const dbA = loadData(pathA)
const dbB = pathB ? loadData(pathB) : null
console.timeEnd('load')
console.time('rmduplicates')
console.time('rmduplicates')
if (!dbB) {
log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`)
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
const res = rmDuplicates(dbA)
console.timeEnd('rmduplicates')
writeData(res, resultDbFileName + '.res')
log('File written')
log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`)
} else {
log(
`Removing questions found in ${C('green')}${pathB}${C()} from ${C(
'green'
)}${pathA}${C()}`
)
const res = difference({ dbA: dbA, dbB: dbB })
console.timeEnd('rmduplicates')
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
writeData(res, resultDbFileName + '.res')
log('File written')
log(
`Removed questions found in ${C('green')}${pathB}${C()} from ${C(
'green'
)}${pathA}${C()}`
)
}
if (!dbB) {
log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`)
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
const res = rmDuplicates(dbA)
console.timeEnd('rmduplicates')
writeData(res, resultDbFileName + '.res')
log('File written')
log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`)
} else {
log(
`Removing questions found in ${C('green')}${pathB}${C()} from ${C(
'green'
)}${pathA}${C()}`
)
const res = difference({ dbA: dbA, dbB: dbB })
console.timeEnd('rmduplicates')
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
writeData(res, resultDbFileName + '.res')
log('File written')
log(
`Removed questions found in ${C('green')}${pathB}${C()} from ${C(
'green'
)}${pathA}${C()}`
)
}
}
// ---------------------------------------------------------------------------------
@ -135,141 +141,141 @@ if (stat.isDirectory()) {
// TODO: dont check every file, only check per directorires
// only compare questions of same subjects
function removePossibleAnswersDuplicates(path) {
const dirs = fs.readdirSync(path)
let count = 0
let currIndex = 1
let delets = 0
const dirs = fs.readdirSync(path)
let count = 0
let currIndex = 1
let delets = 0
iterateDir(path, () => {
count++
})
dirs.forEach((currDir) => {
const contents = fs.readdirSync(path + '/' + currDir)
contents.forEach((currFile) => {
const currPath = path + '/' + currDir + '/' + currFile
if (currPath.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath)) {
return
}
const dataA = utils.ReadJSON(currPath)
currIndex++
printProgressBar(currIndex, count - 1)
contents.forEach((currFile2) => {
const currPath2 = path + '/' + currDir + '/' + currFile2
if (currPath2.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath2)) {
return
}
if (currPath === currPath2) {
return
}
const dataB = utils.ReadJSON(currPath2)
dataA.questions.forEach((q1) => {
dataB.questions.some((q2) => {
const percent = compareQuestionObj(
createQuestion(q1),
'',
createQuestion(q2),
''
)
if (percent.avg === 100) {
utils.deleteFile(currPath2)
count--
delets++
return true
}
})
})
})
iterateDir(path, () => {
count++
})
})
log(`${C('green')}Deleting empty directories ...${C()}`)
count = dirs.length
currIndex = 0
let deletedDirCount = 0
dirs.forEach((dir) => {
currIndex++
const currDirContent = fs.readdirSync(path + '/' + dir)
if (currDirContent.length === 0) {
fs.rmdirSync(path + '/' + dir)
deletedDirCount++
}
printProgressBar(currIndex, count)
})
dirs.forEach((currDir) => {
const contents = fs.readdirSync(path + '/' + currDir)
log(`${C('green')}Updating savedQuestions.json ...${C()}`)
count = dirs.length
currIndex = 0
dirs.forEach((dir) => {
currIndex++
updateSavedQuestionsFile(path + '/' + dir)
printProgressBar(currIndex, count)
})
contents.forEach((currFile) => {
const currPath = path + '/' + currDir + '/' + currFile
if (currPath.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath)) {
return
}
const dataA = utils.ReadJSON(currPath)
log(
`Deleted ${C('green')}${delets}${C()} files, and ${C(
'green'
)}${deletedDirCount}${C()} directories`
)
currIndex++
printProgressBar(currIndex, count - 1)
contents.forEach((currFile2) => {
const currPath2 = path + '/' + currDir + '/' + currFile2
if (currPath2.includes('savedQuestions.json')) {
return
}
if (!utils.FileExists(currPath2)) {
return
}
if (currPath === currPath2) {
return
}
const dataB = utils.ReadJSON(currPath2)
dataA.questions.forEach((q1) => {
dataB.questions.some((q2) => {
const percent = compareQuestionObj(
createQuestion(q1),
'',
createQuestion(q2),
''
)
if (percent.avg === 100) {
utils.deleteFile(currPath2)
count--
delets++
return true
}
})
})
})
})
})
log(`${C('green')}Deleting empty directories ...${C()}`)
count = dirs.length
currIndex = 0
let deletedDirCount = 0
dirs.forEach((dir) => {
currIndex++
const currDirContent = fs.readdirSync(path + '/' + dir)
if (currDirContent.length === 0) {
fs.rmdirSync(path + '/' + dir)
deletedDirCount++
}
printProgressBar(currIndex, count)
})
log(`${C('green')}Updating savedQuestions.json ...${C()}`)
count = dirs.length
currIndex = 0
dirs.forEach((dir) => {
currIndex++
updateSavedQuestionsFile(path + '/' + dir)
printProgressBar(currIndex, count)
})
log(
`Deleted ${C('green')}${delets}${C()} files, and ${C(
'green'
)}${deletedDirCount}${C()} directories`
)
}
function clearPossibleAnswers(path, db) {
let count = 0
let currIndex = 1
let delets = 0
iterateDir(path, () => {
count++
})
iterateDir(path, (currPath) => {
currIndex++
if (currPath.includes('savedQuestions.json')) {
return
}
const { subj, questions } = utils.ReadJSON(currPath)
questions.forEach((question) => {
const searchRes = search({
qdb: db,
subjName: subj,
question: question,
searchTillMatchPercent: 80,
})
if (searchRes.length > 0) {
utils.deleteFile(currPath)
delets++
}
let count = 0
let currIndex = 1
let delets = 0
iterateDir(path, () => {
count++
})
printProgressBar(currIndex, count)
})
log(`Deleted ${C('green')}${delets}${C()} files`)
iterateDir(path, (currPath) => {
currIndex++
if (currPath.includes('savedQuestions.json')) {
return
}
const { subj, questions } = utils.ReadJSON(currPath)
questions.forEach((question) => {
const searchRes = search({
qdb: db,
subjName: subj,
question: question,
searchTillMatchPercent: 80,
})
if (searchRes.length > 0) {
utils.deleteFile(currPath)
delets++
}
})
printProgressBar(currIndex, count)
})
log(`Deleted ${C('green')}${delets}${C()} files`)
}
function updateSavedQuestionsFile(path) {
const filePath = path + '/' + 'savedQuestions.json'
if (!utils.FileExists(filePath)) {
log(`${filePath} does not exists!`)
return
}
const filePath = path + '/' + 'savedQuestions.json'
if (!utils.FileExists(filePath)) {
log(`${filePath} does not exists!`)
return
}
const savedQuestions = utils.ReadJSON(filePath)
const filtered = savedQuestions.filter((sq) => {
return utils.FileExists(path + '/' + sq.fname)
})
const savedQuestions = utils.ReadJSON(filePath)
const filtered = savedQuestions.filter((sq) => {
return utils.FileExists(path + '/' + sq.fname)
})
if (savedQuestions.length !== filtered.length) {
utils.WriteFile(JSON.stringify(filtered), filePath)
}
if (savedQuestions.length !== filtered.length) {
utils.WriteFile(JSON.stringify(filtered), filePath)
}
}
// ---------------------------------------------------------------------------------
@ -277,103 +283,105 @@ function updateSavedQuestionsFile(path) {
// ---------------------------------------------------------------------------------
function rmDuplicates(db) {
return difference({ dbA: db })
return difference({ dbA: db })
}
function difference({ dbA, dbB }) {
const doingDifference = !!dbB
// Stuff only from A
const resultDb = []
let dbLength = 0
let removedTotal = 0
let processedQuestions = 0
const doingDifference = !!dbB
// Stuff only from A
const resultDb = []
let dbLength = 0
let removedTotal = 0
let processedQuestions = 0
iterateSubjects(dbA, () => {
dbLength++
})
currentMaxIndex = dbLength
const getResultDbLength = () => {
let resultDbLength = 0
iterateSubjects(resultDb, () => {
resultDbLength++
iterateSubjects(dbA, () => {
dbLength++
})
return resultDbLength
}
currentMaxIndex = dbLength
for (let i = 0; i < dbA.length; i++) {
const subj = dbA[i]
const subjLogPath = logPath + subj.Name
utils.WriteFile('', subjLogPath)
let removedCount = 0
const getResultDbLength = () => {
let resultDbLength = 0
iterateSubjects(resultDb, () => {
resultDbLength++
})
return resultDbLength
}
for (let i = 0; i < dbA.length; i++) {
const subj = dbA[i]
const subjLogPath = logPath + subj.Name
utils.WriteFile('', subjLogPath)
let removedCount = 0
hr()
log(
`${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${
subj.Name
}, ${C('blue')}${subj.Questions.length}${C(
'green'
)} questions${C()}`
)
printProgressBar(i + 1, dbA.length)
for (let j = 0; j < subj.Questions.length; j++) {
const question = subj.Questions[j]
const searchRes = search({
qdb: doingDifference ? dbB : resultDb,
subjName: subj.Name,
question: question,
searchInAllIfNoResult: doingDifference,
searchTillMatchPercent: minpercent,
})
printProgressBar(processedQuestions, dbLength)
processedQuestions++
currentIndex = processedQuestions
const res = hasRequiredPercent(searchRes, minpercent)
// no result: adding to difference
if (res.length === 0) {
// no result: adding to difference
addQuestion(resultDb, subj.Name, question)
} else {
// has result, not adding to difference
utils.AppendToFile(
line +
'\n' +
line +
'\n' +
JSON.stringify(question, null, 2) +
'\n' +
line +
JSON.stringify(res, null, 2) +
'\n',
subjLogPath
)
removedCount++
removedTotal++
}
}
log(
`${C('yellow')}Removed ${C('red')}${removedCount}${C(
'yellow'
)} questions${C()}`
)
}
hr()
log(
`${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${subj.Name}, ${C(
'blue'
)}${subj.Questions.length}${C('green')} questions${C()}`
`Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions`
)
printProgressBar(i + 1, dbA.length)
for (let j = 0; j < subj.Questions.length; j++) {
const question = subj.Questions[j]
const searchRes = search({
qdb: doingDifference ? dbB : resultDb,
subjName: subj.Name,
question: question,
searchInAllIfNoResult: doingDifference,
searchTillMatchPercent: minpercent,
})
printProgressBar(processedQuestions, dbLength)
processedQuestions++
currentIndex = processedQuestions
const res = hasRequiredPercent(searchRes, minpercent)
// no result: adding to difference
if (res.length === 0) {
// no result: adding to difference
addQuestion(resultDb, subj.Name, question)
} else {
// has result, not adding to difference
utils.AppendToFile(
line +
'\n' +
line +
'\n' +
JSON.stringify(question, null, 2) +
'\n' +
line +
JSON.stringify(res, null, 2) +
'\n',
subjLogPath
)
removedCount++
removedTotal++
}
}
log(
`${C('yellow')}Removed ${C('red')}${removedCount}${C(
'yellow'
)} questions${C()}`
)
}
hr()
log(
`Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions`
)
return resultDb
return resultDb
}
function hasRequiredPercent(result, minpercent) {
return result.reduce((acc, res) => {
if (res.match >= minpercent) {
acc.push(res)
}
return acc
}, [])
return result.reduce((acc, res) => {
if (res.match >= minpercent) {
acc.push(res)
}
return acc
}, [])
}
// ---------------------------------------------------------------------------------
@ -381,22 +389,22 @@ function hasRequiredPercent(result, minpercent) {
// ---------------------------------------------------------------------------------
function search({ qdb, subjName, question, searchInAllIfNoResult }) {
return doSearch(
qdb,
subjName,
question,
null,
minpercent,
searchInAllIfNoResult
)
return doSearch(
qdb,
subjName,
question,
null,
minpercent,
searchInAllIfNoResult
)
}
function iterateSubjects(db, fn) {
db.forEach((subj) => {
subj.Questions.forEach((question) => {
fn(subj, question)
db.forEach((subj) => {
subj.Questions.forEach((question) => {
fn(subj, question)
})
})
})
}
// ---------------------------------------------------------------------------------
@ -404,19 +412,19 @@ function iterateSubjects(db, fn) {
// ---------------------------------------------------------------------------------
function iterateDir(path, action) {
if (!utils.FileExists(path)) {
return
}
if (!utils.FileExists(path)) {
return
}
const stat = fs.lstatSync(path)
if (stat.isDirectory()) {
const content = fs.readdirSync(path)
content.forEach((currContent) => {
iterateDir(`${path}/${currContent}`, action)
})
} else {
action(path)
}
const stat = fs.lstatSync(path)
if (stat.isDirectory()) {
const content = fs.readdirSync(path)
content.forEach((currContent) => {
iterateDir(`${path}/${currContent}`, action)
})
} else {
action(path)
}
}
// ---------------------------------------------------------------------------------
@ -424,69 +432,69 @@ function iterateDir(path, action) {
// ---------------------------------------------------------------------------------
function hr() {
let res = ''
for (let i = 0; i < process.stdout.columns; i++) {
res += '='
}
log(`${C('cyan')}${res}${C()}`)
let res = ''
for (let i = 0; i < process.stdout.columns; i++) {
res += '='
}
log(`${C('cyan')}${res}${C()}`)
}
function log(text) {
utils.AppendToFile(text, globalLog)
if (silenced) return
if (process.stdout.isTTY) {
process.stdout.clearLine()
process.stdout.cursorTo(0)
}
utils.AppendToFile(text, globalLog)
if (silenced) return
if (process.stdout.isTTY) {
process.stdout.clearLine()
process.stdout.cursorTo(0)
}
console.log(text)
console.log(text)
}
function writeInSameLine(text, returnToLineStart) {
if (!process.stdout.isTTY) {
return
}
process.stdout.clearLine()
process.stdout.cursorTo(0)
process.stdout.write(text)
if (returnToLineStart) {
process.stdout.write('\r')
} else {
process.stdout.write('\n')
}
if (!process.stdout.isTTY) {
return
}
process.stdout.clearLine()
process.stdout.cursorTo(0)
process.stdout.write(text)
if (returnToLineStart) {
process.stdout.write('\r')
} else {
process.stdout.write('\n')
}
}
function printProgressBar(current, total) {
if (!process.stdout.isTTY || silenced) {
return
}
const width = process.stdout.columns - 30
if (!process.stdout.isTTY || silenced) {
return
}
const width = process.stdout.columns - 30
if (width <= 0) {
return
}
if (width <= 0) {
return
}
const x = width / total
const xCurrent = Math.floor(current * x)
const xTotal = Math.floor(total * x)
const x = width / total
const xCurrent = Math.floor(current * x)
const xTotal = Math.floor(total * x)
let line = ''
for (let i = 0; i < xCurrent; i++) {
line += '='
}
let line = ''
for (let i = 0; i < xCurrent; i++) {
line += '='
}
for (let i = 0; i < xTotal - xCurrent; i++) {
line += ' '
}
const numbers = `${current} / ${total}`
writeInSameLine(
`${C('magenta')} [${line}]${C('green')} ${numbers}${C()}`,
current !== total
)
for (let i = 0; i < xTotal - xCurrent; i++) {
line += ' '
}
const numbers = `${current} / ${total}`
writeInSameLine(
`${C('magenta')} [${line}]${C('green')} ${numbers}${C()}`,
current !== total
)
}
function C(color) {
return logger.C(color)
return logger.C(color)
}
process.exit()

130
src/standaloneUtils/sqliteTest.js
View file

@ -6,73 +6,73 @@ const dbtools = require('../utils/dbtools.js')
Main()
function Main() {
const cols = {
uname: {
type: 'text',
},
pw: {
type: 'text',
},
notes: {
type: 'text',
},
}
const dbName = 'test'
const cols = {
uname: {
type: 'text',
},
pw: {
type: 'text',
},
notes: {
type: 'text',
},
}
const dbName = 'test'
const db = dbtools.GetDB('./testdb.db')
const db = dbtools.GetDB('./testdb.db')
// Creating table
dbtools.CreateTable(db, dbName, cols)
console.log(dbtools.TableInfo(db, dbName))
dbtools.SelectAll(db, dbName)
// inserting test val to table
dbtools.Insert(db, dbName, {
uname: 'mrfry',
pw: 'dsads',
})
// Selecting a record
console.log(
dbtools.Select(db, dbName, {
uname: 'mrfry',
// Creating table
dbtools.CreateTable(db, dbName, cols)
console.log(dbtools.TableInfo(db, dbName))
dbtools.SelectAll(db, dbName)
// inserting test val to table
dbtools.Insert(db, dbName, {
uname: 'mrfry',
pw: 'dsads',
})
)
console.log(dbtools.TableInfo(db, dbName))
console.log(dbtools.SelectAll(db, dbName))
// Updating record
dbtools.Update(
db,
dbName,
{
pw: 'sspw',
},
{
uname: 'mrfry',
}
)
console.log(dbtools.SelectAll(db, dbName))
// Updating record again
dbtools.Update(
db,
dbName,
{
notes: 'new note!',
},
{
uname: 'mrfry',
}
)
console.log(dbtools.SelectAll(db, dbName))
// Adding new column and
dbtools.AddColumn(db, dbName, {
test: 'text',
})
console.log(dbtools.TableInfo(db, dbName))
console.log(dbtools.SelectAll(db, dbName))
// Deleting stuff
dbtools.Delete(db, dbName, {
uname: 'mrfry',
})
console.log(dbtools.SelectAll(db, dbName))
// Selecting a record
console.log(
dbtools.Select(db, dbName, {
uname: 'mrfry',
})
)
console.log(dbtools.TableInfo(db, dbName))
console.log(dbtools.SelectAll(db, dbName))
// Updating record
dbtools.Update(
db,
dbName,
{
pw: 'sspw',
},
{
uname: 'mrfry',
}
)
console.log(dbtools.SelectAll(db, dbName))
// Updating record again
dbtools.Update(
db,
dbName,
{
notes: 'new note!',
},
{
uname: 'mrfry',
}
)
console.log(dbtools.SelectAll(db, dbName))
// Adding new column and
dbtools.AddColumn(db, dbName, {
test: 'text',
})
console.log(dbtools.TableInfo(db, dbName))
console.log(dbtools.SelectAll(db, dbName))
// Deleting stuff
dbtools.Delete(db, dbName, {
uname: 'mrfry',
})
console.log(dbtools.SelectAll(db, dbName))
dbtools.CloseDB(db)
dbtools.CloseDB(db)
}

64
src/tests/addQuestion.test.ts
View file

@ -4,45 +4,45 @@ import { Subject, Question } from '../types/basicTypes'
const question: Question = createQuestion('asd', 'asd', { type: 'simple' })
test('Adds questions to empty db', () => {
const emptyDb: Subject[] = []
addQuestion(emptyDb, 'test subject', question)
expect(emptyDb.length).toBe(1)
const emptyDb: Subject[] = []
addQuestion(emptyDb, 'test subject', question)
expect(emptyDb.length).toBe(1)
})
test('Adds questions next to existing', () => {
const db: Subject[] = [
{
Name: 'test subject',
Questions: [question],
},
]
addQuestion(db, 'another something', question)
expect(db.length).toBe(2)
const db: Subject[] = [
{
Name: 'test subject',
Questions: [question],
},
]
addQuestion(db, 'another something', question)
expect(db.length).toBe(2)
})
test('Does not add new subject, multiple new questions', () => {
const db: Subject[] = [
{
Name: 'test subject',
Questions: [question],
},
]
addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question)
expect(db.length).toBe(1)
const db: Subject[] = [
{
Name: 'test subject',
Questions: [question],
},
]
addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question)
addQuestion(db, 'test subject', question)
expect(db.length).toBe(1)
})
test('Adds new subjects, multiple new questions', () => {
const db: Subject[] = [
{
Name: 'test subject',
Questions: [question],
},
]
addQuestion(db, 'gfjdkglfd', question)
addQuestion(db, ' somrthing test ', question)
addQuestion(db, 'aaaaaaaa', question)
expect(db.length).toBe(4)
const db: Subject[] = [
{
Name: 'test subject',
Questions: [question],
},
]
addQuestion(db, 'gfjdkglfd', question)
addQuestion(db, ' somrthing test ', question)
addQuestion(db, 'aaaaaaaa', question)
expect(db.length).toBe(4)
})

36
src/tests/base64ToText.test.ts
View file

@ -11,22 +11,22 @@
// const expectedResults = ['Melyik híres zenekar tagja volt Joe Muranyi?']
test('Img text recognition works', async () => {
// TODO: tesseract keeps workers even after terminate(), and jest --detectOpenHandles detects them
expect(true).toBeTruthy()
// await tesseractLoaded
// for (let i = 0; i < imgs.length; i++) {
// const expectedResult = expectedResults[i]
// const img = imgs[i]
//
// const text = await recognizeTextFromBase64(img)
// expect(text.trim() === expectedResult).toBeTruthy()
// }
//
// await terminateWorker()
//
// return new Promise<void>((resolve) => {
// setTimeout(() => {
// resolve()
// }, 1 * 1000)
// })
// TODO: tesseract keeps workers even after terminate(), and jest --detectOpenHandles detects them
expect(true).toBeTruthy()
// await tesseractLoaded
// for (let i = 0; i < imgs.length; i++) {
// const expectedResult = expectedResults[i]
// const img = imgs[i]
//
// const text = await recognizeTextFromBase64(img)
// expect(text.trim() === expectedResult).toBeTruthy()
// }
//
// await terminateWorker()
//
// return new Promise<void>((resolve) => {
// setTimeout(() => {
// resolve()
// }, 1 * 1000)
// })
})

282
src/tests/oldQuestionRemoving.test.ts
View file

@ -6,192 +6,192 @@ import { QuestionDb, Subject, Question } from '../types/basicTypes'
const date = (x?: number) => new Date().getTime() + (x || 0)
const q1 = createQuestion(
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'piaci áruk eltérhet a névértéktől.',
{
type: 'simple',
date: date(-1000),
}
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'piaci áruk eltérhet a névértéktől.',
{
type: 'simple',
date: date(-1000),
}
)
const q2 = createQuestion(
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'afjléa gféda gfdjs légf',
{
type: 'simple',
date: date(-1000),
}
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'afjléa gféda gfdjs légf',
{
type: 'simple',
date: date(-1000),
}
)
const q3 = createQuestion(
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'afjlsd gfds dgfs gf sdgf d',
{
type: 'simple',
date: date(-1000),
}
'A kötvény és a részvény közös tulajdonsága, hogy TOREMOVE',
'afjlsd gfds dgfs gf sdgf d',
{
type: 'simple',
date: date(-1000),
}
)
const q4 = createQuestion(
'A kötvény névértéke',
'A kötvényen feltüntetett meghatározott nagyságú összeg.',
{
type: 'simple',
date: date(-1000),
}
'A kötvény névértéke',
'A kötvényen feltüntetett meghatározott nagyságú összeg.',
{
type: 'simple',
date: date(-1000),
}
)
const q5 = createQuestion(
'Mi az osztalék? asd asd',
'A vállalati profit egy része..',
{
type: 'simple',
date: date(1000),
}
'Mi az osztalék? asd asd',
'A vállalati profit egy része..',
{
type: 'simple',
date: date(1000),
}
)
const q6 = createQuestion(
'valaim nagyon értelmes kérdés asd asd',
'A vállalati profit egy része..',
{
type: 'simple',
date: date(1000),
}
'valaim nagyon értelmes kérdés asd asd',
'A vállalati profit egy része..',
{
type: 'simple',
date: date(1000),
}
)
function setupTest({
newQuestions,
data,
subjToClean,
newQuestions,
data,
subjToClean,
}: {
newQuestions: Question[]
data: Subject[]
subjToClean?: string
newQuestions: Question[]
data: Subject[]
subjToClean?: string
}) {
const recievedQuestions: Question[] = newQuestions.map((x) => {
const recievedQuestions: Question[] = newQuestions.map((x) => {
return {
...x,
data: {
...x.data,
date: date(),
},
}
})
const subjName = subjToClean || 'subject'
const overwriteFromDate = date(-100)
const qdbIndex = 0
const qdbs: QuestionDb[] = [
{
name: 'test',
data: data,
index: 0,
path: '',
shouldSearch: 'asd',
shouldSave: {},
},
]
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
return x.Name.toLowerCase().includes(subjName.toLowerCase())
})
const questionIndexesToRemove = cleanDb(
{
questions: recievedQuestions,
subjToClean: subjName,
overwriteFromDate: overwriteFromDate,
qdbIndex: qdbIndex,
},
qdbs
)
const updatedQuestions = updateQuestionsInArray(
questionIndexesToRemove,
qdbs[qdbIndex].data[subjIndex].Questions,
recievedQuestions
)
return {
...x,
data: {
...x.data,
date: date(),
},
questionIndexesToRemove: questionIndexesToRemove,
updatedQuestions: updatedQuestions,
overwriteFromDate: overwriteFromDate,
subjIndex: subjIndex,
}
})
const subjName = subjToClean || 'subject'
const overwriteFromDate = date(-100)
const qdbIndex = 0
const qdbs: QuestionDb[] = [
{
name: 'test',
data: data,
index: 0,
path: '',
shouldSearch: 'asd',
shouldSave: {},
},
]
const subjIndex = qdbs[qdbIndex].data.findIndex((x) => {
return x.Name.toLowerCase().includes(subjName.toLowerCase())
})
const questionIndexesToRemove = cleanDb(
{
questions: recievedQuestions,
subjToClean: subjName,
overwriteFromDate: overwriteFromDate,
qdbIndex: qdbIndex,
},
qdbs
)
const updatedQuestions = updateQuestionsInArray(
questionIndexesToRemove,
qdbs[qdbIndex].data[subjIndex].Questions,
recievedQuestions
)
return {
questionIndexesToRemove: questionIndexesToRemove,
updatedQuestions: updatedQuestions,
overwriteFromDate: overwriteFromDate,
subjIndex: subjIndex,
}
}
const s1: Subject = { Name: 'test subject', Questions: [q1, q2, q4, q5] }
test('Old and duplicate questions should be removed from the database', () => {
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
setupTest({ newQuestions: [q1, q4, q5], data: [s1] })
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
setupTest({ newQuestions: [q1, q4, q5], data: [s1] })
expect(questionIndexesToRemove.length).toBe(3)
expect(questionIndexesToRemove[0].length).toBe(2)
expect(questionIndexesToRemove.length).toBe(3)
expect(questionIndexesToRemove[0].length).toBe(2)
expect(updatedQuestions.length).toBe(3)
const toremoveCount = updatedQuestions.filter((question) => {
return question.Q.includes('TOREMOVE')
}).length
expect(toremoveCount).toBe(1)
const newQuestion = updatedQuestions.find((question) => {
return question.Q.includes('TOREMOVE')
})
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
expect(updatedQuestions.length).toBe(3)
const toremoveCount = updatedQuestions.filter((question) => {
return question.Q.includes('TOREMOVE')
}).length
expect(toremoveCount).toBe(1)
const newQuestion = updatedQuestions.find((question) => {
return question.Q.includes('TOREMOVE')
})
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
})
const s2: Subject = {
Name: 'test subject',
Questions: [q1, q2, q3, q4, q5, q6],
Name: 'test subject',
Questions: [q1, q2, q3, q4, q5, q6],
}
test('Old and duplicate questions should be removed from the database round 2', () => {
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
setupTest({ newQuestions: [q1, q4, q5], data: [s2] })
const { questionIndexesToRemove, updatedQuestions, overwriteFromDate } =
setupTest({ newQuestions: [q1, q4, q5], data: [s2] })
expect(questionIndexesToRemove.length).toBe(3)
expect(questionIndexesToRemove[0].length).toBe(3)
expect(questionIndexesToRemove.length).toBe(3)
expect(questionIndexesToRemove[0].length).toBe(3)
expect(updatedQuestions.length).toBe(4)
const toremoveCount = updatedQuestions.filter((question) => {
return question.Q.includes('TOREMOVE')
}).length
expect(toremoveCount).toBe(1)
const newQuestion = updatedQuestions.find((question) => {
return question.Q.includes('TOREMOVE')
})
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
expect(updatedQuestions.length).toBe(4)
const toremoveCount = updatedQuestions.filter((question) => {
return question.Q.includes('TOREMOVE')
}).length
expect(toremoveCount).toBe(1)
const newQuestion = updatedQuestions.find((question) => {
return question.Q.includes('TOREMOVE')
})
expect(newQuestion.data.date > overwriteFromDate).toBeTruthy()
})
const s3: Subject = {
Name: 'test subject',
Questions: [q5, q6].map((x) => ({
...x,
data: {
...x.data,
date: date(+50000),
},
})),
Name: 'test subject',
Questions: [q5, q6].map((x) => ({
...x,
data: {
...x.data,
date: date(+50000),
},
})),
}
test('Old and duplicate questions should be removed from the database: questions should be left alone when they are newer', () => {
const { questionIndexesToRemove, updatedQuestions } = setupTest({
newQuestions: [q5, q6],
data: [s3],
})
const { questionIndexesToRemove, updatedQuestions } = setupTest({
newQuestions: [q5, q6],
data: [s3],
})
expect(questionIndexesToRemove.length).toBe(2)
questionIndexesToRemove.forEach((x) => {
expect(x.length).toBe(0)
})
expect(questionIndexesToRemove.length).toBe(2)
questionIndexesToRemove.forEach((x) => {
expect(x.length).toBe(0)
})
expect(updatedQuestions.length).toBe(2)
expect(updatedQuestions.length).toBe(2)
})
const s4: Subject = {
Name: 'something else',
Questions: [q5, q6],
Name: 'something else',
Questions: [q5, q6],
}
test('Old and duplicate questions should be removed from the database:other subjects should be left alone', () => {
const { subjIndex } = setupTest({
newQuestions: [q5, q6],
data: [s2, s1, s4, s3],
subjToClean: 'else',
})
const { subjIndex } = setupTest({
newQuestions: [q5, q6],
data: [s2, s1, s4, s3],
subjToClean: 'else',
})
expect(subjIndex).toBe(2)
expect(subjIndex).toBe(2)
})

466
src/tests/possibleAnswerPenalty.test.ts
View file

@ -1,308 +1,308 @@
import {
setNoPossibleAnswersPenalties,
SearchResultQuestion,
noPossibleAnswerMatchPenalty,
setNoPossibleAnswersPenalties,
SearchResultQuestion,
noPossibleAnswerMatchPenalty,
} from '../utils/classes'
import { Question } from '../types/basicTypes'
const matchPercent = 100
const questionWithNormalPossibleAnswers: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'super laptop' },
{ type: 'txt', val: 'nothing in particular' },
{ type: 'txt', val: 'something giberish' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'super laptop' },
{ type: 'txt', val: 'nothing in particular' },
{ type: 'txt', val: 'something giberish' },
],
},
}
const questionWithNormalPossibleAnswersWithLabels: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'a) nothing in particular' },
{ type: 'txt', val: 'b) super laptop' },
{ type: 'txt', val: 'c) something giberish' },
{ type: 'txt', val: 'd) rubber duck' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'a) nothing in particular' },
{ type: 'txt', val: 'b) super laptop' },
{ type: 'txt', val: 'c) something giberish' },
{ type: 'txt', val: 'd) rubber duck' },
],
},
}
const questionWithNormalPossibleAnswers2: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'cat' },
{ type: 'txt', val: 'nothing in particular' },
{ type: 'txt', val: 'dog' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'cat' },
{ type: 'txt', val: 'nothing in particular' },
{ type: 'txt', val: 'dog' },
],
},
}
const questionWithNormalPossibleAnswers3: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck 2' },
{ type: 'txt', val: 'whale' },
{ type: 'txt', val: 'nothing in particular 2' },
{ type: 'txt', val: 'sea lion' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck 2' },
{ type: 'txt', val: 'whale' },
{ type: 'txt', val: 'nothing in particular 2' },
{ type: 'txt', val: 'sea lion' },
],
},
}
const questionWithNormalPossibleAnswers4: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'super laptop' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'rubber duck' },
{ type: 'txt', val: 'super laptop' },
],
},
}
const questionWithSimilarPossibleAnswers: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'asd' },
{ type: 'txt', val: 'basd' },
{ type: 'txt', val: 'aaa' },
{ type: 'txt', val: 'bbb' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'asd' },
{ type: 'txt', val: 'basd' },
{ type: 'txt', val: 'aaa' },
{ type: 'txt', val: 'bbb' },
],
},
}
const questionWithTrueFalsePossibleAnser: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'true' },
{ type: 'txt', val: 'false' },
],
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
possibleAnswers: [
{ type: 'txt', val: 'true' },
{ type: 'txt', val: 'false' },
],
},
}
const questionWithNoPossibleAnswer: Question = {
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
},
Q: 'asd',
A: 'asd',
data: {
type: 'simple',
},
}
const resNormal: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithNormalPossibleAnswers,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const resNormal2: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers2,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithNormalPossibleAnswers2,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const resNormal3: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers3,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithNormalPossibleAnswers3,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const resNormal4: SearchResultQuestion = {
q: questionWithNormalPossibleAnswers4,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithNormalPossibleAnswers4,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const resSimilar: SearchResultQuestion = {
q: questionWithSimilarPossibleAnswers,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithSimilarPossibleAnswers,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const resTrueFalse: SearchResultQuestion = {
q: questionWithTrueFalsePossibleAnser,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithTrueFalsePossibleAnser,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const resNoPossibleAnswer: SearchResultQuestion = {
q: questionWithNoPossibleAnswer,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
q: questionWithNoPossibleAnswer,
match: matchPercent,
detailedMatch: {
qMatch: matchPercent,
aMatch: matchPercent,
dMatch: matchPercent,
matchedSubjName: 'testSubj',
avg: matchPercent,
},
}
const testFunction = (
question: Question,
searchResult: SearchResultQuestion[],
index: number
question: Question,
searchResult: SearchResultQuestion[],
index: number
) => {
const updated = setNoPossibleAnswersPenalties(
question.data.possibleAnswers,
searchResult
)
const updated = setNoPossibleAnswersPenalties(
question.data.possibleAnswers,
searchResult
)
updated.forEach((x, i) => {
if (i !== index) {
expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty)
expect(x.detailedMatch.qMatch).toBe(
matchPercent - noPossibleAnswerMatchPenalty
)
} else {
expect(x.match).toBe(100)
expect(x.detailedMatch.qMatch).toBe(100)
}
})
updated.forEach((x, i) => {
if (i !== index) {
expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty)
expect(x.detailedMatch.qMatch).toBe(
matchPercent - noPossibleAnswerMatchPenalty
)
} else {
expect(x.match).toBe(100)
expect(x.detailedMatch.qMatch).toBe(100)
}
})
return updated
return updated
}
test('Possible answer penalty applies correctly (normal possible answers)', () => {
testFunction(
questionWithNormalPossibleAnswers,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
0
)
testFunction(
questionWithNormalPossibleAnswers,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
0
)
})
test('Possible answer penalty applies correctly (normal possible answers, with labels)', () => {
testFunction(
questionWithNormalPossibleAnswersWithLabels,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
0
)
testFunction(
questionWithNormalPossibleAnswersWithLabels,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
0
)
})
test('Possible answer penalty applies correctly (similar possible answers)', () => {
testFunction(
questionWithSimilarPossibleAnswers,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
4
)
testFunction(
questionWithSimilarPossibleAnswers,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
4
)
})
test('Possible answer penalty applies correctly (true false possible answers)', () => {
testFunction(
questionWithTrueFalsePossibleAnser,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
5
)
testFunction(
questionWithTrueFalsePossibleAnser,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
],
5
)
})
test('Possible answer penalty applies correctly (no possible answers)', () => {
const updated = setNoPossibleAnswersPenalties(
questionWithNoPossibleAnswer.data.possibleAnswers,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
]
)
const updated = setNoPossibleAnswersPenalties(
questionWithNoPossibleAnswer.data.possibleAnswers,
[
resNormal,
resNormal2,
resNormal3,
resNormal4,
resSimilar,
resTrueFalse,
resNoPossibleAnswer,
]
)
updated.forEach((x) => {
expect(x.match).toBe(100)
expect(x.detailedMatch.qMatch).toBe(100)
})
updated.forEach((x) => {
expect(x.match).toBe(100)
expect(x.detailedMatch.qMatch).toBe(100)
})
})
test('Possible answer penalty applies correctly (empty searchResult)', () => {
const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0)
expect(updated.length).toBe(0)
const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0)
expect(updated.length).toBe(0)
})

12
src/tests/shouldLog.test.ts
View file

@ -6,10 +6,10 @@ const truthy = [1, 2, '10', '40']
const falsey = [5, '55', 47832, 'fhs']
test('ShouldLog works', () => {
truthy.forEach((x) => {
expect(shouldLog(x, noLogIds)).toBeTruthy()
})
falsey.forEach((x) => {
expect(shouldLog(x, noLogIds)).toBeFalsy()
})
truthy.forEach((x) => {
expect(shouldLog(x, noLogIds)).toBeTruthy()
})
falsey.forEach((x) => {
expect(shouldLog(x, noLogIds)).toBeFalsy()
})
})

172
src/types/basicTypes.ts
View file

@ -26,136 +26,136 @@ import http from 'http'
import https from 'https'
export interface QuestionData {
type: string
date?: Date | number
images?: Array<string>
hashedImages?: Array<string>
possibleAnswers?: Array<{
type: string
val: string
selectedByUser?: boolean
}>
base64?: string[]
date?: Date | number
images?: Array<string>
hashedImages?: Array<string>
possibleAnswers?: Array<{
type: string
val: string
selectedByUser?: boolean
}>
base64?: string[]
}
export interface Question {
Q: string
A: string
data: QuestionData
cache?: {
Q: Array<string>
A: Array<string>
}
Q: string
A: string
data: QuestionData
cache?: {
Q: Array<string>
A: Array<string>
}
}
export interface Subject {
Name: string
Questions: Array<Question>
Name: string
Questions: Array<Question>
}
export interface DataFile {
path: string
name: string
locked?: Boolean
overwrites?: Array<{
subjName: string
overwriteFromDate: number
}>
shouldSearch:
| string
| {
path: string
name: string
locked?: Boolean
overwrites?: Array<{
subjName: string
overwriteFromDate: number
}>
shouldSearch:
| string
| {
location?: {
val: string
}
}
shouldSave: {
location?: {
val: string
val: string
}
version?: {
compare: string
val: string
}
}
shouldSave: {
location?: {
val: string
}
version?: {
compare: string
val: string
}
}
}
export interface QuestionDb extends DataFile {
data: Array<Subject>
index: number
data: Array<Subject>
index: number
}
export interface User {
id: number
pw: string
created: Date
id: number
pw: string
created: Date
}
export interface User {
id: number
pw: string
pwRequestCount: number
avaiblePWRequests: number
loginCount: number
createdBy: number
id: number
pw: string
pwRequestCount: number
avaiblePWRequests: number
loginCount: number
createdBy: number
}
export interface Request<T = any> extends express.Request {
body: T
cookies: any
session: {
user?: User
sessionID?: string
isException?: boolean
}
files: any
query: { [key: string]: string }
body: T
cookies: any
session: {
user?: User
sessionID?: string
isException?: boolean
}
files: any
query: { [key: string]: string }
}
export interface SubmoduleData {
app: express.Application
url: string
publicdirs: Array<string>
userDB?: Database
nextdir?: string
moduleSpecificData?: any
httpServer: http.Server
httpsServer: https.Server
app: express.Application
url: string
publicdirs: Array<string>
userDB?: Database
nextdir?: string
moduleSpecificData?: any
httpServer: http.Server
httpsServer: https.Server
}
export interface QuestionFromScript {
questions: Array<Question>
testUrl: string
subj: string
questions: Array<Question>
testUrl: string
subj: string
}
export interface DbSearchResult {
message?: string
recievedData?: string
question: Question
result: SearchResultQuestion[]
success: boolean
message?: string
recievedData?: string
question: Question
result: SearchResultQuestion[]
success: boolean
}
export interface RegisteredUserEntry {
cid: string
version: string
installSource: string
date: string
userAgent: string
loginDate?: string
uid?: number
cid: string
version: string
installSource: string
date: string
userAgent: string
loginDate?: string
uid?: number
}
export interface Submodule {
dailyAction?: () => void
load?: () => void
dailyAction?: () => void
load?: () => void
}
export interface ModuleType {
app: express.Application
dailyAction?: Function
app: express.Application
dailyAction?: Function
}
export interface Socket extends SocketIoSocket {
user: User
user: User
}

1404
src/utils/actions.ts
View file

File diff suppressed because it is too large Load diff

1448
src/utils/classes.ts
View file

File diff suppressed because it is too large Load diff

468
src/utils/dbtools.ts
View file

@ -22,18 +22,18 @@
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
export default {
GetDB: GetDB,
AddColumn: AddColumn,
TableInfo: TableInfo,
Update: Update,
Delete: Delete,
CreateTable: CreateTable,
SelectAll: SelectAll,
Select: Select,
Insert: Insert,
CloseDB: CloseDB,
runStatement: runStatement,
sanitizeQuery: sanitizeQuery,
GetDB: GetDB,
AddColumn: AddColumn,
TableInfo: TableInfo,
Update: Update,
Delete: Delete,
CreateTable: CreateTable,
SelectAll: SelectAll,
Select: Select,
Insert: Insert,
CloseDB: CloseDB,
runStatement: runStatement,
sanitizeQuery: sanitizeQuery,
}
import Sqlite, { Database, RunResult } from 'better-sqlite3'
@ -43,310 +43,310 @@ import utils from '../utils/utils'
const debugLog = process.env.NS_SQL_DEBUG_LOG
function sanitizeQuery(val: string | number): string | number {
if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '')
}
return val
if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '')
}
return val
}
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
function GetSqlQuerry(
conditions: { [key: string]: string | number },
type?: string,
joiner?: string
conditions: { [key: string]: string | number },
type?: string,
joiner?: string
) {
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key])
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key])
if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`)
if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`)
} else {
acc.push(`${conditionKey} = ${condition}`)
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
acc.push(`${conditionKey} = ${condition}`)
return res.join(', ')
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
return res.join(', ')
}
}
// -------------------------------------------------------------------------
function GetDB(path: string): Database {
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
}
function DebugLog(msg: string) {
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
}
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
function AddColumn(
db: Database,
table: string,
col: { [key: string]: string | number }
db: Database,
table: string,
col: { [key: string]: string | number }
): RunResult {
try {
const colName = Object.keys(col)[0]
const colType = col.type
try {
const colName = Object.keys(col)[0]
const colType = col.type
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function TableInfo(
db: Database,
table: string
db: Database,
table: string
): {
columns: any[]
dataCount: number
columns: any[]
dataCount: number
} {
try {
const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command)
try {
const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command)
const infoRes = stmt.all()
const infoRes = stmt.all()
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const countRes = stmt2.get()
const countRes = stmt2.get()
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]],
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]],
}
} catch (err) {
console.error(err)
return null
}
} catch (err) {
console.error(err)
return null
}
}
function Update(
db: Database,
table: string,
newData: { [key: string]: string | number },
conditions: { [key: string]: string | number }
db: Database,
table: string,
newData: { [key: string]: string | number },
conditions: { [key: string]: string | number }
): RunResult {
try {
const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData,
'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command)
try {
const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData,
'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function Delete(
db: Database,
table: string,
conditions: { [key: string]: string | number }
db: Database,
table: string,
conditions: { [key: string]: string | number }
): RunResult {
try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions,
'where'
)}`
const stmt = PrepareStatement(db, command)
try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions,
'where'
)}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
interface DbColumnDescription {
[key: string]: {
type: string
primary?: boolean
autoIncrement?: boolean
notNull?: boolean
defaultZero?: boolean
[key: string]: any
}
[key: string]: {
type: string
primary?: boolean
autoIncrement?: boolean
notNull?: boolean
defaultZero?: boolean
[key: string]: any
}
}
function CreateTable(
db: Database,
name: string,
columns: DbColumnDescription,
foreignKeys: {
keysFrom: string[]
table: string
keysTo: string[]
}[]
db: Database,
name: string,
columns: DbColumnDescription,
foreignKeys: {
keysFrom: string[]
table: string
keysTo: string[]
}[]
): RunResult {
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
try {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0',
try {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0',
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function SelectAll(db: Database, from: string): any[] {
try {
const command = `SELECT * from ${from}`
try {
const command = `SELECT * from ${from}`
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
}
// SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100;
function Select(
db: Database,
from: string,
conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {}
db: Database,
from: string,
conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {}
): any[] {
const { joiner, limit } = options
const { joiner, limit } = options
try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions,
'where',
joiner
)}`
try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions,
'where',
joiner
)}`
if (!isNaN(limit)) {
command += ` LIMIT ${limit}`
if (!isNaN(limit)) {
command += ` LIMIT ${limit}`
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
}
function Insert(
db: Database,
table: string,
data: { [key: string]: number | string }
db: Database,
table: string,
data: { [key: string]: number | string }
): RunResult {
try {
const cols = Object.keys(data)
.reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, [])
.join(', ')
try {
const cols = Object.keys(data)
.reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, [])
.join(', ')
const values = Object.keys(data)
.map((key) => {
const item = data[key]
if (typeof item === 'string') {
return `'${item}'`
} else {
return `${item}`
}
})
.join(', ')
const values = Object.keys(data)
.map((key) => {
const item = data[key]
if (typeof item === 'string') {
return `'${item}'`
} else {
return `${item}`
}
})
.join(', ')
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command)
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function runStatement(db: Database, command: string, runType?: string): any {
const stmt = PrepareStatement(db, command)
if (!runType) {
return stmt.all()
} else if (runType === 'run') {
return stmt.run()
}
return null
const stmt = PrepareStatement(db, command)
if (!runType) {
return stmt.all()
} else if (runType === 'run') {
return stmt.run()
}
return null
}
function CloseDB(db: Database): void {
db.close()
db.close()
}
// -------------------------------------------------------------------------
function PrepareStatement(db: Database, command: string) {
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'
)
}
DebugLog(command)
return db.prepare(command)
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'
)
}
DebugLog(command)
return db.prepare(command)
}

178
src/utils/ids.ts
View file

@ -19,8 +19,8 @@
------------------------------------------------------------------------- */
export default {
LogId: LogId,
Load: Load,
LogId: LogId,
Load: Load,
}
import utils from '../utils/utils'
@ -35,113 +35,113 @@ let idvStatsData = {}
let writes = 0
function Load(): void {
try {
idStatsData = utils.ReadJSON(idStatFile)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
idStatsData = utils.ReadJSON(idStatFile)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
}
function LogId(
id: number,
subj: string,
newQuestions: number,
allQuestions: number
id: number,
subj: string,
newQuestions: number,
allQuestions: number
): void {
Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions)
Save()
Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions)
Save()
}
function AddSubjToList(list: { [key: string]: any }, subj: string) {
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
}
function Inc(
value: number,
subj: string,
newQuestions: number,
allQuestions: number
value: number,
subj: string,
newQuestions: number,
allQuestions: number
) {
if (idStatsData[value] === undefined) {
idStatsData[value] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
if (idStatsData[value] === undefined) {
idStatsData[value] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
}
}
}
idStatsData[value].count++
idStatsData[value].newQuestions += newQuestions
idStatsData[value].allQuestions += allQuestions
AddSubjToList(idStatsData[value].subjs, subj)
idStatsData[value].count++
idStatsData[value].newQuestions += newQuestions
idStatsData[value].allQuestions += allQuestions
AddSubjToList(idStatsData[value].subjs, subj)
}
function AddVisitStat(
name: number,
subj: string,
newQuestions: number,
allQuestions: number
name: number,
subj: string,
newQuestions: number,
allQuestions: number
) {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) {
idvStatsData[now] = {}
}
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) {
idvStatsData[now] = {}
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
}
function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err)
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
}

572
src/utils/logger.ts
View file

@ -19,7 +19,7 @@
------------------------------------------------------------------------- */
const hr =
'---------------------------------------------------------------------------------'
'---------------------------------------------------------------------------------'
const DELIM = C('green') + '|' + C()
@ -50,361 +50,377 @@ let writes = 0
let noLogIds: string[] = []
function getColoredDateString(): string {
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
}
function DebugLog(msg: string | object, name: string, lvl: number): void {
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
}
function Log(msg: string | object, color?: string): void {
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
)
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
)
}
function expandWithSpaces(text: string, count: number) {
while (text.length < count) {
text += ' '
}
return text
while (text.length < count) {
text += ' '
}
return text
}
function LogReq(
req: Request,
toFile?: boolean,
statusCode?: string | number
req: Request,
toFile?: boolean,
statusCode?: string | number
): void {
try {
let logEntry = '' // logHashed(ip)
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
if (req.session && req.session.user && !shouldLog(req.session.user.id, noLogIds)) {
return
}
try {
let logEntry = '' // logHashed(ip)
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
if (
req.session &&
req.session.user &&
!shouldLog(req.session.user.id, noLogIds)
) {
return
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log(
'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg')
)
}
if (!toFile) {
hostname = expandWithSpaces(hostname, 10)
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log(
'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg')
)
}
if (!toFile) {
hostname = expandWithSpaces(hostname, 10)
}
logEntry += logHashed(hostname) + dl
if (toFile) {
logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl
}
logEntry += logHashed(hostname) + dl
if (toFile) {
logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl
}
let uid = ''
if (req.session && req.session.user) {
uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) {
uid = 'EX'
} else {
uid = 'NOUSR'
}
if (!toFile) {
uid = expandWithSpaces(uid, 5)
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
let uid = ''
if (req.session && req.session.user) {
uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) {
uid = 'EX'
} else {
uid = 'NOUSR'
}
if (!toFile) {
uid = expandWithSpaces(uid, 5)
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
if (statusCode !== undefined) {
logEntry += dl + statusCode
if (statusCode !== undefined) {
logEntry += dl + statusCode
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
}
function parseNoLogFile(newData: string) {
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
}
function setNoLogReadInterval() {
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
parseNoLogFile(utils.ReadFile(nologFile))
parseNoLogFile(utils.ReadFile(nologFile))
}
function Load(): void {
Log('Loading logger...')
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
Log('Loading logger...')
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
vData = utils.ReadJSON(statFile)
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
try {
vData = utils.ReadJSON(statFile)
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
dvData = utils.ReadJSON(vStatFile)
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
setNoLogReadInterval()
try {
dvData = utils.ReadJSON(vStatFile)
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
setNoLogReadInterval()
}
export function shouldLog(userId: string | number, nolog: string[]): boolean {
return !nolog.some((noLogId) => {
return noLogId === userId.toString()
})
return !nolog.some((noLogId) => {
return noLogId === userId.toString()
})
}
function LogStat(url: string, hostname: string, userId: number | string): void {
if (!shouldLog(userId, noLogIds)) {
return
}
if (!shouldLog(userId, noLogIds)) {
return
}
url = hostname + url.split('?')[0]
Inc(url)
AddVisitStat(url)
if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
}
Save()
url = hostname + url.split('?')[0]
Inc(url)
AddVisitStat(url)
if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
}
Save()
}
const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
function shouldAddUserStat(url: string) {
return !userStatExcludes.some((x) => url.includes(x))
return !userStatExcludes.some((x) => url.includes(x))
}
function IncUserStat(userId: string) {
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
}
function AddUserIdStat(userId: string) {
try {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) {
udvData[now] = {}
try {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) {
udvData[now] = {}
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
}
function Inc(value: string) {
if (value.startsWith('/?')) {
value = '/'
}
if (vData[value] === undefined) {
vData[value] = 0
}
vData[value]++
if (value.startsWith('/?')) {
value = '/'
}
if (vData[value] === undefined) {
vData[value] = 0
}
vData[value]++
}
function AddVisitStat(name: string) {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) {
dvData[now] = {}
}
if (dvData[now][name] === undefined) {
dvData[now][name] = 0
}
dvData[now][name]++
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) {
dvData[now] = {}
}
if (dvData[now][name] === undefined) {
dvData[now][name] = 0
}
dvData[now][name]++
}
function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log(
'Error at writing visit logs! (more in stderr)',
GetColor('redbg')
)
console.error(err)
}
writes = 0
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
writes = 0
}
}
function logHashed(msg: string): string {
return GetRandomColor(msg.toString()) + msg + C()
return GetRandomColor(msg.toString()) + msg + C()
}
function GetRandomColor(msg: string): string {
if (!msg) {
return 'red'
}
if (!msg) {
return 'red'
}
const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
}
function GetColor(color: string): string {
return color
return color
}
function C(color?: string): string {
if (color !== undefined) {
color = color.toLowerCase()
}
if (color !== undefined) {
color = color.toLowerCase()
}
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
if (color === 'cyanbg') {
return '\x1b[46m'
}
if (color === 'green') {
return '\x1b[32m'
}
if (color === 'red') {
return '\x1b[31m'
}
if (color === 'yellow') {
return '\x1b[33m'
}
if (color === 'blue') {
return '\x1b[34m'
}
if (color === 'magenta') {
return '\x1b[35m'
}
if (color === 'cyan') {
return '\x1b[36m'
}
return '\x1b[0m'
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
if (color === 'cyanbg') {
return '\x1b[46m'
}
if (color === 'green') {
return '\x1b[32m'
}
if (color === 'red') {
return '\x1b[31m'
}
if (color === 'yellow') {
return '\x1b[33m'
}
if (color === 'blue') {
return '\x1b[34m'
}
if (color === 'magenta') {
return '\x1b[35m'
}
if (color === 'cyan') {
return '\x1b[36m'
}
return '\x1b[0m'
}
export default {
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
}

58
src/utils/tesseract.ts
View file

@ -1,7 +1,7 @@
import {
createWorker,
Worker as TesseractWorker,
ConfigResult,
createWorker,
Worker as TesseractWorker,
ConfigResult,
} from 'tesseract.js'
import logger from './logger'
@ -10,44 +10,44 @@ import { isMainThread, workerData } from 'worker_threads'
// https://github.com/naptha/tesseract.js/blob/master/docs/api.md
let tesseractWorker: TesseractWorker = null
export async function initTesseractWorker(): Promise<TesseractWorker> {
const worker = createWorker({
cacheMethod: 'refresh',
// logger: (m) => console.log(m),
})
await worker.load()
await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng')
return worker
// await worker.terminate();
const worker = createWorker({
cacheMethod: 'refresh',
// logger: (m) => console.log(m),
})
await worker.load()
await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng')
return worker
// await worker.terminate();
}
let resolveLoaded: () => void = null
export const tesseractLoaded: Promise<void> = new Promise((resolve) => {
resolveLoaded = resolve
resolveLoaded = resolve
})
initTesseractWorker().then((worker) => {
tesseractWorker = worker
tesseractWorker = worker
if (isMainThread) {
logger.Log('Tesseract loaded on main thread')
} else {
const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
}
resolveLoaded()
if (isMainThread) {
logger.Log('Tesseract loaded on main thread')
} else {
const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
}
resolveLoaded()
})
export async function recognizeTextFromBase64(base64: string): Promise<string> {
const {
data: { text },
} = await tesseractWorker.recognize(base64)
return text
const {
data: { text },
} = await tesseractWorker.recognize(base64)
return text
}
export async function terminateWorker(): Promise<void | ConfigResult> {
if (tesseractWorker) {
return tesseractWorker.terminate()
}
return
if (tesseractWorker) {
return tesseractWorker.terminate()
}
return
}

409
src/utils/utils.ts
View file

@ -19,23 +19,23 @@
------------------------------------------------------------------------- */
export default {
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString,
formatUrl: formatUrl,
deleteFile: deleteFile,
uploadFile: uploadFile,
statFile: statFile,
renameFile: renameFile,
deleteDir: deleteDir,
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString,
formatUrl: formatUrl,
deleteFile: deleteFile,
uploadFile: uploadFile,
statFile: statFile,
renameFile: renameFile,
deleteDir: deleteDir,
}
import fs from 'fs'
@ -45,251 +45,254 @@ import logger from '../utils/logger'
import { Request } from '../types/basicTypes'
interface URLFormatOptions {
pathname?: string
query?: { [key: string]: string }
pathname?: string
query?: { [key: string]: string }
}
function formatUrl(options: URLFormatOptions): string {
const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) {
return path
}
const queryString =
'?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
return path + queryString
const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) {
return path
}
const queryString =
'?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
return path + queryString
}
function GetDateString(
referenceDate?: Date | string,
noTime?: boolean
referenceDate?: Date | string,
noTime?: boolean
): string {
const date = referenceDate ? new Date(referenceDate) : new Date()
const date = referenceDate ? new Date(referenceDate) : new Date()
if (noTime) {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} else {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2) +
' ' +
('0' + date.getHours()).slice(-2) +
':' +
('0' + date.getMinutes()).slice(-2) +
':' +
('0' + date.getSeconds()).slice(-2)
)
}
if (noTime) {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} else {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2) +
' ' +
('0' + date.getHours()).slice(-2) +
':' +
('0' + date.getMinutes()).slice(-2) +
':' +
('0' + date.getSeconds()).slice(-2)
)
}
}
function CopyFile(from: string, to: string): void {
CreatePath(to)
fs.copyFileSync(from, to)
CreatePath(to)
fs.copyFileSync(from, to)
}
function ReadDir(path: string, listHidden?: boolean): Array<string> {
if (listHidden) {
return fs.readdirSync(path)
} else {
return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.')
})
}
if (listHidden) {
return fs.readdirSync(path)
} else {
return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.')
})
}
}
function ReadJSON(name: string): any {
try {
return JSON.parse(ReadFile(name))
} catch (err) {
console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
try {
return JSON.parse(ReadFile(name))
} catch (err) {
console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
}
function ReadFile(name: string): string {
if (!FileExists(name)) {
throw new Error('No such file: ' + name)
}
return fs.readFileSync(name, 'utf8')
if (!FileExists(name)) {
throw new Error('No such file: ' + name)
}
return fs.readFileSync(name, 'utf8')
}
function FileExists(path: string): boolean {
return fs.existsSync(path)
return fs.existsSync(path)
}
function WatchFile(file: string, callback: Function): void {
if (FileExists(file)) {
fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
throw new Error(`${file} does not exits to watch`)
}
if (FileExists(file)) {
fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
throw new Error(`${file} does not exits to watch`)
}
}
function CreatePath(path: string, onlyPath?: boolean): void {
if (FileExists(path)) {
return
}
const spath = path.split('/')
let currDir = spath[0]
for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (err) {
console.log('Failed to make ' + currDir + ' directory... ')
console.error(err)
}
if (FileExists(path)) {
return
}
const spath = path.split('/')
let currDir = spath[0]
for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (err) {
console.log('Failed to make ' + currDir + ' directory... ')
console.error(err)
}
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
}
function WriteFile(content: string, path: string): void {
CreatePath(path)
fs.writeFileSync(path, content)
CreatePath(path)
fs.writeFileSync(path, content)
}
function writeFileAsync(content: string, path: string): void {
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log(
'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg')
)
}
})
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log(
'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg')
)
}
})
}
function AppendToFile(data: string, file: string): void {
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (err) {
logger.Log(
'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg')
)
logger.Log(data)
console.error(err)
}
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (err) {
logger.Log(
'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg')
)
logger.Log(data)
console.error(err)
}
}
function deleteFile(fname: string): Boolean {
if (FileExists(fname)) {
fs.unlinkSync(fname)
return true
}
return false
if (FileExists(fname)) {
fs.unlinkSync(fname)
return true
}
return false
}
function deleteDir(dirName: string): Boolean {
if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true })
return true
}
return false
if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true })
return true
}
return false
}
function uploadFile(
req: Request,
path: string
req: Request,
path: string
): Promise<{
body: Request['body']
fileName: string
filePath: string
body: Request['body']
fileName: string
filePath: string
}> {
return new Promise((resolve, reject) => {
try {
if (!req.files) {
logger.Log(
`Unable to upload file, req.files is undefined`,
logger.GetColor('redbg')
)
return
}
const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
return new Promise((resolve, reject) => {
try {
if (!req.files) {
logger.Log(
`Unable to upload file, req.files is undefined`,
logger.GetColor('redbg')
)
return
}
const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
CreatePath(path, true)
CreatePath(path, true)
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0]
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0]
const temp = file.name.split('.')
const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName
}
const temp = file.name.split('.')
const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName
}
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
console.error(err)
reject(err)
} else {
logger.Log(
`Uploaded: ${fileName} to ${fileDestination}`,
logger.GetColor('blue')
)
resolve({
body: req.body,
fileName: fileName,
filePath: fileDestination,
})
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(
`Unable to upload file!`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
} else {
logger.Log(
`Uploaded: ${fileName} to ${fileDestination}`,
logger.GetColor('blue')
)
resolve({
body: req.body,
fileName: fileName,
filePath: fileDestination,
})
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
})
}
function statFile(file: string): fs.Stats {
if (FileExists(file)) {
return fs.statSync(file)
} else {
return null
}
if (FileExists(file)) {
return fs.statSync(file)
} else {
return null
}
}
function renameFile(oldPath: string, newPath: string): string {
if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath)
return newPath
} else {
return null
}
if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath)
return newPath
} else {
return null
}
}

372
src/utils/workerPool.ts
View file

@ -29,268 +29,274 @@ import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
import type { WorkerResult } from './classes'
interface WorkerObj {
worker: Worker
index: number
free: Boolean
worker: Worker
index: number
free: Boolean
}
export interface TaskObject {
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
qdbIndex: number
}
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
type:
| 'work'
| 'dbEdit'
| 'newQuestions'
| 'newdb'
| 'dbClean'
| 'rmQuestions'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
qdbIndex: number
}
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
}
interface PendingJob {
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
}
interface JobEvent extends EventEmitter {
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
}
interface DoneEvent extends EventEmitter {
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
}
const alertOnPendingCount = 50
const workerFile = './src/utils/classes.ts'
let workers: Array<WorkerObj>
const pendingJobs: {
[id: string]: PendingJob
[id: string]: PendingJob
} = {}
const jobEvents: JobEvent = new EventEmitter()
jobEvents.on('jobDone', () => {
processJob()
processJob()
})
jobEvents.on('newJob', () => {
processJob()
processJob()
})
// ---------------------------------------------------------------------------
function handleWorkerError(worker: WorkerObj, err: Error) {
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker)
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker)
}
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
}
export function doALongTask(
obj: TaskObject,
targetWorkerIndex?: number
obj: TaskObject,
targetWorkerIndex?: number
): Promise<WorkerResult> {
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
)
}
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
)
}
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent,
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent,
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
})
})
})
}
export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> {
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null
}
workers = []
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null
}
workers = []
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
)
}
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
)
}
for (let i = 0; i < threadCount; i++) {
workers.push({
worker: getAWorker(i, initData),
index: i,
free: true,
})
}
for (let i = 0; i < threadCount; i++) {
workers.push({
worker: getAWorker(i, initData),
index: i,
free: true,
})
}
return workers
return workers
}
// ---------------------------------------------------------------------------
function processJob() {
if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job',
1
)
if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job',
1
)
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
}
}
i++
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
if (!freeWorker) {
logger.DebugLog('NO FREE WORKER', 'job', 1)
return
}
}
i++
}
if (!freeWorker) {
logger.DebugLog('NO FREE WORKER', 'job', 1)
return
}
if (freeWorker.free) {
freeWorker.free = false
}
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
if (freeWorker.free) {
freeWorker.free = false
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
}
function getAWorker(i: number, initData: Array<QuestionDb>) {
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
initData: initData,
},
})
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
initData: initData,
},
})
worker.setMaxListeners(50)
worker.setMaxListeners(50)
worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err)
})
worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err)
})
worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
)
})
return worker
worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
)
})
return worker
}
// ---------------------------------------------------------------------------
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
})
})
})
}
const workerTs = (
file: string,
wkOpts: {
workerData: {
workerIndex: number
initData: QuestionDb[]
__filename?: string
file: string,
wkOpts: {
workerData: {
workerIndex: number
initData: QuestionDb[]
__filename?: string
}
eval?: boolean
}
eval?: boolean
}
) => {
wkOpts.eval = true
wkOpts.workerData.__filename = file
return new Worker(
`
wkOpts.eval = true
wkOpts.workerData.__filename = file
return new Worker(
`
const wk = require('worker_threads');
require('ts-node').register();
let file = wk.workerData.__filename;
delete wk.workerData.__filename;
require(file);
`,
wkOpts
)
wkOpts
)
}

2
submodules/qmining-page

@ -1 +1 @@
Subproject commit 281d0e00ce054d46444f377876786b913b8c1a08
Subproject commit ed507dc39f5d34703e53585a75a0138e70bcee3a