albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

added / fixed some types

Browse source
This commit is contained in:
mrfry 2022-03-14 19:35:42 +01:00
parent 5f12284bb8
commit bc5c293539
41 changed files with 4378 additions and 8304 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.eslintrc.js
View file

@ -24,7 +24,6 @@ module.exports = {
'warn',
{ allowArgumentsExplicitlyTypedAsAny: true },
],
'no-unused-vars': 'warn',
'no-prototype-builtins': 'off',
'@typescript-eslint/ban-types': 'off',
'id-length': [

9925
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

38
package.json
View file

@ -2,35 +2,39 @@
"name": "node-ejs",
"main": "src/server.js",
"dependencies": {
"@types/express": "^4.17.9",
"@types/node": "^15.0.1",
"better-sqlite3": "^7.1.5",
"connect-busboy": "0.0.2",
"cookie-parser": "^1.4.5",
"@types/express": "^4.17.13",
"better-sqlite3": "^7.5.0",
"cookie-parser": "^1.4.6",
"cors": "^2.8.5",
"ejs": "^3.1.6",
"express": "^4.6.1",
"express-fileupload": "^1.2.1",
"socket.io": "^4.1.2",
"ts-node": "^9.0.0",
"typescript": "^4.1.2",
"express": "^4.17.3",
"express-fileupload": "^1.3.1",
"socket.io": "^4.4.1",
"ts-node": "^10.7.0",
"typescript": "^4.6.2",
"uuid": "^8.3.2",
"vhost": "^3.0.2"
},
"scripts": {
"start": "node ./dist/server.js",
"dev": "npm run build && NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node ./dist/server.js",
"dev": "npm run build && NS_THREAD_COUNT=2 NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node --inspect ./dist/server.js",
"build": "tsc && bash -c './scripts/postBuild.sh'",
"export": "tsc && bash -c './scripts/postBuild.sh'",
"test": "jest --detectOpenHandles"
},
"devDependencies": {
"@types/jest": "^26.0.23",
"@typescript-eslint/eslint-plugin": "^4.8.1",
"@typescript-eslint/parser": "^4.22.0",
"eslint": "^7.14.0",
"jest": "^26.6.3",
"ts-jest": "^26.5.5"
"@types/better-sqlite3": "^7.5.0",
"@types/cookie-parser": "^1.4.2",
"@types/express-fileupload": "^1.2.2",
"@types/jest": "^27.4.1",
"@types/node": "^17.0.21",
"@types/uuid": "^8.3.4",
"@types/vhost": "^3.0.4",
"@typescript-eslint/eslint-plugin": "^5.15.0",
"@typescript-eslint/parser": "^5.15.0",
"eslint": "^8.11.0",
"jest": "^27.5.1",
"ts-jest": "^27.1.3"
},
"jest": {
"preset": "ts-jest"

22
src/middlewares/auth.middleware.ts
View file

@ -1,3 +1,7 @@
import type { Response, NextFunction } from 'express'
import type { Request } from '../types/basicTypes'
import type { Database } from 'better-sqlite3'
import logger from '../utils/logger'
import utils from '../utils/utils'
import dbtools from '../utils/dbtools'
@ -18,8 +22,8 @@ export const testUser = {
createdBy: 1,
}
function renderLogin(req, res, jsonResponse) {
res.status('401') // Unauthorized
function renderLogin(_req: Request, res: Response, jsonResponse: boolean) {
res.status(401) // Unauthorized
if (jsonResponse) {
res.json({
result: 'nouser',
@ -33,9 +37,17 @@ function renderLogin(req, res, jsonResponse) {
}
export default function (options: Options): any {
const { userDB, jsonResponse, exceptions } = options
const {
userDB,
jsonResponse,
exceptions,
}: {
userDB: Database
jsonResponse: boolean
exceptions: string[]
} = options
return function (req, res, next) {
return function (req: Request, res: Response, next: NextFunction) {
const sessionID = req.cookies.sessionID
const isException = exceptions.some((exc) => {
return req.url.split('?')[0] === exc
@ -44,7 +56,7 @@ export default function (options: Options): any {
if (process.env.NS_NOUSER) {
req.session = {
user: testUser,
sessionID: sessionID || 111111111111111111,
sessionID: sessionID || 11111111111,
isException: false,
}
next()

6
src/middlewares/reqlogger.middleware.ts
View file

@ -1,4 +1,6 @@
import logger from '../utils/logger'
import type { Response, NextFunction } from 'express'
import type { Request } from '../types/basicTypes'
interface Options {
loggableKeywords: Array<string>
@ -13,7 +15,7 @@ export default function (options: Options): any {
const exceptions = options.exceptions || []
const excludeFromStats = options.excludeFromStats || []
return function (req, res, next) {
return function (req: Request, res: Response, next: NextFunction) {
res.on('finish', function () {
// TODO: test this
const isException = exceptions.some((ex) => {
@ -24,7 +26,6 @@ export default function (options: Options): any {
return
}
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
let hostname = 'NOHOST'
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
@ -59,7 +60,6 @@ export default function (options: Options): any {
if (res.statusCode !== 404 && shouldLogStat) {
logger.LogStat(
req.url,
ip,
hostname,
req.session && req.session.user ? req.session.user.id : 'NOUSER'
)

6
src/middlewares/socketAuth.middleware.ts
View file

@ -1,6 +1,8 @@
import cookie from 'cookie'
import logger from '../utils/logger'
import dbtools from '../utils/dbtools'
import cookie from 'cookie'
import { Socket } from '../types/basicTypes'
import { testUser } from './auth.middleware'
@ -11,7 +13,7 @@ interface Options {
export default function SocketAuth(options: Options): any {
const { userDB } = options
return (socket, next) => {
return (socket: Socket, next: (arg0?: any) => void) => {
try {
const cookies = cookie.parse(socket.handshake.headers.cookie || '')
const sessionID = cookies.sessionID

38
src/modules/api/api.ts
View file

@ -19,16 +19,18 @@
------------------------------------------------------------------------- */
// package requires
import express from 'express'
import bodyParser from 'body-parser'
import express, { RequestHandler } from 'express'
import fileUpload from 'express-fileupload'
import type { Database } from 'better-sqlite3'
import http from 'http'
import https from 'https'
// other requires
import logger from '../../utils/logger'
import utils from '../../utils/utils'
import auth from '../../middlewares/auth.middleware'
import { SetupData } from '../../server'
import { ModuleType, Request } from '../../types/basicTypes'
import { ModuleType, Request, Submodule } from '../../types/basicTypes'
// files
const rootRedirectToFile = 'data/apiRootRedirectTo'
@ -37,11 +39,11 @@ const rootRedirectToFile = 'data/apiRootRedirectTo'
const moduleName = 'API'
// stuff gotten from server.js
let userDB
let url
let publicdirs = []
let httpServer
let httpsServer
let userDB: Database
let url: string
let publicdirs: string[] = []
let httpServer: http.Server
let httpsServer: https.Server
function GetApp(): ModuleType {
const app = express()
@ -51,7 +53,7 @@ function GetApp(): ModuleType {
throw new Error(`No public dir! ( API )`)
}
let domain = url.split('.') // [ "https://api", "frylabs", "net" ]
let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
@ -59,15 +61,15 @@ function GetApp(): ModuleType {
// -------------------------------------------------------------------------------------------
app.use(
bodyParser.urlencoded({
express.urlencoded({
limit: '10mb',
extended: true,
})
}) as RequestHandler
)
app.use(
bodyParser.json({
express.json({
limit: '10mb',
})
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/api/views', './src/sharedViews'])
@ -151,11 +153,11 @@ function GetApp(): ModuleType {
// -------------------------------------------------------------------------------------------
app.get('*', function (req: Request, res: any) {
app.get('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
app.post('*', function (req: Request, res: any) {
app.post('*', function (_req: Request, res: any) {
res.status(404).render('404')
})
@ -182,14 +184,14 @@ function GetApp(): ModuleType {
function setupSubModules(
parentApp: express.Application,
moduleSpecificData?: any
): any {
): Submodule[] {
const submoduleDir = './submodules/'
const absolutePath = __dirname + '/' + submoduleDir
if (!utils.FileExists(absolutePath)) {
return
return null
}
const files = utils.ReadDir(absolutePath)
const moduleDatas = []
const moduleDatas: Submodule[] = []
files.forEach((file) => {
if (!file.endsWith('.js')) {
return

32
src/modules/api/msgsDbStruct.json
View file

@ -1,32 +0,0 @@
{
"msgs": {
"tableStruct": {
"id": {
"type": "integer",
"primary": true,
"autoIncrement": true
},
"sender": {
"type": "integer",
"notNull": true
},
"reciever": {
"type": "integer",
"notNull": true
},
"msg": {
"type": "text"
},
"type": {
"type": "text"
},
"date": {
"type": "integer"
},
"unread": {
"type": "integer",
"defaultZero": true
}
}
}
}

34
src/modules/api/msgsDbStruct.ts Normal file
View file

@ -0,0 +1,34 @@
const DbStruct = {
msgs: {
tableStruct: {
id: {
type: 'integer',
primary: true,
autoIncrement: true,
},
sender: {
type: 'integer',
notNull: true,
},
reciever: {
type: 'integer',
notNull: true,
},
msg: {
type: 'text',
},
type: {
type: 'text',
},
date: {
type: 'integer',
},
unread: {
type: 'integer',
defaultZero: true,
},
},
},
}
export default DbStruct

33
src/modules/api/submodules/chat.ts
View file

@ -13,6 +13,16 @@ interface ExtendedSocket extends Socket {
user: User
}
interface Message {
id: number
sender: number
reciever: number
msg: string
type: string
date: number
unread: number
}
function setup(data: SubmoduleData): void {
const { app, httpServer, httpsServer, userDB, publicdirs } = data
const msgDB = dbtools.GetDB(msgDbPath)
@ -28,8 +38,13 @@ function setup(data: SubmoduleData): void {
},
})
function chatMessageRead(data) {
const { sender, reciever } = data
function chatMessageRead({
sender,
reciever,
}: {
sender: number
reciever: number
}) {
dbtools.runStatement(
msgDB,
`update msgs
@ -50,7 +65,7 @@ function setup(data: SubmoduleData): void {
socket.on('join', function (/*data*/) {
socket.join(userid.toString())
const groups = dbtools
const groups: number[] = dbtools
.runStatement(
msgDB,
`select * from
@ -65,14 +80,14 @@ function setup(data: SubmoduleData): void {
)t
order by t.a asc`
)
.reduce((acc, x) => {
.reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a)
return acc
}, [])
socket.emit('prev messages', {
prevMsgs: groups.map((to) => {
const first = dbtools.runStatement(
const first: Message = dbtools.runStatement(
msgDB,
`select * from msgs
where sender = ${userid} and reciever = ${to} or
@ -147,7 +162,7 @@ function setup(data: SubmoduleData): void {
// socket.on('close', () => {})
})
app.post('/postchatfile', function (req: Request, res: any) {
app.post('/postchatfile', function (req: Request, res) {
logger.LogReq(req)
utils
.uploadFile(req, uloadFiles)
@ -189,12 +204,12 @@ function setup(data: SubmoduleData): void {
)t
order by t.a asc`
)
.reduce((acc, x) => {
.reduce((acc: number[], x: { a: number }) => {
if (x.a !== userid) acc.push(x.a)
return acc
}, [])
const prevMsgs = groups.map((to) => {
const prevMsgs = groups.map((to: number) => {
const first = dbtools.runStatement(
msgDB,
`select * from msgs
@ -207,7 +222,7 @@ function setup(data: SubmoduleData): void {
})
res.json({
unreads: prevMsgs.reduce((acc, msg) => {
unreads: prevMsgs.reduce((acc: number[], msg: Message) => {
if (msg && msg.unread === 1 && msg.sender !== userid) {
acc.push(msg.sender)
}

8
src/modules/api/submodules/feedback.ts
View file

@ -1,3 +1,5 @@
import { Response } from 'express'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import { Request, SubmoduleData, User } from '../../../types/basicTypes'
@ -8,7 +10,7 @@ const uloadFiles = 'data/f'
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.post('/postfeedbackfile', function (req: Request, res: any) {
app.post('/postfeedbackfile', function (req: Request, res: Response) {
utils
.uploadFile(req, uloadFiles)
.then(() => {
@ -23,7 +25,7 @@ function setup(data: SubmoduleData): void {
logger.Log('New feedback file', logger.GetColor('bluebg'))
})
app.post('/postfeedback', function (req: Request, res: any) {
app.post('/postfeedback', function (req: Request, res: Response) {
logger.LogReq(req)
if (req.body.fromLogin) {
logger.Log(
@ -55,7 +57,7 @@ function setup(data: SubmoduleData): void {
res.json({ success: true })
})
app.route('/fosuploader').post(function (req: Request, res: any) {
app.route('/fosuploader').post(function (req: Request, res: Response) {
utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
res.redirect('/f/' + fileName)
})

139
src/modules/api/submodules/forum.ts
View file

@ -4,10 +4,40 @@ import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import { Request, SubmoduleData, User } from '../../../types/basicTypes'
interface Comment {
date: string
user: number
content: string
admin: boolean
subComments?: Comment[]
reacts?: {
[key: string]: number[]
}
}
interface ForumEntry {
date: string
user: number
title: string
content: string
admin: boolean
comments: Comment[]
reacts: {
[key: string]: number[]
}
}
interface ForumContents {
user: number
title: string
admin: boolean
commentCount: number
}
const adminUsersFile = 'data/admins.json'
const forumContentsFileName = '.contents.json'
function countComments(comments) {
function countComments(comments: Comment[]) {
return comments.reduce((acc, comment) => {
if (comment.subComments) {
acc += countComments(comment.subComments) + 1
@ -18,7 +48,7 @@ function countComments(comments) {
}, 0)
}
function addComment(obj, path, comment) {
function addComment(obj: Comment[], path: number[], comment: Comment) {
if (path.length === 0) {
obj.push(comment)
} else {
@ -30,7 +60,11 @@ function addComment(obj, path, comment) {
}
}
function deleteComment(obj: any, path: Array<number>, userid: number): boolean {
function deleteComment(
obj: Comment[],
path: number[],
userid: number
): boolean {
if (path.length === 1) {
if (obj[path[0]].user === userid) {
obj.splice(path[0], 1)
@ -45,7 +79,15 @@ function deleteComment(obj: any, path: Array<number>, userid: number): boolean {
}
}
function addReaction(obj, path, { reaction, isDelete, uid }) {
function addReaction(
obj: Comment[],
path: number[],
{
reaction,
isDelete,
uid,
}: { reaction: string; isDelete: boolean; uid: number }
) {
if (path.length === 1) {
const index = path[0]
if (!obj[index].reacts) {
@ -54,8 +96,8 @@ function addReaction(obj, path, { reaction, isDelete, uid }) {
if (isDelete) {
if (obj[index].reacts[reaction]) {
obj[index].reacts[reaction] = obj[index].reacts[reaction].filter(
(uid) => {
return uid !== uid
(currUid: number) => {
return uid !== currUid
}
)
if (obj[index].reacts[reaction].length === 0) {
@ -84,7 +126,7 @@ function addReaction(obj, path, { reaction, isDelete, uid }) {
function getForumData(
forumName: string,
forumDir: string
): { forumPath: string; contentFilePath: string; contents: any } {
): { forumPath: string; contentFilePath: string; contents: ForumContents[] } {
const safeForumName = forumName.replace(/\./g, '').replace(/\/+/g, '')
const forumPath = forumDir + '/' + safeForumName
const contentFilePath = forumPath + '/' + forumContentsFileName
@ -96,7 +138,7 @@ function getForumData(
if (!utils.FileExists(contentFilePath)) {
utils.WriteFile('{}', contentFilePath)
}
const contents = utils.ReadJSON(contentFilePath)
const contents: ForumContents[] = utils.ReadJSON(contentFilePath)
return {
forumPath: forumPath,
contentFilePath: contentFilePath,
@ -104,11 +146,18 @@ function getForumData(
}
}
function getPostData(postKey, contents, forumPath) {
function getPostData(
postKey: string,
contents: ForumContents[],
forumPath: string
): {
postData: ForumEntry
postPath: string
} {
const safePostKey = postKey.replace(/\./g, '').replace(/\/+/g, '')
const postPath = forumPath + '/' + safePostKey
if (!contents[postKey] || !utils.FileExists(postPath)) {
return
return null
}
return { postData: utils.ReadJSON(postPath), postPath: postPath }
}
@ -167,6 +216,7 @@ function setup(data: SubmoduleData): void {
if (passed) {
i++
}
return false
})
res.json({
@ -175,11 +225,20 @@ function setup(data: SubmoduleData): void {
})
})
app.post('/addPost', (req: Request, res) => {
app.post(
'/addPost',
(
req: Request<{
forumName: string
content: string
title: string
}>,
res
) => {
logger.LogReq(req)
const { title, content } = req.body
const forumName: any = req.body.forumName
const forumName = req.body.forumName
if (!forumName) {
res.json({
success: false,
@ -217,9 +276,18 @@ function setup(data: SubmoduleData): void {
newPostKey: newPostKey,
newEntry: { ...postData, content: content },
})
})
}
)
app.post('/rmPost', (req: Request, res) => {
app.post(
'/rmPost',
(
req: Request<{
postKey: string
forumName: string
}>,
res
) => {
logger.LogReq(req)
const { postKey } = req.body
const forumName: any = req.body.forumName
@ -245,14 +313,25 @@ function setup(data: SubmoduleData): void {
} else {
res.json({
success: false,
msg:
'Entry does not exists, or you are not authorized to delete this post',
msg: 'Entry does not exists, or you are not authorized to delete this post',
})
return
}
})
}
)
app.post('/comment', (req: Request, res) => {
app.post(
'/comment',
(
req: Request<{
type: string
path: number[]
postKey: string
forumName: string
content: string
}>,
res
) => {
logger.LogReq(req)
const forumName: any = req.body.forumName
@ -273,7 +352,10 @@ function setup(data: SubmoduleData): void {
: []
const { type, path, postKey } = req.body
if (!postKey || !type || !path) {
res.json({ success: false, msg: 'type or path or postKey is undefined' })
res.json({
success: false,
msg: 'type or path or postKey is undefined',
})
return
}
@ -319,9 +401,21 @@ function setup(data: SubmoduleData): void {
utils.WriteFile(JSON.stringify(postData, null, 2), postPath)
utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath)
res.json({ success: true, postKey: postKey, postData: postData })
})
}
)
app.post('/react', (req: Request, res) => {
app.post(
'/react',
(
req: Request<{
forumName: string
postKey: string
path: number[]
reaction: string
isDelete: boolean
}>,
res
) => {
logger.LogReq(req)
const forumName: any = req.body.forumName
@ -386,7 +480,8 @@ function setup(data: SubmoduleData): void {
utils.WriteFile(JSON.stringify(postData, null, 2), postPath)
res.json({ success: true, postData: postData })
})
}
)
}
export default {

260
src/modules/api/submodules/qminingapi.ts
View file

@ -1,5 +1,7 @@
import fs from 'fs'
import { fork } from 'child_process'
import { Response } from 'express'
import { fork, ChildProcess } from 'child_process'
import type { Database } from 'better-sqlite3'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
@ -9,6 +11,11 @@ import {
Request,
QuestionDb,
SubmoduleData,
Question,
QuestionFromScript,
DbSearchResult,
RegisteredUserEntry,
Submodule,
} from '../../../types/basicTypes'
import {
processIncomingRequest,
@ -24,6 +31,7 @@ import {
import {
dataToString,
getSubjNameWithoutYear,
WorkerResult,
// compareQuestionObj,
} from '../../../utils/classes'
import {
@ -33,6 +41,22 @@ import {
} from '../../../utils/workerPool'
import dbtools from '../../../utils/dbtools'
interface SavedQuestionData {
fname: string
subj: string
userid: number
testUrl: string
date: string | Date
}
// interface SavedQuestion {
// Questions: Question[]
// subj: string
// userid: number
// testUrl: string
// date: string
// }
const line = '====================================================' // lol
const registeredScriptsFile = 'stats/registeredScripts.json'
const testUsersFile = 'data/testUsers.json'
@ -43,13 +67,13 @@ const oldMotdFile = 'publicDirs/qminingPublic/oldMotd'
const dailyDataCountFile = 'stats/dailyDataCount'
const dataEditsLog = 'stats/dataEdits'
function getSubjCount(qdbs) {
function getSubjCount(qdbs: QuestionDb[]): number {
return qdbs.reduce((acc, qdb) => {
return acc + qdb.data.length
}, 0)
}
function getQuestionCount(qdbs) {
function getQuestionCount(qdbs: QuestionDb[]): number {
return qdbs.reduce((acc, qdb) => {
return (
acc +
@ -60,7 +84,7 @@ function getQuestionCount(qdbs) {
}, 0)
}
function ExportDailyDataCount(questionDbs, userDB) {
function ExportDailyDataCount(questionDbs: QuestionDb[], userDB: Database) {
logger.Log('Saving daily data count ...')
utils.AppendToFile(
JSON.stringify({
@ -76,9 +100,9 @@ function ExportDailyDataCount(questionDbs, userDB) {
function getDbIndexesToSearchIn(
testUrl: string,
questionDbs,
questionDbs: Array<QuestionDb>,
trueIfAlways?: boolean
) {
): number[] {
return testUrl
? questionDbs.reduce((acc, qdb, i) => {
if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) {
@ -89,14 +113,17 @@ function getDbIndexesToSearchIn(
: []
}
function getSimplreRes(questionDbs) {
function getSimplreRes(questionDbs: QuestionDb[]): {
subjects: number
questions: number
} {
return {
subjects: getSubjCount(questionDbs),
questions: getQuestionCount(questionDbs),
}
}
function getDetailedRes(questionDbs) {
function getDetailedRes(questionDbs: QuestionDb[]) {
return questionDbs.map((qdb) => {
return {
dbName: qdb.name,
@ -110,7 +137,7 @@ function getDetailedRes(questionDbs) {
})
}
function getMotd(version, motd) {
function getMotd(version: any, motd: string) {
if (version) {
if (version.startsWith('2.0.')) {
if (utils.FileExists(oldMotdFile)) {
@ -122,13 +149,11 @@ function getMotd(version, motd) {
}
function searchInDbs(
question,
subj,
recData,
recievedData,
searchIn,
testUrl?
) {
question: Question,
subj: string,
searchIn: number[],
testUrl?: string
): Promise<DbSearchResult> {
// searchIn could be [0], [1], ... to search every db in different thread. Put this into a
// forEach(qdbs) to achieve this
return new Promise((resolve) => {
@ -139,17 +164,11 @@ function searchInDbs(
testUrl: testUrl,
question: question,
subjName: subj,
questionData: recData,
searchInAllIfNoResult: true,
},
})
.then((taskResult) => {
.then((taskResult: WorkerResult) => {
try {
logger.DebugLog(
`Question result length: ${taskResult.length}`,
'ask',
1
)
logger.DebugLog(taskResult, 'ask', 2)
resolve({
question: question,
@ -168,22 +187,27 @@ function searchInDbs(
logger.Log('Search Data error!', logger.GetColor('redbg'))
console.error(err)
resolve({
mesage: `There was an error processing the question: ${err.message}`,
question: question,
message: `There was an error processing the question: ${err.message}`,
result: [],
recievedData: JSON.stringify(recievedData),
success: false,
})
})
})
}
function getResult(data) {
const { question, subj, recData, recievedData, questionDbs, testUrl } = data
function getResult(data: {
question: Question
subj: string
questionDbs: Array<QuestionDb>
testUrl: string
}): Promise<DbSearchResult> {
const { question, subj, questionDbs, testUrl } = data
return new Promise((resolve) => {
const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false)
searchInDbs(question, subj, recData, recievedData, searchIn, testUrl).then(
(res: any) => {
searchInDbs(question, subj, searchIn, testUrl).then(
(res: DbSearchResult) => {
if (res.result.length === 0) {
logger.DebugLog(
`No result while searching specific question db ${testUrl}`,
@ -197,14 +221,7 @@ function getResult(data) {
).filter((x) => {
return !searchIn.includes(x)
})
searchInDbs(
question,
subj,
recData,
recievedData,
searchInMore,
testUrl
).then((res) => {
searchInDbs(question, subj, searchInMore, testUrl).then((res) => {
resolve(res)
})
} else {
@ -215,13 +232,13 @@ function getResult(data) {
})
}
function dbExists(location, qdbs: Array<QuestionDb>) {
function dbExists(location: string, qdbs: Array<QuestionDb>) {
return qdbs.some((qdb) => {
return qdb.name === location
})
}
function writeAskData(body) {
function writeAskData(body: QuestionFromScript) {
try {
let towrite = utils.GetDateString() + '\n'
towrite +=
@ -236,7 +253,13 @@ function writeAskData(body) {
}
}
function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) {
function saveQuestion(
questions: Question[],
subj: string,
testUrl: string,
userid: number,
savedQuestionsDir: string
) {
// TODO: clear folder every now and then, check if saved questions exist
if (!subj) {
logger.Log('No subj name to save test question')
@ -259,7 +282,9 @@ function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) {
utils.WriteFile('[]', savedSubjQuestionsFilePath)
}
const savedQuestions = utils.ReadJSON(savedSubjQuestionsFilePath)
const savedQuestions: SavedQuestionData[] = utils.ReadJSON(
savedSubjQuestionsFilePath
)
const testExists = false
// TODO: do this on another thread?
@ -327,7 +352,7 @@ function loadSupportedSites() {
function LoadVersion() {
const scriptContent = utils.ReadFile(userScriptFile)
let temp: any = scriptContent.split('\n').find((x) => {
let temp: string | string[] = scriptContent.split('\n').find((x) => {
return x.includes('@version')
})
temp = temp.split(' ')
@ -336,19 +361,10 @@ function LoadVersion() {
return temp
}
function LoadMOTD(motdFile) {
function LoadMOTD(motdFile: string) {
return utils.ReadFile(motdFile)
}
function LoadUserSpecificMOTD(userSpecificMotdFile) {
try {
return utils.ReadJSON(userSpecificMotdFile)
} catch (err) {
logger.Log('Couldnt parse user specific motd!', logger.GetColor('redbg'))
console.error(err)
}
}
function LoadTestUsers() {
let testUsers = utils.ReadJSON(testUsersFile)
if (testUsers) {
@ -357,7 +373,13 @@ function LoadTestUsers() {
return testUsers
}
function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) {
function getNewQdb(
location: string,
maxIndex: number,
dbsFile: string,
publicDir: string,
questionDbs: QuestionDb[]
) {
logger.Log(
`No suitable questiondbs found for ${location}, creating a new one...`
)
@ -399,40 +421,31 @@ function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) {
questionDbs.push(loadedNewDb)
msgAllWorker({
newdb: loadedNewDb,
data: loadedNewDb,
type: 'newdb',
})
return loadedNewDb
}
function setup(data: SubmoduleData): any {
function setup(data: SubmoduleData): Submodule {
const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data
const publicDir = publicdirs[0]
const motdFile = publicDir + 'motd'
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
const dbsFile = publicDir + 'questionDbs.json'
const savedQuestionsDir = publicDir + 'savedQuestions'
let version = LoadVersion()
let supportedSites = loadSupportedSites()
let motd = LoadMOTD(motdFile)
let userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
let testUsers: any = LoadTestUsers()
let testUsers: number[] = LoadTestUsers()
const dataFiles: Array<DataFile> = utils.ReadJSON(dbsFile)
const questionDbs: Array<QuestionDb> = loadJSON(dataFiles, publicDir)
initWorkerPool(questionDbs)
const filesToWatch = [
{
fname: userSpecificMotdFile,
logMsg: 'User Specific Motd updated',
action: () => {
userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
},
},
{
fname: motdFile,
logMsg: 'Motd updated',
@ -457,7 +470,7 @@ function setup(data: SubmoduleData): any {
},
]
app.get('/getDbs', (req: Request, res: any) => {
app.get('/getDbs', (req: Request, res: Response) => {
logger.LogReq(req)
res.json(
questionDbs.map((qdb) => {
@ -470,7 +483,7 @@ function setup(data: SubmoduleData): any {
)
})
app.get('/allqr.txt', function (req: Request, res: any) {
app.get('/allqr.txt', function (req: Request, res: Response) {
logger.LogReq(req)
const db: any = req.query.db
let stringifiedData = ''
@ -508,7 +521,7 @@ function setup(data: SubmoduleData): any {
res.end(stringifiedData)
})
app.post('/isAdding', function (req: Request, res: any) {
app.post('/isAdding', function (req: Request, res: Response) {
logger.LogReq(req)
const user: User = req.session.user
const dryRun = testUsers.includes(user.id)
@ -596,39 +609,39 @@ function setup(data: SubmoduleData): any {
}
})
app.post('/ask', function (req: Request, res) {
app.post('/ask', function (req: Request<QuestionFromScript>, res) {
const user: User = req.session.user
if (!req.body.questions) {
res.json({
message: `ask something! { questions:'' ,subject:'', location:'' }`,
result: [],
recievedData: JSON.stringify(req.body),
success: false,
})
return
}
const subj: any = req.body.subj || ''
const subj: string = req.body.subj || ''
// TODO: test if testUrl is undefined (it shouldnt)
const testUrl = req.body.testUrl
const testUrl: string = req.body.testUrl
? req.body.testUrl.split('/')[2]
: undefined
writeAskData(req.body)
// every question in a different thread
const resultPromises = req.body.questions.map((question) => {
const resultPromises: Promise<DbSearchResult>[] = req.body.questions.map(
(question: Question) => {
return getResult({
question: question,
subj: subj,
recData: req.body,
testUrl: testUrl,
questionDbs: questionDbs,
})
})
}
)
Promise.all(resultPromises).then((results) => {
const response = results.map((result: any) => {
Promise.all(resultPromises).then((results: DbSearchResult[]) => {
const response = results.map((result: DbSearchResult) => {
return {
answers: result.result,
question: result.question,
@ -659,50 +672,13 @@ function setup(data: SubmoduleData): any {
})
})
app.get('/ask', function (req: Request, res) {
if (Object.keys(req.query).length === 0) {
logger.DebugLog(`No query params /ask GET`, 'ask', 1)
res.json({
message:
'ask something! ?q=[question]&subj=[subject]&data=[question data]. "subj" is optimal for faster result',
result: [],
recievedData: JSON.stringify(req.query),
success: false,
})
return
}
if (req.query.q && req.query.data) {
const subj: any = req.query.subj || ''
const question = req.query.q
const recData: any = req.query.data
getResult({
question: question,
subj: subj,
recData: recData,
recievedData: req.query,
questionDbs: questionDbs,
}).then((result) => {
res.json(result)
})
} else {
logger.DebugLog(`Invalid question`, 'ask', 1)
res.json({
message: `Invalid question :(`,
result: [],
recievedData: JSON.stringify(req.query),
success: false,
})
}
})
app.get('/supportedSites', function (req: Request, res: any) {
app.get('/supportedSites', function (req: Request, res: Response) {
logger.LogReq(req)
res.json(supportedSites)
})
app.get('/datacount', function (req: Request, res: any) {
app.get('/datacount', function (req: Request, res: Response) {
logger.LogReq(req)
if (req.query.detailed === 'all') {
res.json({
@ -719,7 +695,16 @@ function setup(data: SubmoduleData): any {
app.get('/infos', function (req: Request, res) {
const user: User = req.session.user
const result: any = {
const result: {
result: string
uid: number
version?: string
subjinfo?: {
subjects: number
questions: number
}
motd?: string
} = {
result: 'success',
uid: user.id,
}
@ -732,34 +717,9 @@ function setup(data: SubmoduleData): any {
}
if (req.query.motd) {
result.motd = getMotd(req.query.cversion, motd)
if (userSpecificMotd[user.id]) {
result.userSpecificMotd = {
msg: userSpecificMotd[user.id].msg,
seen: userSpecificMotd[user.id].seen,
}
}
}
res.json(result)
})
app.post('/infos', (req: Request, res) => {
const user: User = req.session.user
if (req.body.userSpecificMotdSeen && !userSpecificMotd[user.id].seen) {
userSpecificMotd[user.id].seen = true
logger.Log(
`User #${user.id}'s user specific motd is now seen:`,
logger.GetColor('bluebg')
)
logger.Log(userSpecificMotd[user.id].msg)
utils.WriteFile(
JSON.stringify(userSpecificMotd, null, 2),
userSpecificMotdFile
)
}
res.json({ msg: 'done' })
})
app.post('/registerscript', function (req: Request, res) {
logger.LogReq(req)
@ -769,7 +729,9 @@ function setup(data: SubmoduleData): any {
}
const ua: any = req.headers['user-agent']
const registeredScripts = utils.ReadJSON(registeredScriptsFile)
const registeredScripts: RegisteredUserEntry[] = utils.ReadJSON(
registeredScriptsFile
)
const { cid, uid, version, installSource, date } = req.body
const index = registeredScripts.findIndex((registration) => {
@ -816,7 +778,7 @@ function setup(data: SubmoduleData): any {
res.json({ msg: 'done' })
})
app.get('/possibleAnswers', (req: Request, res: any) => {
app.get('/possibleAnswers', (req: Request, res: Response) => {
logger.LogReq(req)
const files = utils.ReadDir(savedQuestionsDir)
@ -838,16 +800,16 @@ function setup(data: SubmoduleData): any {
})
})
app.post('/rmPossibleAnswer', (req: Request, res: any) => {
app.post('/rmPossibleAnswer', (req: Request, res: Response) => {
logger.LogReq(req)
const user: User = req.session.user
const subj = req.body.subj
const file = req.body.file
const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}`
const savedQuestions = utils.ReadJSON(savedQuestionsPath)
const savedQuestions: SavedQuestionData[] =
utils.ReadJSON(savedQuestionsPath)
let path = `${savedQuestionsDir}/${subj}/${file}`
// to prevent deleting ../../../ ... /etc/shadow
while (path.includes('..')) {
path = path.replace(/\.\./g, '.')
}
@ -1007,7 +969,7 @@ function setup(data: SubmoduleData): any {
})
})
let questionCleaner = null
let questionCleaner: ChildProcess = null
app.get('/clearQuestions', (req: Request, res) => {
// TODO: dont allow multiple instances
// TODO: get status of it cleaning
@ -1046,7 +1008,7 @@ function setup(data: SubmoduleData): any {
`${process.cwd()}/src/standaloneUtils/rmDuplicates.js`,
['-s', `${process.cwd()}/${questionDbs[0].path}`]
)
questionCleaner.on('exit', function (code) {
questionCleaner.on('exit', function (code: number) {
console.log('EXIT', code)
questionCleaner = null
})

18
src/modules/api/submodules/quickvote.ts
View file

@ -4,12 +4,24 @@ import { Request, SubmoduleData, User } from '../../../types/basicTypes'
const quickVoteResultsDir = 'stats/qvote'
const quickVotes = 'stats/qvote/votes.json'
interface QuickVotes {
voteNames?: string[]
}
interface QuickVote {
votes: {
[key: string]: string
}
sum: {
[key: string]: number
}
}
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/quickvote', (req: Request, res: any) => {
const key = req.query.key
const key = req.query.key.toString()
const val: any = req.query.val
const user: User = req.session.user
@ -22,7 +34,7 @@ function setup(data: SubmoduleData): void {
}
// FIXME: check vote type in file
let votes: any = {}
let votes: QuickVotes = {}
if (utils.FileExists(quickVotes)) {
votes = utils.ReadJSON(quickVotes)
} else {
@ -49,7 +61,7 @@ function setup(data: SubmoduleData): void {
const voteFile = quickVoteResultsDir + '/' + key + '.json'
let voteData = {
let voteData: QuickVote = {
votes: {},
sum: {},
}

25
src/modules/api/submodules/ranklist.ts
View file

@ -2,10 +2,29 @@ import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import { Request, SubmoduleData, User } from '../../../types/basicTypes'
interface Subjects {
[key: string]: number
}
interface IdStat {
count: number
newQuestions: number
allQuestions: number
subjs: Subjects
}
interface IdStats {
[key: string]: IdStat
}
interface IdStatWithUID extends IdStat {
userId: number
}
const idStatFile = 'stats/idstats'
const idvStatFile = 'stats/idvstats'
function mergeObjSum(a, b) {
function mergeObjSum(a: Subjects, b: Subjects) {
const res = { ...b }
Object.keys(a).forEach((key) => {
if (res[key]) {
@ -23,7 +42,7 @@ function setup(data: SubmoduleData): void {
app.get('/ranklist', (req: Request, res: any) => {
logger.LogReq(req)
let result
let result: IdStats
const querySince: any = req.query.since
const user: User = req.session.user
@ -66,7 +85,7 @@ function setup(data: SubmoduleData): void {
}
}
const list = []
const list: Array<IdStatWithUID> = []
const sum = {
count: 0,
newQuestions: 0,

55
src/modules/api/submodules/todos.ts
View file

@ -1,17 +1,66 @@
import { Response } from 'express'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
import { Request, SubmoduleData } from '../../../types/basicTypes'
interface Categories {
[key: string]: {
name: string
color: string
}
}
enum CardState {
TODO = 'todo',
INPROGRESS = 'inprogress',
TESTING = 'testing',
DONE = 'done',
INPROD = 'inprod',
NOTPOSSIBLE = 'notpossible',
}
interface Card {
id: number
name: string
description: string
category: string
points: number
state: CardState
votes: number[]
}
type Columns = {
[key in CardState]: {
name: string
clickable: boolean
}
}
interface Groups {
[key: string]: {
name: string
description: string
}
}
interface Todos {
categories: Categories
cards: Card[]
columns: Columns
groups: Groups
}
const todosFile = 'data/todos.json'
function setup(data: SubmoduleData): void {
const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
app.get('/voteTodo', (req: Request, res: any) => {
app.get('/voteTodo', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const id: any = req.query.id
const todos = utils.ReadJSON(todosFile)
const todos: Todos = utils.ReadJSON(todosFile)
if (!id) {
res.json({
@ -47,7 +96,7 @@ function setup(data: SubmoduleData): void {
})
})
app.get('/todos', (req: Request, res: any) => {
app.get('/todos', (req: Request, res: Response) => {
logger.LogReq(req)
const userId = req.session.user.id
const todos = utils.ReadJSON(todosFile)

8
src/modules/api/submodules/userFiles.ts
View file

@ -6,7 +6,7 @@ import { Request, SubmoduleData, User } from '../../../types/basicTypes'
const dataFileName = '.data.json'
function listDir(publicDir, subdir, userFilesDir) {
function listDir(publicDir: string, subdir: string, userFilesDir: string) {
const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '')
const dir = userFilesDir + '/' + safeSubdir
const usersFile = dir + '/' + dataFileName
@ -16,7 +16,6 @@ function listDir(publicDir, subdir, userFilesDir) {
success: false,
msg: `Directory ${subdir} does not exists`,
}
return
}
if (!utils.FileExists(usersFile)) {
utils.WriteFile('{}', usersFile)
@ -28,7 +27,6 @@ function listDir(publicDir, subdir, userFilesDir) {
success: false,
msg: `Path '${safeSubdir}' does not exists`,
}
return
}
return {
@ -65,7 +63,7 @@ function listDir(publicDir, subdir, userFilesDir) {
function setup(data: SubmoduleData): void {
const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data
app.use((req: Request, res, next) => {
app.use((req: Request, _res, next) => {
// /userFiles/test/2021-04-28_10-59.png
try {
if (req.url.includes('/userFiles/')) {
@ -264,7 +262,7 @@ function setup(data: SubmoduleData): void {
data[fname].downvotes = []
}
const removeVote = (from, uid) => {
const removeVote = (from: number[], uid: number) => {
if (!from.includes(uid)) {
return from
}

24
src/modules/api/submodules/userManagement.ts
View file

@ -1,4 +1,5 @@
import { v4 as uuidv4 } from 'uuid'
import type { Database } from 'better-sqlite3'
import logger from '../../../utils/logger'
import utils from '../../../utils/utils'
@ -11,7 +12,15 @@ const usersDbBackupPath = 'data/dbs/backup'
const maxPWCount = 3
const daysAfterUserGetsPWs = 7 // days after user gets pw-s
function BackupDB(usersDbBackupPath, userDB) {
interface Session {
id: string
userId: number
createDate: string
lastAccess: string
isScript: number
}
function BackupDB(usersDbBackupPath: string, userDB: Database) {
logger.Log('Backing up auth DB ...')
utils.CreatePath(usersDbBackupPath, true)
userDB
@ -23,7 +32,7 @@ function BackupDB(usersDbBackupPath, userDB) {
.then(() => {
logger.Log('Auth DB backup complete!')
})
.catch((err) => {
.catch((err: Error) => {
logger.Log('Auth DB backup failed!', logger.GetColor('redbg'))
console.error(err)
})
@ -62,8 +71,7 @@ function setup(data: SubmoduleData): any {
res.json({
result: 'error',
success: false,
msg:
'Too many passwords requested or cant request password yet, try later',
msg: 'Too many passwords requested or cant request password yet, try later',
})
logger.Log(
`User #${requestingUser.id} requested too much passwords`,
@ -128,8 +136,10 @@ function setup(data: SubmoduleData): any {
userID: user.id,
isScript: isScript ? 1 : 0,
})
.sort((a, b) => {
return new Date(a).getTime() - new Date(b).getTime()
.sort((a: Session, b: Session) => {
return (
new Date(a.lastAccess).getTime() - new Date(b.lastAccess).getTime()
)
})
const diff = existingSessions.length - minimumAlowwedSessions
@ -238,7 +248,7 @@ function setup(data: SubmoduleData): any {
})
})
function getDayDiff(dateString) {
function getDayDiff(dateString: string | Date) {
const msdiff = new Date().getTime() - new Date(dateString).getTime()
return Math.floor(msdiff / (1000 * 3600 * 24))
}

79
src/modules/api/usersDBStruct.json
View file

@ -1,79 +0,0 @@
{
"users": {
"tableStruct": {
"id": {
"type": "integer",
"primary": true,
"autoIncrement": true
},
"pw": {
"type": "text",
"notNull": true,
"unique": true
},
"notes": {
"type": "text"
},
"loginCount": {
"type": "number",
"defaultZero": true
},
"created": {
"type": "text",
"notNull": true
},
"lastLogin": {
"type": "text"
},
"lastAccess": {
"type": "text"
},
"avaiblePWRequests": {
"type": "number",
"defaultZero": true
},
"pwRequestCount": {
"type": "number",
"defaultZero": true
},
"createdBy": {
"type": "number"
}
}
},
"sessions": {
"foreignKey": [
{
"keysFrom": [
"userID"
],
"table": "users",
"keysTo": [
"id"
]
}
],
"tableStruct": {
"id": {
"type": "text",
"primary": true,
"notNull": true
},
"userID": {
"type": "number",
"notNull": true
},
"createDate": {
"type": "text",
"notNull": true
},
"lastAccess": {
"type": "text"
},
"isScript": {
"type": "number",
"notNull": true
}
}
}
}

78
src/modules/api/usersDBStruct.ts Normal file
View file

@ -0,0 +1,78 @@
export interface Users {}
const DBStruct = {
users: {
tableStruct: {
id: {
type: 'integer',
primary: true,
autoIncrement: true,
},
pw: {
type: 'text',
notNull: true,
unique: true,
},
notes: {
type: 'text',
},
loginCount: {
type: 'number',
defaultZero: true,
},
created: {
type: 'text',
notNull: true,
},
lastLogin: {
type: 'text',
},
lastAccess: {
type: 'text',
},
avaiblePWRequests: {
type: 'number',
defaultZero: true,
},
pwRequestCount: {
type: 'number',
defaultZero: true,
},
createdBy: {
type: 'number',
},
},
},
sessions: {
foreignKey: [
{
keysFrom: ['userID'],
table: 'users',
keysTo: ['id'],
},
],
tableStruct: {
id: {
type: 'text',
primary: true,
notNull: true,
},
userID: {
type: 'number',
notNull: true,
},
createDate: {
type: 'text',
notNull: true,
},
lastAccess: {
type: 'text',
},
isScript: {
type: 'number',
notNull: true,
},
},
},
}
export default DBStruct

34
src/modules/dataEditor/dataEditor.ts
View file

@ -19,9 +19,8 @@
------------------------------------------------------------------------- */
// package requires
import express from 'express'
import bodyParser from 'body-parser'
import busboy from 'connect-busboy'
import express, { RequestHandler } from 'express'
import type { Database } from 'better-sqlite3'
const app = express()
// other requires
@ -32,21 +31,21 @@ import { SetupData } from '../../server'
import { ModuleType, Request } from '../../types/basicTypes'
// stuff gotten from server.js
let userDB
let publicdirs = []
let userDB: Database
let publicdirs: string[] = []
let nextdir = ''
function GetApp(): ModuleType {
app.use(
bodyParser.urlencoded({
express.urlencoded({
limit: '5mb',
extended: true,
})
}) as RequestHandler
)
app.use(
bodyParser.json({
express.json({
limit: '5mb',
})
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/dataEditor/views', './src/sharedViews'])
@ -57,7 +56,7 @@ function GetApp(): ModuleType {
exceptions: ['/favicon.ico'],
})
)
app.use((req: Request, res, next) => {
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
@ -69,17 +68,10 @@ function GetApp(): ModuleType {
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
app.use(
busboy({
limits: {
fileSize: 10000 * 1024 * 1024,
},
})
)
// --------------------------------------------------------------
function AddHtmlRoutes(files) {
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
@ -90,7 +82,7 @@ function GetApp(): ModuleType {
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1)
app.get(`/${route}`, function (req: Request, res) {
app.get(`/${route}`, function (_req: Request, res) {
res.redirect(`${route}.html`)
})
})
@ -104,11 +96,11 @@ function GetApp(): ModuleType {
logger.LogReq(req)
})
app.get('*', function (req: Request, res) {
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.post('*', function (req: Request, res) {
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})

31
src/modules/main/main.ts
View file

@ -19,9 +19,7 @@
------------------------------------------------------------------------- */
// package requires
import express from 'express'
import bodyParser from 'body-parser'
import busboy from 'connect-busboy'
import express, { RequestHandler } from 'express'
const app = express()
// other requires
@ -30,7 +28,7 @@ import { SetupData } from '../../server'
import { ModuleType } from '../../types/basicTypes'
// stuff gotten from server.js
let publicdirs = []
let publicdirs: string[] = []
let url = '' // http(s)//asd.basd
function GetApp(): ModuleType {
@ -40,39 +38,28 @@ function GetApp(): ModuleType {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.json() as RequestHandler)
app.use(
busboy({
limits: {
fileSize: 10000 * 1024 * 1024,
},
})
)
app.use(bodyParser.json())
app.use(
bodyParser.urlencoded({
express.urlencoded({
limit: '5mb',
extended: true,
})
)
app.use(
bodyParser.json({
limit: '5mb',
})
}) as RequestHandler
)
// --------------------------------------------------------------
app.get('/', function(req, res) {
app.get('/', function (_req, res) {
res.render('main', {
siteurl: url,
})
})
app.get('*', function(req, res) {
app.get('*', function (_req, res) {
res.status(404).render('404')
})
app.post('*', function(req, res) {
app.post('*', function (_req, res) {
res.status(404).render('404')
})

38
src/modules/qmining/qmining.ts
View file

@ -19,9 +19,8 @@
------------------------------------------------------------------------- */
// package requires
import express from 'express'
import bodyParser from 'body-parser'
import busboy from 'connect-busboy'
import express, { RequestHandler } from 'express'
import type { Database } from 'better-sqlite3'
const app = express()
// other requires
@ -32,21 +31,21 @@ import { SetupData } from '../../server'
import { ModuleType, Request } from '../../types/basicTypes'
// stuff gotten from server.js
let publicdirs = []
let userDB
let publicdirs: string[] = []
let userDB: Database
let nextdir = ''
function GetApp(): ModuleType {
app.use(
bodyParser.urlencoded({
express.urlencoded({
limit: '5mb',
extended: true,
})
}) as RequestHandler
)
app.use(
bodyParser.json({
express.json({
limit: '5mb',
})
}) as RequestHandler
)
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/qmining/views', './src/sharedViews'])
@ -57,7 +56,7 @@ function GetApp(): ModuleType {
exceptions: ['/favicon.ico', '/img/frylabs-logo_large_transparent.png'],
})
)
app.use((req: Request, res, next) => {
app.use((req: Request, _res, next) => {
const url = req.url.split('?')[0]
if (url.includes('.html') || url === '/') {
logger.LogReq(req)
@ -69,13 +68,6 @@ function GetApp(): ModuleType {
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
app.use(
busboy({
limits: {
fileSize: 10000 * 1024 * 1024,
},
})
)
const linksFile = 'data/links.json'
let links: any = {}
@ -91,7 +83,7 @@ function GetApp(): ModuleType {
loadDonateURL()
if (utils.FileExists(linksFile)) {
utils.WatchFile(linksFile, (newData) => {
utils.WatchFile(linksFile, (newData: string) => {
logger.Log(`Donate URL changed: ${newData.replace(/\/n/g, '')}`)
loadDonateURL()
})
@ -175,7 +167,7 @@ function GetApp(): ModuleType {
},
{
from: '/irc',
to: links.irc,
to: '/chat',
},
{
from: '/patreon',
@ -201,7 +193,7 @@ function GetApp(): ModuleType {
// --------------------------------------------------------------
function AddHtmlRoutes(files) {
function AddHtmlRoutes(files: string[]) {
const routes = files.reduce((acc, file) => {
if (file.includes('html')) {
acc.push(file.split('.')[0])
@ -210,7 +202,7 @@ function GetApp(): ModuleType {
return acc
}, [])
routes.forEach((route) => {
routes.forEach((route: string) => {
logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1)
app.get(`/${route}`, function (req: Request, res) {
res.redirect(
@ -231,11 +223,11 @@ function GetApp(): ModuleType {
logger.LogReq(req)
})
app.get('*', function (req: Request, res) {
app.get('*', function (_req: Request, res) {
res.status(404).render('404')
})
app.post('*', function (req: Request, res) {
app.post('*', function (_req: Request, res) {
res.status(404).render('404')
})

135
src/modules/sio/sio.ts
View file

@ -1,135 +0,0 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
import express from 'express'
import bodyParser from 'body-parser'
import busboy from 'connect-busboy'
import fs from 'fs'
const app = express()
// other requires
import logger from '../../utils/logger'
import utils from '../../utils/utils'
import { SetupData } from '../../server'
import { ModuleType } from '../../types/basicTypes'
// stuff gotten from server.js
let publicdirs = []
function GetApp(): ModuleType {
const publicdir = publicdirs[0]
if (!publicdir) {
throw new Error(`No public dir! ( SIO )`)
}
// files in public dirs
const uloadFiles = publicdir + 'f'
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/sio/views', './src/sharedViews'])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(
busboy({
limits: {
fileSize: 10000 * 1024 * 1024,
},
})
)
app.use(bodyParser.json())
app.use(
bodyParser.urlencoded({
limit: '5mb',
extended: true,
})
)
app.use(
bodyParser.json({
limit: '5mb',
})
)
// --------------------------------------------------------------
app.get('/', function(req, res) {
res.render('uload')
res.end()
})
function UploadFile(req, res, path, next) {
req.pipe(req.busboy)
req.busboy.on('file', function(fieldname, file, filename) {
logger.Log('Uploading: ' + filename, logger.GetColor('blue'))
utils.CreatePath(path, true)
const date = new Date()
const fn =
date.getHours() +
'' +
date.getMinutes() +
'' +
date.getSeconds() +
'_' +
filename
const fstream = fs.createWriteStream(path + '/' + fn)
file.pipe(fstream)
fstream.on('close', function() {
logger.Log(
'Upload Finished of ' + path + '/' + fn,
logger.GetColor('blue')
)
next(fn)
})
fstream.on('error', function(err) {
console.log(err)
res.end('something bad happened :s')
})
})
}
app.route('/fosuploader').post(function(req, res) {
UploadFile(req, res, uloadFiles, (fn) => {
res.redirect('/f/' + fn)
})
})
app.get('*', function(req, res) {
res.status(404).render('404')
})
app.post('*', function(req, res) {
res.status(404).render('404')
})
return {
app: app,
}
}
export default {
name: 'Sio',
getApp: GetApp,
setup: (data: SetupData): void => {
publicdirs = data.publicdirs
},
}

39
src/modules/sio/views/uload.ejs
View file

@ -1,39 +0,0 @@
<html>
<body bgcolor="#212127">
<head>
<title>Shit uploader</title>
<meta charset="UTF-8">
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top
}
textarea {
font: normal 14px Verdana;
color: #999999;
background-color: #212127;
width: 100%;
height: 700;
}
a {
color: #9999ff;
}
</style>
</head>
<form action="/fosuploader" enctype=multipart/form-data method="post">
<input type="file" name="dasfile" />
<input type="submit" value="Upload" />
</form>
</body>
</html>

253
src/modules/stuff/stuff.ts
View file

@ -1,253 +0,0 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
import express from 'express'
import bodyParser from 'body-parser'
import busboy from 'connect-busboy'
import fs from 'fs'
const app = express()
// other requires
import logger from '../../utils/logger'
import { SetupData } from '../../server'
import { ModuleType, Request } from '../../types/basicTypes'
// stuff gotten from server.js
let publicdirs = []
let url = ''
function GetApp(): ModuleType {
const publicDir = publicdirs[0]
if (!publicDir) {
throw new Error(`No public dir! ( Stuff )`)
}
// files in public dirs
const listedFiles = './' + publicDir + 'files'
app.set('view engine', 'ejs')
app.set('views', ['./src/modules/stuff/views', './src/sharedViews'])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(
busboy({
limits: {
fileSize: 10000 * 1024 * 1024,
},
})
)
app.use(bodyParser.json())
app.use(
bodyParser.urlencoded({
limit: '5mb',
extended: true,
})
)
app.use(
bodyParser.json({
limit: '5mb',
})
)
// --------------------------------------------------------------
// app, '/*.mp4', 'video/mp4', 'stuff/video'
function appGetFileType(app, wildcard, contentType, pageToRender) {
app.get(wildcard, function(req, res) {
const path = decodeURIComponent(req.url)
let fp: any = path
if (path.includes('?')) {
fp = path.split('?')
fp.pop()
fp = fp.join('/')
}
const fpath = listedFiles + fp
if (!fs.existsSync(fpath)) {
res.render('nofile', {
missingFile: fp,
url,
})
return
}
if (req.query.stream || !pageToRender) {
const stat = fs.statSync(fpath)
const fileSize = stat.size
const range = req.headers.range
if (range) {
const parts = range.replace(/bytes=/, '').split('-')
const start = parseInt(parts[0], 10)
const end = parts[1] ? parseInt(parts[1], 10) : fileSize - 1
const chunksize = end - start + 1
const file = fs.createReadStream(fpath, { start, end })
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': contentType,
}
res.writeHead(206, head)
file.pipe(res)
} else {
const head = {
'Content-Length': fileSize,
'Content-Type': contentType,
}
res.writeHead(200, head)
fs.createReadStream(fpath).pipe(res)
}
} else {
logger.LogReq(req)
const fname = fpath.split('/').pop()
res.render(pageToRender, {
path: fp,
fname,
url,
contentType,
albumArt: GetAlbumArt(path),
})
}
})
}
const fileTypes = [
['/*.mp4', 'video/mp4', 'video'],
['/*.mkv', 'audio/x-matroska', 'video'],
['/*.mp3', 'audio/mpeg', 'audio'],
['/*.pdf', 'application/pdf'],
['/*.zip', 'application/zip'],
]
function GetAlbumArt(path) {
let tmp = path.split('.')
tmp.pop()
tmp = tmp.join('.').split('/')
const last = tmp.pop()
return tmp.join('/') + '/.' + last + '.png'
}
fileTypes.forEach((t) => {
appGetFileType(app, t[0], t[1], t[2])
})
app.get('/*', function(req: Request, res) {
const parsedUrl = decodeURIComponent(req.url)
let curr =
listedFiles +
'/' +
parsedUrl.substring('/'.length, parsedUrl.length).split('?')[0]
let relPath = curr.substring(listedFiles.length, curr.length)
if (relPath[relPath.length - 1] !== '/') {
relPath += '/'
}
const temp = relPath.split('/')
let prevDir = ''
for (let i = 0; i < temp.length - 2; i++) {
prevDir += temp[i] + '/'
}
// curr = curr.replace(/\//g, "/");
// relPath = relPath.replace(/\//g, "/");
logger.LogReq(req)
try {
const stat = fs.lstatSync(curr)
if (stat.isDirectory() || stat.isSymbolicLink()) {
if (curr[curr.length - 1] !== '/') {
curr += '/'
}
const folders = []
const files = fs.readdirSync(curr)
files.sort(function(a, b) {
return (
fs.statSync(curr + b).mtime.getTime() -
fs.statSync(curr + a).mtime.getTime()
)
})
files.forEach((item) => {
if (item[0] !== '.') {
const res: any = { name: item }
const stat = fs.statSync(curr + '/' + item)
const fileSizeInBytes = stat['size']
res.size = Math.round(fileSizeInBytes / 1000000)
res.path = relPath
if (res.path[res.path.length - 1] !== '/') {
res.path += '/'
}
res.path += item
res.mtime = stat['mtime'].toLocaleString()
res.isDir = stat.isDirectory()
folders.push(res)
}
})
res.render('folders', {
folders: folders,
dirname: relPath,
prevDir,
url,
})
} else {
const fileStream = fs.createReadStream(curr)
fileStream.pipe(res)
}
} catch (err) {
res.render('nofile', {
missingFile: relPath,
url,
})
}
})
// -----------------------------------------------------------------------------------------------
app.get('*', function(req: Request, res) {
res.status(404).render('404')
})
app.post('*', function(req: Request, res) {
res.status(404).render('404')
})
return {
app: app,
}
}
export default {
name: 'Stuff',
getApp: GetApp,
setup: (data: SetupData): void => {
url = data.url
publicdirs = data.publicdirs
},
}

39
src/modules/stuff/views/audio.ejs
View file

@ -1,39 +0,0 @@
<html>
<body bgcolor="#212127">
<head>
<title><%= fname %></title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
video {
width: 100%
}
</style>
</head>
<center>
<h2>
<%= fname %>
</h2>
<img
id="coverArt"
style="width:auto; max-height: 100%;"
src="<%= url + albumArt %>"
alt=' '
/>
<audio id="audioPlayer" controls style="width:100%">
<source src="<%= url %><%= path %>?stream=true" type=<%= contentType %>>
</audio>
</center>
</body>
<script>
console.log('a')
document.getElementById('coverArt').style.height = window.innerHeight - 140
</script>
</html>

130
src/modules/stuff/views/folders.ejs
View file

@ -1,130 +0,0 @@
<html>
<body bgcolor="#212127">
<head>
<title><%=dirname%></title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top;
word-wrap: break-word;
word-break: break-all;
table-layout: fixed;
padding: 0 12;
vertical-align: middle;
}
textarea {
font: normal 14px Verdana;
color: #999999;
background-color: #212127;
width: 100%;
height: 700;
}
a {
color: #9999ff;
}
.subtable {
border-collapse: collapse;
table-layout:fixed;
width:100%
}
.maintable {
border-collapse: collapse;
table-layout:fixed;
width:100%
padding:0; margin:0;
border: none !important;
}
tr {
width:32%;
}
.butt {
background-color: #212127;
color: #999999;
cursor: pointer;
width: 100%;
border: none;
text-align: left;
outline: none;
font-size: 13px;
}
</style>
</head>
<center>
<h1>
<%=dirname%>
</h1>
</center>
<h2>
<a href="<%= url + prevDir%>" > Up one level </a>
</h2>
</p>
<table class="maintable">
<% for (var i = 0; i < folders.length; i++) { %>
<tr>
<td>
<a
href="<%= url + folders[i].path %>"
style="font-size: 0px"
>
<button
class="butt"
style='<%= i % 2 === 0 ? "background-color: #2f2f37" : "" %>'
onmouseenter='mouseEnter(this, <%= i %>)'
onmouseleave='mouseLeave(this, <%= i %>)'
>
<table class="subtable">
<tr height="62">
<td style='width:30%;'>
<%=folders[i].name %>
</td>
<td style='width:20%;'>
<%=folders[i].mtime %>
</td>
<td style='width:10%;'>
<%
if (folders[i].isDir) {
%> <%= "DIR" %> <%
} else {
if (folders[i].size === 0) {
%> <%= "~0 MB" %> <%
} else {
%> <%= folders[i].size + ' MB' %> <%
}
}
%>
</td>
</tr>
</table>
</button>
</a>
</td>
</tr>
<% } %>
</table>
</body>
<script>
console.log('hi')
function mouseEnter (e, i) {
e.style.backgroundColor = "#555"
}
function mouseLeave (e, i) {
if (i % 2 == 0) {
e.style.backgroundColor = "#2f2f37"
} else {
e.style.backgroundColor = "#212127"
}
}
</script>
</html>

94
src/modules/stuff/views/nofile.ejs
View file

@ -1,94 +0,0 @@
<html>
<body bgcolor="#212127">
<head>
<title>No such file/folder</title>
<meta charset="UTF-8">
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top
}
textarea {
font: normal 14px Verdana;
color: #999999;
background-color: #212127;
width: 100%;
height: 700;
}
a {
color: #9999ff;
}
.subtable {
border-collapse: collapse;
table-layout:fixed;
width:100%
}
.maintable {
border-collapse: collapse;
table-layout:fixed;
width:100%
padding:0; margin:0;
border: none !important;
}
tr {
line-height: 29px;
width:32%;
}
.butt {
background-color: #212127;
color: #999999;
cursor: pointer;
width: 100%;
border: none;
text-align: left;
outline: none;
font-size: 13px;
}
.active,
.butt:hover {
background-color: #555;
}
</style>
</head>
<center>
<h1>
No such file / folder:
</br>
<%= missingFile %>
</br>
<a href="<%= url %>" > Back to root </a>
</br>
<a
onclick='goBack("<%=missingFile%>")'
href="#"
>
Go back
</a>
</h1>
</center>
</body>
<script>
function goBack(path) {
path = path.replace('./public/files', '')
if (path[path.length - 1] == '/') {
path = path.substring(0, path.length -2)
}
let p = path.split('/')
p.pop()
if (p.length > 1) {
location.href = p.join('/')
} else {
location.href = '/'
}
}
</script>
</html>

34
src/modules/stuff/views/video.ejs
View file

@ -1,34 +0,0 @@
<html>
<body bgcolor="#212127">
<head>
<title><%= fname %></title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
video {
width: 100%
}
</style>
</head>
<center>
<h2>
<%= fname %>
</h2>
</center>
<video id="videoPlayer" controls>
<source src="<%= url %><%= path %>?stream=true" type="video/mp4">
</video>
</body>
<script>
var v = document.getElementsByTagName('video')[0]
v.style.maxHeight = window.innerHeight - 100
v.maxWidth = window.innerWidth
console.log('a')
</script>
</html>

16
src/server.ts
View file

@ -38,6 +38,7 @@ import https from 'https'
import cors from 'cors'
import cookieParser from 'cookie-parser'
import { v4 as uuidv4 } from 'uuid'
import type { Database } from 'better-sqlite3'
import logger from './utils/logger'
import utils from './utils/utils'
@ -51,7 +52,7 @@ const usersDBPath = './data/dbs/users.db'
const logFile = logger.logDir + logger.logFileName
const vlogFile = logger.vlogDir + logger.logFileName
function moveLogIfNotFromToday(path, to) {
function moveLogIfNotFromToday(path: string, to: string) {
if (utils.FileExists(path)) {
const today = new Date()
const stat = utils.statFile(path)
@ -89,10 +90,10 @@ interface Module {
export interface SetupData {
url: string
publicdirs: Array<string>
userDB?: any
userDB?: Database
nextdir?: string
httpServer: any
httpsServer: any
httpServer: http.Server
httpsServer: https.Server
}
if (!utils.FileExists(usersDBPath)) {
@ -122,7 +123,7 @@ try {
process.on('SIGINT', () => exit('SIGINT'))
process.on('SIGTERM', () => exit('SIGTERM'))
function exit(reason) {
function exit(reason: string) {
console.log()
logger.Log(`Exiting, reason: ${reason}`)
Object.keys(modules).forEach((key) => {
@ -152,7 +153,8 @@ const fullchainFile = '/etc/letsencrypt/live/frylabs.net/fullchain.pem'
const chainFile = '/etc/letsencrypt/live/frylabs.net/chain.pem'
let certsLoaded = false
let certs
let certs: { key: string; cert: string; ca: string }
if (
startHTTPS &&
utils.FileExists(privkeyFile) &&
@ -177,7 +179,7 @@ if (
const app = express()
const httpServer = http.createServer(app)
let httpsServer
let httpsServer: https.Server
if (certsLoaded) {
httpsServer = https.createServer(certs, app)
logger.Log('Listening on port: ' + httpsport + ' (https)')

62
src/types/basicTypes.ts
View file

@ -1,12 +1,16 @@
import express from 'express'
import { SearchResultQuestion } from '../utils/classes'
import type { Database } from 'better-sqlite3'
import type { Socket as SocketIoSocket } from 'socket.io'
import http from 'http'
import https from 'https'
export interface QuestionData {
type: string
date?: Date | number
images?: Array<string>
hashedImages?: Array<string>
possibleAnswers?:
| Array<string>
| Array<{
possibleAnswers?: Array<{
text: string
selectedByUser: boolean
}>
@ -60,11 +64,6 @@ export interface User {
created: Date
}
export interface ModuleType {
app: express.Application
dailyAction?: Function
}
export interface User {
id: number
pw: string
@ -74,7 +73,8 @@ export interface User {
createdBy: number
}
export interface Request extends express.Request {
export interface Request<T = any> extends express.Request {
body: T
cookies: any
session: any
busboy: any
@ -85,9 +85,47 @@ export interface SubmoduleData {
app: express.Application
url: string
publicdirs: Array<string>
userDB?: any
userDB?: Database
nextdir?: string
httpServer: any
moduleSpecificData?: any
httpsServer?: any
httpServer: http.Server
httpsServer: https.Server
}
export interface QuestionFromScript {
questions: Array<Question>
testUrl: string
subj: string
}
export interface DbSearchResult {
message?: string
recievedData?: string
question: Question
result: SearchResultQuestion[]
success: boolean
}
export interface RegisteredUserEntry {
cid: string
version: string
installSource: string
date: string
userAgent: string
uid: number
loginDate: string
}
export interface Submodule {
dailyAction?: () => void
load?: () => void
}
export interface ModuleType {
app: express.Application
dailyAction?: Function
}
export interface Socket extends SocketIoSocket {
user: User
}

87
src/utils/actions.ts
View file

@ -21,11 +21,15 @@ const recDataFile = './stats/recdata'
const dataLockFile = './data/lockData'
import logger from '../utils/logger'
import { createQuestion } from '../utils/classes'
import {
createQuestion,
WorkerResult,
SearchResultQuestion,
} from '../utils/classes'
import { doALongTask } from './workerPool'
import idStats from '../utils/ids'
import utils from '../utils/utils'
import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
import { addQuestion, getSubjNameWithoutYear } from './classes'
// types
import {
@ -152,12 +156,12 @@ function processIncomingRequestUsingDb(
): Promise<Result> {
return new Promise((resolve, reject) => {
try {
const recievedQuestions = []
const recievedQuestions: Question[] = []
logger.DebugLog('recievedData JSON parsed', 'isadding', 1)
logger.DebugLog(recievedData, 'isadding', 3)
const allQLength = recievedData.quiz.length
const questionSearchPromises = []
const questionSearchPromises: Promise<WorkerResult>[] = []
recievedData.quiz.forEach((question) => {
logger.DebugLog('Question:', 'isadding', 2)
logger.DebugLog(question, 'isadding', 2)
@ -193,10 +197,10 @@ function processIncomingRequestUsingDb(
})
Promise.all(questionSearchPromises)
.then((results: Array<SearchResult>) => {
const allQuestions = [] // all new questions here that do not have result
results.forEach((result, i) => {
const add = result.result.every((res) => {
.then((results: Array<WorkerResult>) => {
const allQuestions: Question[] = [] // all new questions here that do not have result
results.forEach((result: WorkerResult, i) => {
const add = result.result.every((res: SearchResultQuestion) => {
return res.match < minMatchAmmountToAdd
})
if (add) {
@ -216,7 +220,10 @@ function processIncomingRequestUsingDb(
logger.DebugLog(currentQuestion, 'isadding', 3)
addQuestion(qdb.data, sName, {
...currentQuestion,
date: new Date(),
data: {
...currentQuestion.data,
date: new Date().getTime(),
},
})
})
@ -279,7 +286,7 @@ function processIncomingRequestUsingDb(
})
}
export function isQuestionValid(question: Question): Boolean {
export function isQuestionValid(question: Question): boolean {
if (!question.Q) {
return false
}
@ -292,8 +299,8 @@ export function isQuestionValid(question: Question): Boolean {
export function shouldSearchDataFile(
df: DataFile,
testUrl: string,
trueIfAlways?: Boolean
): Boolean {
trueIfAlways?: boolean
): boolean {
if (
typeof df.shouldSearch === 'string' &&
df.shouldSearch === 'always' &&
@ -316,7 +323,7 @@ export function shouldSearchDataFile(
export function shouldSaveDataFile(
df: DataFile,
recievedData: RecievedData
): Boolean {
): boolean {
if (df.locked) {
return false
}
@ -348,17 +355,14 @@ export function shouldSaveDataFile(
}
export function loadData(path: string): Array<Subject> {
return JSON.parse(utils.ReadFile(path)).reduce((acc, subj) => {
return [
...acc,
{
return JSON.parse(utils.ReadFile(path)).map((subj: Subject) => {
return {
Name: subj.Name,
Questions: subj.Questions.map((question) => {
Questions: subj.Questions.map((question: Question) => {
return createQuestion(question)
}),
},
]
}, [])
}
})
}
export function loadJSON(
@ -455,7 +459,7 @@ function deleteFromDb(
selectedDb: { path: string; name: string }
}
): {
success: Boolean
success: boolean
msg: string
deletedQuestion?: Question
resultDb?: QuestionDb
@ -470,7 +474,7 @@ function deleteFromDb(
// }
// }
const { index, subjName } = edits
let deletedQuestion
let deletedQuestion: Question
if (isNaN(index) || !subjName) {
return {
success: false,
@ -511,10 +515,10 @@ function editQuestionInDb(
subjName: string
type: string
selectedDb: { path: string; name: string }
newVal?: Question
newVal: Question
}
): {
success: Boolean
success: boolean
msg: string
newVal?: Question
oldVal?: Question
@ -543,7 +547,8 @@ function editQuestionInDb(
// }
// }
const { index, subjName, newVal } = edits
let oldVal
let oldVal: Question
if (isNaN(index) || !subjName) {
return {
success: false,
@ -598,10 +603,10 @@ function editSubjInDb(
}>
}
): {
success: Boolean
success: boolean
msg: string
deletedQuestions?: Array<Question>
changedQuestions?: Array<Question>
changedQuestions?: { oldVal: Question; newVal: Question }[]
resultDb?: QuestionDb
} {
// {
@ -632,8 +637,8 @@ function editSubjInDb(
// }
// }
const { subjName, changedQuestions, deletedQuestions } = edits
const deletedQuestionsToWrite = []
const changedQuestionsToWrite = []
const deletedQuestionsToWrite: Question[] = []
const changedQuestionsToWrite: { oldVal: Question; newVal: Question }[] = []
if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) {
return {
success: false,
@ -695,29 +700,32 @@ function editSubjInDb(
}
}
export function editDb(
questionDb: QuestionDb,
edits: {
// FIXME: newVal is optional in some places but not in others
export interface Edits {
index: number
subjName: string
selectedDb: { path: string; name: string }
type: string
newVal?: Question
newVal: Question
deletedQuestion?: Array<number>
changedQuestions?: Array<{
index: number
value: Question
}>
}
}
export function editDb(
questionDb: QuestionDb,
edits: Edits
): {
success: Boolean
success: boolean
msg: string
resultDb?: QuestionDb
deletedQuestion?: Question
newVal?: Question
oldVal?: Question
deletedQuestions?: Array<Question>
changedQuestions?: Array<Question>
changedQuestions?: { oldVal: Question; newVal: Question }[]
} {
if (edits.type === 'delete') {
return deleteFromDb(questionDb, edits)
@ -729,4 +737,9 @@ export function editDb(
if (edits.type === 'subjEdit') {
return editSubjInDb(questionDb, edits)
}
return {
success: false,
msg: 'DB edit error, no matched type',
}
}

237
src/utils/classes.ts
View file

@ -8,21 +8,26 @@ import {
QuestionDb,
Subject,
} from '../types/basicTypes'
import { editDb } from './actions'
import { editDb, Edits } from './actions'
interface SearchResultQuestion extends Question {
export interface WorkerResult {
msg: string
workerIndex: number
result: SearchResultQuestion[]
}
interface DetailedMatch {
qMatch: number
aMatch: number
dMatch: number
matchedSubjName: string
avg: number
}
export interface SearchResultQuestion {
q: Question
match: number
}
export interface SearchResult {
result: Array<SearchResultQuestion>
dbName: string
}
const assert = (val) => {
if (!val) {
throw new Error('Assertion failed')
}
detailedMatch: DetailedMatch
}
const commonUselessAnswerParts = [
@ -65,7 +70,7 @@ function getSubjNameWithoutYear(subjName: string): string {
// Not exported
// ---------------------------------------------------------------------------------------------------------
function simplifyString(toremove) {
function simplifyString(toremove: string): string {
return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase()
}
@ -73,7 +78,7 @@ function removeStuff(
value: string,
removableStrings: Array<string>,
toReplace?: string
) {
): string {
removableStrings.forEach((removableString) => {
const regex = new RegExp(removableString, 'g')
value = value.replace(regex, toReplace || '')
@ -82,11 +87,11 @@ function removeStuff(
}
// damn nonbreaking space
function normalizeSpaces(input) {
function normalizeSpaces(input: string): string {
return input.replace(/\s/g, ' ')
}
function removeUnnecesarySpaces(toremove: string) {
function removeUnnecesarySpaces(toremove: string): string {
return normalizeSpaces(toremove).replace(/\s+/g, ' ')
}
@ -141,7 +146,7 @@ export function compareString(
return percent
}
function answerPreProcessor(value: string) {
function answerPreProcessor(value: string): string {
if (!value) {
return value
}
@ -150,9 +155,9 @@ function answerPreProcessor(value: string) {
}
// 'a. pécsi sör' -> 'pécsi sör'
function removeAnswerLetters(value: string) {
function removeAnswerLetters(value: string): string {
if (!value) {
return
return value
}
const val = value.split('. ')
@ -164,9 +169,9 @@ function removeAnswerLetters(value: string) {
}
}
function simplifyQA(value: string, mods: Array<Function>) {
function simplifyQA(value: string, mods: Array<Function>): string {
if (!value) {
return
return value
}
return mods.reduce((res, fn) => {
@ -174,7 +179,7 @@ function simplifyQA(value: string, mods: Array<Function>) {
}, value)
}
function simplifyAnswer(value: string) {
function simplifyAnswer(value: string): string {
if (!value) {
return value
}
@ -185,13 +190,17 @@ function simplifyAnswer(value: string) {
])
}
function simplifyQuestion(question: Question | string) {
function simplifyQuestion(question: string): string {
if (!question) {
return
return question
}
if (typeof question === 'string') {
return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters])
} else {
}
function simplifyQuestionObj(question: Question): Question {
if (!question) {
return question
}
if (question.Q) {
question.Q = simplifyQA(question.Q, [
removeUnnecesarySpaces,
@ -205,7 +214,6 @@ function simplifyQuestion(question: Question | string) {
])
}
return question
}
}
// ---------------------------------------------------------------------------------------------------------
@ -241,10 +249,11 @@ function createQuestion(
logger.Log('Error creating question', logger.GetColor('redbg'))
console.error(question, answer, data)
console.error(err)
return null
}
}
function compareImage(data: QuestionData, data2: QuestionData) {
function compareImage(data: QuestionData, data2: QuestionData): number {
if (data.hashedImages && data2.hashedImages) {
return compareString(
data.hashedImages.join(' '),
@ -266,7 +275,7 @@ function compareImage(data: QuestionData, data2: QuestionData) {
}
}
function compareData(q1: Question, q2: Question) {
function compareData(q1: Question, q2: Question): number {
try {
if (q1.data.type === q2.data.type) {
const dataType = q1.data.type
@ -294,7 +303,7 @@ function compareData(q1: Question, q2: Question) {
return 0
}
function compareQuestion(q1: Question, q2: Question) {
function compareQuestion(q1: Question, q2: Question): number {
return compareString(q1.Q, q1.cache.Q, q2.Q, q2.cache.Q)
// return compareString(
// q1.Q,
@ -304,7 +313,7 @@ function compareQuestion(q1: Question, q2: Question) {
// )
}
function compareAnswer(q1: Question, q2: Question) {
function compareAnswer(q1: Question, q2: Question): number {
return compareString(q1.A, q1.cache.A, q2.A, q2.cache.A)
// return compareString(
// q1.A,
@ -316,15 +325,10 @@ function compareAnswer(q1: Question, q2: Question) {
function compareQuestionObj(
q1: Question,
q1subjName: string,
_q1subjName: string,
q2: Question,
q2subjName: string
): any {
assert(q1)
assert(typeof q1 === 'object')
assert(q2)
assert(typeof q2 === 'object')
): DetailedMatch {
const qMatch = compareQuestion(q1, q2)
const aMatch = q2.A ? compareAnswer(q1, q2) : 0
// -1 if botth questions are simple
@ -354,7 +358,7 @@ function compareQuestionObj(
}
}
function questionToString(question: Question) {
function questionToString(question: Question): string {
const { Q, A, data } = question
if (data.type !== 'simple') {
@ -372,10 +376,8 @@ function searchSubject(
question: Question,
subjName: string,
searchTillMatchPercent?: number
) {
assert(question)
let result = []
): SearchResultQuestion[] {
let result: SearchResultQuestion[] = []
let stopSearch = false
let i = subj.Questions.length - 1
@ -416,10 +418,10 @@ function searchSubject(
return result
}
function subjectToString(subj: Subject) {
function subjectToString(subj: Subject): string {
const { Questions, Name } = subj
const result = []
const result: string[] = []
Questions.forEach((question) => {
result.push(questionToString(question))
})
@ -437,23 +439,14 @@ function addQuestion(
): void {
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
logger.DebugLog(question, 'qdb add', 3)
assert(data)
assert(subj)
assert(question)
assert(typeof question === 'object')
let i = 0
// FIXME: this only adds to the first matched subject name. Check if this doesnt cause any bugs
while (
i < data.length &&
!subj
const i = data.findIndex((x) => {
return !subj
.toLowerCase()
.includes(getSubjNameWithoutYear(data[i].Name).toLowerCase())
) {
i++
}
.includes(getSubjNameWithoutYear(x.Name).toLowerCase())
})
if (i < data.length) {
if (i !== -1) {
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
data[i].Questions.push(question)
} else {
@ -465,34 +458,12 @@ function addQuestion(
}
}
function prepareQuestion(
question: string | Question,
data: string | QuestionData
): any {
let preparedQuestion: Question
if (typeof question === 'object') {
preparedQuestion = createQuestion(question)
} else {
let parsedData: any
if (typeof data === 'string') {
try {
parsedData = JSON.parse(data)
} catch (err) {
// asd
}
}
if (typeof data === 'object') {
parsedData = data
}
preparedQuestion = createQuestion(question, null, parsedData)
}
return simplifyQuestion(preparedQuestion)
function prepareQuestion(question: Question): Question {
return simplifyQuestionObj(createQuestion(question))
}
function dataToString(data: Array<Subject>): string {
const result = []
const result: string[] = []
data.forEach((subj) => {
result.push(subjectToString(subj))
})
@ -502,16 +473,13 @@ function dataToString(data: Array<Subject>): string {
function doSearch(
data: Array<Subject>,
subjName: string,
question: Question | string,
questionData?: QuestionData,
question: Question,
searchTillMatchPercent?: number,
searchInAllIfNoResult?: Boolean
): any {
let result = []
): SearchResultQuestion[] {
let result: SearchResultQuestion[] = []
const questionToSearch = prepareQuestion(question, questionData)
assert(questionToSearch.data)
const questionToSearch = prepareQuestion(question)
data.every((subj) => {
if (
@ -569,7 +537,7 @@ function doSearch(
}
result = setNoPossibleAnswersPenalties(
questionToSearch.possibleAnswers,
questionToSearch.data.possibleAnswers,
result
)
@ -587,8 +555,8 @@ function doSearch(
}
function setNoPossibleAnswersPenalties(
possibleAnswers: any,
result: any[]
possibleAnswers: QuestionData['possibleAnswers'],
result: SearchResultQuestion[]
): any {
if (!Array.isArray(possibleAnswers)) {
return result
@ -602,7 +570,8 @@ function setNoPossibleAnswersPenalties(
const updated = result.map((result) => {
const hasMatch = result.q.data.possibleAnswers.some((possibleAnswer) => {
return possibleAnswers.some((questionPossibleAnswer) => {
return questionPossibleAnswer.includes(questionPossibleAnswer)
// FIXME: this could be object: questionPossibleAnswer
return questionPossibleAnswer.text.includes(possibleAnswer.text)
})
})
if (hasMatch) {
@ -626,12 +595,24 @@ function setNoPossibleAnswersPenalties(
// Multi threaded stuff
// ---------------------------------------------------------------------------------------------------------
interface WorkData {
subjName: string
question: Question
searchTillMatchPercent: number
searchInAllIfNoResult: boolean
searchIn: number[]
index: number
}
if (!isMainThread) {
// os.setPriority(10)
// logger.Log(`Worker thread priority set to ${os.getPriority()}`)
const { workerIndex } = workerData
let qdbs: Array<QuestionDb> = workerData.initData
const {
workerIndex,
initData,
}: { workerIndex: number; initData: Array<QuestionDb> } = workerData
let qdbs: Array<QuestionDb> = initData
logger.Log(
`[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
@ -642,30 +623,21 @@ if (!isMainThread) {
const {
subjName,
question,
questionData,
searchTillMatchPercent,
searchInAllIfNoResult,
searchIn,
index,
} = msg.data
}: WorkData = msg.data
// console.log(
// `[THREAD #${workerIndex}]: staring work${
// !isNaN(index) ? ` on job index #${index}` : ''
// }`
// )
let searchResult = []
let searchResult: SearchResultQuestion[] = []
try {
qdbs.forEach((qdb) => {
// FIXME: === 0?
if (searchIn.length === 0 || searchIn.includes(qdb.index)) {
if (searchIn.includes(qdb.index)) {
const res = doSearch(
qdb.data,
subjName,
question,
questionData,
searchTillMatchPercent,
searchInAllIfNoResult
)
@ -686,17 +658,19 @@ if (!isMainThread) {
} catch (err) {
logger.Log('Error in worker thread!', logger.GetColor('redbg'))
console.error(err)
console.error('subjName', subjName)
console.error('question', question)
console.error('questionData', questionData)
console.error('searchTillMatchPercent', searchTillMatchPercent)
console.error('searchInAllIfNoResult', searchInAllIfNoResult)
console.error('searchIn', searchIn)
console.error('index', index)
console.error({
subjName: subjName,
question: question,
searchTillMatchPercent: searchTillMatchPercent,
searchInAllIfNoResult: searchInAllIfNoResult,
searchIn: searchIn,
index: index,
})
}
// sorting
const sortedResult = searchResult.sort((q1, q2) => {
const sortedResult: SearchResultQuestion[] = searchResult.sort(
(q1, q2) => {
if (q1.match < q2.match) {
return 1
} else if (q1.match > q2.match) {
@ -704,16 +678,19 @@ if (!isMainThread) {
} else {
return 0
}
})
}
)
// ONDONE:
parentPort.postMessage({
const workerResult: WorkerResult = {
msg: `From thread #${workerIndex}: job ${
!isNaN(index) ? `#${index}` : ''
}done`,
workerIndex: workerIndex,
result: sortedResult,
})
}
// ONDONE:
parentPort.postMessage(workerResult)
// console.log(
// `[THREAD #${workerIndex}]: Work ${
@ -721,7 +698,7 @@ if (!isMainThread) {
// }done!`
// )
} else if (msg.type === 'dbEdit') {
const { dbIndex, edits } = msg.data
const { dbIndex, edits }: { dbIndex: number; edits: Edits } = msg.data
const { resultDb } = editDb(qdbs[dbIndex], edits)
qdbs[dbIndex] = resultDb
logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1)
@ -731,7 +708,16 @@ if (!isMainThread) {
workerIndex: workerIndex,
})
} else if (msg.type === 'newQuestions') {
const { subjName, qdbIndex, newQuestions } = msg.data
const {
subjName,
qdbIndex,
newQuestions,
}: {
subjName: string
qdbIndex: number
newQuestions: Question[]
} = msg.data
let added = false
qdbs = qdbs.map((qdb) => {
if (qdb.index === qdbIndex) {
@ -781,7 +767,8 @@ if (!isMainThread) {
// console.log(`[THREAD #${workerIndex}]: update`)
} else if (msg.type === 'newdb') {
qdbs.push(msg.newdb)
const { data }: { data: QuestionDb } = msg
qdbs.push(data)
parentPort.postMessage({
msg: `From thread #${workerIndex}: new db add done`,

22
src/utils/dbtools.ts
View file

@ -15,7 +15,7 @@ export default {
runStatement: runStatement,
}
import Sqlite from 'better-sqlite3'
import Sqlite, { Database } from 'better-sqlite3'
import logger from '../utils/logger'
import utils from '../utils/utils'
@ -52,7 +52,7 @@ function GetDB(path: string): any {
return res
}
function DebugLog(msg) {
function DebugLog(msg: string) {
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
@ -131,7 +131,7 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags = []
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
@ -150,16 +150,22 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
}, [])
.join(', ')
const fKeys = []
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
foreignKeys.forEach(
(foreignKey: {
keysFrom: string[]
table: string
keysTo: string[]
}) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
}
)
}
// IF NOT EXISTS
@ -244,7 +250,7 @@ function runStatement(db: any, command: string, runType?: string): any {
}
function CloseDB(db: any): void {
db.close((err) => {
db.close((err: Error) => {
if (err) {
return console.error(err.message)
}
@ -254,7 +260,7 @@ function CloseDB(db: any): void {
// -------------------------------------------------------------------------
function PrepareStatement(db, command) {
function PrepareStatement(db: Database, command: string) {
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'

2
src/utils/ids.ts
View file

@ -68,7 +68,7 @@ function LogId(
Save()
}
function AddSubjToList(list: Array<string>, subj: string) {
function AddSubjToList(list: { [key: string]: any }, subj: string) {
if (!list[subj]) {
list[subj] = 0
}

38
src/utils/logger.ts
View file

@ -52,7 +52,7 @@ let uvData = {} // visit data, but per user
let udvData = {} // visit data, but per user and daily
let writes = 0
let noLogips = []
let noLogips: string[] = []
function getColoredDateString(): string {
const date = new Date()
@ -93,14 +93,18 @@ function Log(msg: string | object, color?: string): void {
)
}
function expandWithSpaces(text, count) {
function expandWithSpaces(text: string, count: number) {
while (text.length < count) {
text += ' '
}
return text
}
function LogReq(req: Request, toFile?: boolean, statusCode?: string): void {
function LogReq(
req: Request,
toFile?: boolean,
statusCode?: string | number
): void {
try {
const ip: any =
req.headers['cf-connecting-ip'] || req.connection.remoteAddress
@ -173,7 +177,7 @@ function LogReq(req: Request, toFile?: boolean, statusCode?: string): void {
}
}
function parseNoLogFile(newData) {
function parseNoLogFile(newData: string) {
noLogips = newData.split('\n')
if (noLogips[noLogips.length - 1] === '') {
noLogips.pop()
@ -185,7 +189,7 @@ function parseNoLogFile(newData) {
}
function setNoLogReadInterval() {
utils.WatchFile(nologFile, (newData) => {
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
@ -227,19 +231,7 @@ function Load(): void {
setNoLogReadInterval()
}
function LogStat(
url: string,
ip: string,
hostname: string,
userId: number
): void {
const nolog = noLogips.some((noLogips) => {
return ip.includes(noLogips)
})
if (nolog) {
return
}
function LogStat(url: string, hostname: string, userId: number): void {
url = hostname + url.split('?')[0]
Inc(url)
AddUserIdStat(userId)
@ -248,7 +240,7 @@ function LogStat(
Save()
}
function IncUserStat(userId) {
function IncUserStat(userId: number) {
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
@ -260,7 +252,7 @@ function IncUserStat(userId) {
}
}
function AddUserIdStat(userId) {
function AddUserIdStat(userId: number) {
try {
const date = new Date()
const now =
@ -282,7 +274,7 @@ function AddUserIdStat(userId) {
}
}
function Inc(value) {
function Inc(value: string) {
if (value.startsWith('/?')) {
value = '/'
}
@ -292,7 +284,7 @@ function Inc(value) {
vData[value]++
}
function AddVisitStat(name) {
function AddVisitStat(name: string) {
const date = new Date()
const now =
date.getFullYear() +
@ -346,7 +338,7 @@ function logHashed(msg: string): string {
return GetRandomColor(msg.toString()) + msg + C()
}
function GetRandomColor(msg): string {
function GetRandomColor(msg: string): string {
if (!msg) {
return 'red'
}

2
src/utils/utils.ts
View file

@ -211,7 +211,7 @@ function uploadFile(req: Request, path: string): Promise<any> {
fileDestination = path + '/' + fileName
}
file.mv(fileDestination, (err) => {
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
console.error(err)

67
src/utils/workerPool.ts
View file

@ -4,19 +4,52 @@ import { EventEmitter } from 'events'
import os from 'os'
import logger from './logger'
import { Result } from './actions'
import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
import type { WorkerResult } from './classes'
interface WorkerObj {
worker: any
worker: Worker
index: number
free: Boolean
}
interface TaskObject {
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: any }
| QuestionDb
| Result
}
interface PendingJob {
workData: any
doneEvent: any
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
}
interface JobEvent extends EventEmitter {
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
}
interface DoneEvent extends EventEmitter {
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
}
const alertOnPendingCount = 50
const workerFile = './src/utils/classes.ts'
let workers: Array<WorkerObj>
@ -24,7 +57,7 @@ const pendingJobs: {
[id: string]: PendingJob
} = {}
const jobEvents = new EventEmitter()
const jobEvents: JobEvent = new EventEmitter()
jobEvents.on('jobDone', () => {
processJob()
@ -36,17 +69,17 @@ jobEvents.on('newJob', () => {
// ---------------------------------------------------------------------------
function handleWorkerError(worker: WorkerObj, err) {
function handleWorkerError(worker: WorkerObj, err: Error) {
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err)
console.error(err, worker)
}
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: any): Promise<any> {
export function msgAllWorker(data: TaskObject): Promise<any> {
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises = []
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
})
@ -58,7 +91,7 @@ export function msgAllWorker(data: any): Promise<any> {
}
export function doALongTask(
obj: any,
obj: TaskObject,
targetWorkerIndex?: number
): Promise<any> {
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
@ -72,7 +105,7 @@ export function doALongTask(
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent = new EventEmitter()
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
@ -80,7 +113,7 @@ export function doALongTask(
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
})
@ -90,7 +123,7 @@ export function doALongTask(
export function initWorkerPool(initData: any): Array<WorkerObj> {
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return
return null
}
workers = []
@ -119,7 +152,7 @@ function processJob() {
if (Object.keys(pendingJobs).length > 0) {
// FIXME: FIFO OR ANYTHING ELSE (JOB PROCESSING ORDER)
const keys = Object.keys(pendingJobs)
let jobKey, freeWorker
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
@ -159,7 +192,7 @@ function processJob() {
delete pendingJobs[jobKey]
doSomething(freeWorker, job.workData)
.then((res) => {
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
@ -169,7 +202,7 @@ function processJob() {
}
}
function getAWorker(i, initData) {
function getAWorker(i: number, initData: Array<QuestionDb>) {
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
@ -196,11 +229,11 @@ function getAWorker(i, initData) {
// ---------------------------------------------------------------------------
function doSomething(currWorker, obj) {
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg) => {
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
})
})

2
submodules/qmining-page

@ -1 +1 @@
Subproject commit 57a7ab9fb7e6148e488cc492a8d8fb55de211bf6
Subproject commit 8e4e24bed641960f75156301f29ad6fc29d9c754

7
tsconfig.json
View file

@ -1,5 +1,10 @@
{
"compilerOptions": {
"noUnusedLocals": true,
"noUnusedParameters": true,
"noImplicitReturns":true,
"noFallthroughCasesInSwitch": true,
"suppressImplicitAnyIndexErrors": true,
"moduleResolution": "node",
"esModuleInterop": true,
"target": "es6",
@ -9,7 +14,7 @@
"preserveConstEnums": true,
"sourceMap": true,
"outDir": "dist",
"noImplicitAny": false,
"noImplicitAny": true,
"lib": [
"ES2020"
]