prettier 4 tabwidth

This commit is contained in:
mrfry 2022-12-10 15:34:54 +01:00
parent 00ec614f1d
commit 96b413a365
42 changed files with 7034 additions and 6905 deletions

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -22,18 +22,18 @@
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
export default {
GetDB: GetDB,
AddColumn: AddColumn,
TableInfo: TableInfo,
Update: Update,
Delete: Delete,
CreateTable: CreateTable,
SelectAll: SelectAll,
Select: Select,
Insert: Insert,
CloseDB: CloseDB,
runStatement: runStatement,
sanitizeQuery: sanitizeQuery,
GetDB: GetDB,
AddColumn: AddColumn,
TableInfo: TableInfo,
Update: Update,
Delete: Delete,
CreateTable: CreateTable,
SelectAll: SelectAll,
Select: Select,
Insert: Insert,
CloseDB: CloseDB,
runStatement: runStatement,
sanitizeQuery: sanitizeQuery,
}
import Sqlite, { Database, RunResult } from 'better-sqlite3'
@ -43,310 +43,310 @@ import utils from '../utils/utils'
const debugLog = process.env.NS_SQL_DEBUG_LOG
function sanitizeQuery(val: string | number): string | number {
if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '')
}
return val
if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '')
}
return val
}
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
function GetSqlQuerry(
conditions: { [key: string]: string | number },
type?: string,
joiner?: string
conditions: { [key: string]: string | number },
type?: string,
joiner?: string
) {
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key])
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key])
if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`)
if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`)
} else {
acc.push(`${conditionKey} = ${condition}`)
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
acc.push(`${conditionKey} = ${condition}`)
return res.join(', ')
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
return res.join(', ')
}
}
// -------------------------------------------------------------------------
function GetDB(path: string): Database {
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
}
function DebugLog(msg: string) {
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
}
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
function AddColumn(
db: Database,
table: string,
col: { [key: string]: string | number }
db: Database,
table: string,
col: { [key: string]: string | number }
): RunResult {
try {
const colName = Object.keys(col)[0]
const colType = col.type
try {
const colName = Object.keys(col)[0]
const colType = col.type
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function TableInfo(
db: Database,
table: string
db: Database,
table: string
): {
columns: any[]
dataCount: number
columns: any[]
dataCount: number
} {
try {
const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command)
try {
const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command)
const infoRes = stmt.all()
const infoRes = stmt.all()
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const countRes = stmt2.get()
const countRes = stmt2.get()
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]],
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]],
}
} catch (err) {
console.error(err)
return null
}
} catch (err) {
console.error(err)
return null
}
}
function Update(
db: Database,
table: string,
newData: { [key: string]: string | number },
conditions: { [key: string]: string | number }
db: Database,
table: string,
newData: { [key: string]: string | number },
conditions: { [key: string]: string | number }
): RunResult {
try {
const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData,
'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command)
try {
const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData,
'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function Delete(
db: Database,
table: string,
conditions: { [key: string]: string | number }
db: Database,
table: string,
conditions: { [key: string]: string | number }
): RunResult {
try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions,
'where'
)}`
const stmt = PrepareStatement(db, command)
try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions,
'where'
)}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
interface DbColumnDescription {
[key: string]: {
type: string
primary?: boolean
autoIncrement?: boolean
notNull?: boolean
defaultZero?: boolean
[key: string]: any
}
[key: string]: {
type: string
primary?: boolean
autoIncrement?: boolean
notNull?: boolean
defaultZero?: boolean
[key: string]: any
}
}
function CreateTable(
db: Database,
name: string,
columns: DbColumnDescription,
foreignKeys: {
keysFrom: string[]
table: string
keysTo: string[]
}[]
db: Database,
name: string,
columns: DbColumnDescription,
foreignKeys: {
keysFrom: string[]
table: string
keysTo: string[]
}[]
): RunResult {
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
try {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0',
try {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0',
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function SelectAll(db: Database, from: string): any[] {
try {
const command = `SELECT * from ${from}`
try {
const command = `SELECT * from ${from}`
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
}
// SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100;
function Select(
db: Database,
from: string,
conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {}
db: Database,
from: string,
conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {}
): any[] {
const { joiner, limit } = options
const { joiner, limit } = options
try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions,
'where',
joiner
)}`
try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions,
'where',
joiner
)}`
if (!isNaN(limit)) {
command += ` LIMIT ${limit}`
if (!isNaN(limit)) {
command += ` LIMIT ${limit}`
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
}
function Insert(
db: Database,
table: string,
data: { [key: string]: number | string }
db: Database,
table: string,
data: { [key: string]: number | string }
): RunResult {
try {
const cols = Object.keys(data)
.reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, [])
.join(', ')
try {
const cols = Object.keys(data)
.reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, [])
.join(', ')
const values = Object.keys(data)
.map((key) => {
const item = data[key]
if (typeof item === 'string') {
return `'${item}'`
} else {
return `${item}`
}
})
.join(', ')
const values = Object.keys(data)
.map((key) => {
const item = data[key]
if (typeof item === 'string') {
return `'${item}'`
} else {
return `${item}`
}
})
.join(', ')
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command)
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function runStatement(db: Database, command: string, runType?: string): any {
const stmt = PrepareStatement(db, command)
if (!runType) {
return stmt.all()
} else if (runType === 'run') {
return stmt.run()
}
return null
const stmt = PrepareStatement(db, command)
if (!runType) {
return stmt.all()
} else if (runType === 'run') {
return stmt.run()
}
return null
}
function CloseDB(db: Database): void {
db.close()
db.close()
}
// -------------------------------------------------------------------------
function PrepareStatement(db: Database, command: string) {
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'
)
}
DebugLog(command)
return db.prepare(command)
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'
)
}
DebugLog(command)
return db.prepare(command)
}

View file

@ -19,8 +19,8 @@
------------------------------------------------------------------------- */
export default {
LogId: LogId,
Load: Load,
LogId: LogId,
Load: Load,
}
import utils from '../utils/utils'
@ -35,113 +35,113 @@ let idvStatsData = {}
let writes = 0
function Load(): void {
try {
idStatsData = utils.ReadJSON(idStatFile)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
idStatsData = utils.ReadJSON(idStatFile)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
}
function LogId(
id: number,
subj: string,
newQuestions: number,
allQuestions: number
id: number,
subj: string,
newQuestions: number,
allQuestions: number
): void {
Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions)
Save()
Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions)
Save()
}
function AddSubjToList(list: { [key: string]: any }, subj: string) {
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
}
function Inc(
value: number,
subj: string,
newQuestions: number,
allQuestions: number
value: number,
subj: string,
newQuestions: number,
allQuestions: number
) {
if (idStatsData[value] === undefined) {
idStatsData[value] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
if (idStatsData[value] === undefined) {
idStatsData[value] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
}
}
}
idStatsData[value].count++
idStatsData[value].newQuestions += newQuestions
idStatsData[value].allQuestions += allQuestions
AddSubjToList(idStatsData[value].subjs, subj)
idStatsData[value].count++
idStatsData[value].newQuestions += newQuestions
idStatsData[value].allQuestions += allQuestions
AddSubjToList(idStatsData[value].subjs, subj)
}
function AddVisitStat(
name: number,
subj: string,
newQuestions: number,
allQuestions: number
name: number,
subj: string,
newQuestions: number,
allQuestions: number
) {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) {
idvStatsData[now] = {}
}
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) {
idvStatsData[now] = {}
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
}
function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err)
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
}

View file

@ -19,7 +19,7 @@
------------------------------------------------------------------------- */
const hr =
'---------------------------------------------------------------------------------'
'---------------------------------------------------------------------------------'
const DELIM = C('green') + '|' + C()
@ -50,361 +50,377 @@ let writes = 0
let noLogIds: string[] = []
function getColoredDateString(): string {
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
}
function DebugLog(msg: string | object, name: string, lvl: number): void {
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
}
function Log(msg: string | object, color?: string): void {
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
)
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
)
}
function expandWithSpaces(text: string, count: number) {
while (text.length < count) {
text += ' '
}
return text
while (text.length < count) {
text += ' '
}
return text
}
function LogReq(
req: Request,
toFile?: boolean,
statusCode?: string | number
req: Request,
toFile?: boolean,
statusCode?: string | number
): void {
try {
let logEntry = '' // logHashed(ip)
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
if (req.session && req.session.user && !shouldLog(req.session.user.id, noLogIds)) {
return
}
try {
let logEntry = '' // logHashed(ip)
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
if (
req.session &&
req.session.user &&
!shouldLog(req.session.user.id, noLogIds)
) {
return
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log(
'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg')
)
}
if (!toFile) {
hostname = expandWithSpaces(hostname, 10)
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log(
'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg')
)
}
if (!toFile) {
hostname = expandWithSpaces(hostname, 10)
}
logEntry += logHashed(hostname) + dl
if (toFile) {
logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl
}
logEntry += logHashed(hostname) + dl
if (toFile) {
logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl
}
let uid = ''
if (req.session && req.session.user) {
uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) {
uid = 'EX'
} else {
uid = 'NOUSR'
}
if (!toFile) {
uid = expandWithSpaces(uid, 5)
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
let uid = ''
if (req.session && req.session.user) {
uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) {
uid = 'EX'
} else {
uid = 'NOUSR'
}
if (!toFile) {
uid = expandWithSpaces(uid, 5)
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
if (statusCode !== undefined) {
logEntry += dl + statusCode
if (statusCode !== undefined) {
logEntry += dl + statusCode
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
}
function parseNoLogFile(newData: string) {
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
}
function setNoLogReadInterval() {
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
parseNoLogFile(utils.ReadFile(nologFile))
parseNoLogFile(utils.ReadFile(nologFile))
}
function Load(): void {
Log('Loading logger...')
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
Log('Loading logger...')
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
vData = utils.ReadJSON(statFile)
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
try {
vData = utils.ReadJSON(statFile)
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
dvData = utils.ReadJSON(vStatFile)
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
setNoLogReadInterval()
try {
dvData = utils.ReadJSON(vStatFile)
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
setNoLogReadInterval()
}
export function shouldLog(userId: string | number, nolog: string[]): boolean {
return !nolog.some((noLogId) => {
return noLogId === userId.toString()
})
return !nolog.some((noLogId) => {
return noLogId === userId.toString()
})
}
function LogStat(url: string, hostname: string, userId: number | string): void {
if (!shouldLog(userId, noLogIds)) {
return
}
if (!shouldLog(userId, noLogIds)) {
return
}
url = hostname + url.split('?')[0]
Inc(url)
AddVisitStat(url)
if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
}
Save()
url = hostname + url.split('?')[0]
Inc(url)
AddVisitStat(url)
if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
}
Save()
}
const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
function shouldAddUserStat(url: string) {
return !userStatExcludes.some((x) => url.includes(x))
return !userStatExcludes.some((x) => url.includes(x))
}
function IncUserStat(userId: string) {
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
}
function AddUserIdStat(userId: string) {
try {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) {
udvData[now] = {}
try {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) {
udvData[now] = {}
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
}
function Inc(value: string) {
if (value.startsWith('/?')) {
value = '/'
}
if (vData[value] === undefined) {
vData[value] = 0
}
vData[value]++
if (value.startsWith('/?')) {
value = '/'
}
if (vData[value] === undefined) {
vData[value] = 0
}
vData[value]++
}
function AddVisitStat(name: string) {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) {
dvData[now] = {}
}
if (dvData[now][name] === undefined) {
dvData[now][name] = 0
}
dvData[now][name]++
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) {
dvData[now] = {}
}
if (dvData[now][name] === undefined) {
dvData[now][name] = 0
}
dvData[now][name]++
}
function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log(
'Error at writing visit logs! (more in stderr)',
GetColor('redbg')
)
console.error(err)
}
writes = 0
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
writes = 0
}
}
function logHashed(msg: string): string {
return GetRandomColor(msg.toString()) + msg + C()
return GetRandomColor(msg.toString()) + msg + C()
}
function GetRandomColor(msg: string): string {
if (!msg) {
return 'red'
}
if (!msg) {
return 'red'
}
const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
}
function GetColor(color: string): string {
return color
return color
}
function C(color?: string): string {
if (color !== undefined) {
color = color.toLowerCase()
}
if (color !== undefined) {
color = color.toLowerCase()
}
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
if (color === 'cyanbg') {
return '\x1b[46m'
}
if (color === 'green') {
return '\x1b[32m'
}
if (color === 'red') {
return '\x1b[31m'
}
if (color === 'yellow') {
return '\x1b[33m'
}
if (color === 'blue') {
return '\x1b[34m'
}
if (color === 'magenta') {
return '\x1b[35m'
}
if (color === 'cyan') {
return '\x1b[36m'
}
return '\x1b[0m'
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
if (color === 'cyanbg') {
return '\x1b[46m'
}
if (color === 'green') {
return '\x1b[32m'
}
if (color === 'red') {
return '\x1b[31m'
}
if (color === 'yellow') {
return '\x1b[33m'
}
if (color === 'blue') {
return '\x1b[34m'
}
if (color === 'magenta') {
return '\x1b[35m'
}
if (color === 'cyan') {
return '\x1b[36m'
}
return '\x1b[0m'
}
export default {
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
}

View file

@ -1,7 +1,7 @@
import {
createWorker,
Worker as TesseractWorker,
ConfigResult,
createWorker,
Worker as TesseractWorker,
ConfigResult,
} from 'tesseract.js'
import logger from './logger'
@ -10,44 +10,44 @@ import { isMainThread, workerData } from 'worker_threads'
// https://github.com/naptha/tesseract.js/blob/master/docs/api.md
let tesseractWorker: TesseractWorker = null
export async function initTesseractWorker(): Promise<TesseractWorker> {
const worker = createWorker({
cacheMethod: 'refresh',
// logger: (m) => console.log(m),
})
await worker.load()
await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng')
return worker
// await worker.terminate();
const worker = createWorker({
cacheMethod: 'refresh',
// logger: (m) => console.log(m),
})
await worker.load()
await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng')
return worker
// await worker.terminate();
}
let resolveLoaded: () => void = null
export const tesseractLoaded: Promise<void> = new Promise((resolve) => {
resolveLoaded = resolve
resolveLoaded = resolve
})
initTesseractWorker().then((worker) => {
tesseractWorker = worker
tesseractWorker = worker
if (isMainThread) {
logger.Log('Tesseract loaded on main thread')
} else {
const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
}
resolveLoaded()
if (isMainThread) {
logger.Log('Tesseract loaded on main thread')
} else {
const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
}
resolveLoaded()
})
export async function recognizeTextFromBase64(base64: string): Promise<string> {
const {
data: { text },
} = await tesseractWorker.recognize(base64)
return text
const {
data: { text },
} = await tesseractWorker.recognize(base64)
return text
}
export async function terminateWorker(): Promise<void | ConfigResult> {
if (tesseractWorker) {
return tesseractWorker.terminate()
}
return
if (tesseractWorker) {
return tesseractWorker.terminate()
}
return
}

View file

@ -19,23 +19,23 @@
------------------------------------------------------------------------- */
export default {
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString,
formatUrl: formatUrl,
deleteFile: deleteFile,
uploadFile: uploadFile,
statFile: statFile,
renameFile: renameFile,
deleteDir: deleteDir,
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString,
formatUrl: formatUrl,
deleteFile: deleteFile,
uploadFile: uploadFile,
statFile: statFile,
renameFile: renameFile,
deleteDir: deleteDir,
}
import fs from 'fs'
@ -45,251 +45,254 @@ import logger from '../utils/logger'
import { Request } from '../types/basicTypes'
interface URLFormatOptions {
pathname?: string
query?: { [key: string]: string }
pathname?: string
query?: { [key: string]: string }
}
function formatUrl(options: URLFormatOptions): string {
const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) {
return path
}
const queryString =
'?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
return path + queryString
const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) {
return path
}
const queryString =
'?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
return path + queryString
}
function GetDateString(
referenceDate?: Date | string,
noTime?: boolean
referenceDate?: Date | string,
noTime?: boolean
): string {
const date = referenceDate ? new Date(referenceDate) : new Date()
const date = referenceDate ? new Date(referenceDate) : new Date()
if (noTime) {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} else {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2) +
' ' +
('0' + date.getHours()).slice(-2) +
':' +
('0' + date.getMinutes()).slice(-2) +
':' +
('0' + date.getSeconds()).slice(-2)
)
}
if (noTime) {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} else {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2) +
' ' +
('0' + date.getHours()).slice(-2) +
':' +
('0' + date.getMinutes()).slice(-2) +
':' +
('0' + date.getSeconds()).slice(-2)
)
}
}
function CopyFile(from: string, to: string): void {
CreatePath(to)
fs.copyFileSync(from, to)
CreatePath(to)
fs.copyFileSync(from, to)
}
function ReadDir(path: string, listHidden?: boolean): Array<string> {
if (listHidden) {
return fs.readdirSync(path)
} else {
return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.')
})
}
if (listHidden) {
return fs.readdirSync(path)
} else {
return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.')
})
}
}
function ReadJSON(name: string): any {
try {
return JSON.parse(ReadFile(name))
} catch (err) {
console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
try {
return JSON.parse(ReadFile(name))
} catch (err) {
console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
}
function ReadFile(name: string): string {
if (!FileExists(name)) {
throw new Error('No such file: ' + name)
}
return fs.readFileSync(name, 'utf8')
if (!FileExists(name)) {
throw new Error('No such file: ' + name)
}
return fs.readFileSync(name, 'utf8')
}
function FileExists(path: string): boolean {
return fs.existsSync(path)
return fs.existsSync(path)
}
function WatchFile(file: string, callback: Function): void {
if (FileExists(file)) {
fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
throw new Error(`${file} does not exits to watch`)
}
if (FileExists(file)) {
fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
throw new Error(`${file} does not exits to watch`)
}
}
function CreatePath(path: string, onlyPath?: boolean): void {
if (FileExists(path)) {
return
}
const spath = path.split('/')
let currDir = spath[0]
for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (err) {
console.log('Failed to make ' + currDir + ' directory... ')
console.error(err)
}
if (FileExists(path)) {
return
}
const spath = path.split('/')
let currDir = spath[0]
for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (err) {
console.log('Failed to make ' + currDir + ' directory... ')
console.error(err)
}
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
}
function WriteFile(content: string, path: string): void {
CreatePath(path)
fs.writeFileSync(path, content)
CreatePath(path)
fs.writeFileSync(path, content)
}
function writeFileAsync(content: string, path: string): void {
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log(
'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg')
)
}
})
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log(
'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg')
)
}
})
}
function AppendToFile(data: string, file: string): void {
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (err) {
logger.Log(
'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg')
)
logger.Log(data)
console.error(err)
}
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (err) {
logger.Log(
'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg')
)
logger.Log(data)
console.error(err)
}
}
function deleteFile(fname: string): Boolean {
if (FileExists(fname)) {
fs.unlinkSync(fname)
return true
}
return false
if (FileExists(fname)) {
fs.unlinkSync(fname)
return true
}
return false
}
function deleteDir(dirName: string): Boolean {
if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true })
return true
}
return false
if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true })
return true
}
return false
}
function uploadFile(
req: Request,
path: string
req: Request,
path: string
): Promise<{
body: Request['body']
fileName: string
filePath: string
body: Request['body']
fileName: string
filePath: string
}> {
return new Promise((resolve, reject) => {
try {
if (!req.files) {
logger.Log(
`Unable to upload file, req.files is undefined`,
logger.GetColor('redbg')
)
return
}
const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
return new Promise((resolve, reject) => {
try {
if (!req.files) {
logger.Log(
`Unable to upload file, req.files is undefined`,
logger.GetColor('redbg')
)
return
}
const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
CreatePath(path, true)
CreatePath(path, true)
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0]
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0]
const temp = file.name.split('.')
const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName
}
const temp = file.name.split('.')
const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName
}
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
console.error(err)
reject(err)
} else {
logger.Log(
`Uploaded: ${fileName} to ${fileDestination}`,
logger.GetColor('blue')
)
resolve({
body: req.body,
fileName: fileName,
filePath: fileDestination,
})
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(
`Unable to upload file!`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
} else {
logger.Log(
`Uploaded: ${fileName} to ${fileDestination}`,
logger.GetColor('blue')
)
resolve({
body: req.body,
fileName: fileName,
filePath: fileDestination,
})
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
})
}
function statFile(file: string): fs.Stats {
if (FileExists(file)) {
return fs.statSync(file)
} else {
return null
}
if (FileExists(file)) {
return fs.statSync(file)
} else {
return null
}
}
function renameFile(oldPath: string, newPath: string): string {
if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath)
return newPath
} else {
return null
}
if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath)
return newPath
} else {
return null
}
}

View file

@ -29,268 +29,274 @@ import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
import type { WorkerResult } from './classes'
interface WorkerObj {
worker: Worker
index: number
free: Boolean
worker: Worker
index: number
free: Boolean
}
export interface TaskObject {
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
qdbIndex: number
}
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
type:
| 'work'
| 'dbEdit'
| 'newQuestions'
| 'newdb'
| 'dbClean'
| 'rmQuestions'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
qdbIndex: number
}
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
}
interface PendingJob {
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
}
interface JobEvent extends EventEmitter {
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
}
interface DoneEvent extends EventEmitter {
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
}
const alertOnPendingCount = 50
const workerFile = './src/utils/classes.ts'
let workers: Array<WorkerObj>
const pendingJobs: {
[id: string]: PendingJob
[id: string]: PendingJob
} = {}
const jobEvents: JobEvent = new EventEmitter()
jobEvents.on('jobDone', () => {
processJob()
processJob()
})
jobEvents.on('newJob', () => {
processJob()
processJob()
})
// ---------------------------------------------------------------------------
function handleWorkerError(worker: WorkerObj, err: Error) {
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker)
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker)
}
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
}
export function doALongTask(
obj: TaskObject,
targetWorkerIndex?: number
obj: TaskObject,
targetWorkerIndex?: number
): Promise<WorkerResult> {
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
)
}
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
)
}
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent,
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent,
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
})
})
})
}
export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> {
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null
}
workers = []
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null
}
workers = []
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
)
}
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
)
}
for (let i = 0; i < threadCount; i++) {
workers.push({
worker: getAWorker(i, initData),
index: i,
free: true,
})
}
for (let i = 0; i < threadCount; i++) {
workers.push({
worker: getAWorker(i, initData),
index: i,
free: true,
})
}
return workers
return workers
}
// ---------------------------------------------------------------------------
function processJob() {
if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job',
1
)
if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job',
1
)
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
}
}
i++
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
if (!freeWorker) {
logger.DebugLog('NO FREE WORKER', 'job', 1)
return
}
}
i++
}
if (!freeWorker) {
logger.DebugLog('NO FREE WORKER', 'job', 1)
return
}
if (freeWorker.free) {
freeWorker.free = false
}
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
if (freeWorker.free) {
freeWorker.free = false
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
}
function getAWorker(i: number, initData: Array<QuestionDb>) {
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
initData: initData,
},
})
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
initData: initData,
},
})
worker.setMaxListeners(50)
worker.setMaxListeners(50)
worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err)
})
worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err)
})
worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
)
})
return worker
worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
)
})
return worker
}
// ---------------------------------------------------------------------------
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
})
})
})
}
const workerTs = (
file: string,
wkOpts: {
workerData: {
workerIndex: number
initData: QuestionDb[]
__filename?: string
file: string,
wkOpts: {
workerData: {
workerIndex: number
initData: QuestionDb[]
__filename?: string
}
eval?: boolean
}
eval?: boolean
}
) => {
wkOpts.eval = true
wkOpts.workerData.__filename = file
return new Worker(
`
wkOpts.eval = true
wkOpts.workerData.__filename = file
return new Worker(
`
const wk = require('worker_threads');
require('ts-node').register();
let file = wk.workerData.__filename;
delete wk.workerData.__filename;
require(file);
`,
wkOpts
)
wkOpts
)
}