prettier 4 tabwidth

This commit is contained in:
mrfry
2022-12-10 15:34:54 +01:00
parent 00ec614f1d
commit 96b413a365
42 changed files with 7034 additions and 6905 deletions
+710 -694
View File
File diff suppressed because it is too large Load Diff
+733 -715
View File
File diff suppressed because it is too large Load Diff
+234 -234
View File
@@ -22,18 +22,18 @@
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
export default {
GetDB: GetDB,
AddColumn: AddColumn,
TableInfo: TableInfo,
Update: Update,
Delete: Delete,
CreateTable: CreateTable,
SelectAll: SelectAll,
Select: Select,
Insert: Insert,
CloseDB: CloseDB,
runStatement: runStatement,
sanitizeQuery: sanitizeQuery,
GetDB: GetDB,
AddColumn: AddColumn,
TableInfo: TableInfo,
Update: Update,
Delete: Delete,
CreateTable: CreateTable,
SelectAll: SelectAll,
Select: Select,
Insert: Insert,
CloseDB: CloseDB,
runStatement: runStatement,
sanitizeQuery: sanitizeQuery,
}
import Sqlite, { Database, RunResult } from 'better-sqlite3'
@@ -43,310 +43,310 @@ import utils from '../utils/utils'
const debugLog = process.env.NS_SQL_DEBUG_LOG
function sanitizeQuery(val: string | number): string | number {
if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '')
}
return val
if (typeof val === 'string') {
return val.replace(/'/g, '').replace(/;/g, '')
}
return val
}
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
function GetSqlQuerry(
conditions: { [key: string]: string | number },
type?: string,
joiner?: string
conditions: { [key: string]: string | number },
type?: string,
joiner?: string
) {
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key])
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
const conditionKey = sanitizeQuery(key)
const condition = sanitizeQuery(conditions[key])
if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`)
if (typeof item === 'string') {
acc.push(`${conditionKey} = '${condition}'`)
} else {
acc.push(`${conditionKey} = ${condition}`)
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
acc.push(`${conditionKey} = ${condition}`)
return res.join(', ')
}
return acc
}, [])
if (type === 'where') {
if (joiner) {
return res.join(` ${joiner} `)
} else {
return res.join(' AND ')
}
} else {
return res.join(', ')
}
}
// -------------------------------------------------------------------------
function GetDB(path: string): Database {
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
}
function DebugLog(msg: string) {
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
}
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
function AddColumn(
db: Database,
table: string,
col: { [key: string]: string | number }
db: Database,
table: string,
col: { [key: string]: string | number }
): RunResult {
try {
const colName = Object.keys(col)[0]
const colType = col.type
try {
const colName = Object.keys(col)[0]
const colType = col.type
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function TableInfo(
db: Database,
table: string
db: Database,
table: string
): {
columns: any[]
dataCount: number
columns: any[]
dataCount: number
} {
try {
const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command)
try {
const command = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, command)
const infoRes = stmt.all()
const infoRes = stmt.all()
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const countRes = stmt2.get()
const countRes = stmt2.get()
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]],
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]],
}
} catch (err) {
console.error(err)
return null
}
} catch (err) {
console.error(err)
return null
}
}
function Update(
db: Database,
table: string,
newData: { [key: string]: string | number },
conditions: { [key: string]: string | number }
db: Database,
table: string,
newData: { [key: string]: string | number },
conditions: { [key: string]: string | number }
): RunResult {
try {
const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData,
'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command)
try {
const command = `UPDATE ${table} SET ${GetSqlQuerry(
newData,
'set'
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function Delete(
db: Database,
table: string,
conditions: { [key: string]: string | number }
db: Database,
table: string,
conditions: { [key: string]: string | number }
): RunResult {
try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions,
'where'
)}`
const stmt = PrepareStatement(db, command)
try {
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
conditions,
'where'
)}`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
interface DbColumnDescription {
[key: string]: {
type: string
primary?: boolean
autoIncrement?: boolean
notNull?: boolean
defaultZero?: boolean
[key: string]: any
}
[key: string]: {
type: string
primary?: boolean
autoIncrement?: boolean
notNull?: boolean
defaultZero?: boolean
[key: string]: any
}
}
function CreateTable(
db: Database,
name: string,
columns: DbColumnDescription,
foreignKeys: {
keysFrom: string[]
table: string
keysTo: string[]
}[]
db: Database,
name: string,
columns: DbColumnDescription,
foreignKeys: {
keysFrom: string[]
table: string
keysTo: string[]
}[]
): RunResult {
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
try {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0',
try {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
const flags: string[] = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0',
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, [])
.join(', ')
const fKeys: string[] = []
if (foreignKeys) {
foreignKeys.forEach((foreignKey) => {
const { keysFrom, table, keysTo } = foreignKey
fKeys.push(
`, FOREIGN KEY(${keysFrom.join(
', '
)}) REFERENCES ${table}(${keysTo.join(', ')})`
)
})
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
// IF NOT EXISTS
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function SelectAll(db: Database, from: string): any[] {
try {
const command = `SELECT * from ${from}`
try {
const command = `SELECT * from ${from}`
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
}
// SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100;
function Select(
db: Database,
from: string,
conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {}
db: Database,
from: string,
conditions: { [key: string]: string | number },
options: { joiner?: string; limit?: number } = {}
): any[] {
const { joiner, limit } = options
const { joiner, limit } = options
try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions,
'where',
joiner
)}`
try {
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
conditions,
'where',
joiner
)}`
if (!isNaN(limit)) {
command += ` LIMIT ${limit}`
if (!isNaN(limit)) {
command += ` LIMIT ${limit}`
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
const stmt = PrepareStatement(db, command)
return stmt.all()
} catch (err) {
console.error(err)
return null
}
}
function Insert(
db: Database,
table: string,
data: { [key: string]: number | string }
db: Database,
table: string,
data: { [key: string]: number | string }
): RunResult {
try {
const cols = Object.keys(data)
.reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, [])
.join(', ')
try {
const cols = Object.keys(data)
.reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, [])
.join(', ')
const values = Object.keys(data)
.map((key) => {
const item = data[key]
if (typeof item === 'string') {
return `'${item}'`
} else {
return `${item}`
}
})
.join(', ')
const values = Object.keys(data)
.map((key) => {
const item = data[key]
if (typeof item === 'string') {
return `'${item}'`
} else {
return `${item}`
}
})
.join(', ')
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command)
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, command)
return stmt.run()
} catch (err) {
console.error(err)
return null
}
return stmt.run()
} catch (err) {
console.error(err)
return null
}
}
function runStatement(db: Database, command: string, runType?: string): any {
const stmt = PrepareStatement(db, command)
if (!runType) {
return stmt.all()
} else if (runType === 'run') {
return stmt.run()
}
return null
const stmt = PrepareStatement(db, command)
if (!runType) {
return stmt.all()
} else if (runType === 'run') {
return stmt.run()
}
return null
}
function CloseDB(db: Database): void {
db.close()
db.close()
}
// -------------------------------------------------------------------------
function PrepareStatement(db: Database, command: string) {
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'
)
}
DebugLog(command)
return db.prepare(command)
if (!db) {
throw new Error(
'DB is undefined in prepare statement! DB action called with undefined db'
)
}
DebugLog(command)
return db.prepare(command)
}
+89 -89
View File
@@ -19,8 +19,8 @@
------------------------------------------------------------------------- */
export default {
LogId: LogId,
Load: Load,
LogId: LogId,
Load: Load,
}
import utils from '../utils/utils'
@@ -35,113 +35,113 @@ let idvStatsData = {}
let writes = 0
function Load(): void {
try {
idStatsData = utils.ReadJSON(idStatFile)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
idStatsData = utils.ReadJSON(idStatFile)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
try {
const prevVData = utils.ReadFile(idVStatFile)
idvStatsData = JSON.parse(prevVData)
} catch (err) {
logger.Log(
'Error at loading id logs! (@ first run its normal)',
logger.GetColor('redbg')
)
console.error(err)
}
}
function LogId(
id: number,
subj: string,
newQuestions: number,
allQuestions: number
id: number,
subj: string,
newQuestions: number,
allQuestions: number
): void {
Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions)
Save()
Inc(id, subj, newQuestions, allQuestions)
AddVisitStat(id, subj, newQuestions, allQuestions)
Save()
}
function AddSubjToList(list: { [key: string]: any }, subj: string) {
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
}
function Inc(
value: number,
subj: string,
newQuestions: number,
allQuestions: number
value: number,
subj: string,
newQuestions: number,
allQuestions: number
) {
if (idStatsData[value] === undefined) {
idStatsData[value] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
if (idStatsData[value] === undefined) {
idStatsData[value] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
}
}
}
idStatsData[value].count++
idStatsData[value].newQuestions += newQuestions
idStatsData[value].allQuestions += allQuestions
AddSubjToList(idStatsData[value].subjs, subj)
idStatsData[value].count++
idStatsData[value].newQuestions += newQuestions
idStatsData[value].allQuestions += allQuestions
AddSubjToList(idStatsData[value].subjs, subj)
}
function AddVisitStat(
name: number,
subj: string,
newQuestions: number,
allQuestions: number
name: number,
subj: string,
newQuestions: number,
allQuestions: number
) {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) {
idvStatsData[now] = {}
}
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (idvStatsData[now] === undefined) {
idvStatsData[now] = {}
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
if (idvStatsData[now][name] === undefined) {
idvStatsData[now][name] = {
count: 0,
newQuestions: 0,
allQuestions: 0,
subjs: {},
}
}
idvStatsData[now][name].count++
idvStatsData[now][name].newQuestions += newQuestions
idvStatsData[now][name].allQuestions += allQuestions
AddSubjToList(idvStatsData[now][name].subjs, subj)
}
function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err)
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
try {
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
// Log("Stats wrote.");
} catch (err) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.error(err)
}
writes = 0
}
}
+294 -278
View File
@@ -19,7 +19,7 @@
------------------------------------------------------------------------- */
const hr =
'---------------------------------------------------------------------------------'
'---------------------------------------------------------------------------------'
const DELIM = C('green') + '|' + C()
@@ -50,361 +50,377 @@ let writes = 0
let noLogIds: string[] = []
function getColoredDateString(): string {
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
}
function DebugLog(msg: string | object, name: string, lvl: number): void {
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
let res = msg
const header = `${C('red')}#DEBUG${lvl}#${C(
'yellow'
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
res = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
res = msg
}
Log(res, 'yellow')
}
}
function Log(msg: string | object, color?: string): void {
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
)
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
)
}
function expandWithSpaces(text: string, count: number) {
while (text.length < count) {
text += ' '
}
return text
while (text.length < count) {
text += ' '
}
return text
}
function LogReq(
req: Request,
toFile?: boolean,
statusCode?: string | number
req: Request,
toFile?: boolean,
statusCode?: string | number
): void {
try {
let logEntry = '' // logHashed(ip)
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
if (req.session && req.session.user && !shouldLog(req.session.user.id, noLogIds)) {
return
}
try {
let logEntry = '' // logHashed(ip)
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
if (
req.session &&
req.session.user &&
!shouldLog(req.session.user.id, noLogIds)
) {
return
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log(
'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg')
)
}
if (!toFile) {
hostname = expandWithSpaces(hostname, 10)
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log(
'req.hostname is undefined! req.hostname: ' + req.hostname,
GetColor('redbg')
)
}
if (!toFile) {
hostname = expandWithSpaces(hostname, 10)
}
logEntry += logHashed(hostname) + dl
if (toFile) {
logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl
}
logEntry += logHashed(hostname) + dl
if (toFile) {
logEntry += req.headers['user-agent'] + dl
logEntry += req.method + dl
}
let uid = ''
if (req.session && req.session.user) {
uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) {
uid = 'EX'
} else {
uid = 'NOUSR'
}
if (!toFile) {
uid = expandWithSpaces(uid, 5)
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
let uid = ''
if (req.session && req.session.user) {
uid = req.session.user.id.toString()
} else if (req.session && req.session.isException === true) {
uid = 'EX'
} else {
uid = 'NOUSR'
}
if (!toFile) {
uid = expandWithSpaces(uid, 5)
}
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
if (statusCode !== undefined) {
logEntry += dl + statusCode
if (statusCode !== undefined) {
logEntry += dl + statusCode
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
const defLogs = utils.GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
}
function parseNoLogFile(newData: string) {
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
}
function setNoLogReadInterval() {
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
utils.WatchFile(nologFile, (newData: string) => {
parseNoLogFile(newData)
})
parseNoLogFile(utils.ReadFile(nologFile))
parseNoLogFile(utils.ReadFile(nologFile))
}
function Load(): void {
Log('Loading logger...')
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
Log('Loading logger...')
try {
uvData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
try {
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
vData = utils.ReadJSON(statFile)
} catch (err) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.error(err)
}
try {
vData = utils.ReadJSON(statFile)
} catch (err) {
Log(
'Error at loading logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
try {
dvData = utils.ReadJSON(vStatFile)
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
setNoLogReadInterval()
try {
dvData = utils.ReadJSON(vStatFile)
} catch (err) {
Log(
'Error at loading visit logs! (@ first run its normal)',
GetColor('redbg')
)
console.error(err)
}
setNoLogReadInterval()
}
export function shouldLog(userId: string | number, nolog: string[]): boolean {
return !nolog.some((noLogId) => {
return noLogId === userId.toString()
})
return !nolog.some((noLogId) => {
return noLogId === userId.toString()
})
}
function LogStat(url: string, hostname: string, userId: number | string): void {
if (!shouldLog(userId, noLogIds)) {
return
}
if (!shouldLog(userId, noLogIds)) {
return
}
url = hostname + url.split('?')[0]
Inc(url)
AddVisitStat(url)
if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
}
Save()
url = hostname + url.split('?')[0]
Inc(url)
AddVisitStat(url)
if (shouldAddUserStat(url)) {
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
}
Save()
}
const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
function shouldAddUserStat(url: string) {
return !userStatExcludes.some((x) => url.includes(x))
return !userStatExcludes.some((x) => url.includes(x))
}
function IncUserStat(userId: string) {
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
uvData[userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
}
function AddUserIdStat(userId: string) {
try {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) {
udvData[now] = {}
try {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (udvData[now] === undefined) {
udvData[now] = {}
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
if (udvData[now][userId] === undefined) {
udvData[now][userId] = 0
}
udvData[now][userId]++
} catch (err) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(err)
}
}
function Inc(value: string) {
if (value.startsWith('/?')) {
value = '/'
}
if (vData[value] === undefined) {
vData[value] = 0
}
vData[value]++
if (value.startsWith('/?')) {
value = '/'
}
if (vData[value] === undefined) {
vData[value] = 0
}
vData[value]++
}
function AddVisitStat(name: string) {
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) {
dvData[now] = {}
}
if (dvData[now][name] === undefined) {
dvData[now][name] = 0
}
dvData[now][name]++
const date = new Date()
const now =
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
if (dvData[now] === undefined) {
dvData[now] = {}
}
if (dvData[now][name] === undefined) {
dvData[now][name] = 0
}
dvData[now][name]++
}
function Save() {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log(
'Error at writing visit logs! (more in stderr)',
GetColor('redbg')
)
console.error(err)
}
writes = 0
}
try {
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(vData), statFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
try {
utils.WriteFile(JSON.stringify(dvData), vStatFile)
// Log("Stats wrote.");
} catch (err) {
Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
console.error(err)
}
writes = 0
}
}
function logHashed(msg: string): string {
return GetRandomColor(msg.toString()) + msg + C()
return GetRandomColor(msg.toString()) + msg + C()
}
function GetRandomColor(msg: string): string {
if (!msg) {
return 'red'
}
if (!msg) {
return 'red'
}
const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
const res = msg.split('').reduce((res, character) => {
return res + character.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
}
function GetColor(color: string): string {
return color
return color
}
function C(color?: string): string {
if (color !== undefined) {
color = color.toLowerCase()
}
if (color !== undefined) {
color = color.toLowerCase()
}
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
if (color === 'cyanbg') {
return '\x1b[46m'
}
if (color === 'green') {
return '\x1b[32m'
}
if (color === 'red') {
return '\x1b[31m'
}
if (color === 'yellow') {
return '\x1b[33m'
}
if (color === 'blue') {
return '\x1b[34m'
}
if (color === 'magenta') {
return '\x1b[35m'
}
if (color === 'cyan') {
return '\x1b[36m'
}
return '\x1b[0m'
if (color === 'redbg') {
return '\x1b[41m'
}
if (color === 'bluebg') {
return '\x1b[44m'
}
if (color === 'cyanbg') {
return '\x1b[46m'
}
if (color === 'green') {
return '\x1b[32m'
}
if (color === 'red') {
return '\x1b[31m'
}
if (color === 'yellow') {
return '\x1b[33m'
}
if (color === 'blue') {
return '\x1b[34m'
}
if (color === 'magenta') {
return '\x1b[35m'
}
if (color === 'cyan') {
return '\x1b[36m'
}
return '\x1b[0m'
}
export default {
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C,
logFileName: logFileName,
logDir: logDir,
vlogDir: vlogDir,
}
+29 -29
View File
@@ -1,7 +1,7 @@
import {
createWorker,
Worker as TesseractWorker,
ConfigResult,
createWorker,
Worker as TesseractWorker,
ConfigResult,
} from 'tesseract.js'
import logger from './logger'
@@ -10,44 +10,44 @@ import { isMainThread, workerData } from 'worker_threads'
// https://github.com/naptha/tesseract.js/blob/master/docs/api.md
let tesseractWorker: TesseractWorker = null
export async function initTesseractWorker(): Promise<TesseractWorker> {
const worker = createWorker({
cacheMethod: 'refresh',
// logger: (m) => console.log(m),
})
await worker.load()
await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng')
return worker
// await worker.terminate();
const worker = createWorker({
cacheMethod: 'refresh',
// logger: (m) => console.log(m),
})
await worker.load()
await worker.loadLanguage('hun+eng')
await worker.initialize('hun+eng')
return worker
// await worker.terminate();
}
let resolveLoaded: () => void = null
export const tesseractLoaded: Promise<void> = new Promise((resolve) => {
resolveLoaded = resolve
resolveLoaded = resolve
})
initTesseractWorker().then((worker) => {
tesseractWorker = worker
tesseractWorker = worker
if (isMainThread) {
logger.Log('Tesseract loaded on main thread')
} else {
const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
}
resolveLoaded()
if (isMainThread) {
logger.Log('Tesseract loaded on main thread')
} else {
const { workerIndex }: { workerIndex: number } = workerData
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
}
resolveLoaded()
})
export async function recognizeTextFromBase64(base64: string): Promise<string> {
const {
data: { text },
} = await tesseractWorker.recognize(base64)
return text
const {
data: { text },
} = await tesseractWorker.recognize(base64)
return text
}
export async function terminateWorker(): Promise<void | ConfigResult> {
if (tesseractWorker) {
return tesseractWorker.terminate()
}
return
if (tesseractWorker) {
return tesseractWorker.terminate()
}
return
}
+206 -203
View File
@@ -19,23 +19,23 @@
------------------------------------------------------------------------- */
export default {
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString,
formatUrl: formatUrl,
deleteFile: deleteFile,
uploadFile: uploadFile,
statFile: statFile,
renameFile: renameFile,
deleteDir: deleteDir,
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: writeFileAsync,
AppendToFile: AppendToFile,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString,
formatUrl: formatUrl,
deleteFile: deleteFile,
uploadFile: uploadFile,
statFile: statFile,
renameFile: renameFile,
deleteDir: deleteDir,
}
import fs from 'fs'
@@ -45,251 +45,254 @@ import logger from '../utils/logger'
import { Request } from '../types/basicTypes'
interface URLFormatOptions {
pathname?: string
query?: { [key: string]: string }
pathname?: string
query?: { [key: string]: string }
}
function formatUrl(options: URLFormatOptions): string {
const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) {
return path
}
const queryString =
'?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
return path + queryString
const path = options.pathname || ''
if (!options.query || Object.keys(options.query).length === 0) {
return path
}
const queryString =
'?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
return path + queryString
}
function GetDateString(
referenceDate?: Date | string,
noTime?: boolean
referenceDate?: Date | string,
noTime?: boolean
): string {
const date = referenceDate ? new Date(referenceDate) : new Date()
const date = referenceDate ? new Date(referenceDate) : new Date()
if (noTime) {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} else {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2) +
' ' +
('0' + date.getHours()).slice(-2) +
':' +
('0' + date.getMinutes()).slice(-2) +
':' +
('0' + date.getSeconds()).slice(-2)
)
}
if (noTime) {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2)
)
} else {
return (
date.getFullYear() +
'-' +
('0' + (date.getMonth() + 1)).slice(-2) +
'-' +
('0' + date.getDate()).slice(-2) +
' ' +
('0' + date.getHours()).slice(-2) +
':' +
('0' + date.getMinutes()).slice(-2) +
':' +
('0' + date.getSeconds()).slice(-2)
)
}
}
function CopyFile(from: string, to: string): void {
CreatePath(to)
fs.copyFileSync(from, to)
CreatePath(to)
fs.copyFileSync(from, to)
}
function ReadDir(path: string, listHidden?: boolean): Array<string> {
if (listHidden) {
return fs.readdirSync(path)
} else {
return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.')
})
}
if (listHidden) {
return fs.readdirSync(path)
} else {
return fs.readdirSync(path).filter((file) => {
return !file.startsWith('.')
})
}
}
function ReadJSON(name: string): any {
try {
return JSON.parse(ReadFile(name))
} catch (err) {
console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
try {
return JSON.parse(ReadFile(name))
} catch (err) {
console.error(err)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
}
function ReadFile(name: string): string {
if (!FileExists(name)) {
throw new Error('No such file: ' + name)
}
return fs.readFileSync(name, 'utf8')
if (!FileExists(name)) {
throw new Error('No such file: ' + name)
}
return fs.readFileSync(name, 'utf8')
}
function FileExists(path: string): boolean {
return fs.existsSync(path)
return fs.existsSync(path)
}
function WatchFile(file: string, callback: Function): void {
if (FileExists(file)) {
fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
throw new Error(`${file} does not exits to watch`)
}
if (FileExists(file)) {
fs.watchFile(file, () => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
throw new Error(`${file} does not exits to watch`)
}
}
function CreatePath(path: string, onlyPath?: boolean): void {
if (FileExists(path)) {
return
}
const spath = path.split('/')
let currDir = spath[0]
for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (err) {
console.log('Failed to make ' + currDir + ' directory... ')
console.error(err)
}
if (FileExists(path)) {
return
}
const spath = path.split('/')
let currDir = spath[0]
for (let i = 1; i < spath.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (err) {
console.log('Failed to make ' + currDir + ' directory... ')
console.error(err)
}
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
currDir += '/' + spath[i]
}
if (onlyPath) {
fs.mkdirSync(path)
}
}
function WriteFile(content: string, path: string): void {
CreatePath(path)
fs.writeFileSync(path, content)
CreatePath(path)
fs.writeFileSync(path, content)
}
function writeFileAsync(content: string, path: string): void {
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log(
'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg')
)
}
})
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log(
'Error writing file: ' + path + ' (sync)',
logger.GetColor('redbg')
)
}
})
}
function AppendToFile(data: string, file: string): void {
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (err) {
logger.Log(
'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg')
)
logger.Log(data)
console.error(err)
}
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (err) {
logger.Log(
'Error appendig to file log file: ' + file + ' (sync)',
logger.GetColor('redbg')
)
logger.Log(data)
console.error(err)
}
}
function deleteFile(fname: string): Boolean {
if (FileExists(fname)) {
fs.unlinkSync(fname)
return true
}
return false
if (FileExists(fname)) {
fs.unlinkSync(fname)
return true
}
return false
}
function deleteDir(dirName: string): Boolean {
if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true })
return true
}
return false
if (FileExists(dirName)) {
fs.rmSync(dirName, { recursive: true })
return true
}
return false
}
function uploadFile(
req: Request,
path: string
req: Request,
path: string
): Promise<{
body: Request['body']
fileName: string
filePath: string
body: Request['body']
fileName: string
filePath: string
}> {
return new Promise((resolve, reject) => {
try {
if (!req.files) {
logger.Log(
`Unable to upload file, req.files is undefined`,
logger.GetColor('redbg')
)
return
}
const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
return new Promise((resolve, reject) => {
try {
if (!req.files) {
logger.Log(
`Unable to upload file, req.files is undefined`,
logger.GetColor('redbg')
)
return
}
const file = req.files.file
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
CreatePath(path, true)
CreatePath(path, true)
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0]
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
let fileDestination = path + '/' + fileName
if (FileExists(fileDestination)) {
const id = uuidv4().split('-')[0]
const temp = file.name.split('.')
const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName
}
const temp = file.name.split('.')
const extension = temp.pop()
fileName = temp.join('.') + '_' + id + '.' + extension
fileDestination = path + '/' + fileName
}
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
console.error(err)
reject(err)
} else {
logger.Log(
`Uploaded: ${fileName} to ${fileDestination}`,
logger.GetColor('blue')
)
resolve({
body: req.body,
fileName: fileName,
filePath: fileDestination,
})
file.mv(fileDestination, (err: Error) => {
if (err) {
logger.Log(
`Unable to upload file!`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
} else {
logger.Log(
`Uploaded: ${fileName} to ${fileDestination}`,
logger.GetColor('blue')
)
resolve({
body: req.body,
fileName: fileName,
filePath: fileDestination,
})
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
} catch (err) {
logger.Log(
`Unable to upload file, error on stderr`,
logger.GetColor('redbg')
)
console.error(err)
reject(err)
}
})
})
}
function statFile(file: string): fs.Stats {
if (FileExists(file)) {
return fs.statSync(file)
} else {
return null
}
if (FileExists(file)) {
return fs.statSync(file)
} else {
return null
}
}
function renameFile(oldPath: string, newPath: string): string {
if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath)
return newPath
} else {
return null
}
if (FileExists(oldPath)) {
fs.renameSync(oldPath, newPath)
return newPath
} else {
return null
}
}
+189 -183
View File
@@ -29,268 +29,274 @@ import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
import type { WorkerResult } from './classes'
interface WorkerObj {
worker: Worker
index: number
free: Boolean
worker: Worker
index: number
free: Boolean
}
export interface TaskObject {
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
qdbIndex: number
}
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
type:
| 'work'
| 'dbEdit'
| 'newQuestions'
| 'newdb'
| 'dbClean'
| 'rmQuestions'
data:
| {
searchIn: number[]
question: Question
subjName: string
testUrl?: string
questionData?: QuestionData
searchInAllIfNoResult?: boolean
searchTillMatchPercent?: number
[key: string]: any
}
| { dbIndex: number; edits: Edits }
| QuestionDb
| Result
| {
questions: Question[]
subjToClean: string
overwriteFromDate: number
qdbIndex: number
}
| {
questionIndexesToRemove: number[][]
subjIndex: number
qdbIndex: number
recievedQuestions: Question[]
}
}
interface PendingJob {
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
workData: TaskObject
doneEvent: DoneEvent
targetWorkerIndex?: number
}
interface JobEvent extends EventEmitter {
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
on(event: 'jobDone', listener: () => void): this
on(event: 'newJob', listener: () => void): this
emit(event: 'newJob'): boolean
emit(event: 'jobDone'): boolean
}
interface DoneEvent extends EventEmitter {
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
once(event: 'done', listener: (result: WorkerResult) => void): this
emit(event: 'done', res: WorkerResult): boolean
}
const alertOnPendingCount = 50
const workerFile = './src/utils/classes.ts'
let workers: Array<WorkerObj>
const pendingJobs: {
[id: string]: PendingJob
[id: string]: PendingJob
} = {}
const jobEvents: JobEvent = new EventEmitter()
jobEvents.on('jobDone', () => {
processJob()
processJob()
})
jobEvents.on('newJob', () => {
processJob()
processJob()
})
// ---------------------------------------------------------------------------
function handleWorkerError(worker: WorkerObj, err: Error) {
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker)
// TODO: restart worker if exited or things like that
logger.Log('resourcePromise error', logger.GetColor('redbg'))
console.error(err, worker)
}
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
return new Promise((resolve) => {
const promises: Promise<WorkerResult>[] = []
workers.forEach((worker) => {
promises.push(doALongTask(data, worker.index))
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
Promise.all(promises).then((res) => {
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
resolve(res)
})
})
}
export function doALongTask(
obj: TaskObject,
targetWorkerIndex?: number
obj: TaskObject,
targetWorkerIndex?: number
): Promise<WorkerResult> {
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
)
}
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
logger.Log(
`More than ${alertOnPendingCount} callers waiting for free resource! (${
Object.keys(pendingJobs).length
})`,
logger.GetColor('redbg')
)
}
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent,
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
const jobId = uuidv4()
// FIXME: delete doneEvent?
const doneEvent: DoneEvent = new EventEmitter()
pendingJobs[jobId] = {
workData: obj,
targetWorkerIndex: targetWorkerIndex,
doneEvent: doneEvent,
}
jobEvents.emit('newJob')
return new Promise((resolve) => {
doneEvent.once('done', (result: WorkerResult) => {
jobEvents.emit('jobDone')
resolve(result)
})
})
})
}
export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> {
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null
}
workers = []
if (workers) {
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
return null
}
workers = []
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
)
}
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
if (process.env.NS_THREAD_COUNT) {
logger.Log(
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
logger.GetColor('red')
)
}
for (let i = 0; i < threadCount; i++) {
workers.push({
worker: getAWorker(i, initData),
index: i,
free: true,
})
}
for (let i = 0; i < threadCount; i++) {
workers.push({
worker: getAWorker(i, initData),
index: i,
free: true,
})
}
return workers
return workers
}
// ---------------------------------------------------------------------------
function processJob() {
if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job',
1
)
if (Object.keys(pendingJobs).length > 0) {
const keys = Object.keys(pendingJobs)
let jobKey: string, freeWorker: WorkerObj
let i = 0
while (!freeWorker && i < keys.length) {
jobKey = keys[i]
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
logger.DebugLog(
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
'job',
1
)
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
}
}
i++
}
} else {
freeWorker = workers.find((worker) => {
return worker.free
})
if (freeWorker) {
logger.DebugLog(
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
'job',
1
)
if (!freeWorker) {
logger.DebugLog('NO FREE WORKER', 'job', 1)
return
}
}
i++
}
if (!freeWorker) {
logger.DebugLog('NO FREE WORKER', 'job', 1)
return
}
if (freeWorker.free) {
freeWorker.free = false
}
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
if (freeWorker.free) {
freeWorker.free = false
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
const job = pendingJobs[jobKey]
delete pendingJobs[jobKey]
doSomething(freeWorker, job.workData)
.then((res: WorkerResult) => {
freeWorker.free = true
job.doneEvent.emit('done', res)
})
.catch(function (err) {
handleWorkerError(freeWorker, err)
})
}
}
function getAWorker(i: number, initData: Array<QuestionDb>) {
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
initData: initData,
},
})
const worker = workerTs(workerFile, {
workerData: {
workerIndex: i,
initData: initData,
},
})
worker.setMaxListeners(50)
worker.setMaxListeners(50)
worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err)
})
worker.on('error', (err) => {
logger.Log('Worker error!', logger.GetColor('redbg'))
console.error(err)
})
worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
)
})
return worker
worker.on('exit', (code) => {
// TODO: this is critical, whole server should stop, or child threads should be restarted
logger.Log(
`[MAIN]: worker #${i} exit code: ${code}`,
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
)
})
return worker
}
// ---------------------------------------------------------------------------
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
const { /* index, */ worker } = currWorker
return new Promise((resolve) => {
worker.postMessage(obj)
worker.once('message', (msg: WorkerResult) => {
resolve(msg)
})
})
})
}
const workerTs = (
file: string,
wkOpts: {
workerData: {
workerIndex: number
initData: QuestionDb[]
__filename?: string
file: string,
wkOpts: {
workerData: {
workerIndex: number
initData: QuestionDb[]
__filename?: string
}
eval?: boolean
}
eval?: boolean
}
) => {
wkOpts.eval = true
wkOpts.workerData.__filename = file
return new Worker(
`
wkOpts.eval = true
wkOpts.workerData.__filename = file
return new Worker(
`
const wk = require('worker_threads');
require('ts-node').register();
let file = wk.workerData.__filename;
delete wk.workerData.__filename;
require(file);
`,
wkOpts
)
wkOpts
)
}