mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
prettier 4 tabwidth
This commit is contained in:
parent
00ec614f1d
commit
96b413a365
42 changed files with 7034 additions and 6905 deletions
1404
src/utils/actions.ts
1404
src/utils/actions.ts
File diff suppressed because it is too large
Load diff
1448
src/utils/classes.ts
1448
src/utils/classes.ts
File diff suppressed because it is too large
Load diff
|
@ -22,18 +22,18 @@
|
|||
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
|
||||
|
||||
export default {
|
||||
GetDB: GetDB,
|
||||
AddColumn: AddColumn,
|
||||
TableInfo: TableInfo,
|
||||
Update: Update,
|
||||
Delete: Delete,
|
||||
CreateTable: CreateTable,
|
||||
SelectAll: SelectAll,
|
||||
Select: Select,
|
||||
Insert: Insert,
|
||||
CloseDB: CloseDB,
|
||||
runStatement: runStatement,
|
||||
sanitizeQuery: sanitizeQuery,
|
||||
GetDB: GetDB,
|
||||
AddColumn: AddColumn,
|
||||
TableInfo: TableInfo,
|
||||
Update: Update,
|
||||
Delete: Delete,
|
||||
CreateTable: CreateTable,
|
||||
SelectAll: SelectAll,
|
||||
Select: Select,
|
||||
Insert: Insert,
|
||||
CloseDB: CloseDB,
|
||||
runStatement: runStatement,
|
||||
sanitizeQuery: sanitizeQuery,
|
||||
}
|
||||
|
||||
import Sqlite, { Database, RunResult } from 'better-sqlite3'
|
||||
|
@ -43,310 +43,310 @@ import utils from '../utils/utils'
|
|||
const debugLog = process.env.NS_SQL_DEBUG_LOG
|
||||
|
||||
function sanitizeQuery(val: string | number): string | number {
|
||||
if (typeof val === 'string') {
|
||||
return val.replace(/'/g, '').replace(/;/g, '')
|
||||
}
|
||||
return val
|
||||
if (typeof val === 'string') {
|
||||
return val.replace(/'/g, '').replace(/;/g, '')
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
|
||||
function GetSqlQuerry(
|
||||
conditions: { [key: string]: string | number },
|
||||
type?: string,
|
||||
joiner?: string
|
||||
conditions: { [key: string]: string | number },
|
||||
type?: string,
|
||||
joiner?: string
|
||||
) {
|
||||
const res = Object.keys(conditions).reduce((acc, key) => {
|
||||
const item = conditions[key]
|
||||
const conditionKey = sanitizeQuery(key)
|
||||
const condition = sanitizeQuery(conditions[key])
|
||||
const res = Object.keys(conditions).reduce((acc, key) => {
|
||||
const item = conditions[key]
|
||||
const conditionKey = sanitizeQuery(key)
|
||||
const condition = sanitizeQuery(conditions[key])
|
||||
|
||||
if (typeof item === 'string') {
|
||||
acc.push(`${conditionKey} = '${condition}'`)
|
||||
if (typeof item === 'string') {
|
||||
acc.push(`${conditionKey} = '${condition}'`)
|
||||
} else {
|
||||
acc.push(`${conditionKey} = ${condition}`)
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
if (type === 'where') {
|
||||
if (joiner) {
|
||||
return res.join(` ${joiner} `)
|
||||
} else {
|
||||
return res.join(' AND ')
|
||||
}
|
||||
} else {
|
||||
acc.push(`${conditionKey} = ${condition}`)
|
||||
return res.join(', ')
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
if (type === 'where') {
|
||||
if (joiner) {
|
||||
return res.join(` ${joiner} `)
|
||||
} else {
|
||||
return res.join(' AND ')
|
||||
}
|
||||
} else {
|
||||
return res.join(', ')
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function GetDB(path: string): Database {
|
||||
utils.CreatePath(path)
|
||||
const res = new Sqlite(path)
|
||||
res.pragma('synchronous = OFF')
|
||||
return res
|
||||
utils.CreatePath(path)
|
||||
const res = new Sqlite(path)
|
||||
res.pragma('synchronous = OFF')
|
||||
return res
|
||||
}
|
||||
|
||||
function DebugLog(msg: string) {
|
||||
if (debugLog) {
|
||||
logger.DebugLog(msg, 'sql', 0)
|
||||
}
|
||||
if (debugLog) {
|
||||
logger.DebugLog(msg, 'sql', 0)
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
|
||||
function AddColumn(
|
||||
db: Database,
|
||||
table: string,
|
||||
col: { [key: string]: string | number }
|
||||
db: Database,
|
||||
table: string,
|
||||
col: { [key: string]: string | number }
|
||||
): RunResult {
|
||||
try {
|
||||
const colName = Object.keys(col)[0]
|
||||
const colType = col.type
|
||||
try {
|
||||
const colName = Object.keys(col)[0]
|
||||
const colType = col.type
|
||||
|
||||
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
const command = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function TableInfo(
|
||||
db: Database,
|
||||
table: string
|
||||
db: Database,
|
||||
table: string
|
||||
): {
|
||||
columns: any[]
|
||||
dataCount: number
|
||||
columns: any[]
|
||||
dataCount: number
|
||||
} {
|
||||
try {
|
||||
const command = `PRAGMA table_info(${table})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
try {
|
||||
const command = `PRAGMA table_info(${table})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
const infoRes = stmt.all()
|
||||
const infoRes = stmt.all()
|
||||
|
||||
const s2 = `SELECT COUNT(*) FROM ${table}`
|
||||
const stmt2 = PrepareStatement(db, s2)
|
||||
const s2 = `SELECT COUNT(*) FROM ${table}`
|
||||
const stmt2 = PrepareStatement(db, s2)
|
||||
|
||||
const countRes = stmt2.get()
|
||||
const countRes = stmt2.get()
|
||||
|
||||
return {
|
||||
columns: infoRes,
|
||||
dataCount: countRes[Object.keys(countRes)[0]],
|
||||
return {
|
||||
columns: infoRes,
|
||||
dataCount: countRes[Object.keys(countRes)[0]],
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function Update(
|
||||
db: Database,
|
||||
table: string,
|
||||
newData: { [key: string]: string | number },
|
||||
conditions: { [key: string]: string | number }
|
||||
db: Database,
|
||||
table: string,
|
||||
newData: { [key: string]: string | number },
|
||||
conditions: { [key: string]: string | number }
|
||||
): RunResult {
|
||||
try {
|
||||
const command = `UPDATE ${table} SET ${GetSqlQuerry(
|
||||
newData,
|
||||
'set'
|
||||
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
try {
|
||||
const command = `UPDATE ${table} SET ${GetSqlQuerry(
|
||||
newData,
|
||||
'set'
|
||||
)} WHERE ${GetSqlQuerry(conditions, 'where')}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function Delete(
|
||||
db: Database,
|
||||
table: string,
|
||||
conditions: { [key: string]: string | number }
|
||||
db: Database,
|
||||
table: string,
|
||||
conditions: { [key: string]: string | number }
|
||||
): RunResult {
|
||||
try {
|
||||
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
|
||||
conditions,
|
||||
'where'
|
||||
)}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
try {
|
||||
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
|
||||
conditions,
|
||||
'where'
|
||||
)}`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
interface DbColumnDescription {
|
||||
[key: string]: {
|
||||
type: string
|
||||
primary?: boolean
|
||||
autoIncrement?: boolean
|
||||
notNull?: boolean
|
||||
defaultZero?: boolean
|
||||
[key: string]: any
|
||||
}
|
||||
[key: string]: {
|
||||
type: string
|
||||
primary?: boolean
|
||||
autoIncrement?: boolean
|
||||
notNull?: boolean
|
||||
defaultZero?: boolean
|
||||
[key: string]: any
|
||||
}
|
||||
}
|
||||
|
||||
function CreateTable(
|
||||
db: Database,
|
||||
name: string,
|
||||
columns: DbColumnDescription,
|
||||
foreignKeys: {
|
||||
keysFrom: string[]
|
||||
table: string
|
||||
keysTo: string[]
|
||||
}[]
|
||||
db: Database,
|
||||
name: string,
|
||||
columns: DbColumnDescription,
|
||||
foreignKeys: {
|
||||
keysFrom: string[]
|
||||
table: string
|
||||
keysTo: string[]
|
||||
}[]
|
||||
): RunResult {
|
||||
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
|
||||
// number, lastLogin text, lastAccess text
|
||||
//
|
||||
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
|
||||
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
|
||||
// number, lastLogin text, lastAccess text
|
||||
//
|
||||
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
|
||||
|
||||
try {
|
||||
const cols = Object.keys(columns)
|
||||
.reduce((acc, key) => {
|
||||
const item = columns[key]
|
||||
const flags: string[] = []
|
||||
const toCheck = {
|
||||
primary: 'PRIMARY KEY',
|
||||
notNull: 'NOT NULL',
|
||||
unique: 'UNIQUE',
|
||||
autoIncrement: 'AUTOINCREMENT',
|
||||
defaultZero: 'DEFAULT 0',
|
||||
try {
|
||||
const cols = Object.keys(columns)
|
||||
.reduce((acc, key) => {
|
||||
const item = columns[key]
|
||||
const flags: string[] = []
|
||||
const toCheck = {
|
||||
primary: 'PRIMARY KEY',
|
||||
notNull: 'NOT NULL',
|
||||
unique: 'UNIQUE',
|
||||
autoIncrement: 'AUTOINCREMENT',
|
||||
defaultZero: 'DEFAULT 0',
|
||||
}
|
||||
Object.keys(toCheck).forEach((key) => {
|
||||
if (item[key]) {
|
||||
flags.push(toCheck[key])
|
||||
}
|
||||
})
|
||||
|
||||
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
|
||||
return acc
|
||||
}, [])
|
||||
.join(', ')
|
||||
|
||||
const fKeys: string[] = []
|
||||
if (foreignKeys) {
|
||||
foreignKeys.forEach((foreignKey) => {
|
||||
const { keysFrom, table, keysTo } = foreignKey
|
||||
fKeys.push(
|
||||
`, FOREIGN KEY(${keysFrom.join(
|
||||
', '
|
||||
)}) REFERENCES ${table}(${keysTo.join(', ')})`
|
||||
)
|
||||
})
|
||||
}
|
||||
Object.keys(toCheck).forEach((key) => {
|
||||
if (item[key]) {
|
||||
flags.push(toCheck[key])
|
||||
}
|
||||
})
|
||||
|
||||
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
|
||||
return acc
|
||||
}, [])
|
||||
.join(', ')
|
||||
|
||||
const fKeys: string[] = []
|
||||
if (foreignKeys) {
|
||||
foreignKeys.forEach((foreignKey) => {
|
||||
const { keysFrom, table, keysTo } = foreignKey
|
||||
fKeys.push(
|
||||
`, FOREIGN KEY(${keysFrom.join(
|
||||
', '
|
||||
)}) REFERENCES ${table}(${keysTo.join(', ')})`
|
||||
)
|
||||
})
|
||||
// IF NOT EXISTS
|
||||
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
|
||||
// IF NOT EXISTS
|
||||
const command = `CREATE TABLE ${name}(${cols}${fKeys.join(' ')})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function SelectAll(db: Database, from: string): any[] {
|
||||
try {
|
||||
const command = `SELECT * from ${from}`
|
||||
try {
|
||||
const command = `SELECT * from ${from}`
|
||||
|
||||
const stmt = PrepareStatement(db, command)
|
||||
return stmt.all()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
const stmt = PrepareStatement(db, command)
|
||||
return stmt.all()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100;
|
||||
function Select(
|
||||
db: Database,
|
||||
from: string,
|
||||
conditions: { [key: string]: string | number },
|
||||
options: { joiner?: string; limit?: number } = {}
|
||||
db: Database,
|
||||
from: string,
|
||||
conditions: { [key: string]: string | number },
|
||||
options: { joiner?: string; limit?: number } = {}
|
||||
): any[] {
|
||||
const { joiner, limit } = options
|
||||
const { joiner, limit } = options
|
||||
|
||||
try {
|
||||
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
|
||||
conditions,
|
||||
'where',
|
||||
joiner
|
||||
)}`
|
||||
try {
|
||||
let command = `SELECT * from ${from} WHERE ${GetSqlQuerry(
|
||||
conditions,
|
||||
'where',
|
||||
joiner
|
||||
)}`
|
||||
|
||||
if (!isNaN(limit)) {
|
||||
command += ` LIMIT ${limit}`
|
||||
if (!isNaN(limit)) {
|
||||
command += ` LIMIT ${limit}`
|
||||
}
|
||||
|
||||
const stmt = PrepareStatement(db, command)
|
||||
return stmt.all()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
|
||||
const stmt = PrepareStatement(db, command)
|
||||
return stmt.all()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function Insert(
|
||||
db: Database,
|
||||
table: string,
|
||||
data: { [key: string]: number | string }
|
||||
db: Database,
|
||||
table: string,
|
||||
data: { [key: string]: number | string }
|
||||
): RunResult {
|
||||
try {
|
||||
const cols = Object.keys(data)
|
||||
.reduce((acc, key) => {
|
||||
acc.push(`${key}`)
|
||||
return acc
|
||||
}, [])
|
||||
.join(', ')
|
||||
try {
|
||||
const cols = Object.keys(data)
|
||||
.reduce((acc, key) => {
|
||||
acc.push(`${key}`)
|
||||
return acc
|
||||
}, [])
|
||||
.join(', ')
|
||||
|
||||
const values = Object.keys(data)
|
||||
.map((key) => {
|
||||
const item = data[key]
|
||||
if (typeof item === 'string') {
|
||||
return `'${item}'`
|
||||
} else {
|
||||
return `${item}`
|
||||
}
|
||||
})
|
||||
.join(', ')
|
||||
const values = Object.keys(data)
|
||||
.map((key) => {
|
||||
const item = data[key]
|
||||
if (typeof item === 'string') {
|
||||
return `'${item}'`
|
||||
} else {
|
||||
return `${item}`
|
||||
}
|
||||
})
|
||||
.join(', ')
|
||||
|
||||
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function runStatement(db: Database, command: string, runType?: string): any {
|
||||
const stmt = PrepareStatement(db, command)
|
||||
if (!runType) {
|
||||
return stmt.all()
|
||||
} else if (runType === 'run') {
|
||||
return stmt.run()
|
||||
}
|
||||
return null
|
||||
const stmt = PrepareStatement(db, command)
|
||||
if (!runType) {
|
||||
return stmt.all()
|
||||
} else if (runType === 'run') {
|
||||
return stmt.run()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function CloseDB(db: Database): void {
|
||||
db.close()
|
||||
db.close()
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function PrepareStatement(db: Database, command: string) {
|
||||
if (!db) {
|
||||
throw new Error(
|
||||
'DB is undefined in prepare statement! DB action called with undefined db'
|
||||
)
|
||||
}
|
||||
DebugLog(command)
|
||||
return db.prepare(command)
|
||||
if (!db) {
|
||||
throw new Error(
|
||||
'DB is undefined in prepare statement! DB action called with undefined db'
|
||||
)
|
||||
}
|
||||
DebugLog(command)
|
||||
return db.prepare(command)
|
||||
}
|
||||
|
|
178
src/utils/ids.ts
178
src/utils/ids.ts
|
@ -19,8 +19,8 @@
|
|||
------------------------------------------------------------------------- */
|
||||
|
||||
export default {
|
||||
LogId: LogId,
|
||||
Load: Load,
|
||||
LogId: LogId,
|
||||
Load: Load,
|
||||
}
|
||||
|
||||
import utils from '../utils/utils'
|
||||
|
@ -35,113 +35,113 @@ let idvStatsData = {}
|
|||
let writes = 0
|
||||
|
||||
function Load(): void {
|
||||
try {
|
||||
idStatsData = utils.ReadJSON(idStatFile)
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error at loading id logs! (@ first run its normal)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
idStatsData = utils.ReadJSON(idStatFile)
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error at loading id logs! (@ first run its normal)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
|
||||
try {
|
||||
const prevVData = utils.ReadFile(idVStatFile)
|
||||
idvStatsData = JSON.parse(prevVData)
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error at loading id logs! (@ first run its normal)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
const prevVData = utils.ReadFile(idVStatFile)
|
||||
idvStatsData = JSON.parse(prevVData)
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error at loading id logs! (@ first run its normal)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
function LogId(
|
||||
id: number,
|
||||
subj: string,
|
||||
newQuestions: number,
|
||||
allQuestions: number
|
||||
id: number,
|
||||
subj: string,
|
||||
newQuestions: number,
|
||||
allQuestions: number
|
||||
): void {
|
||||
Inc(id, subj, newQuestions, allQuestions)
|
||||
AddVisitStat(id, subj, newQuestions, allQuestions)
|
||||
Save()
|
||||
Inc(id, subj, newQuestions, allQuestions)
|
||||
AddVisitStat(id, subj, newQuestions, allQuestions)
|
||||
Save()
|
||||
}
|
||||
|
||||
function AddSubjToList(list: { [key: string]: any }, subj: string) {
|
||||
if (!list[subj]) {
|
||||
list[subj] = 0
|
||||
}
|
||||
list[subj]++
|
||||
if (!list[subj]) {
|
||||
list[subj] = 0
|
||||
}
|
||||
list[subj]++
|
||||
}
|
||||
|
||||
function Inc(
|
||||
value: number,
|
||||
subj: string,
|
||||
newQuestions: number,
|
||||
allQuestions: number
|
||||
value: number,
|
||||
subj: string,
|
||||
newQuestions: number,
|
||||
allQuestions: number
|
||||
) {
|
||||
if (idStatsData[value] === undefined) {
|
||||
idStatsData[value] = {
|
||||
count: 0,
|
||||
newQuestions: 0,
|
||||
allQuestions: 0,
|
||||
subjs: {},
|
||||
if (idStatsData[value] === undefined) {
|
||||
idStatsData[value] = {
|
||||
count: 0,
|
||||
newQuestions: 0,
|
||||
allQuestions: 0,
|
||||
subjs: {},
|
||||
}
|
||||
}
|
||||
}
|
||||
idStatsData[value].count++
|
||||
idStatsData[value].newQuestions += newQuestions
|
||||
idStatsData[value].allQuestions += allQuestions
|
||||
AddSubjToList(idStatsData[value].subjs, subj)
|
||||
idStatsData[value].count++
|
||||
idStatsData[value].newQuestions += newQuestions
|
||||
idStatsData[value].allQuestions += allQuestions
|
||||
AddSubjToList(idStatsData[value].subjs, subj)
|
||||
}
|
||||
|
||||
function AddVisitStat(
|
||||
name: number,
|
||||
subj: string,
|
||||
newQuestions: number,
|
||||
allQuestions: number
|
||||
name: number,
|
||||
subj: string,
|
||||
newQuestions: number,
|
||||
allQuestions: number
|
||||
) {
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
if (idvStatsData[now] === undefined) {
|
||||
idvStatsData[now] = {}
|
||||
}
|
||||
if (idvStatsData[now][name] === undefined) {
|
||||
idvStatsData[now][name] = {
|
||||
count: 0,
|
||||
newQuestions: 0,
|
||||
allQuestions: 0,
|
||||
subjs: {},
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
if (idvStatsData[now] === undefined) {
|
||||
idvStatsData[now] = {}
|
||||
}
|
||||
}
|
||||
idvStatsData[now][name].count++
|
||||
idvStatsData[now][name].newQuestions += newQuestions
|
||||
idvStatsData[now][name].allQuestions += allQuestions
|
||||
AddSubjToList(idvStatsData[now][name].subjs, subj)
|
||||
if (idvStatsData[now][name] === undefined) {
|
||||
idvStatsData[now][name] = {
|
||||
count: 0,
|
||||
newQuestions: 0,
|
||||
allQuestions: 0,
|
||||
subjs: {},
|
||||
}
|
||||
}
|
||||
idvStatsData[now][name].count++
|
||||
idvStatsData[now][name].newQuestions += newQuestions
|
||||
idvStatsData[now][name].allQuestions += allQuestions
|
||||
AddSubjToList(idvStatsData[now][name].subjs, subj)
|
||||
}
|
||||
|
||||
function Save() {
|
||||
writes++
|
||||
if (writes === writeInterval) {
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
writes++
|
||||
if (writes === writeInterval) {
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(idStatsData), idStatFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
writes = 0
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(idvStatsData), idVStatFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
writes = 0
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,7 +19,7 @@
|
|||
------------------------------------------------------------------------- */
|
||||
|
||||
const hr =
|
||||
'---------------------------------------------------------------------------------'
|
||||
'---------------------------------------------------------------------------------'
|
||||
|
||||
const DELIM = C('green') + '|' + C()
|
||||
|
||||
|
@ -50,361 +50,377 @@ let writes = 0
|
|||
let noLogIds: string[] = []
|
||||
|
||||
function getColoredDateString(): string {
|
||||
const date = new Date()
|
||||
const dateString = utils.GetDateString()
|
||||
return GetRandomColor(date.getHours().toString()) + dateString + C()
|
||||
const date = new Date()
|
||||
const dateString = utils.GetDateString()
|
||||
return GetRandomColor(date.getHours().toString()) + dateString + C()
|
||||
}
|
||||
|
||||
function DebugLog(msg: string | object, name: string, lvl: number): void {
|
||||
if (lvl <= debugLevel) {
|
||||
if (msg === 'hr') {
|
||||
msg = hr
|
||||
if (lvl <= debugLevel) {
|
||||
if (msg === 'hr') {
|
||||
msg = hr
|
||||
}
|
||||
let res = msg
|
||||
const header = `${C('red')}#DEBUG${lvl}#${C(
|
||||
'yellow'
|
||||
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
|
||||
if (typeof msg !== 'object') {
|
||||
res = header + msg
|
||||
} else {
|
||||
Log(header + 'OBJECT:', 'yellow')
|
||||
res = msg
|
||||
}
|
||||
Log(res, 'yellow')
|
||||
}
|
||||
let res = msg
|
||||
const header = `${C('red')}#DEBUG${lvl}#${C(
|
||||
'yellow'
|
||||
)}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
|
||||
if (typeof msg !== 'object') {
|
||||
res = header + msg
|
||||
} else {
|
||||
Log(header + 'OBJECT:', 'yellow')
|
||||
res = msg
|
||||
}
|
||||
Log(res, 'yellow')
|
||||
}
|
||||
}
|
||||
|
||||
function Log(msg: string | object, color?: string): void {
|
||||
let log = msg
|
||||
if (typeof msg !== 'object') {
|
||||
const delimiter = DELIM + C(color)
|
||||
log = getColoredDateString() + delimiter + C(color) + msg + C()
|
||||
}
|
||||
let log = msg
|
||||
if (typeof msg !== 'object') {
|
||||
const delimiter = DELIM + C(color)
|
||||
log = getColoredDateString() + delimiter + C(color) + msg + C()
|
||||
}
|
||||
|
||||
if (!process.env.NS_NOLOG) {
|
||||
console.log(log)
|
||||
}
|
||||
utils.AppendToFile(
|
||||
typeof log === 'string' ? log : JSON.stringify(log),
|
||||
logDir + logFileName
|
||||
)
|
||||
if (!process.env.NS_NOLOG) {
|
||||
console.log(log)
|
||||
}
|
||||
utils.AppendToFile(
|
||||
typeof log === 'string' ? log : JSON.stringify(log),
|
||||
logDir + logFileName
|
||||
)
|
||||
}
|
||||
|
||||
function expandWithSpaces(text: string, count: number) {
|
||||
while (text.length < count) {
|
||||
text += ' '
|
||||
}
|
||||
return text
|
||||
while (text.length < count) {
|
||||
text += ' '
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
function LogReq(
|
||||
req: Request,
|
||||
toFile?: boolean,
|
||||
statusCode?: string | number
|
||||
req: Request,
|
||||
toFile?: boolean,
|
||||
statusCode?: string | number
|
||||
): void {
|
||||
try {
|
||||
let logEntry = '' // logHashed(ip)
|
||||
let dl = DELIM
|
||||
if (req.url.includes('lred')) {
|
||||
dl += C('red')
|
||||
}
|
||||
if (req.session && req.session.user && !shouldLog(req.session.user.id, noLogIds)) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
let logEntry = '' // logHashed(ip)
|
||||
let dl = DELIM
|
||||
if (req.url.includes('lred')) {
|
||||
dl += C('red')
|
||||
}
|
||||
if (
|
||||
req.session &&
|
||||
req.session.user &&
|
||||
!shouldLog(req.session.user.id, noLogIds)
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
let hostname
|
||||
if (req.hostname) {
|
||||
hostname = req.hostname.replace('www.', '').split('.')[0]
|
||||
} else {
|
||||
hostname = 'NOHOST'
|
||||
Log(
|
||||
'req.hostname is undefined! req.hostname: ' + req.hostname,
|
||||
GetColor('redbg')
|
||||
)
|
||||
}
|
||||
if (!toFile) {
|
||||
hostname = expandWithSpaces(hostname, 10)
|
||||
}
|
||||
let hostname
|
||||
if (req.hostname) {
|
||||
hostname = req.hostname.replace('www.', '').split('.')[0]
|
||||
} else {
|
||||
hostname = 'NOHOST'
|
||||
Log(
|
||||
'req.hostname is undefined! req.hostname: ' + req.hostname,
|
||||
GetColor('redbg')
|
||||
)
|
||||
}
|
||||
if (!toFile) {
|
||||
hostname = expandWithSpaces(hostname, 10)
|
||||
}
|
||||
|
||||
logEntry += logHashed(hostname) + dl
|
||||
if (toFile) {
|
||||
logEntry += req.headers['user-agent'] + dl
|
||||
logEntry += req.method + dl
|
||||
}
|
||||
logEntry += logHashed(hostname) + dl
|
||||
if (toFile) {
|
||||
logEntry += req.headers['user-agent'] + dl
|
||||
logEntry += req.method + dl
|
||||
}
|
||||
|
||||
let uid = ''
|
||||
if (req.session && req.session.user) {
|
||||
uid = req.session.user.id.toString()
|
||||
} else if (req.session && req.session.isException === true) {
|
||||
uid = 'EX'
|
||||
} else {
|
||||
uid = 'NOUSR'
|
||||
}
|
||||
if (!toFile) {
|
||||
uid = expandWithSpaces(uid, 5)
|
||||
}
|
||||
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
|
||||
let uid = ''
|
||||
if (req.session && req.session.user) {
|
||||
uid = req.session.user.id.toString()
|
||||
} else if (req.session && req.session.isException === true) {
|
||||
uid = 'EX'
|
||||
} else {
|
||||
uid = 'NOUSR'
|
||||
}
|
||||
if (!toFile) {
|
||||
uid = expandWithSpaces(uid, 5)
|
||||
}
|
||||
logEntry += GetRandomColor(uid.toString()) + uid + C() + dl
|
||||
|
||||
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
|
||||
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url + C()
|
||||
|
||||
if (statusCode !== undefined) {
|
||||
logEntry += dl + statusCode
|
||||
if (statusCode !== undefined) {
|
||||
logEntry += dl + statusCode
|
||||
}
|
||||
|
||||
logEntry += C()
|
||||
if (!toFile) {
|
||||
Log(logEntry)
|
||||
} else {
|
||||
const defLogs = utils.GetDateString() + dl + logEntry
|
||||
|
||||
utils.AppendToFile(defLogs, vlogDir + logFileName)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
Log('Error at logging lol', GetColor('redbg'))
|
||||
}
|
||||
|
||||
logEntry += C()
|
||||
if (!toFile) {
|
||||
Log(logEntry)
|
||||
} else {
|
||||
const defLogs = utils.GetDateString() + dl + logEntry
|
||||
|
||||
utils.AppendToFile(defLogs, vlogDir + logFileName)
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
Log('Error at logging lol', GetColor('redbg'))
|
||||
}
|
||||
}
|
||||
|
||||
function parseNoLogFile(newData: string) {
|
||||
noLogIds = newData.split('\n')
|
||||
if (noLogIds[noLogIds.length - 1] === '') {
|
||||
noLogIds.pop()
|
||||
}
|
||||
noLogIds = noLogIds.filter((noLogId) => {
|
||||
return noLogId !== ''
|
||||
})
|
||||
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
|
||||
noLogIds = newData.split('\n')
|
||||
if (noLogIds[noLogIds.length - 1] === '') {
|
||||
noLogIds.pop()
|
||||
}
|
||||
noLogIds = noLogIds.filter((noLogId) => {
|
||||
return noLogId !== ''
|
||||
})
|
||||
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
|
||||
}
|
||||
|
||||
function setNoLogReadInterval() {
|
||||
utils.WatchFile(nologFile, (newData: string) => {
|
||||
parseNoLogFile(newData)
|
||||
})
|
||||
utils.WatchFile(nologFile, (newData: string) => {
|
||||
parseNoLogFile(newData)
|
||||
})
|
||||
|
||||
parseNoLogFile(utils.ReadFile(nologFile))
|
||||
parseNoLogFile(utils.ReadFile(nologFile))
|
||||
}
|
||||
|
||||
function Load(): void {
|
||||
Log('Loading logger...')
|
||||
try {
|
||||
uvData = JSON.parse(utils.ReadFile(uStatsFile))
|
||||
} catch (err) {
|
||||
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
Log('Loading logger...')
|
||||
try {
|
||||
uvData = JSON.parse(utils.ReadFile(uStatsFile))
|
||||
} catch (err) {
|
||||
Log(
|
||||
'Error at loading logs! (@ first run its normal)',
|
||||
GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
|
||||
try {
|
||||
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
|
||||
} catch (err) {
|
||||
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
udvData = JSON.parse(utils.ReadFile(uvStatsFile))
|
||||
} catch (err) {
|
||||
Log(
|
||||
'Error at loading logs! (@ first run its normal)',
|
||||
GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
|
||||
try {
|
||||
vData = utils.ReadJSON(statFile)
|
||||
} catch (err) {
|
||||
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
vData = utils.ReadJSON(statFile)
|
||||
} catch (err) {
|
||||
Log(
|
||||
'Error at loading logs! (@ first run its normal)',
|
||||
GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
|
||||
try {
|
||||
dvData = utils.ReadJSON(vStatFile)
|
||||
} catch (err) {
|
||||
Log(
|
||||
'Error at loading visit logs! (@ first run its normal)',
|
||||
GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
setNoLogReadInterval()
|
||||
try {
|
||||
dvData = utils.ReadJSON(vStatFile)
|
||||
} catch (err) {
|
||||
Log(
|
||||
'Error at loading visit logs! (@ first run its normal)',
|
||||
GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
setNoLogReadInterval()
|
||||
}
|
||||
|
||||
export function shouldLog(userId: string | number, nolog: string[]): boolean {
|
||||
return !nolog.some((noLogId) => {
|
||||
return noLogId === userId.toString()
|
||||
})
|
||||
return !nolog.some((noLogId) => {
|
||||
return noLogId === userId.toString()
|
||||
})
|
||||
}
|
||||
|
||||
function LogStat(url: string, hostname: string, userId: number | string): void {
|
||||
if (!shouldLog(userId, noLogIds)) {
|
||||
return
|
||||
}
|
||||
if (!shouldLog(userId, noLogIds)) {
|
||||
return
|
||||
}
|
||||
|
||||
url = hostname + url.split('?')[0]
|
||||
Inc(url)
|
||||
AddVisitStat(url)
|
||||
if (shouldAddUserStat(url)) {
|
||||
AddUserIdStat(userId.toString())
|
||||
IncUserStat(userId.toString())
|
||||
}
|
||||
Save()
|
||||
url = hostname + url.split('?')[0]
|
||||
Inc(url)
|
||||
AddVisitStat(url)
|
||||
if (shouldAddUserStat(url)) {
|
||||
AddUserIdStat(userId.toString())
|
||||
IncUserStat(userId.toString())
|
||||
}
|
||||
Save()
|
||||
}
|
||||
|
||||
const userStatExcludes = ['stable.user.js', 'infos', 'hasNewMsg']
|
||||
function shouldAddUserStat(url: string) {
|
||||
return !userStatExcludes.some((x) => url.includes(x))
|
||||
return !userStatExcludes.some((x) => url.includes(x))
|
||||
}
|
||||
|
||||
function IncUserStat(userId: string) {
|
||||
try {
|
||||
if (uvData[userId] === undefined) {
|
||||
uvData[userId] = 0
|
||||
try {
|
||||
if (uvData[userId] === undefined) {
|
||||
uvData[userId] = 0
|
||||
}
|
||||
uvData[userId]++
|
||||
} catch (err) {
|
||||
Log('Error at making user ID stats!', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
uvData[userId]++
|
||||
} catch (err) {
|
||||
Log('Error at making user ID stats!', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
function AddUserIdStat(userId: string) {
|
||||
try {
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
if (udvData[now] === undefined) {
|
||||
udvData[now] = {}
|
||||
try {
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
if (udvData[now] === undefined) {
|
||||
udvData[now] = {}
|
||||
}
|
||||
if (udvData[now][userId] === undefined) {
|
||||
udvData[now][userId] = 0
|
||||
}
|
||||
udvData[now][userId]++
|
||||
} catch (err) {
|
||||
Log('Error at making user ID stats!', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
if (udvData[now][userId] === undefined) {
|
||||
udvData[now][userId] = 0
|
||||
}
|
||||
udvData[now][userId]++
|
||||
} catch (err) {
|
||||
Log('Error at making user ID stats!', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
function Inc(value: string) {
|
||||
if (value.startsWith('/?')) {
|
||||
value = '/'
|
||||
}
|
||||
if (vData[value] === undefined) {
|
||||
vData[value] = 0
|
||||
}
|
||||
vData[value]++
|
||||
if (value.startsWith('/?')) {
|
||||
value = '/'
|
||||
}
|
||||
if (vData[value] === undefined) {
|
||||
vData[value] = 0
|
||||
}
|
||||
vData[value]++
|
||||
}
|
||||
|
||||
function AddVisitStat(name: string) {
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
if (dvData[now] === undefined) {
|
||||
dvData[now] = {}
|
||||
}
|
||||
if (dvData[now][name] === undefined) {
|
||||
dvData[now][name] = 0
|
||||
}
|
||||
dvData[now][name]++
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
if (dvData[now] === undefined) {
|
||||
dvData[now] = {}
|
||||
}
|
||||
if (dvData[now][name] === undefined) {
|
||||
dvData[now][name] = 0
|
||||
}
|
||||
dvData[now][name]++
|
||||
}
|
||||
|
||||
function Save() {
|
||||
writes++
|
||||
if (writes === writeInterval) {
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
|
||||
} catch (err) {
|
||||
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
writes++
|
||||
if (writes === writeInterval) {
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(uvData), uStatsFile)
|
||||
} catch (err) {
|
||||
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
|
||||
} catch (err) {
|
||||
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(vData), statFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(dvData), vStatFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
Log(
|
||||
'Error at writing visit logs! (more in stderr)',
|
||||
GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
}
|
||||
writes = 0
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(udvData), uvStatsFile)
|
||||
} catch (err) {
|
||||
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(vData), statFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
try {
|
||||
utils.WriteFile(JSON.stringify(dvData), vStatFile)
|
||||
// Log("Stats wrote.");
|
||||
} catch (err) {
|
||||
Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
|
||||
console.error(err)
|
||||
}
|
||||
writes = 0
|
||||
}
|
||||
}
|
||||
|
||||
function logHashed(msg: string): string {
|
||||
return GetRandomColor(msg.toString()) + msg + C()
|
||||
return GetRandomColor(msg.toString()) + msg + C()
|
||||
}
|
||||
|
||||
function GetRandomColor(msg: string): string {
|
||||
if (!msg) {
|
||||
return 'red'
|
||||
}
|
||||
if (!msg) {
|
||||
return 'red'
|
||||
}
|
||||
|
||||
const res = msg.split('').reduce((res, character) => {
|
||||
return res + character.charCodeAt(0)
|
||||
}, 0)
|
||||
return C(colors[res % colors.length])
|
||||
const res = msg.split('').reduce((res, character) => {
|
||||
return res + character.charCodeAt(0)
|
||||
}, 0)
|
||||
return C(colors[res % colors.length])
|
||||
}
|
||||
|
||||
function GetColor(color: string): string {
|
||||
return color
|
||||
return color
|
||||
}
|
||||
|
||||
function C(color?: string): string {
|
||||
if (color !== undefined) {
|
||||
color = color.toLowerCase()
|
||||
}
|
||||
if (color !== undefined) {
|
||||
color = color.toLowerCase()
|
||||
}
|
||||
|
||||
if (color === 'redbg') {
|
||||
return '\x1b[41m'
|
||||
}
|
||||
if (color === 'bluebg') {
|
||||
return '\x1b[44m'
|
||||
}
|
||||
if (color === 'cyanbg') {
|
||||
return '\x1b[46m'
|
||||
}
|
||||
if (color === 'green') {
|
||||
return '\x1b[32m'
|
||||
}
|
||||
if (color === 'red') {
|
||||
return '\x1b[31m'
|
||||
}
|
||||
if (color === 'yellow') {
|
||||
return '\x1b[33m'
|
||||
}
|
||||
if (color === 'blue') {
|
||||
return '\x1b[34m'
|
||||
}
|
||||
if (color === 'magenta') {
|
||||
return '\x1b[35m'
|
||||
}
|
||||
if (color === 'cyan') {
|
||||
return '\x1b[36m'
|
||||
}
|
||||
return '\x1b[0m'
|
||||
if (color === 'redbg') {
|
||||
return '\x1b[41m'
|
||||
}
|
||||
if (color === 'bluebg') {
|
||||
return '\x1b[44m'
|
||||
}
|
||||
if (color === 'cyanbg') {
|
||||
return '\x1b[46m'
|
||||
}
|
||||
if (color === 'green') {
|
||||
return '\x1b[32m'
|
||||
}
|
||||
if (color === 'red') {
|
||||
return '\x1b[31m'
|
||||
}
|
||||
if (color === 'yellow') {
|
||||
return '\x1b[33m'
|
||||
}
|
||||
if (color === 'blue') {
|
||||
return '\x1b[34m'
|
||||
}
|
||||
if (color === 'magenta') {
|
||||
return '\x1b[35m'
|
||||
}
|
||||
if (color === 'cyan') {
|
||||
return '\x1b[36m'
|
||||
}
|
||||
return '\x1b[0m'
|
||||
}
|
||||
|
||||
export default {
|
||||
getColoredDateString: getColoredDateString,
|
||||
Log: Log,
|
||||
DebugLog: DebugLog,
|
||||
GetColor: GetColor,
|
||||
LogReq: LogReq,
|
||||
LogStat: LogStat,
|
||||
Load: Load,
|
||||
logHashed: logHashed,
|
||||
hr: hr,
|
||||
C: C,
|
||||
logFileName: logFileName,
|
||||
logDir: logDir,
|
||||
vlogDir: vlogDir,
|
||||
getColoredDateString: getColoredDateString,
|
||||
Log: Log,
|
||||
DebugLog: DebugLog,
|
||||
GetColor: GetColor,
|
||||
LogReq: LogReq,
|
||||
LogStat: LogStat,
|
||||
Load: Load,
|
||||
logHashed: logHashed,
|
||||
hr: hr,
|
||||
C: C,
|
||||
logFileName: logFileName,
|
||||
logDir: logDir,
|
||||
vlogDir: vlogDir,
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import {
|
||||
createWorker,
|
||||
Worker as TesseractWorker,
|
||||
ConfigResult,
|
||||
createWorker,
|
||||
Worker as TesseractWorker,
|
||||
ConfigResult,
|
||||
} from 'tesseract.js'
|
||||
|
||||
import logger from './logger'
|
||||
|
@ -10,44 +10,44 @@ import { isMainThread, workerData } from 'worker_threads'
|
|||
// https://github.com/naptha/tesseract.js/blob/master/docs/api.md
|
||||
let tesseractWorker: TesseractWorker = null
|
||||
export async function initTesseractWorker(): Promise<TesseractWorker> {
|
||||
const worker = createWorker({
|
||||
cacheMethod: 'refresh',
|
||||
// logger: (m) => console.log(m),
|
||||
})
|
||||
await worker.load()
|
||||
await worker.loadLanguage('hun+eng')
|
||||
await worker.initialize('hun+eng')
|
||||
return worker
|
||||
// await worker.terminate();
|
||||
const worker = createWorker({
|
||||
cacheMethod: 'refresh',
|
||||
// logger: (m) => console.log(m),
|
||||
})
|
||||
await worker.load()
|
||||
await worker.loadLanguage('hun+eng')
|
||||
await worker.initialize('hun+eng')
|
||||
return worker
|
||||
// await worker.terminate();
|
||||
}
|
||||
|
||||
let resolveLoaded: () => void = null
|
||||
export const tesseractLoaded: Promise<void> = new Promise((resolve) => {
|
||||
resolveLoaded = resolve
|
||||
resolveLoaded = resolve
|
||||
})
|
||||
|
||||
initTesseractWorker().then((worker) => {
|
||||
tesseractWorker = worker
|
||||
tesseractWorker = worker
|
||||
|
||||
if (isMainThread) {
|
||||
logger.Log('Tesseract loaded on main thread')
|
||||
} else {
|
||||
const { workerIndex }: { workerIndex: number } = workerData
|
||||
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
|
||||
}
|
||||
resolveLoaded()
|
||||
if (isMainThread) {
|
||||
logger.Log('Tesseract loaded on main thread')
|
||||
} else {
|
||||
const { workerIndex }: { workerIndex: number } = workerData
|
||||
logger.Log(`[THREAD #${workerIndex}]: Tesseract loaded`)
|
||||
}
|
||||
resolveLoaded()
|
||||
})
|
||||
|
||||
export async function recognizeTextFromBase64(base64: string): Promise<string> {
|
||||
const {
|
||||
data: { text },
|
||||
} = await tesseractWorker.recognize(base64)
|
||||
return text
|
||||
const {
|
||||
data: { text },
|
||||
} = await tesseractWorker.recognize(base64)
|
||||
return text
|
||||
}
|
||||
|
||||
export async function terminateWorker(): Promise<void | ConfigResult> {
|
||||
if (tesseractWorker) {
|
||||
return tesseractWorker.terminate()
|
||||
}
|
||||
return
|
||||
if (tesseractWorker) {
|
||||
return tesseractWorker.terminate()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
|
@ -19,23 +19,23 @@
|
|||
------------------------------------------------------------------------- */
|
||||
|
||||
export default {
|
||||
ReadFile: ReadFile,
|
||||
ReadJSON: ReadJSON,
|
||||
WriteFile: WriteFile,
|
||||
writeFileAsync: writeFileAsync,
|
||||
AppendToFile: AppendToFile,
|
||||
FileExists: FileExists,
|
||||
CreatePath: CreatePath,
|
||||
WatchFile: WatchFile,
|
||||
ReadDir: ReadDir,
|
||||
CopyFile: CopyFile,
|
||||
GetDateString: GetDateString,
|
||||
formatUrl: formatUrl,
|
||||
deleteFile: deleteFile,
|
||||
uploadFile: uploadFile,
|
||||
statFile: statFile,
|
||||
renameFile: renameFile,
|
||||
deleteDir: deleteDir,
|
||||
ReadFile: ReadFile,
|
||||
ReadJSON: ReadJSON,
|
||||
WriteFile: WriteFile,
|
||||
writeFileAsync: writeFileAsync,
|
||||
AppendToFile: AppendToFile,
|
||||
FileExists: FileExists,
|
||||
CreatePath: CreatePath,
|
||||
WatchFile: WatchFile,
|
||||
ReadDir: ReadDir,
|
||||
CopyFile: CopyFile,
|
||||
GetDateString: GetDateString,
|
||||
formatUrl: formatUrl,
|
||||
deleteFile: deleteFile,
|
||||
uploadFile: uploadFile,
|
||||
statFile: statFile,
|
||||
renameFile: renameFile,
|
||||
deleteDir: deleteDir,
|
||||
}
|
||||
|
||||
import fs from 'fs'
|
||||
|
@ -45,251 +45,254 @@ import logger from '../utils/logger'
|
|||
import { Request } from '../types/basicTypes'
|
||||
|
||||
interface URLFormatOptions {
|
||||
pathname?: string
|
||||
query?: { [key: string]: string }
|
||||
pathname?: string
|
||||
query?: { [key: string]: string }
|
||||
}
|
||||
|
||||
function formatUrl(options: URLFormatOptions): string {
|
||||
const path = options.pathname || ''
|
||||
if (!options.query || Object.keys(options.query).length === 0) {
|
||||
return path
|
||||
}
|
||||
const queryString =
|
||||
'?' +
|
||||
Object.keys(options.query)
|
||||
.map((key) => {
|
||||
return `${key}=${encodeURIComponent(options.query[key])}`
|
||||
})
|
||||
.join('&')
|
||||
return path + queryString
|
||||
const path = options.pathname || ''
|
||||
if (!options.query || Object.keys(options.query).length === 0) {
|
||||
return path
|
||||
}
|
||||
const queryString =
|
||||
'?' +
|
||||
Object.keys(options.query)
|
||||
.map((key) => {
|
||||
return `${key}=${encodeURIComponent(options.query[key])}`
|
||||
})
|
||||
.join('&')
|
||||
return path + queryString
|
||||
}
|
||||
|
||||
function GetDateString(
|
||||
referenceDate?: Date | string,
|
||||
noTime?: boolean
|
||||
referenceDate?: Date | string,
|
||||
noTime?: boolean
|
||||
): string {
|
||||
const date = referenceDate ? new Date(referenceDate) : new Date()
|
||||
const date = referenceDate ? new Date(referenceDate) : new Date()
|
||||
|
||||
if (noTime) {
|
||||
return (
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
)
|
||||
} else {
|
||||
return (
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2) +
|
||||
' ' +
|
||||
('0' + date.getHours()).slice(-2) +
|
||||
':' +
|
||||
('0' + date.getMinutes()).slice(-2) +
|
||||
':' +
|
||||
('0' + date.getSeconds()).slice(-2)
|
||||
)
|
||||
}
|
||||
if (noTime) {
|
||||
return (
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2)
|
||||
)
|
||||
} else {
|
||||
return (
|
||||
date.getFullYear() +
|
||||
'-' +
|
||||
('0' + (date.getMonth() + 1)).slice(-2) +
|
||||
'-' +
|
||||
('0' + date.getDate()).slice(-2) +
|
||||
' ' +
|
||||
('0' + date.getHours()).slice(-2) +
|
||||
':' +
|
||||
('0' + date.getMinutes()).slice(-2) +
|
||||
':' +
|
||||
('0' + date.getSeconds()).slice(-2)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function CopyFile(from: string, to: string): void {
|
||||
CreatePath(to)
|
||||
fs.copyFileSync(from, to)
|
||||
CreatePath(to)
|
||||
fs.copyFileSync(from, to)
|
||||
}
|
||||
|
||||
function ReadDir(path: string, listHidden?: boolean): Array<string> {
|
||||
if (listHidden) {
|
||||
return fs.readdirSync(path)
|
||||
} else {
|
||||
return fs.readdirSync(path).filter((file) => {
|
||||
return !file.startsWith('.')
|
||||
})
|
||||
}
|
||||
if (listHidden) {
|
||||
return fs.readdirSync(path)
|
||||
} else {
|
||||
return fs.readdirSync(path).filter((file) => {
|
||||
return !file.startsWith('.')
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function ReadJSON(name: string): any {
|
||||
try {
|
||||
return JSON.parse(ReadFile(name))
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
|
||||
}
|
||||
try {
|
||||
return JSON.parse(ReadFile(name))
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
|
||||
}
|
||||
}
|
||||
|
||||
function ReadFile(name: string): string {
|
||||
if (!FileExists(name)) {
|
||||
throw new Error('No such file: ' + name)
|
||||
}
|
||||
return fs.readFileSync(name, 'utf8')
|
||||
if (!FileExists(name)) {
|
||||
throw new Error('No such file: ' + name)
|
||||
}
|
||||
return fs.readFileSync(name, 'utf8')
|
||||
}
|
||||
|
||||
function FileExists(path: string): boolean {
|
||||
return fs.existsSync(path)
|
||||
return fs.existsSync(path)
|
||||
}
|
||||
|
||||
function WatchFile(file: string, callback: Function): void {
|
||||
if (FileExists(file)) {
|
||||
fs.watchFile(file, () => {
|
||||
fs.readFile(file, 'utf8', (err, data) => {
|
||||
if (err) {
|
||||
// console.log(err)
|
||||
} else {
|
||||
callback(data)
|
||||
}
|
||||
})
|
||||
})
|
||||
} else {
|
||||
throw new Error(`${file} does not exits to watch`)
|
||||
}
|
||||
if (FileExists(file)) {
|
||||
fs.watchFile(file, () => {
|
||||
fs.readFile(file, 'utf8', (err, data) => {
|
||||
if (err) {
|
||||
// console.log(err)
|
||||
} else {
|
||||
callback(data)
|
||||
}
|
||||
})
|
||||
})
|
||||
} else {
|
||||
throw new Error(`${file} does not exits to watch`)
|
||||
}
|
||||
}
|
||||
|
||||
function CreatePath(path: string, onlyPath?: boolean): void {
|
||||
if (FileExists(path)) {
|
||||
return
|
||||
}
|
||||
|
||||
const spath = path.split('/')
|
||||
let currDir = spath[0]
|
||||
for (let i = 1; i < spath.length; i++) {
|
||||
if (currDir !== '' && !fs.existsSync(currDir)) {
|
||||
try {
|
||||
fs.mkdirSync(currDir)
|
||||
} catch (err) {
|
||||
console.log('Failed to make ' + currDir + ' directory... ')
|
||||
console.error(err)
|
||||
}
|
||||
if (FileExists(path)) {
|
||||
return
|
||||
}
|
||||
|
||||
const spath = path.split('/')
|
||||
let currDir = spath[0]
|
||||
for (let i = 1; i < spath.length; i++) {
|
||||
if (currDir !== '' && !fs.existsSync(currDir)) {
|
||||
try {
|
||||
fs.mkdirSync(currDir)
|
||||
} catch (err) {
|
||||
console.log('Failed to make ' + currDir + ' directory... ')
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
currDir += '/' + spath[i]
|
||||
}
|
||||
if (onlyPath) {
|
||||
fs.mkdirSync(path)
|
||||
}
|
||||
currDir += '/' + spath[i]
|
||||
}
|
||||
if (onlyPath) {
|
||||
fs.mkdirSync(path)
|
||||
}
|
||||
}
|
||||
|
||||
function WriteFile(content: string, path: string): void {
|
||||
CreatePath(path)
|
||||
fs.writeFileSync(path, content)
|
||||
CreatePath(path)
|
||||
fs.writeFileSync(path, content)
|
||||
}
|
||||
|
||||
function writeFileAsync(content: string, path: string): void {
|
||||
CreatePath(path)
|
||||
fs.writeFile(path, content, function (err) {
|
||||
if (err) {
|
||||
logger.Log(
|
||||
'Error writing file: ' + path + ' (sync)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
})
|
||||
CreatePath(path)
|
||||
fs.writeFile(path, content, function (err) {
|
||||
if (err) {
|
||||
logger.Log(
|
||||
'Error writing file: ' + path + ' (sync)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function AppendToFile(data: string, file: string): void {
|
||||
CreatePath(file)
|
||||
try {
|
||||
fs.appendFileSync(file, '\n' + data)
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error appendig to file log file: ' + file + ' (sync)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
logger.Log(data)
|
||||
console.error(err)
|
||||
}
|
||||
CreatePath(file)
|
||||
try {
|
||||
fs.appendFileSync(file, '\n' + data)
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error appendig to file log file: ' + file + ' (sync)',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
logger.Log(data)
|
||||
console.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
function deleteFile(fname: string): Boolean {
|
||||
if (FileExists(fname)) {
|
||||
fs.unlinkSync(fname)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
if (FileExists(fname)) {
|
||||
fs.unlinkSync(fname)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function deleteDir(dirName: string): Boolean {
|
||||
if (FileExists(dirName)) {
|
||||
fs.rmSync(dirName, { recursive: true })
|
||||
return true
|
||||
}
|
||||
return false
|
||||
if (FileExists(dirName)) {
|
||||
fs.rmSync(dirName, { recursive: true })
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function uploadFile(
|
||||
req: Request,
|
||||
path: string
|
||||
req: Request,
|
||||
path: string
|
||||
): Promise<{
|
||||
body: Request['body']
|
||||
fileName: string
|
||||
filePath: string
|
||||
body: Request['body']
|
||||
fileName: string
|
||||
filePath: string
|
||||
}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
if (!req.files) {
|
||||
logger.Log(
|
||||
`Unable to upload file, req.files is undefined`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
return
|
||||
}
|
||||
const file = req.files.file
|
||||
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
if (!req.files) {
|
||||
logger.Log(
|
||||
`Unable to upload file, req.files is undefined`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
return
|
||||
}
|
||||
const file = req.files.file
|
||||
// FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
|
||||
|
||||
CreatePath(path, true)
|
||||
CreatePath(path, true)
|
||||
|
||||
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
|
||||
let fileDestination = path + '/' + fileName
|
||||
if (FileExists(fileDestination)) {
|
||||
const id = uuidv4().split('-')[0]
|
||||
let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
|
||||
let fileDestination = path + '/' + fileName
|
||||
if (FileExists(fileDestination)) {
|
||||
const id = uuidv4().split('-')[0]
|
||||
|
||||
const temp = file.name.split('.')
|
||||
const extension = temp.pop()
|
||||
fileName = temp.join('.') + '_' + id + '.' + extension
|
||||
fileDestination = path + '/' + fileName
|
||||
}
|
||||
const temp = file.name.split('.')
|
||||
const extension = temp.pop()
|
||||
fileName = temp.join('.') + '_' + id + '.' + extension
|
||||
fileDestination = path + '/' + fileName
|
||||
}
|
||||
|
||||
file.mv(fileDestination, (err: Error) => {
|
||||
if (err) {
|
||||
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
reject(err)
|
||||
} else {
|
||||
logger.Log(
|
||||
`Uploaded: ${fileName} to ${fileDestination}`,
|
||||
logger.GetColor('blue')
|
||||
)
|
||||
resolve({
|
||||
body: req.body,
|
||||
fileName: fileName,
|
||||
filePath: fileDestination,
|
||||
})
|
||||
file.mv(fileDestination, (err: Error) => {
|
||||
if (err) {
|
||||
logger.Log(
|
||||
`Unable to upload file!`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
reject(err)
|
||||
} else {
|
||||
logger.Log(
|
||||
`Uploaded: ${fileName} to ${fileDestination}`,
|
||||
logger.GetColor('blue')
|
||||
)
|
||||
resolve({
|
||||
body: req.body,
|
||||
fileName: fileName,
|
||||
filePath: fileDestination,
|
||||
})
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
`Unable to upload file, error on stderr`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
reject(err)
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
`Unable to upload file, error on stderr`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
reject(err)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function statFile(file: string): fs.Stats {
|
||||
if (FileExists(file)) {
|
||||
return fs.statSync(file)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
if (FileExists(file)) {
|
||||
return fs.statSync(file)
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function renameFile(oldPath: string, newPath: string): string {
|
||||
if (FileExists(oldPath)) {
|
||||
fs.renameSync(oldPath, newPath)
|
||||
return newPath
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
if (FileExists(oldPath)) {
|
||||
fs.renameSync(oldPath, newPath)
|
||||
return newPath
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
@ -29,268 +29,274 @@ import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
|
|||
import type { WorkerResult } from './classes'
|
||||
|
||||
interface WorkerObj {
|
||||
worker: Worker
|
||||
index: number
|
||||
free: Boolean
|
||||
worker: Worker
|
||||
index: number
|
||||
free: Boolean
|
||||
}
|
||||
|
||||
export interface TaskObject {
|
||||
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb' | 'dbClean' | 'rmQuestions'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
question: Question
|
||||
subjName: string
|
||||
testUrl?: string
|
||||
questionData?: QuestionData
|
||||
searchInAllIfNoResult?: boolean
|
||||
searchTillMatchPercent?: number
|
||||
[key: string]: any
|
||||
}
|
||||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Result
|
||||
| {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteFromDate: number
|
||||
qdbIndex: number
|
||||
}
|
||||
| {
|
||||
questionIndexesToRemove: number[][]
|
||||
subjIndex: number
|
||||
qdbIndex: number
|
||||
recievedQuestions: Question[]
|
||||
}
|
||||
type:
|
||||
| 'work'
|
||||
| 'dbEdit'
|
||||
| 'newQuestions'
|
||||
| 'newdb'
|
||||
| 'dbClean'
|
||||
| 'rmQuestions'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
question: Question
|
||||
subjName: string
|
||||
testUrl?: string
|
||||
questionData?: QuestionData
|
||||
searchInAllIfNoResult?: boolean
|
||||
searchTillMatchPercent?: number
|
||||
[key: string]: any
|
||||
}
|
||||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Result
|
||||
| {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteFromDate: number
|
||||
qdbIndex: number
|
||||
}
|
||||
| {
|
||||
questionIndexesToRemove: number[][]
|
||||
subjIndex: number
|
||||
qdbIndex: number
|
||||
recievedQuestions: Question[]
|
||||
}
|
||||
}
|
||||
|
||||
interface PendingJob {
|
||||
workData: TaskObject
|
||||
doneEvent: DoneEvent
|
||||
targetWorkerIndex?: number
|
||||
workData: TaskObject
|
||||
doneEvent: DoneEvent
|
||||
targetWorkerIndex?: number
|
||||
}
|
||||
|
||||
interface JobEvent extends EventEmitter {
|
||||
on(event: 'jobDone', listener: () => void): this
|
||||
on(event: 'newJob', listener: () => void): this
|
||||
emit(event: 'newJob'): boolean
|
||||
emit(event: 'jobDone'): boolean
|
||||
on(event: 'jobDone', listener: () => void): this
|
||||
on(event: 'newJob', listener: () => void): this
|
||||
emit(event: 'newJob'): boolean
|
||||
emit(event: 'jobDone'): boolean
|
||||
}
|
||||
|
||||
interface DoneEvent extends EventEmitter {
|
||||
once(event: 'done', listener: (result: WorkerResult) => void): this
|
||||
emit(event: 'done', res: WorkerResult): boolean
|
||||
once(event: 'done', listener: (result: WorkerResult) => void): this
|
||||
emit(event: 'done', res: WorkerResult): boolean
|
||||
}
|
||||
|
||||
const alertOnPendingCount = 50
|
||||
const workerFile = './src/utils/classes.ts'
|
||||
let workers: Array<WorkerObj>
|
||||
const pendingJobs: {
|
||||
[id: string]: PendingJob
|
||||
[id: string]: PendingJob
|
||||
} = {}
|
||||
|
||||
const jobEvents: JobEvent = new EventEmitter()
|
||||
|
||||
jobEvents.on('jobDone', () => {
|
||||
processJob()
|
||||
processJob()
|
||||
})
|
||||
|
||||
jobEvents.on('newJob', () => {
|
||||
processJob()
|
||||
processJob()
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function handleWorkerError(worker: WorkerObj, err: Error) {
|
||||
// TODO: restart worker if exited or things like that
|
||||
logger.Log('resourcePromise error', logger.GetColor('redbg'))
|
||||
console.error(err, worker)
|
||||
// TODO: restart worker if exited or things like that
|
||||
logger.Log('resourcePromise error', logger.GetColor('redbg'))
|
||||
console.error(err, worker)
|
||||
}
|
||||
|
||||
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
|
||||
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
|
||||
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
|
||||
return new Promise((resolve) => {
|
||||
const promises: Promise<WorkerResult>[] = []
|
||||
workers.forEach((worker) => {
|
||||
promises.push(doALongTask(data, worker.index))
|
||||
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
|
||||
return new Promise((resolve) => {
|
||||
const promises: Promise<WorkerResult>[] = []
|
||||
workers.forEach((worker) => {
|
||||
promises.push(doALongTask(data, worker.index))
|
||||
})
|
||||
Promise.all(promises).then((res) => {
|
||||
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
|
||||
resolve(res)
|
||||
})
|
||||
})
|
||||
Promise.all(promises).then((res) => {
|
||||
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
|
||||
resolve(res)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export function doALongTask(
|
||||
obj: TaskObject,
|
||||
targetWorkerIndex?: number
|
||||
obj: TaskObject,
|
||||
targetWorkerIndex?: number
|
||||
): Promise<WorkerResult> {
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
logger.Log(
|
||||
`More than ${alertOnPendingCount} callers waiting for free resource! (${
|
||||
Object.keys(pendingJobs).length
|
||||
})`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
logger.Log(
|
||||
`More than ${alertOnPendingCount} callers waiting for free resource! (${
|
||||
Object.keys(pendingJobs).length
|
||||
})`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
|
||||
const jobId = uuidv4()
|
||||
// FIXME: delete doneEvent?
|
||||
const doneEvent: DoneEvent = new EventEmitter()
|
||||
pendingJobs[jobId] = {
|
||||
workData: obj,
|
||||
targetWorkerIndex: targetWorkerIndex,
|
||||
doneEvent: doneEvent,
|
||||
}
|
||||
jobEvents.emit('newJob')
|
||||
return new Promise((resolve) => {
|
||||
doneEvent.once('done', (result: WorkerResult) => {
|
||||
jobEvents.emit('jobDone')
|
||||
resolve(result)
|
||||
const jobId = uuidv4()
|
||||
// FIXME: delete doneEvent?
|
||||
const doneEvent: DoneEvent = new EventEmitter()
|
||||
pendingJobs[jobId] = {
|
||||
workData: obj,
|
||||
targetWorkerIndex: targetWorkerIndex,
|
||||
doneEvent: doneEvent,
|
||||
}
|
||||
jobEvents.emit('newJob')
|
||||
return new Promise((resolve) => {
|
||||
doneEvent.once('done', (result: WorkerResult) => {
|
||||
jobEvents.emit('jobDone')
|
||||
resolve(result)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> {
|
||||
if (workers) {
|
||||
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
|
||||
return null
|
||||
}
|
||||
workers = []
|
||||
if (workers) {
|
||||
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
|
||||
return null
|
||||
}
|
||||
workers = []
|
||||
|
||||
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
|
||||
if (process.env.NS_THREAD_COUNT) {
|
||||
logger.Log(
|
||||
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
|
||||
logger.GetColor('red')
|
||||
)
|
||||
}
|
||||
const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
|
||||
if (process.env.NS_THREAD_COUNT) {
|
||||
logger.Log(
|
||||
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
|
||||
logger.GetColor('red')
|
||||
)
|
||||
}
|
||||
|
||||
for (let i = 0; i < threadCount; i++) {
|
||||
workers.push({
|
||||
worker: getAWorker(i, initData),
|
||||
index: i,
|
||||
free: true,
|
||||
})
|
||||
}
|
||||
for (let i = 0; i < threadCount; i++) {
|
||||
workers.push({
|
||||
worker: getAWorker(i, initData),
|
||||
index: i,
|
||||
free: true,
|
||||
})
|
||||
}
|
||||
|
||||
return workers
|
||||
return workers
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function processJob() {
|
||||
if (Object.keys(pendingJobs).length > 0) {
|
||||
const keys = Object.keys(pendingJobs)
|
||||
let jobKey: string, freeWorker: WorkerObj
|
||||
let i = 0
|
||||
while (!freeWorker && i < keys.length) {
|
||||
jobKey = keys[i]
|
||||
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
|
||||
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
|
||||
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
|
||||
logger.DebugLog(
|
||||
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
|
||||
'job',
|
||||
1
|
||||
)
|
||||
if (Object.keys(pendingJobs).length > 0) {
|
||||
const keys = Object.keys(pendingJobs)
|
||||
let jobKey: string, freeWorker: WorkerObj
|
||||
let i = 0
|
||||
while (!freeWorker && i < keys.length) {
|
||||
jobKey = keys[i]
|
||||
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
|
||||
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
|
||||
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
|
||||
logger.DebugLog(
|
||||
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
|
||||
'job',
|
||||
1
|
||||
)
|
||||
}
|
||||
} else {
|
||||
freeWorker = workers.find((worker) => {
|
||||
return worker.free
|
||||
})
|
||||
if (freeWorker) {
|
||||
logger.DebugLog(
|
||||
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
|
||||
'job',
|
||||
1
|
||||
)
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
} else {
|
||||
freeWorker = workers.find((worker) => {
|
||||
return worker.free
|
||||
})
|
||||
if (freeWorker) {
|
||||
logger.DebugLog(
|
||||
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
|
||||
'job',
|
||||
1
|
||||
)
|
||||
|
||||
if (!freeWorker) {
|
||||
logger.DebugLog('NO FREE WORKER', 'job', 1)
|
||||
return
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
if (!freeWorker) {
|
||||
logger.DebugLog('NO FREE WORKER', 'job', 1)
|
||||
return
|
||||
}
|
||||
if (freeWorker.free) {
|
||||
freeWorker.free = false
|
||||
}
|
||||
const job = pendingJobs[jobKey]
|
||||
delete pendingJobs[jobKey]
|
||||
|
||||
if (freeWorker.free) {
|
||||
freeWorker.free = false
|
||||
doSomething(freeWorker, job.workData)
|
||||
.then((res: WorkerResult) => {
|
||||
freeWorker.free = true
|
||||
job.doneEvent.emit('done', res)
|
||||
})
|
||||
.catch(function (err) {
|
||||
handleWorkerError(freeWorker, err)
|
||||
})
|
||||
}
|
||||
const job = pendingJobs[jobKey]
|
||||
delete pendingJobs[jobKey]
|
||||
|
||||
doSomething(freeWorker, job.workData)
|
||||
.then((res: WorkerResult) => {
|
||||
freeWorker.free = true
|
||||
job.doneEvent.emit('done', res)
|
||||
})
|
||||
.catch(function (err) {
|
||||
handleWorkerError(freeWorker, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function getAWorker(i: number, initData: Array<QuestionDb>) {
|
||||
const worker = workerTs(workerFile, {
|
||||
workerData: {
|
||||
workerIndex: i,
|
||||
initData: initData,
|
||||
},
|
||||
})
|
||||
const worker = workerTs(workerFile, {
|
||||
workerData: {
|
||||
workerIndex: i,
|
||||
initData: initData,
|
||||
},
|
||||
})
|
||||
|
||||
worker.setMaxListeners(50)
|
||||
worker.setMaxListeners(50)
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
})
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
// TODO: this is critical, whole server should stop, or child threads should be restarted
|
||||
logger.Log(
|
||||
`[MAIN]: worker #${i} exit code: ${code}`,
|
||||
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
|
||||
)
|
||||
})
|
||||
return worker
|
||||
worker.on('exit', (code) => {
|
||||
// TODO: this is critical, whole server should stop, or child threads should be restarted
|
||||
logger.Log(
|
||||
`[MAIN]: worker #${i} exit code: ${code}`,
|
||||
code === 0 ? logger.GetColor('redbg') : logger.GetColor('green')
|
||||
)
|
||||
})
|
||||
return worker
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
|
||||
const { /* index, */ worker } = currWorker
|
||||
return new Promise((resolve) => {
|
||||
worker.postMessage(obj)
|
||||
worker.once('message', (msg: WorkerResult) => {
|
||||
resolve(msg)
|
||||
const { /* index, */ worker } = currWorker
|
||||
return new Promise((resolve) => {
|
||||
worker.postMessage(obj)
|
||||
worker.once('message', (msg: WorkerResult) => {
|
||||
resolve(msg)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const workerTs = (
|
||||
file: string,
|
||||
wkOpts: {
|
||||
workerData: {
|
||||
workerIndex: number
|
||||
initData: QuestionDb[]
|
||||
__filename?: string
|
||||
file: string,
|
||||
wkOpts: {
|
||||
workerData: {
|
||||
workerIndex: number
|
||||
initData: QuestionDb[]
|
||||
__filename?: string
|
||||
}
|
||||
eval?: boolean
|
||||
}
|
||||
eval?: boolean
|
||||
}
|
||||
) => {
|
||||
wkOpts.eval = true
|
||||
wkOpts.workerData.__filename = file
|
||||
return new Worker(
|
||||
`
|
||||
wkOpts.eval = true
|
||||
wkOpts.workerData.__filename = file
|
||||
return new Worker(
|
||||
`
|
||||
const wk = require('worker_threads');
|
||||
require('ts-node').register();
|
||||
let file = wk.workerData.__filename;
|
||||
delete wk.workerData.__filename;
|
||||
require(file);
|
||||
`,
|
||||
wkOpts
|
||||
)
|
||||
wkOpts
|
||||
)
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue