mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Merged type fixes
This commit is contained in:
commit
85b5acc692
22 changed files with 583 additions and 150 deletions
|
@ -1,5 +1,6 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
Question Server
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
|
@ -437,7 +438,7 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
|
|||
logger.Log(`Backing up ${data.name}...`)
|
||||
writeData(
|
||||
data.data,
|
||||
`${path}${data.name}_${utils.GetDateString(true)}.json`
|
||||
`${path}${data.name}_${utils.GetDateString(undefined, true)}.json`
|
||||
)
|
||||
logger.Log('Done')
|
||||
} catch (err) {
|
||||
|
|
|
@ -1,4 +1,23 @@
|
|||
// import os from 'os'
|
||||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
import { isMainThread, parentPort, workerData } from 'worker_threads'
|
||||
|
||||
import logger from './logger'
|
||||
|
@ -13,7 +32,7 @@ import { editDb, Edits } from './actions'
|
|||
export interface WorkerResult {
|
||||
msg: string
|
||||
workerIndex: number
|
||||
result: SearchResultQuestion[]
|
||||
result?: SearchResultQuestion[]
|
||||
}
|
||||
|
||||
interface DetailedMatch {
|
||||
|
@ -557,7 +576,7 @@ function doSearch(
|
|||
function setNoPossibleAnswersPenalties(
|
||||
possibleAnswers: QuestionData['possibleAnswers'],
|
||||
result: SearchResultQuestion[]
|
||||
): any {
|
||||
): SearchResultQuestion[] {
|
||||
if (!Array.isArray(possibleAnswers)) {
|
||||
return result
|
||||
}
|
||||
|
@ -605,9 +624,6 @@ interface WorkData {
|
|||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
// os.setPriority(10)
|
||||
// logger.Log(`Worker thread priority set to ${os.getPriority()}`)
|
||||
|
||||
const {
|
||||
workerIndex,
|
||||
initData,
|
||||
|
|
|
@ -1,3 +1,23 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
// https://www.sqlitetutorial.net/sqlite-nodejs/
|
||||
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
|
||||
|
||||
|
@ -16,18 +36,25 @@ export default {
|
|||
sanitizeQuery: sanitizeQuery,
|
||||
}
|
||||
|
||||
import Sqlite, { Database } from 'better-sqlite3'
|
||||
import Sqlite, { Database, RunResult } from 'better-sqlite3'
|
||||
import logger from '../utils/logger'
|
||||
import utils from '../utils/utils'
|
||||
|
||||
const debugLog = process.env.NS_SQL_DEBUG_LOG
|
||||
|
||||
function sanitizeQuery(val: string): string {
|
||||
return val.replace(/'/g, '').replace(/;/g, '')
|
||||
function sanitizeQuery(val: string | number): string | number {
|
||||
if (typeof val === 'string') {
|
||||
return val.replace(/'/g, '').replace(/;/g, '')
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
|
||||
function GetSqlQuerry(conditions: any, type: string, joiner?: string) {
|
||||
function GetSqlQuerry(
|
||||
conditions: { [key: string]: string | number },
|
||||
type?: string,
|
||||
joiner?: string
|
||||
) {
|
||||
const res = Object.keys(conditions).reduce((acc, key) => {
|
||||
const item = conditions[key]
|
||||
const conditionKey = sanitizeQuery(key)
|
||||
|
@ -53,7 +80,7 @@ function GetSqlQuerry(conditions: any, type: string, joiner?: string) {
|
|||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function GetDB(path: string): any {
|
||||
function GetDB(path: string): Database {
|
||||
utils.CreatePath(path)
|
||||
const res = new Sqlite(path)
|
||||
res.pragma('synchronous = OFF')
|
||||
|
@ -66,7 +93,12 @@ function DebugLog(msg: string) {
|
|||
}
|
||||
}
|
||||
|
||||
function AddColumn(db: any, table: any, col: any): any {
|
||||
// FIXME: this might not work: what is col exactly, and how we use AddColumn?
|
||||
function AddColumn(
|
||||
db: Database,
|
||||
table: string,
|
||||
col: { [key: string]: string | number }
|
||||
): RunResult {
|
||||
try {
|
||||
const colName = Object.keys(col)[0]
|
||||
const colType = col.type
|
||||
|
@ -77,10 +109,17 @@ function AddColumn(db: any, table: any, col: any): any {
|
|||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function TableInfo(db: any, table: any): any {
|
||||
function TableInfo(
|
||||
db: Database,
|
||||
table: string
|
||||
): {
|
||||
columns: any[]
|
||||
dataCount: number
|
||||
} {
|
||||
try {
|
||||
const command = `PRAGMA table_info(${table})`
|
||||
const stmt = PrepareStatement(db, command)
|
||||
|
@ -98,10 +137,16 @@ function TableInfo(db: any, table: any): any {
|
|||
}
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function Update(db: any, table: any, newData: any, conditions: any): any {
|
||||
function Update(
|
||||
db: Database,
|
||||
table: string,
|
||||
newData: { [key: string]: string | number },
|
||||
conditions: { [key: string]: string | number }
|
||||
): RunResult {
|
||||
try {
|
||||
const command = `UPDATE ${table} SET ${GetSqlQuerry(
|
||||
newData,
|
||||
|
@ -112,10 +157,15 @@ function Update(db: any, table: any, newData: any, conditions: any): any {
|
|||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function Delete(db: any, table: any, conditions: any): any {
|
||||
function Delete(
|
||||
db: Database,
|
||||
table: string,
|
||||
conditions: { [key: string]: string | number }
|
||||
): RunResult {
|
||||
try {
|
||||
const command = `DELETE FROM ${table} WHERE ${GetSqlQuerry(
|
||||
conditions,
|
||||
|
@ -126,10 +176,31 @@ function Delete(db: any, table: any, conditions: any): any {
|
|||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
|
||||
interface DbColumnDescription {
|
||||
[key: string]: {
|
||||
type: string
|
||||
primary?: boolean
|
||||
autoIncrement?: boolean
|
||||
notNull?: boolean
|
||||
defaultZero?: boolean
|
||||
[key: string]: any
|
||||
}
|
||||
}
|
||||
|
||||
function CreateTable(
|
||||
db: Database,
|
||||
name: string,
|
||||
columns: DbColumnDescription,
|
||||
foreignKeys: {
|
||||
keysFrom: string[]
|
||||
table: string
|
||||
keysTo: string[]
|
||||
}[]
|
||||
): RunResult {
|
||||
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
|
||||
// number, lastLogin text, lastAccess text
|
||||
//
|
||||
|
@ -160,20 +231,14 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
|
|||
|
||||
const fKeys: string[] = []
|
||||
if (foreignKeys) {
|
||||
foreignKeys.forEach(
|
||||
(foreignKey: {
|
||||
keysFrom: string[]
|
||||
table: string
|
||||
keysTo: string[]
|
||||
}) => {
|
||||
const { keysFrom, table, keysTo } = foreignKey
|
||||
fKeys.push(
|
||||
`, FOREIGN KEY(${keysFrom.join(
|
||||
', '
|
||||
)}) REFERENCES ${table}(${keysTo.join(', ')})`
|
||||
)
|
||||
}
|
||||
)
|
||||
foreignKeys.forEach((foreignKey) => {
|
||||
const { keysFrom, table, keysTo } = foreignKey
|
||||
fKeys.push(
|
||||
`, FOREIGN KEY(${keysFrom.join(
|
||||
', '
|
||||
)}) REFERENCES ${table}(${keysTo.join(', ')})`
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
// IF NOT EXISTS
|
||||
|
@ -182,10 +247,11 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
|
|||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function SelectAll(db: any, from: any): any {
|
||||
function SelectAll(db: Database, from: string): any[] {
|
||||
try {
|
||||
const command = `SELECT * from ${from}`
|
||||
|
||||
|
@ -193,11 +259,17 @@ function SelectAll(db: any, from: any): any {
|
|||
return stmt.all()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
// SELECT * FROM MyTable WHERE SomeColumn > LastValue ORDER BY SomeColumn LIMIT 100;
|
||||
function Select(db: any, from: any, conditions: any, options: any = {}): any {
|
||||
function Select(
|
||||
db: Database,
|
||||
from: string,
|
||||
conditions: { [key: string]: string | number },
|
||||
options: { joiner?: string; limit?: number } = {}
|
||||
): any[] {
|
||||
const { joiner, limit } = options
|
||||
|
||||
try {
|
||||
|
@ -215,10 +287,15 @@ function Select(db: any, from: any, conditions: any, options: any = {}): any {
|
|||
return stmt.all()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function Insert(db: any, table: any, data: any): any {
|
||||
function Insert(
|
||||
db: Database,
|
||||
table: string,
|
||||
data: { [key: string]: number | string }
|
||||
): RunResult {
|
||||
try {
|
||||
const cols = Object.keys(data)
|
||||
.reduce((acc, key) => {
|
||||
|
@ -228,15 +305,13 @@ function Insert(db: any, table: any, data: any): any {
|
|||
.join(', ')
|
||||
|
||||
const values = Object.keys(data)
|
||||
.reduce((acc, key) => {
|
||||
const item = data[key]
|
||||
.map((item) => {
|
||||
if (typeof item === 'string') {
|
||||
acc.push(`'${item}'`)
|
||||
return `'${item}'`
|
||||
} else {
|
||||
acc.push(`${item}`)
|
||||
return `${item}`
|
||||
}
|
||||
return acc
|
||||
}, [])
|
||||
})
|
||||
.join(', ')
|
||||
|
||||
const command = `INSERT INTO ${table} (${cols}) VALUES (${values})`
|
||||
|
@ -245,25 +320,22 @@ function Insert(db: any, table: any, data: any): any {
|
|||
return stmt.run()
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function runStatement(db: any, command: string, runType?: string): any {
|
||||
function runStatement(db: Database, command: string, runType?: string): any {
|
||||
const stmt = PrepareStatement(db, command)
|
||||
if (!runType) {
|
||||
return stmt.all()
|
||||
} else if (runType === 'run') {
|
||||
return stmt.run()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function CloseDB(db: any): void {
|
||||
db.close((err: Error) => {
|
||||
if (err) {
|
||||
return console.error(err.message)
|
||||
}
|
||||
DebugLog('Close the database connection.')
|
||||
})
|
||||
function CloseDB(db: Database): void {
|
||||
db.close()
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
|
|
@ -106,18 +106,6 @@ function LogReq(
|
|||
statusCode?: string | number
|
||||
): void {
|
||||
try {
|
||||
const ip: any =
|
||||
req.headers['cf-connecting-ip'] || req.connection.remoteAddress
|
||||
// if (!toFile) {
|
||||
// ip = expandWithSpaces(ip, 39)
|
||||
// }
|
||||
const nolog = noLogips.some((noLogip) => {
|
||||
return ip.includes(noLogip)
|
||||
})
|
||||
if (nolog) {
|
||||
return
|
||||
}
|
||||
|
||||
let logEntry = '' // logHashed(ip)
|
||||
let dl = DELIM
|
||||
if (req.url.includes('lred')) {
|
||||
|
|
|
@ -1,3 +1,23 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
export default {
|
||||
ReadFile: ReadFile,
|
||||
ReadJSON: ReadJSON,
|
||||
|
@ -24,7 +44,7 @@ import { Request } from '../types/basicTypes'
|
|||
|
||||
interface URLFormatOptions {
|
||||
pathname?: string
|
||||
query?: any
|
||||
query?: { [key: string]: string }
|
||||
}
|
||||
|
||||
function formatUrl(options: URLFormatOptions): string {
|
||||
|
@ -42,8 +62,11 @@ function formatUrl(options: URLFormatOptions): string {
|
|||
return path + queryString
|
||||
}
|
||||
|
||||
function GetDateString(noTime?: boolean): string {
|
||||
const date = new Date()
|
||||
function GetDateString(
|
||||
referenceDate?: Date | string,
|
||||
noTime?: boolean
|
||||
): string {
|
||||
const date = referenceDate ? new Date(referenceDate) : new Date()
|
||||
|
||||
if (noTime) {
|
||||
return (
|
||||
|
@ -183,7 +206,14 @@ function deleteFile(fname: string): Boolean {
|
|||
return false
|
||||
}
|
||||
|
||||
function uploadFile(req: Request, path: string): Promise<any> {
|
||||
function uploadFile(
|
||||
req: Request,
|
||||
path: string
|
||||
): Promise<{
|
||||
body: Request['body']
|
||||
fileName: string
|
||||
filePath: string
|
||||
}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
if (!req.files) {
|
||||
|
@ -235,7 +265,7 @@ function uploadFile(req: Request, path: string): Promise<any> {
|
|||
})
|
||||
}
|
||||
|
||||
function statFile(file: string): any {
|
||||
function statFile(file: string): fs.Stats {
|
||||
if (FileExists(file)) {
|
||||
return fs.statSync(file)
|
||||
} else {
|
||||
|
@ -243,7 +273,7 @@ function statFile(file: string): any {
|
|||
}
|
||||
}
|
||||
|
||||
function renameFile(oldPath: string, newPath: string): any {
|
||||
function renameFile(oldPath: string, newPath: string): string {
|
||||
if (FileExists(oldPath)) {
|
||||
fs.renameSync(oldPath, newPath)
|
||||
return newPath
|
||||
|
|
|
@ -1,10 +1,30 @@
|
|||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
import { Worker } from 'worker_threads'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { EventEmitter } from 'events'
|
||||
import os from 'os'
|
||||
|
||||
import logger from './logger'
|
||||
import { Result } from './actions'
|
||||
import { Result, Edits } from './actions'
|
||||
import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
|
||||
import type { WorkerResult } from './classes'
|
||||
|
||||
|
@ -27,7 +47,7 @@ interface TaskObject {
|
|||
searchTillMatchPercent?: number
|
||||
[key: string]: any
|
||||
}
|
||||
| { dbIndex: number; edits: any }
|
||||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Result
|
||||
}
|
||||
|
@ -76,7 +96,7 @@ function handleWorkerError(worker: WorkerObj, err: Error) {
|
|||
}
|
||||
|
||||
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
|
||||
export function msgAllWorker(data: TaskObject): Promise<any> {
|
||||
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
|
||||
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
|
||||
return new Promise((resolve) => {
|
||||
const promises: Promise<WorkerResult>[] = []
|
||||
|
@ -93,7 +113,7 @@ export function msgAllWorker(data: TaskObject): Promise<any> {
|
|||
export function doALongTask(
|
||||
obj: TaskObject,
|
||||
targetWorkerIndex?: number
|
||||
): Promise<any> {
|
||||
): Promise<WorkerResult> {
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
logger.Log(
|
||||
`More than ${alertOnPendingCount} callers waiting for free resource! (${
|
||||
|
@ -120,7 +140,7 @@ export function doALongTask(
|
|||
})
|
||||
}
|
||||
|
||||
export function initWorkerPool(initData: any): Array<WorkerObj> {
|
||||
export function initWorkerPool(initData: Array<QuestionDb>): Array<WorkerObj> {
|
||||
if (workers) {
|
||||
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
|
||||
return null
|
||||
|
@ -150,7 +170,6 @@ export function initWorkerPool(initData: any): Array<WorkerObj> {
|
|||
|
||||
function processJob() {
|
||||
if (Object.keys(pendingJobs).length > 0) {
|
||||
// FIXME: FIFO OR ANYTHING ELSE (JOB PROCESSING ORDER)
|
||||
const keys = Object.keys(pendingJobs)
|
||||
let jobKey: string, freeWorker: WorkerObj
|
||||
let i = 0
|
||||
|
@ -239,11 +258,18 @@ function doSomething(currWorker: WorkerObj, obj: TaskObject) {
|
|||
})
|
||||
}
|
||||
|
||||
const workerTs = (file: string, wkOpts: any) => {
|
||||
wkOpts.eval = true
|
||||
if (!wkOpts.workerData) {
|
||||
wkOpts.workerData = {}
|
||||
const workerTs = (
|
||||
file: string,
|
||||
wkOpts: {
|
||||
workerData: {
|
||||
workerIndex: number
|
||||
initData: QuestionDb[]
|
||||
__filename?: string
|
||||
}
|
||||
eval?: boolean
|
||||
}
|
||||
) => {
|
||||
wkOpts.eval = true
|
||||
wkOpts.workerData.__filename = file
|
||||
return new Worker(
|
||||
`
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue