mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
added / fixed some types
This commit is contained in:
mrfry
parent
5f12284bb8
commit
bc5c293539
41 changed files with 4378 additions and 8304 deletions
|
|
@ -21,11 +21,15 @@ const recDataFile = './stats/recdata'
|
|||
const dataLockFile = './data/lockData'
|
||||
|
||||
import logger from '../utils/logger'
|
||||
import { createQuestion } from '../utils/classes'
|
||||
import {
|
||||
createQuestion,
|
||||
WorkerResult,
|
||||
SearchResultQuestion,
|
||||
} from '../utils/classes'
|
||||
import { doALongTask } from './workerPool'
|
||||
import idStats from '../utils/ids'
|
||||
import utils from '../utils/utils'
|
||||
import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
import { addQuestion, getSubjNameWithoutYear } from './classes'
|
||||
|
||||
// types
|
||||
import {
|
||||
|
|
@ -152,12 +156,12 @@ function processIncomingRequestUsingDb(
|
|||
): Promise<Result> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const recievedQuestions = []
|
||||
const recievedQuestions: Question[] = []
|
||||
|
||||
logger.DebugLog('recievedData JSON parsed', 'isadding', 1)
|
||||
logger.DebugLog(recievedData, 'isadding', 3)
|
||||
const allQLength = recievedData.quiz.length
|
||||
const questionSearchPromises = []
|
||||
const questionSearchPromises: Promise<WorkerResult>[] = []
|
||||
recievedData.quiz.forEach((question) => {
|
||||
logger.DebugLog('Question:', 'isadding', 2)
|
||||
logger.DebugLog(question, 'isadding', 2)
|
||||
|
|
@ -193,10 +197,10 @@ function processIncomingRequestUsingDb(
|
|||
})
|
||||
|
||||
Promise.all(questionSearchPromises)
|
||||
.then((results: Array<SearchResult>) => {
|
||||
const allQuestions = [] // all new questions here that do not have result
|
||||
results.forEach((result, i) => {
|
||||
const add = result.result.every((res) => {
|
||||
.then((results: Array<WorkerResult>) => {
|
||||
const allQuestions: Question[] = [] // all new questions here that do not have result
|
||||
results.forEach((result: WorkerResult, i) => {
|
||||
const add = result.result.every((res: SearchResultQuestion) => {
|
||||
return res.match < minMatchAmmountToAdd
|
||||
})
|
||||
if (add) {
|
||||
|
|
@ -216,7 +220,10 @@ function processIncomingRequestUsingDb(
|
|||
logger.DebugLog(currentQuestion, 'isadding', 3)
|
||||
addQuestion(qdb.data, sName, {
|
||||
...currentQuestion,
|
||||
date: new Date(),
|
||||
data: {
|
||||
...currentQuestion.data,
|
||||
date: new Date().getTime(),
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
|
|
@ -279,7 +286,7 @@ function processIncomingRequestUsingDb(
|
|||
})
|
||||
}
|
||||
|
||||
export function isQuestionValid(question: Question): Boolean {
|
||||
export function isQuestionValid(question: Question): boolean {
|
||||
if (!question.Q) {
|
||||
return false
|
||||
}
|
||||
|
|
@ -292,8 +299,8 @@ export function isQuestionValid(question: Question): Boolean {
|
|||
export function shouldSearchDataFile(
|
||||
df: DataFile,
|
||||
testUrl: string,
|
||||
trueIfAlways?: Boolean
|
||||
): Boolean {
|
||||
trueIfAlways?: boolean
|
||||
): boolean {
|
||||
if (
|
||||
typeof df.shouldSearch === 'string' &&
|
||||
df.shouldSearch === 'always' &&
|
||||
|
|
@ -316,7 +323,7 @@ export function shouldSearchDataFile(
|
|||
export function shouldSaveDataFile(
|
||||
df: DataFile,
|
||||
recievedData: RecievedData
|
||||
): Boolean {
|
||||
): boolean {
|
||||
if (df.locked) {
|
||||
return false
|
||||
}
|
||||
|
|
@ -348,17 +355,14 @@ export function shouldSaveDataFile(
|
|||
}
|
||||
|
||||
export function loadData(path: string): Array<Subject> {
|
||||
return JSON.parse(utils.ReadFile(path)).reduce((acc, subj) => {
|
||||
return [
|
||||
...acc,
|
||||
{
|
||||
Name: subj.Name,
|
||||
Questions: subj.Questions.map((question) => {
|
||||
return createQuestion(question)
|
||||
}),
|
||||
},
|
||||
]
|
||||
}, [])
|
||||
return JSON.parse(utils.ReadFile(path)).map((subj: Subject) => {
|
||||
return {
|
||||
Name: subj.Name,
|
||||
Questions: subj.Questions.map((question: Question) => {
|
||||
return createQuestion(question)
|
||||
}),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function loadJSON(
|
||||
|
|
@ -455,7 +459,7 @@ function deleteFromDb(
|
|||
selectedDb: { path: string; name: string }
|
||||
}
|
||||
): {
|
||||
success: Boolean
|
||||
success: boolean
|
||||
msg: string
|
||||
deletedQuestion?: Question
|
||||
resultDb?: QuestionDb
|
||||
|
|
@ -470,7 +474,7 @@ function deleteFromDb(
|
|||
// }
|
||||
// }
|
||||
const { index, subjName } = edits
|
||||
let deletedQuestion
|
||||
let deletedQuestion: Question
|
||||
if (isNaN(index) || !subjName) {
|
||||
return {
|
||||
success: false,
|
||||
|
|
@ -511,10 +515,10 @@ function editQuestionInDb(
|
|||
subjName: string
|
||||
type: string
|
||||
selectedDb: { path: string; name: string }
|
||||
newVal?: Question
|
||||
newVal: Question
|
||||
}
|
||||
): {
|
||||
success: Boolean
|
||||
success: boolean
|
||||
msg: string
|
||||
newVal?: Question
|
||||
oldVal?: Question
|
||||
|
|
@ -543,7 +547,8 @@ function editQuestionInDb(
|
|||
// }
|
||||
// }
|
||||
const { index, subjName, newVal } = edits
|
||||
let oldVal
|
||||
|
||||
let oldVal: Question
|
||||
if (isNaN(index) || !subjName) {
|
||||
return {
|
||||
success: false,
|
||||
|
|
@ -598,10 +603,10 @@ function editSubjInDb(
|
|||
}>
|
||||
}
|
||||
): {
|
||||
success: Boolean
|
||||
success: boolean
|
||||
msg: string
|
||||
deletedQuestions?: Array<Question>
|
||||
changedQuestions?: Array<Question>
|
||||
changedQuestions?: { oldVal: Question; newVal: Question }[]
|
||||
resultDb?: QuestionDb
|
||||
} {
|
||||
// {
|
||||
|
|
@ -632,8 +637,8 @@ function editSubjInDb(
|
|||
// }
|
||||
// }
|
||||
const { subjName, changedQuestions, deletedQuestions } = edits
|
||||
const deletedQuestionsToWrite = []
|
||||
const changedQuestionsToWrite = []
|
||||
const deletedQuestionsToWrite: Question[] = []
|
||||
const changedQuestionsToWrite: { oldVal: Question; newVal: Question }[] = []
|
||||
if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) {
|
||||
return {
|
||||
success: false,
|
||||
|
|
@ -695,29 +700,32 @@ function editSubjInDb(
|
|||
}
|
||||
}
|
||||
|
||||
// FIXME: newVal is optional in some places but not in others
|
||||
export interface Edits {
|
||||
index: number
|
||||
subjName: string
|
||||
selectedDb: { path: string; name: string }
|
||||
type: string
|
||||
newVal: Question
|
||||
deletedQuestion?: Array<number>
|
||||
changedQuestions?: Array<{
|
||||
index: number
|
||||
value: Question
|
||||
}>
|
||||
}
|
||||
|
||||
export function editDb(
|
||||
questionDb: QuestionDb,
|
||||
edits: {
|
||||
index: number
|
||||
subjName: string
|
||||
selectedDb: { path: string; name: string }
|
||||
type: string
|
||||
newVal?: Question
|
||||
deletedQuestion?: Array<number>
|
||||
changedQuestions?: Array<{
|
||||
index: number
|
||||
value: Question
|
||||
}>
|
||||
}
|
||||
edits: Edits
|
||||
): {
|
||||
success: Boolean
|
||||
success: boolean
|
||||
msg: string
|
||||
resultDb?: QuestionDb
|
||||
deletedQuestion?: Question
|
||||
newVal?: Question
|
||||
oldVal?: Question
|
||||
deletedQuestions?: Array<Question>
|
||||
changedQuestions?: Array<Question>
|
||||
changedQuestions?: { oldVal: Question; newVal: Question }[]
|
||||
} {
|
||||
if (edits.type === 'delete') {
|
||||
return deleteFromDb(questionDb, edits)
|
||||
|
|
@ -729,4 +737,9 @@ export function editDb(
|
|||
if (edits.type === 'subjEdit') {
|
||||
return editSubjInDb(questionDb, edits)
|
||||
}
|
||||
|
||||
return {
|
||||
success: false,
|
||||
msg: 'DB edit error, no matched type',
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,21 +8,26 @@ import {
|
|||
QuestionDb,
|
||||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
import { editDb } from './actions'
|
||||
import { editDb, Edits } from './actions'
|
||||
|
||||
interface SearchResultQuestion extends Question {
|
||||
export interface WorkerResult {
|
||||
msg: string
|
||||
workerIndex: number
|
||||
result: SearchResultQuestion[]
|
||||
}
|
||||
|
||||
interface DetailedMatch {
|
||||
qMatch: number
|
||||
aMatch: number
|
||||
dMatch: number
|
||||
matchedSubjName: string
|
||||
avg: number
|
||||
}
|
||||
|
||||
export interface SearchResultQuestion {
|
||||
q: Question
|
||||
match: number
|
||||
}
|
||||
|
||||
export interface SearchResult {
|
||||
result: Array<SearchResultQuestion>
|
||||
dbName: string
|
||||
}
|
||||
|
||||
const assert = (val) => {
|
||||
if (!val) {
|
||||
throw new Error('Assertion failed')
|
||||
}
|
||||
detailedMatch: DetailedMatch
|
||||
}
|
||||
|
||||
const commonUselessAnswerParts = [
|
||||
|
|
@ -65,7 +70,7 @@ function getSubjNameWithoutYear(subjName: string): string {
|
|||
// Not exported
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
function simplifyString(toremove) {
|
||||
function simplifyString(toremove: string): string {
|
||||
return toremove.replace(/\s/g, ' ').replace(/\s+/g, ' ').toLowerCase()
|
||||
}
|
||||
|
||||
|
|
@ -73,7 +78,7 @@ function removeStuff(
|
|||
value: string,
|
||||
removableStrings: Array<string>,
|
||||
toReplace?: string
|
||||
) {
|
||||
): string {
|
||||
removableStrings.forEach((removableString) => {
|
||||
const regex = new RegExp(removableString, 'g')
|
||||
value = value.replace(regex, toReplace || '')
|
||||
|
|
@ -82,11 +87,11 @@ function removeStuff(
|
|||
}
|
||||
|
||||
// damn nonbreaking space
|
||||
function normalizeSpaces(input) {
|
||||
function normalizeSpaces(input: string): string {
|
||||
return input.replace(/\s/g, ' ')
|
||||
}
|
||||
|
||||
function removeUnnecesarySpaces(toremove: string) {
|
||||
function removeUnnecesarySpaces(toremove: string): string {
|
||||
return normalizeSpaces(toremove).replace(/\s+/g, ' ')
|
||||
}
|
||||
|
||||
|
|
@ -141,7 +146,7 @@ export function compareString(
|
|||
return percent
|
||||
}
|
||||
|
||||
function answerPreProcessor(value: string) {
|
||||
function answerPreProcessor(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
|
@ -150,9 +155,9 @@ function answerPreProcessor(value: string) {
|
|||
}
|
||||
|
||||
// 'a. pécsi sör' -> 'pécsi sör'
|
||||
function removeAnswerLetters(value: string) {
|
||||
function removeAnswerLetters(value: string): string {
|
||||
if (!value) {
|
||||
return
|
||||
return value
|
||||
}
|
||||
|
||||
const val = value.split('. ')
|
||||
|
|
@ -164,9 +169,9 @@ function removeAnswerLetters(value: string) {
|
|||
}
|
||||
}
|
||||
|
||||
function simplifyQA(value: string, mods: Array<Function>) {
|
||||
function simplifyQA(value: string, mods: Array<Function>): string {
|
||||
if (!value) {
|
||||
return
|
||||
return value
|
||||
}
|
||||
|
||||
return mods.reduce((res, fn) => {
|
||||
|
|
@ -174,7 +179,7 @@ function simplifyQA(value: string, mods: Array<Function>) {
|
|||
}, value)
|
||||
}
|
||||
|
||||
function simplifyAnswer(value: string) {
|
||||
function simplifyAnswer(value: string): string {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
|
|
@ -185,27 +190,30 @@ function simplifyAnswer(value: string) {
|
|||
])
|
||||
}
|
||||
|
||||
function simplifyQuestion(question: Question | string) {
|
||||
function simplifyQuestion(question: string): string {
|
||||
if (!question) {
|
||||
return
|
||||
}
|
||||
if (typeof question === 'string') {
|
||||
return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters])
|
||||
} else {
|
||||
if (question.Q) {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
if (question.A) {
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
return question
|
||||
}
|
||||
return simplifyQA(question, [removeUnnecesarySpaces, removeAnswerLetters])
|
||||
}
|
||||
|
||||
function simplifyQuestionObj(question: Question): Question {
|
||||
if (!question) {
|
||||
return question
|
||||
}
|
||||
if (question.Q) {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
if (question.A) {
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
return question
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
|
@ -241,10 +249,11 @@ function createQuestion(
|
|||
logger.Log('Error creating question', logger.GetColor('redbg'))
|
||||
console.error(question, answer, data)
|
||||
console.error(err)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function compareImage(data: QuestionData, data2: QuestionData) {
|
||||
function compareImage(data: QuestionData, data2: QuestionData): number {
|
||||
if (data.hashedImages && data2.hashedImages) {
|
||||
return compareString(
|
||||
data.hashedImages.join(' '),
|
||||
|
|
@ -266,7 +275,7 @@ function compareImage(data: QuestionData, data2: QuestionData) {
|
|||
}
|
||||
}
|
||||
|
||||
function compareData(q1: Question, q2: Question) {
|
||||
function compareData(q1: Question, q2: Question): number {
|
||||
try {
|
||||
if (q1.data.type === q2.data.type) {
|
||||
const dataType = q1.data.type
|
||||
|
|
@ -294,7 +303,7 @@ function compareData(q1: Question, q2: Question) {
|
|||
return 0
|
||||
}
|
||||
|
||||
function compareQuestion(q1: Question, q2: Question) {
|
||||
function compareQuestion(q1: Question, q2: Question): number {
|
||||
return compareString(q1.Q, q1.cache.Q, q2.Q, q2.cache.Q)
|
||||
// return compareString(
|
||||
// q1.Q,
|
||||
|
|
@ -304,7 +313,7 @@ function compareQuestion(q1: Question, q2: Question) {
|
|||
// )
|
||||
}
|
||||
|
||||
function compareAnswer(q1: Question, q2: Question) {
|
||||
function compareAnswer(q1: Question, q2: Question): number {
|
||||
return compareString(q1.A, q1.cache.A, q2.A, q2.cache.A)
|
||||
// return compareString(
|
||||
// q1.A,
|
||||
|
|
@ -316,15 +325,10 @@ function compareAnswer(q1: Question, q2: Question) {
|
|||
|
||||
function compareQuestionObj(
|
||||
q1: Question,
|
||||
q1subjName: string,
|
||||
_q1subjName: string,
|
||||
q2: Question,
|
||||
q2subjName: string
|
||||
): any {
|
||||
assert(q1)
|
||||
assert(typeof q1 === 'object')
|
||||
assert(q2)
|
||||
assert(typeof q2 === 'object')
|
||||
|
||||
): DetailedMatch {
|
||||
const qMatch = compareQuestion(q1, q2)
|
||||
const aMatch = q2.A ? compareAnswer(q1, q2) : 0
|
||||
// -1 if botth questions are simple
|
||||
|
|
@ -354,7 +358,7 @@ function compareQuestionObj(
|
|||
}
|
||||
}
|
||||
|
||||
function questionToString(question: Question) {
|
||||
function questionToString(question: Question): string {
|
||||
const { Q, A, data } = question
|
||||
|
||||
if (data.type !== 'simple') {
|
||||
|
|
@ -372,10 +376,8 @@ function searchSubject(
|
|||
question: Question,
|
||||
subjName: string,
|
||||
searchTillMatchPercent?: number
|
||||
) {
|
||||
assert(question)
|
||||
|
||||
let result = []
|
||||
): SearchResultQuestion[] {
|
||||
let result: SearchResultQuestion[] = []
|
||||
|
||||
let stopSearch = false
|
||||
let i = subj.Questions.length - 1
|
||||
|
|
@ -416,10 +418,10 @@ function searchSubject(
|
|||
return result
|
||||
}
|
||||
|
||||
function subjectToString(subj: Subject) {
|
||||
function subjectToString(subj: Subject): string {
|
||||
const { Questions, Name } = subj
|
||||
|
||||
const result = []
|
||||
const result: string[] = []
|
||||
Questions.forEach((question) => {
|
||||
result.push(questionToString(question))
|
||||
})
|
||||
|
|
@ -437,23 +439,14 @@ function addQuestion(
|
|||
): void {
|
||||
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
logger.DebugLog(question, 'qdb add', 3)
|
||||
assert(data)
|
||||
assert(subj)
|
||||
assert(question)
|
||||
assert(typeof question === 'object')
|
||||
|
||||
let i = 0
|
||||
// FIXME: this only adds to the first matched subject name. Check if this doesnt cause any bugs
|
||||
while (
|
||||
i < data.length &&
|
||||
!subj
|
||||
const i = data.findIndex((x) => {
|
||||
return !subj
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(data[i].Name).toLowerCase())
|
||||
) {
|
||||
i++
|
||||
}
|
||||
.includes(getSubjNameWithoutYear(x.Name).toLowerCase())
|
||||
})
|
||||
|
||||
if (i < data.length) {
|
||||
if (i !== -1) {
|
||||
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
data[i].Questions.push(question)
|
||||
} else {
|
||||
|
|
@ -465,34 +458,12 @@ function addQuestion(
|
|||
}
|
||||
}
|
||||
|
||||
function prepareQuestion(
|
||||
question: string | Question,
|
||||
data: string | QuestionData
|
||||
): any {
|
||||
let preparedQuestion: Question
|
||||
|
||||
if (typeof question === 'object') {
|
||||
preparedQuestion = createQuestion(question)
|
||||
} else {
|
||||
let parsedData: any
|
||||
if (typeof data === 'string') {
|
||||
try {
|
||||
parsedData = JSON.parse(data)
|
||||
} catch (err) {
|
||||
// asd
|
||||
}
|
||||
}
|
||||
if (typeof data === 'object') {
|
||||
parsedData = data
|
||||
}
|
||||
preparedQuestion = createQuestion(question, null, parsedData)
|
||||
}
|
||||
|
||||
return simplifyQuestion(preparedQuestion)
|
||||
function prepareQuestion(question: Question): Question {
|
||||
return simplifyQuestionObj(createQuestion(question))
|
||||
}
|
||||
|
||||
function dataToString(data: Array<Subject>): string {
|
||||
const result = []
|
||||
const result: string[] = []
|
||||
data.forEach((subj) => {
|
||||
result.push(subjectToString(subj))
|
||||
})
|
||||
|
|
@ -502,16 +473,13 @@ function dataToString(data: Array<Subject>): string {
|
|||
function doSearch(
|
||||
data: Array<Subject>,
|
||||
subjName: string,
|
||||
question: Question | string,
|
||||
questionData?: QuestionData,
|
||||
question: Question,
|
||||
searchTillMatchPercent?: number,
|
||||
searchInAllIfNoResult?: Boolean
|
||||
): any {
|
||||
let result = []
|
||||
): SearchResultQuestion[] {
|
||||
let result: SearchResultQuestion[] = []
|
||||
|
||||
const questionToSearch = prepareQuestion(question, questionData)
|
||||
|
||||
assert(questionToSearch.data)
|
||||
const questionToSearch = prepareQuestion(question)
|
||||
|
||||
data.every((subj) => {
|
||||
if (
|
||||
|
|
@ -569,7 +537,7 @@ function doSearch(
|
|||
}
|
||||
|
||||
result = setNoPossibleAnswersPenalties(
|
||||
questionToSearch.possibleAnswers,
|
||||
questionToSearch.data.possibleAnswers,
|
||||
result
|
||||
)
|
||||
|
||||
|
|
@ -587,8 +555,8 @@ function doSearch(
|
|||
}
|
||||
|
||||
function setNoPossibleAnswersPenalties(
|
||||
possibleAnswers: any,
|
||||
result: any[]
|
||||
possibleAnswers: QuestionData['possibleAnswers'],
|
||||
result: SearchResultQuestion[]
|
||||
): any {
|
||||
if (!Array.isArray(possibleAnswers)) {
|
||||
return result
|
||||
|
|
@ -602,7 +570,8 @@ function setNoPossibleAnswersPenalties(
|
|||
const updated = result.map((result) => {
|
||||
const hasMatch = result.q.data.possibleAnswers.some((possibleAnswer) => {
|
||||
return possibleAnswers.some((questionPossibleAnswer) => {
|
||||
return questionPossibleAnswer.includes(questionPossibleAnswer)
|
||||
// FIXME: this could be object: questionPossibleAnswer
|
||||
return questionPossibleAnswer.text.includes(possibleAnswer.text)
|
||||
})
|
||||
})
|
||||
if (hasMatch) {
|
||||
|
|
@ -626,12 +595,24 @@ function setNoPossibleAnswersPenalties(
|
|||
// Multi threaded stuff
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
interface WorkData {
|
||||
subjName: string
|
||||
question: Question
|
||||
searchTillMatchPercent: number
|
||||
searchInAllIfNoResult: boolean
|
||||
searchIn: number[]
|
||||
index: number
|
||||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
// os.setPriority(10)
|
||||
// logger.Log(`Worker thread priority set to ${os.getPriority()}`)
|
||||
|
||||
const { workerIndex } = workerData
|
||||
let qdbs: Array<QuestionDb> = workerData.initData
|
||||
const {
|
||||
workerIndex,
|
||||
initData,
|
||||
}: { workerIndex: number; initData: Array<QuestionDb> } = workerData
|
||||
let qdbs: Array<QuestionDb> = initData
|
||||
|
||||
logger.Log(
|
||||
`[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
|
||||
|
|
@ -642,30 +623,21 @@ if (!isMainThread) {
|
|||
const {
|
||||
subjName,
|
||||
question,
|
||||
questionData,
|
||||
searchTillMatchPercent,
|
||||
searchInAllIfNoResult,
|
||||
searchIn,
|
||||
index,
|
||||
} = msg.data
|
||||
}: WorkData = msg.data
|
||||
|
||||
// console.log(
|
||||
// `[THREAD #${workerIndex}]: staring work${
|
||||
// !isNaN(index) ? ` on job index #${index}` : ''
|
||||
// }`
|
||||
// )
|
||||
|
||||
let searchResult = []
|
||||
let searchResult: SearchResultQuestion[] = []
|
||||
|
||||
try {
|
||||
qdbs.forEach((qdb) => {
|
||||
// FIXME: === 0?
|
||||
if (searchIn.length === 0 || searchIn.includes(qdb.index)) {
|
||||
if (searchIn.includes(qdb.index)) {
|
||||
const res = doSearch(
|
||||
qdb.data,
|
||||
subjName,
|
||||
question,
|
||||
questionData,
|
||||
searchTillMatchPercent,
|
||||
searchInAllIfNoResult
|
||||
)
|
||||
|
|
@ -686,34 +658,39 @@ if (!isMainThread) {
|
|||
} catch (err) {
|
||||
logger.Log('Error in worker thread!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
console.error('subjName', subjName)
|
||||
console.error('question', question)
|
||||
console.error('questionData', questionData)
|
||||
console.error('searchTillMatchPercent', searchTillMatchPercent)
|
||||
console.error('searchInAllIfNoResult', searchInAllIfNoResult)
|
||||
console.error('searchIn', searchIn)
|
||||
console.error('index', index)
|
||||
console.error({
|
||||
subjName: subjName,
|
||||
question: question,
|
||||
searchTillMatchPercent: searchTillMatchPercent,
|
||||
searchInAllIfNoResult: searchInAllIfNoResult,
|
||||
searchIn: searchIn,
|
||||
index: index,
|
||||
})
|
||||
}
|
||||
|
||||
// sorting
|
||||
const sortedResult = searchResult.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
const sortedResult: SearchResultQuestion[] = searchResult.sort(
|
||||
(q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
// ONDONE:
|
||||
parentPort.postMessage({
|
||||
const workerResult: WorkerResult = {
|
||||
msg: `From thread #${workerIndex}: job ${
|
||||
!isNaN(index) ? `#${index}` : ''
|
||||
}done`,
|
||||
workerIndex: workerIndex,
|
||||
result: sortedResult,
|
||||
})
|
||||
}
|
||||
|
||||
// ONDONE:
|
||||
parentPort.postMessage(workerResult)
|
||||
|
||||
// console.log(
|
||||
// `[THREAD #${workerIndex}]: Work ${
|
||||
|
|
@ -721,7 +698,7 @@ if (!isMainThread) {
|
|||
// }done!`
|
||||
// )
|
||||
} else if (msg.type === 'dbEdit') {
|
||||
const { dbIndex, edits } = msg.data
|
||||
const { dbIndex, edits }: { dbIndex: number; edits: Edits } = msg.data
|
||||
const { resultDb } = editDb(qdbs[dbIndex], edits)
|
||||
qdbs[dbIndex] = resultDb
|
||||
logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1)
|
||||
|
|
@ -731,7 +708,16 @@ if (!isMainThread) {
|
|||
workerIndex: workerIndex,
|
||||
})
|
||||
} else if (msg.type === 'newQuestions') {
|
||||
const { subjName, qdbIndex, newQuestions } = msg.data
|
||||
const {
|
||||
subjName,
|
||||
qdbIndex,
|
||||
newQuestions,
|
||||
}: {
|
||||
subjName: string
|
||||
qdbIndex: number
|
||||
newQuestions: Question[]
|
||||
} = msg.data
|
||||
|
||||
let added = false
|
||||
qdbs = qdbs.map((qdb) => {
|
||||
if (qdb.index === qdbIndex) {
|
||||
|
|
@ -781,7 +767,8 @@ if (!isMainThread) {
|
|||
|
||||
// console.log(`[THREAD #${workerIndex}]: update`)
|
||||
} else if (msg.type === 'newdb') {
|
||||
qdbs.push(msg.newdb)
|
||||
const { data }: { data: QuestionDb } = msg
|
||||
qdbs.push(data)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: new db add done`,
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ export default {
|
|||
runStatement: runStatement,
|
||||
}
|
||||
|
||||
import Sqlite from 'better-sqlite3'
|
||||
import Sqlite, { Database } from 'better-sqlite3'
|
||||
import logger from '../utils/logger'
|
||||
import utils from '../utils/utils'
|
||||
|
||||
|
|
@ -52,7 +52,7 @@ function GetDB(path: string): any {
|
|||
return res
|
||||
}
|
||||
|
||||
function DebugLog(msg) {
|
||||
function DebugLog(msg: string) {
|
||||
if (debugLog) {
|
||||
logger.DebugLog(msg, 'sql', 0)
|
||||
}
|
||||
|
|
@ -131,7 +131,7 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
|
|||
const cols = Object.keys(columns)
|
||||
.reduce((acc, key) => {
|
||||
const item = columns[key]
|
||||
const flags = []
|
||||
const flags: string[] = []
|
||||
const toCheck = {
|
||||
primary: 'PRIMARY KEY',
|
||||
notNull: 'NOT NULL',
|
||||
|
|
@ -150,16 +150,22 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
|
|||
}, [])
|
||||
.join(', ')
|
||||
|
||||
const fKeys = []
|
||||
const fKeys: string[] = []
|
||||
if (foreignKeys) {
|
||||
foreignKeys.forEach((foreignKey) => {
|
||||
const { keysFrom, table, keysTo } = foreignKey
|
||||
fKeys.push(
|
||||
`, FOREIGN KEY(${keysFrom.join(
|
||||
', '
|
||||
)}) REFERENCES ${table}(${keysTo.join(', ')})`
|
||||
)
|
||||
})
|
||||
foreignKeys.forEach(
|
||||
(foreignKey: {
|
||||
keysFrom: string[]
|
||||
table: string
|
||||
keysTo: string[]
|
||||
}) => {
|
||||
const { keysFrom, table, keysTo } = foreignKey
|
||||
fKeys.push(
|
||||
`, FOREIGN KEY(${keysFrom.join(
|
||||
', '
|
||||
)}) REFERENCES ${table}(${keysTo.join(', ')})`
|
||||
)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
// IF NOT EXISTS
|
||||
|
|
@ -244,7 +250,7 @@ function runStatement(db: any, command: string, runType?: string): any {
|
|||
}
|
||||
|
||||
function CloseDB(db: any): void {
|
||||
db.close((err) => {
|
||||
db.close((err: Error) => {
|
||||
if (err) {
|
||||
return console.error(err.message)
|
||||
}
|
||||
|
|
@ -254,7 +260,7 @@ function CloseDB(db: any): void {
|
|||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
function PrepareStatement(db, command) {
|
||||
function PrepareStatement(db: Database, command: string) {
|
||||
if (!db) {
|
||||
throw new Error(
|
||||
'DB is undefined in prepare statement! DB action called with undefined db'
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ function LogId(
|
|||
Save()
|
||||
}
|
||||
|
||||
function AddSubjToList(list: Array<string>, subj: string) {
|
||||
function AddSubjToList(list: { [key: string]: any }, subj: string) {
|
||||
if (!list[subj]) {
|
||||
list[subj] = 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ let uvData = {} // visit data, but per user
|
|||
let udvData = {} // visit data, but per user and daily
|
||||
let writes = 0
|
||||
|
||||
let noLogips = []
|
||||
let noLogips: string[] = []
|
||||
|
||||
function getColoredDateString(): string {
|
||||
const date = new Date()
|
||||
|
|
@ -93,14 +93,18 @@ function Log(msg: string | object, color?: string): void {
|
|||
)
|
||||
}
|
||||
|
||||
function expandWithSpaces(text, count) {
|
||||
function expandWithSpaces(text: string, count: number) {
|
||||
while (text.length < count) {
|
||||
text += ' '
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
function LogReq(req: Request, toFile?: boolean, statusCode?: string): void {
|
||||
function LogReq(
|
||||
req: Request,
|
||||
toFile?: boolean,
|
||||
statusCode?: string | number
|
||||
): void {
|
||||
try {
|
||||
const ip: any =
|
||||
req.headers['cf-connecting-ip'] || req.connection.remoteAddress
|
||||
|
|
@ -173,7 +177,7 @@ function LogReq(req: Request, toFile?: boolean, statusCode?: string): void {
|
|||
}
|
||||
}
|
||||
|
||||
function parseNoLogFile(newData) {
|
||||
function parseNoLogFile(newData: string) {
|
||||
noLogips = newData.split('\n')
|
||||
if (noLogips[noLogips.length - 1] === '') {
|
||||
noLogips.pop()
|
||||
|
|
@ -185,7 +189,7 @@ function parseNoLogFile(newData) {
|
|||
}
|
||||
|
||||
function setNoLogReadInterval() {
|
||||
utils.WatchFile(nologFile, (newData) => {
|
||||
utils.WatchFile(nologFile, (newData: string) => {
|
||||
parseNoLogFile(newData)
|
||||
})
|
||||
|
||||
|
|
@ -227,19 +231,7 @@ function Load(): void {
|
|||
setNoLogReadInterval()
|
||||
}
|
||||
|
||||
function LogStat(
|
||||
url: string,
|
||||
ip: string,
|
||||
hostname: string,
|
||||
userId: number
|
||||
): void {
|
||||
const nolog = noLogips.some((noLogips) => {
|
||||
return ip.includes(noLogips)
|
||||
})
|
||||
if (nolog) {
|
||||
return
|
||||
}
|
||||
|
||||
function LogStat(url: string, hostname: string, userId: number): void {
|
||||
url = hostname + url.split('?')[0]
|
||||
Inc(url)
|
||||
AddUserIdStat(userId)
|
||||
|
|
@ -248,7 +240,7 @@ function LogStat(
|
|||
Save()
|
||||
}
|
||||
|
||||
function IncUserStat(userId) {
|
||||
function IncUserStat(userId: number) {
|
||||
try {
|
||||
if (uvData[userId] === undefined) {
|
||||
uvData[userId] = 0
|
||||
|
|
@ -260,7 +252,7 @@ function IncUserStat(userId) {
|
|||
}
|
||||
}
|
||||
|
||||
function AddUserIdStat(userId) {
|
||||
function AddUserIdStat(userId: number) {
|
||||
try {
|
||||
const date = new Date()
|
||||
const now =
|
||||
|
|
@ -282,7 +274,7 @@ function AddUserIdStat(userId) {
|
|||
}
|
||||
}
|
||||
|
||||
function Inc(value) {
|
||||
function Inc(value: string) {
|
||||
if (value.startsWith('/?')) {
|
||||
value = '/'
|
||||
}
|
||||
|
|
@ -292,7 +284,7 @@ function Inc(value) {
|
|||
vData[value]++
|
||||
}
|
||||
|
||||
function AddVisitStat(name) {
|
||||
function AddVisitStat(name: string) {
|
||||
const date = new Date()
|
||||
const now =
|
||||
date.getFullYear() +
|
||||
|
|
@ -346,7 +338,7 @@ function logHashed(msg: string): string {
|
|||
return GetRandomColor(msg.toString()) + msg + C()
|
||||
}
|
||||
|
||||
function GetRandomColor(msg): string {
|
||||
function GetRandomColor(msg: string): string {
|
||||
if (!msg) {
|
||||
return 'red'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -211,7 +211,7 @@ function uploadFile(req: Request, path: string): Promise<any> {
|
|||
fileDestination = path + '/' + fileName
|
||||
}
|
||||
|
||||
file.mv(fileDestination, (err) => {
|
||||
file.mv(fileDestination, (err: Error) => {
|
||||
if (err) {
|
||||
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
|
|
|
|||
|
|
@ -4,19 +4,52 @@ import { EventEmitter } from 'events'
|
|||
import os from 'os'
|
||||
|
||||
import logger from './logger'
|
||||
import { Result } from './actions'
|
||||
import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
|
||||
import type { WorkerResult } from './classes'
|
||||
|
||||
interface WorkerObj {
|
||||
worker: any
|
||||
worker: Worker
|
||||
index: number
|
||||
free: Boolean
|
||||
}
|
||||
|
||||
interface TaskObject {
|
||||
type: 'work' | 'dbEdit' | 'newQuestions' | 'newdb'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
question: Question
|
||||
subjName: string
|
||||
testUrl?: string
|
||||
questionData?: QuestionData
|
||||
searchInAllIfNoResult?: boolean
|
||||
searchTillMatchPercent?: number
|
||||
[key: string]: any
|
||||
}
|
||||
| { dbIndex: number; edits: any }
|
||||
| QuestionDb
|
||||
| Result
|
||||
}
|
||||
|
||||
interface PendingJob {
|
||||
workData: any
|
||||
doneEvent: any
|
||||
workData: TaskObject
|
||||
doneEvent: DoneEvent
|
||||
targetWorkerIndex?: number
|
||||
}
|
||||
|
||||
interface JobEvent extends EventEmitter {
|
||||
on(event: 'jobDone', listener: () => void): this
|
||||
on(event: 'newJob', listener: () => void): this
|
||||
emit(event: 'newJob'): boolean
|
||||
emit(event: 'jobDone'): boolean
|
||||
}
|
||||
|
||||
interface DoneEvent extends EventEmitter {
|
||||
once(event: 'done', listener: (result: WorkerResult) => void): this
|
||||
emit(event: 'done', res: WorkerResult): boolean
|
||||
}
|
||||
|
||||
const alertOnPendingCount = 50
|
||||
const workerFile = './src/utils/classes.ts'
|
||||
let workers: Array<WorkerObj>
|
||||
|
|
@ -24,7 +57,7 @@ const pendingJobs: {
|
|||
[id: string]: PendingJob
|
||||
} = {}
|
||||
|
||||
const jobEvents = new EventEmitter()
|
||||
const jobEvents: JobEvent = new EventEmitter()
|
||||
|
||||
jobEvents.on('jobDone', () => {
|
||||
processJob()
|
||||
|
|
@ -36,17 +69,17 @@ jobEvents.on('newJob', () => {
|
|||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function handleWorkerError(worker: WorkerObj, err) {
|
||||
function handleWorkerError(worker: WorkerObj, err: Error) {
|
||||
// TODO: restart worker if exited or things like that
|
||||
logger.Log('resourcePromise error', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
console.error(err, worker)
|
||||
}
|
||||
|
||||
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
|
||||
export function msgAllWorker(data: any): Promise<any> {
|
||||
export function msgAllWorker(data: TaskObject): Promise<any> {
|
||||
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
|
||||
return new Promise((resolve) => {
|
||||
const promises = []
|
||||
const promises: Promise<WorkerResult>[] = []
|
||||
workers.forEach((worker) => {
|
||||
promises.push(doALongTask(data, worker.index))
|
||||
})
|
||||
|
|
@ -58,7 +91,7 @@ export function msgAllWorker(data: any): Promise<any> {
|
|||
}
|
||||
|
||||
export function doALongTask(
|
||||
obj: any,
|
||||
obj: TaskObject,
|
||||
targetWorkerIndex?: number
|
||||
): Promise<any> {
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
|
|
@ -72,7 +105,7 @@ export function doALongTask(
|
|||
|
||||
const jobId = uuidv4()
|
||||
// FIXME: delete doneEvent?
|
||||
const doneEvent = new EventEmitter()
|
||||
const doneEvent: DoneEvent = new EventEmitter()
|
||||
pendingJobs[jobId] = {
|
||||
workData: obj,
|
||||
targetWorkerIndex: targetWorkerIndex,
|
||||
|
|
@ -80,7 +113,7 @@ export function doALongTask(
|
|||
}
|
||||
jobEvents.emit('newJob')
|
||||
return new Promise((resolve) => {
|
||||
doneEvent.once('done', (result) => {
|
||||
doneEvent.once('done', (result: WorkerResult) => {
|
||||
jobEvents.emit('jobDone')
|
||||
resolve(result)
|
||||
})
|
||||
|
|
@ -90,7 +123,7 @@ export function doALongTask(
|
|||
export function initWorkerPool(initData: any): Array<WorkerObj> {
|
||||
if (workers) {
|
||||
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
|
||||
return
|
||||
return null
|
||||
}
|
||||
workers = []
|
||||
|
||||
|
|
@ -119,7 +152,7 @@ function processJob() {
|
|||
if (Object.keys(pendingJobs).length > 0) {
|
||||
// FIXME: FIFO OR ANYTHING ELSE (JOB PROCESSING ORDER)
|
||||
const keys = Object.keys(pendingJobs)
|
||||
let jobKey, freeWorker
|
||||
let jobKey: string, freeWorker: WorkerObj
|
||||
let i = 0
|
||||
while (!freeWorker && i < keys.length) {
|
||||
jobKey = keys[i]
|
||||
|
|
@ -159,7 +192,7 @@ function processJob() {
|
|||
delete pendingJobs[jobKey]
|
||||
|
||||
doSomething(freeWorker, job.workData)
|
||||
.then((res) => {
|
||||
.then((res: WorkerResult) => {
|
||||
freeWorker.free = true
|
||||
job.doneEvent.emit('done', res)
|
||||
})
|
||||
|
|
@ -169,7 +202,7 @@ function processJob() {
|
|||
}
|
||||
}
|
||||
|
||||
function getAWorker(i, initData) {
|
||||
function getAWorker(i: number, initData: Array<QuestionDb>) {
|
||||
const worker = workerTs(workerFile, {
|
||||
workerData: {
|
||||
workerIndex: i,
|
||||
|
|
@ -196,11 +229,11 @@ function getAWorker(i, initData) {
|
|||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function doSomething(currWorker, obj) {
|
||||
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
|
||||
const { /* index, */ worker } = currWorker
|
||||
return new Promise((resolve) => {
|
||||
worker.postMessage(obj)
|
||||
worker.once('message', (msg) => {
|
||||
worker.once('message', (msg: WorkerResult) => {
|
||||
resolve(msg)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue