mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2026-04-28 11:17:38 +02:00
worker file split, sending new questions to peers instantly
This commit is contained in:
mrfry
+13
-12
@@ -22,8 +22,6 @@ const recDataFile = './stats/recdata'
|
||||
const dataLockFile = './data/lockData'
|
||||
|
||||
import logger from '../utils/logger'
|
||||
import { WorkerResult } from '../utils/classes'
|
||||
import { doALongTask, msgAllWorker } from './workerPool'
|
||||
import idStats from '../utils/ids'
|
||||
import utils from '../utils/utils'
|
||||
import {
|
||||
@@ -42,6 +40,8 @@ import {
|
||||
DataFile,
|
||||
} from '../types/basicTypes'
|
||||
import { countOfQdbs } from './qdbUtils'
|
||||
import { WorkerResult } from '../worker/worker'
|
||||
import { queueWork, msgAllWorker } from '../worker/workerPool'
|
||||
|
||||
// if a recievend question doesnt match at least this % to any other question in the db it gets
|
||||
// added to db
|
||||
@@ -56,6 +56,7 @@ export interface RecievedData {
|
||||
id: string
|
||||
version: string
|
||||
location: string
|
||||
fromPeer?: boolean
|
||||
}
|
||||
|
||||
export interface Result {
|
||||
@@ -183,8 +184,8 @@ function processIncomingRequestUsingDb(
|
||||
recievedQuestions.push(currentQuestion)
|
||||
// This here searches only in relevant subjects, and not all subjects
|
||||
questionSearchPromises.push(
|
||||
doALongTask({
|
||||
type: 'work',
|
||||
queueWork({
|
||||
type: 'search',
|
||||
data: {
|
||||
searchIn: [qdb.index],
|
||||
question: currentQuestion,
|
||||
@@ -201,7 +202,7 @@ function processIncomingRequestUsingDb(
|
||||
|
||||
Promise.all(questionSearchPromises)
|
||||
.then((results: Array<WorkerResult>) => {
|
||||
const allQuestions: Question[] = [] // all new questions here that do not have result
|
||||
const newQuestions: Question[] = [] // all new questions here that do not have result
|
||||
results.forEach((result: WorkerResult, i) => {
|
||||
const add = (
|
||||
result.result as SearchResultQuestion[]
|
||||
@@ -209,7 +210,7 @@ function processIncomingRequestUsingDb(
|
||||
return res.match < minMatchAmmountToAdd
|
||||
})
|
||||
if (add && !result.error) {
|
||||
allQuestions.push(recievedQuestions[i])
|
||||
newQuestions.push(recievedQuestions[i])
|
||||
}
|
||||
})
|
||||
|
||||
@@ -223,8 +224,8 @@ function processIncomingRequestUsingDb(
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
if (allQuestions.length > 0 && subjName) {
|
||||
addQuestionsToDb(allQuestions, subjName, qdb)
|
||||
if (newQuestions.length > 0 && subjName) {
|
||||
addQuestionsToDb(newQuestions, subjName, qdb)
|
||||
|
||||
currWrites++
|
||||
logger.DebugLog(
|
||||
@@ -246,12 +247,12 @@ function processIncomingRequestUsingDb(
|
||||
idStats.LogId(
|
||||
user.id,
|
||||
recievedData.subj,
|
||||
allQuestions.length,
|
||||
newQuestions.length,
|
||||
allQLength
|
||||
)
|
||||
|
||||
logger.DebugLog('New Questions:', 'isadding', 2)
|
||||
logger.DebugLog(allQuestions, 'isadding', 2)
|
||||
logger.DebugLog(newQuestions, 'isadding', 2)
|
||||
|
||||
logger.DebugLog(
|
||||
'ProcessIncomingRequest done',
|
||||
@@ -259,7 +260,7 @@ function processIncomingRequestUsingDb(
|
||||
1
|
||||
)
|
||||
resolve({
|
||||
newQuestions: allQuestions,
|
||||
newQuestions: newQuestions,
|
||||
subjName: recievedData.subj,
|
||||
qdbIndex: qdb.index,
|
||||
qdbName: qdb.name,
|
||||
@@ -343,7 +344,7 @@ function runCleanWorker(
|
||||
// }${logger.C('')}")`
|
||||
// )
|
||||
// pass recieved questions to a worker
|
||||
doALongTask({
|
||||
queueWork({
|
||||
type: 'dbClean',
|
||||
data: {
|
||||
questions: recievedQuesitons,
|
||||
|
||||
@@ -1,539 +0,0 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
// FIXME: this should be renamed to worker.ts or something
|
||||
|
||||
import { isMainThread, parentPort, workerData } from 'worker_threads'
|
||||
|
||||
import { recognizeTextFromBase64, tesseractLoaded } from './tesseract'
|
||||
import logger from './logger'
|
||||
import {
|
||||
Question,
|
||||
QuestionData,
|
||||
QuestionDb,
|
||||
Subject,
|
||||
} from '../types/basicTypes'
|
||||
import { editDb, Edits, updateQuestionsInArray } from './actions'
|
||||
import {
|
||||
cleanDb,
|
||||
countOfQdbs,
|
||||
createQuestion,
|
||||
getSubjectDifference,
|
||||
getSubjNameWithoutYear,
|
||||
minMatchToNotSearchOtherSubjects,
|
||||
noPossibleAnswerMatchPenalty,
|
||||
prepareQuestion,
|
||||
SearchResultQuestion,
|
||||
searchSubject,
|
||||
} from './qdbUtils'
|
||||
// import { TaskObject } from './workerPool'
|
||||
|
||||
export interface WorkerResult {
|
||||
msg: string
|
||||
workerIndex: number
|
||||
result?: SearchResultQuestion[] | number[][]
|
||||
error?: boolean
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// String Utils
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// Exported
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// Not exported
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Question
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
async function recognizeQuestionImage(question: Question): Promise<Question> {
|
||||
const base64Data = question.data.base64
|
||||
if (Array.isArray(base64Data) && base64Data.length) {
|
||||
const res: string[] = []
|
||||
for (let i = 0; i < base64Data.length; i++) {
|
||||
const base64 = base64Data[i]
|
||||
const text = await recognizeTextFromBase64(base64)
|
||||
if (text && text.trim()) {
|
||||
res.push(text)
|
||||
}
|
||||
}
|
||||
|
||||
if (res.length) {
|
||||
return {
|
||||
...question,
|
||||
Q: res.join(' '),
|
||||
data: {
|
||||
...question.data,
|
||||
type: 'simple',
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return question
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Subject
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// QuestionDB
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
function doSearch(
|
||||
data: Array<Subject>,
|
||||
subjName: string,
|
||||
question: Question,
|
||||
searchTillMatchPercent?: number,
|
||||
searchInAllIfNoResult?: Boolean
|
||||
): SearchResultQuestion[] {
|
||||
let result: SearchResultQuestion[] = []
|
||||
|
||||
const questionToSearch = prepareQuestion(question)
|
||||
|
||||
data.every((subj) => {
|
||||
if (
|
||||
subjName
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
||||
) {
|
||||
logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2)
|
||||
const subjRes = searchSubject(
|
||||
subj,
|
||||
questionToSearch,
|
||||
subjName,
|
||||
searchTillMatchPercent
|
||||
)
|
||||
result = result.concat(subjRes)
|
||||
if (searchTillMatchPercent) {
|
||||
return !subjRes.some((sr) => {
|
||||
return sr.match >= searchTillMatchPercent
|
||||
})
|
||||
}
|
||||
return true
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if (searchInAllIfNoResult) {
|
||||
// FIXME: dont research subject searched above
|
||||
if (
|
||||
result.length === 0 ||
|
||||
result[0].match < minMatchToNotSearchOtherSubjects
|
||||
) {
|
||||
logger.DebugLog(
|
||||
'Reqults length is zero when comparing names, trying all subjects',
|
||||
'searchworker',
|
||||
1
|
||||
)
|
||||
data.every((subj) => {
|
||||
const subjRes = searchSubject(
|
||||
subj,
|
||||
questionToSearch,
|
||||
subjName,
|
||||
searchTillMatchPercent
|
||||
)
|
||||
result = result.concat(subjRes)
|
||||
|
||||
if (searchTillMatchPercent) {
|
||||
const continueSearching = !subjRes.some((sr) => {
|
||||
return sr.match >= searchTillMatchPercent
|
||||
})
|
||||
return continueSearching
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
result = setNoPossibleAnswersPenalties(
|
||||
questionToSearch.data.possibleAnswers,
|
||||
result
|
||||
)
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function setNoPossibleAnswersPenalties(
|
||||
questionPossibleAnswers: QuestionData['possibleAnswers'],
|
||||
results: SearchResultQuestion[]
|
||||
): SearchResultQuestion[] {
|
||||
if (!Array.isArray(questionPossibleAnswers)) {
|
||||
return results
|
||||
}
|
||||
const noneHasPossibleAnswers = results.every((x) => {
|
||||
return !Array.isArray(x.q.data.possibleAnswers)
|
||||
})
|
||||
if (noneHasPossibleAnswers) return results
|
||||
|
||||
let possibleAnswerMatch = false
|
||||
const updated = results.map((result) => {
|
||||
const matchCount = Array.isArray(result.q.data.possibleAnswers)
|
||||
? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => {
|
||||
return questionPossibleAnswers.some(
|
||||
(questionPossibleAnswer) => {
|
||||
if (
|
||||
questionPossibleAnswer.val &&
|
||||
resultPossibleAnswer.val
|
||||
) {
|
||||
return questionPossibleAnswer.val.includes(
|
||||
resultPossibleAnswer.val
|
||||
)
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
)
|
||||
}).length
|
||||
: 0
|
||||
|
||||
if (matchCount === questionPossibleAnswers.length) {
|
||||
possibleAnswerMatch = true
|
||||
return result
|
||||
} else {
|
||||
return {
|
||||
...result,
|
||||
match: result.match - noPossibleAnswerMatchPenalty,
|
||||
detailedMatch: {
|
||||
...result.detailedMatch,
|
||||
qMatch:
|
||||
result.detailedMatch.qMatch -
|
||||
noPossibleAnswerMatchPenalty,
|
||||
},
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
if (possibleAnswerMatch) {
|
||||
return updated
|
||||
} else {
|
||||
return results
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Multi threaded stuff
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
interface WorkData {
|
||||
subjName: string
|
||||
question: Question
|
||||
searchTillMatchPercent: number
|
||||
searchInAllIfNoResult: boolean
|
||||
searchIn: number[]
|
||||
index: number
|
||||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
handleWorkerData()
|
||||
}
|
||||
|
||||
function handleWorkerData() {
|
||||
const {
|
||||
workerIndex,
|
||||
initData,
|
||||
}: { workerIndex: number; initData: Array<QuestionDb> } = workerData
|
||||
let qdbs: Array<QuestionDb> = initData
|
||||
|
||||
const qdbCount = initData.length
|
||||
const { subjCount, questionCount } = countOfQdbs(initData)
|
||||
|
||||
logger.Log(
|
||||
`[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty! qdbs: ${qdbCount}, subjects: ${subjCount.toLocaleString()}, questions: ${questionCount.toLocaleString()}`
|
||||
)
|
||||
|
||||
parentPort.on('message', async (msg /*: TaskObject */) => {
|
||||
try {
|
||||
await tesseractLoaded
|
||||
|
||||
if (msg.type === 'work') {
|
||||
const {
|
||||
subjName,
|
||||
question: originalQuestion,
|
||||
searchTillMatchPercent,
|
||||
searchInAllIfNoResult,
|
||||
searchIn,
|
||||
index,
|
||||
}: WorkData = msg.data
|
||||
|
||||
let searchResult: SearchResultQuestion[] = []
|
||||
let error = false
|
||||
|
||||
const question = await recognizeQuestionImage(originalQuestion)
|
||||
|
||||
try {
|
||||
qdbs.forEach((qdb) => {
|
||||
if (searchIn.includes(qdb.index)) {
|
||||
const res = doSearch(
|
||||
qdb.data,
|
||||
subjName,
|
||||
question,
|
||||
searchTillMatchPercent,
|
||||
searchInAllIfNoResult
|
||||
)
|
||||
searchResult = [
|
||||
...searchResult,
|
||||
...res.map((x) => {
|
||||
return {
|
||||
...x,
|
||||
detailedMatch: {
|
||||
...x.detailedMatch,
|
||||
qdb: qdb.name,
|
||||
},
|
||||
}
|
||||
}),
|
||||
]
|
||||
}
|
||||
})
|
||||
} catch (err) {
|
||||
logger.Log(
|
||||
'Error in worker thread!',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
console.error(err)
|
||||
console.error(
|
||||
JSON.stringify(
|
||||
{
|
||||
subjName: subjName,
|
||||
question: question,
|
||||
searchTillMatchPercent: searchTillMatchPercent,
|
||||
searchInAllIfNoResult: searchInAllIfNoResult,
|
||||
searchIn: searchIn,
|
||||
index: index,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
)
|
||||
error = true
|
||||
}
|
||||
|
||||
// sorting
|
||||
const sortedResult: SearchResultQuestion[] = searchResult.sort(
|
||||
(q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const workerResult: WorkerResult = {
|
||||
msg: `From thread #${workerIndex}: job ${
|
||||
!isNaN(index) ? `#${index}` : ''
|
||||
}done`,
|
||||
workerIndex: workerIndex,
|
||||
result: sortedResult,
|
||||
error: error,
|
||||
}
|
||||
|
||||
// ONDONE:
|
||||
parentPort.postMessage(workerResult)
|
||||
|
||||
// console.log(
|
||||
// `[THREAD #${workerIndex}]: Work ${
|
||||
// !isNaN(index) ? `#${index}` : ''
|
||||
// }done!`
|
||||
// )
|
||||
} else if (msg.type === 'merge') {
|
||||
const {
|
||||
localQdbIndex,
|
||||
remoteQdb,
|
||||
}: { localQdbIndex: number; remoteQdb: QuestionDb } = msg.data
|
||||
const localQdb = qdbs.find((qdb) => qdb.index === localQdbIndex)
|
||||
|
||||
const { newData, newSubjects } = getSubjectDifference(
|
||||
localQdb.data,
|
||||
remoteQdb.data
|
||||
)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: merge done`,
|
||||
workerIndex: workerIndex,
|
||||
newData: newData,
|
||||
newSubjects: newSubjects,
|
||||
localQdbIndex: localQdbIndex,
|
||||
})
|
||||
} else if (msg.type === 'dbEdit') {
|
||||
const { dbIndex, edits }: { dbIndex: number; edits: Edits } =
|
||||
msg.data
|
||||
const { resultDb } = editDb(qdbs[dbIndex], edits)
|
||||
qdbs[dbIndex] = resultDb
|
||||
logger.DebugLog(
|
||||
`Worker db edit ${workerIndex}`,
|
||||
'worker update',
|
||||
1
|
||||
)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: db edit`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
} else if (msg.type === 'newQuestions') {
|
||||
const {
|
||||
subjName,
|
||||
qdbIndex,
|
||||
newQuestions,
|
||||
}: {
|
||||
subjName: string
|
||||
qdbIndex: number
|
||||
newQuestions: Question[]
|
||||
} = msg.data
|
||||
|
||||
const newQuestionsWithCache = newQuestions.map((question) => {
|
||||
if (!question.cache) {
|
||||
return createQuestion(question)
|
||||
} else {
|
||||
return question
|
||||
}
|
||||
})
|
||||
|
||||
let added = false
|
||||
qdbs = qdbs.map((qdb) => {
|
||||
if (qdb.index === qdbIndex) {
|
||||
return {
|
||||
...qdb,
|
||||
data: qdb.data.map((subj) => {
|
||||
if (subj.Name === subjName) {
|
||||
added = true
|
||||
return {
|
||||
Name: subj.Name,
|
||||
Questions: [
|
||||
...subj.Questions,
|
||||
...newQuestionsWithCache,
|
||||
],
|
||||
}
|
||||
} else {
|
||||
return subj
|
||||
}
|
||||
}),
|
||||
}
|
||||
} else {
|
||||
return qdb
|
||||
}
|
||||
})
|
||||
|
||||
if (!added) {
|
||||
qdbs = qdbs.map((qdb) => {
|
||||
if (qdb.index === qdbIndex) {
|
||||
return {
|
||||
...qdb,
|
||||
data: [
|
||||
...qdb.data,
|
||||
{
|
||||
Name: subjName,
|
||||
Questions: [...newQuestionsWithCache],
|
||||
},
|
||||
],
|
||||
}
|
||||
} else {
|
||||
return qdb
|
||||
}
|
||||
})
|
||||
}
|
||||
logger.DebugLog(
|
||||
`Worker new question ${workerIndex}`,
|
||||
'worker update',
|
||||
1
|
||||
)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: update done`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
|
||||
// console.log(`[THREAD #${workerIndex}]: update`)
|
||||
} else if (msg.type === 'newdb') {
|
||||
const { data }: { data: QuestionDb } = msg
|
||||
qdbs.push(data)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: new db add done`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
// console.log(`[THREAD #${workerIndex}]: newdb`)
|
||||
} else if (msg.type === 'dbClean') {
|
||||
const removedIndexes = cleanDb(msg.data, qdbs)
|
||||
|
||||
const workerResult: WorkerResult = {
|
||||
msg: `From thread #${workerIndex}: db clean done`,
|
||||
workerIndex: workerIndex,
|
||||
result: removedIndexes,
|
||||
}
|
||||
|
||||
parentPort.postMessage(workerResult)
|
||||
} else if (msg.type === 'rmQuestions') {
|
||||
const {
|
||||
questionIndexesToRemove,
|
||||
subjIndex,
|
||||
qdbIndex,
|
||||
recievedQuestions,
|
||||
} = msg.data
|
||||
|
||||
qdbs[qdbIndex].data[subjIndex].Questions =
|
||||
updateQuestionsInArray(
|
||||
questionIndexesToRemove,
|
||||
qdbs[qdbIndex].data[subjIndex].Questions,
|
||||
recievedQuestions
|
||||
)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: rm question done`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
} else {
|
||||
logger.Log(`Invalid msg type!`, logger.GetColor('redbg'))
|
||||
console.error(msg)
|
||||
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`,
|
||||
workerIndex: workerIndex,
|
||||
})
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
parentPort.postMessage({
|
||||
msg: `From thread #${workerIndex}: unhandled error occured!`,
|
||||
workerIndex: workerIndex,
|
||||
e: e,
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
export { doSearch, setNoPossibleAnswersPenalties }
|
||||
@@ -23,7 +23,6 @@ export function get<T = any>(
|
||||
try {
|
||||
resolve({ data: JSON.parse(body) })
|
||||
} catch (e) {
|
||||
console.log(body)
|
||||
resolve({ error: e, options: options })
|
||||
}
|
||||
})
|
||||
@@ -46,6 +45,7 @@ interface PostParams {
|
||||
port: number
|
||||
bodyObject: any
|
||||
http?: boolean
|
||||
cookies?: string
|
||||
}
|
||||
|
||||
// https://nodejs.org/api/http.html#httprequesturl-options-callback
|
||||
@@ -55,6 +55,7 @@ export function post<T = any>({
|
||||
port,
|
||||
bodyObject,
|
||||
http,
|
||||
cookies,
|
||||
}: PostParams): Promise<PostResult<T>> {
|
||||
const provider = http ? httpRequest : httpsRequest
|
||||
const body = JSON.stringify(bodyObject)
|
||||
@@ -69,6 +70,11 @@ export function post<T = any>({
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Content-Length': Buffer.byteLength(body),
|
||||
...(cookies
|
||||
? {
|
||||
cookie: cookies,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
},
|
||||
(res) => {
|
||||
@@ -85,7 +91,6 @@ export function post<T = any>({
|
||||
cookies: res.headers['set-cookie'],
|
||||
})
|
||||
} catch (e) {
|
||||
console.log(body)
|
||||
resolve({ error: e })
|
||||
}
|
||||
})
|
||||
@@ -100,3 +105,20 @@ export function post<T = any>({
|
||||
req.end()
|
||||
})
|
||||
}
|
||||
|
||||
export function parseCookies(responseCookies: string[]): {
|
||||
[key: string]: string
|
||||
} {
|
||||
const cookiesArray = responseCookies.join('; ').split('; ')
|
||||
const parsedCookies: { [key: string]: string } = cookiesArray.reduce(
|
||||
(acc, cookieString) => {
|
||||
const [key, val] = cookieString.split('=')
|
||||
return {
|
||||
...acc,
|
||||
[key]: val || true,
|
||||
}
|
||||
},
|
||||
{}
|
||||
)
|
||||
return parsedCookies
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { PeerInfo } from '../types/basicTypes'
|
||||
import { paths } from './files'
|
||||
import { parseCookies, post } from './networkUtils'
|
||||
import utils from './utils'
|
||||
|
||||
export function peerToString(peer: {
|
||||
host: string
|
||||
port: string | number
|
||||
}): string {
|
||||
return `${peer.host}:${peer.port}`
|
||||
}
|
||||
|
||||
export function isPeerSameAs(peer1: PeerInfo, peer2: PeerInfo): boolean {
|
||||
return peer1.host === peer2.host && peer1.port === peer2.port
|
||||
}
|
||||
|
||||
export function updatePeersFile(
|
||||
peers: PeerInfo[],
|
||||
updatedPeer: PeerInfo
|
||||
): void {
|
||||
const updatedPeers = peers.map((x) => {
|
||||
if (isPeerSameAs(updatedPeer, x)) {
|
||||
return {
|
||||
...x,
|
||||
...updatedPeer,
|
||||
}
|
||||
} else {
|
||||
return x
|
||||
}
|
||||
})
|
||||
|
||||
utils.WriteFile(JSON.stringify(updatedPeers, null, 2), paths.peersFile)
|
||||
}
|
||||
|
||||
export async function loginToPeer(peer: PeerInfo): Promise<string | Error> {
|
||||
const { data, error, cookies } = await post<{
|
||||
result: string
|
||||
msg: string
|
||||
}>({
|
||||
hostname: peer.host,
|
||||
path: '/api/login',
|
||||
port: peer.port,
|
||||
bodyObject: { pw: peer.pw },
|
||||
http: peer.http,
|
||||
})
|
||||
|
||||
if (error || !data || data.result !== 'success') {
|
||||
return data ? new Error(data.msg) : error
|
||||
}
|
||||
|
||||
const parsedCookies = parseCookies(cookies)
|
||||
return parsedCookies.sessionID
|
||||
}
|
||||
@@ -1,346 +0,0 @@
|
||||
/* ----------------------------------------------------------------------------
|
||||
|
||||
Question Server
|
||||
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
------------------------------------------------------------------------- */
|
||||
|
||||
import { Worker } from 'worker_threads'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { EventEmitter } from 'events'
|
||||
import os from 'os'
|
||||
|
||||
import logger from './logger'
|
||||
import { Result, Edits } from './actions'
|
||||
import type { Question, QuestionDb, QuestionData } from '../types/basicTypes'
|
||||
import type { WorkerResult } from './classes'
|
||||
|
||||
const threadCount = +process.env.NS_THREAD_COUNT || os.cpus().length
|
||||
|
||||
interface WorkerObj {
|
||||
worker: Worker
|
||||
index: number
|
||||
free: Boolean
|
||||
}
|
||||
|
||||
// FIXME: type depending on type
|
||||
export interface TaskObject {
|
||||
type:
|
||||
| 'work'
|
||||
| 'dbEdit'
|
||||
| 'newQuestions'
|
||||
| 'newdb'
|
||||
| 'dbClean'
|
||||
| 'rmQuestions'
|
||||
| 'merge'
|
||||
data:
|
||||
| {
|
||||
searchIn: number[]
|
||||
question: Question
|
||||
subjName: string
|
||||
testUrl?: string
|
||||
questionData?: QuestionData
|
||||
searchInAllIfNoResult?: boolean
|
||||
searchTillMatchPercent?: number
|
||||
[key: string]: any
|
||||
}
|
||||
| { dbIndex: number; edits: Edits }
|
||||
| QuestionDb
|
||||
| Omit<Result, 'qdbName'>
|
||||
| {
|
||||
questions: Question[]
|
||||
subjToClean: string
|
||||
overwriteBeforeDate: number
|
||||
qdbIndex: number
|
||||
}
|
||||
| {
|
||||
questionIndexesToRemove: number[][]
|
||||
subjIndex: number
|
||||
qdbIndex: number
|
||||
recievedQuestions: Question[]
|
||||
}
|
||||
| {
|
||||
localQdbIndex: number
|
||||
remoteQdb: QuestionDb
|
||||
}
|
||||
}
|
||||
|
||||
interface PendingJob {
|
||||
workData: TaskObject
|
||||
doneEvent: DoneEvent
|
||||
targetWorkerIndex?: number
|
||||
}
|
||||
|
||||
interface JobEvent extends EventEmitter {
|
||||
on(event: 'jobDone', listener: () => void): this
|
||||
on(event: 'newJob', listener: () => void): this
|
||||
emit(event: 'newJob'): boolean
|
||||
emit(event: 'jobDone'): boolean
|
||||
}
|
||||
|
||||
interface DoneEvent extends EventEmitter {
|
||||
once(event: 'done', listener: (result: WorkerResult) => void): this
|
||||
emit(event: 'done', res: WorkerResult): boolean
|
||||
}
|
||||
|
||||
export const defaultAlertOnPendingCount = 100
|
||||
let alertOnPendingCount = defaultAlertOnPendingCount
|
||||
const workerFile = './src/utils/classes.ts'
|
||||
let workers: Array<WorkerObj>
|
||||
let getInitData: () => Array<QuestionDb> = null
|
||||
const pendingJobs: {
|
||||
[id: string]: PendingJob
|
||||
} = {}
|
||||
|
||||
const jobEvents: JobEvent = new EventEmitter()
|
||||
|
||||
jobEvents.on('jobDone', () => {
|
||||
processJob()
|
||||
})
|
||||
|
||||
jobEvents.on('newJob', () => {
|
||||
processJob()
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function handleWorkerError(worker: WorkerObj, err: Error) {
|
||||
// TODO: restart worker if exited or things like that
|
||||
logger.Log('resourcePromise error', logger.GetColor('redbg'))
|
||||
console.error(err, worker)
|
||||
}
|
||||
|
||||
// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
|
||||
export function msgAllWorker(data: TaskObject): Promise<WorkerResult[]> {
|
||||
logger.DebugLog('MSGING ALL WORKER', 'job', 1)
|
||||
return new Promise((resolve) => {
|
||||
const promises: Promise<WorkerResult>[] = []
|
||||
workers.forEach((worker) => {
|
||||
promises.push(doALongTask(data, worker.index))
|
||||
})
|
||||
Promise.all(promises).then((res) => {
|
||||
logger.DebugLog('MSGING ALL WORKER DONE', 'job', 1)
|
||||
resolve(res)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export function setPendingJobsAlertCount(newVal?: number): void {
|
||||
const count = newVal != null ? newVal : defaultAlertOnPendingCount
|
||||
logger.DebugLog('setPendingJobsAlertCount: ' + count, 'job', 1)
|
||||
alertOnPendingCount = count
|
||||
}
|
||||
|
||||
export function doALongTask(
|
||||
obj: TaskObject,
|
||||
targetWorkerIndex?: number
|
||||
): Promise<WorkerResult> {
|
||||
if (Object.keys(pendingJobs).length > alertOnPendingCount) {
|
||||
console.error(
|
||||
`More than ${alertOnPendingCount} callers waiting for free resource! (${
|
||||
Object.keys(pendingJobs).length
|
||||
})`
|
||||
)
|
||||
}
|
||||
|
||||
const jobId = uuidv4()
|
||||
// FIXME: delete doneEvent?
|
||||
const doneEvent: DoneEvent = new EventEmitter()
|
||||
pendingJobs[jobId] = {
|
||||
workData: obj,
|
||||
targetWorkerIndex: targetWorkerIndex,
|
||||
doneEvent: doneEvent,
|
||||
}
|
||||
jobEvents.emit('newJob')
|
||||
return new Promise((resolve) => {
|
||||
doneEvent.once('done', (result: WorkerResult) => {
|
||||
jobEvents.emit('jobDone')
|
||||
resolve(result)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
export function initWorkerPool(
|
||||
initDataGetter: () => Array<QuestionDb>
|
||||
): Array<WorkerObj> {
|
||||
getInitData = initDataGetter
|
||||
if (workers) {
|
||||
logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
|
||||
return null
|
||||
}
|
||||
workers = []
|
||||
|
||||
if (process.env.NS_THREAD_COUNT) {
|
||||
logger.Log(
|
||||
`Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
|
||||
'yellowbg'
|
||||
)
|
||||
}
|
||||
|
||||
for (let i = 0; i < threadCount; i++) {
|
||||
workers.push({
|
||||
worker: getAWorker(i, getInitData()),
|
||||
index: i,
|
||||
free: true,
|
||||
})
|
||||
}
|
||||
|
||||
return workers
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function processJob() {
|
||||
if (Object.keys(pendingJobs).length > 0) {
|
||||
const keys = Object.keys(pendingJobs)
|
||||
let jobKey: string, freeWorker: WorkerObj
|
||||
let i = 0
|
||||
while (!freeWorker && i < keys.length) {
|
||||
jobKey = keys[i]
|
||||
if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
|
||||
if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
|
||||
freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
|
||||
logger.DebugLog(
|
||||
`RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`,
|
||||
'job',
|
||||
1
|
||||
)
|
||||
}
|
||||
} else {
|
||||
freeWorker = workers.find((worker) => {
|
||||
return worker.free
|
||||
})
|
||||
if (freeWorker) {
|
||||
logger.DebugLog(
|
||||
`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`,
|
||||
'job',
|
||||
1
|
||||
)
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
if (!freeWorker) {
|
||||
logger.DebugLog('NO FREE WORKER', 'job', 1)
|
||||
return
|
||||
}
|
||||
|
||||
if (freeWorker.free) {
|
||||
freeWorker.free = false
|
||||
}
|
||||
const job = pendingJobs[jobKey]
|
||||
delete pendingJobs[jobKey]
|
||||
|
||||
doSomething(freeWorker, job.workData)
|
||||
.then((res: WorkerResult) => {
|
||||
freeWorker.free = true
|
||||
job.doneEvent.emit('done', res)
|
||||
})
|
||||
.catch(function (err) {
|
||||
handleWorkerError(freeWorker, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function getAWorker(workerIndex: number, initData: Array<QuestionDb>) {
|
||||
const worker = workerTs(workerFile, {
|
||||
workerData: {
|
||||
workerIndex: workerIndex,
|
||||
initData: initData,
|
||||
},
|
||||
})
|
||||
|
||||
worker.setMaxListeners(50)
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (exitCode) => {
|
||||
handleWorkerExit(workerIndex, exitCode)
|
||||
})
|
||||
return worker
|
||||
}
|
||||
|
||||
function handleWorkerExit(exitedWorkerIndex: number, exitCode: number) {
|
||||
logger.Log(
|
||||
`[THREAD #${exitedWorkerIndex}]: exit code: ${exitCode}`,
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
|
||||
const exitedWorker = workers.find((worker) => {
|
||||
return worker.index === exitedWorkerIndex
|
||||
})
|
||||
try {
|
||||
exitedWorker.worker.removeAllListeners()
|
||||
exitedWorker.worker.terminate()
|
||||
} catch (e) {
|
||||
console.log(e)
|
||||
}
|
||||
|
||||
workers = workers.filter((worker) => {
|
||||
return worker.index !== exitedWorkerIndex
|
||||
})
|
||||
|
||||
if (workers.length < threadCount) {
|
||||
logger.Log(`[THREAD #${exitedWorkerIndex}]: Restarting ... `)
|
||||
|
||||
workers.push({
|
||||
worker: getAWorker(exitedWorkerIndex, getInitData()),
|
||||
index: exitedWorkerIndex,
|
||||
free: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function doSomething(currWorker: WorkerObj, obj: TaskObject) {
|
||||
const { /* index, */ worker } = currWorker
|
||||
return new Promise((resolve) => {
|
||||
worker.postMessage(obj)
|
||||
worker.once('message', (msg: WorkerResult) => {
|
||||
resolve(msg)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
const workerTs = (
|
||||
file: string,
|
||||
wkOpts: {
|
||||
workerData: {
|
||||
workerIndex: number
|
||||
initData: QuestionDb[]
|
||||
__filename?: string
|
||||
}
|
||||
eval?: boolean
|
||||
}
|
||||
) => {
|
||||
wkOpts.eval = true
|
||||
wkOpts.workerData.__filename = file
|
||||
return new Worker(
|
||||
`
|
||||
const wk = require('worker_threads');
|
||||
require('ts-node').register();
|
||||
let file = wk.workerData.__filename;
|
||||
delete wk.workerData.__filename;
|
||||
require(file);
|
||||
`,
|
||||
wkOpts
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user