mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Problems converting process incoming request to use new searchData worker
This commit is contained in:
mrfry
parent
c846d2591e
commit
63b08244dc
4 changed files with 233 additions and 228 deletions
|
|
@ -1,14 +1,12 @@
|
|||
var debugLogger = null
|
||||
const {
|
||||
Worker,
|
||||
isMainThread,
|
||||
parentPort,
|
||||
workerData,
|
||||
} = require('worker_threads')
|
||||
const searchDataWorkerFile = './src/utils/searchData.js'
|
||||
|
||||
function initLogger(logger) {
|
||||
debugLogger = logger
|
||||
}
|
||||
|
||||
function debugLog(msg, name, lvl) {
|
||||
if (debugLogger) {
|
||||
debugLogger(msg, name, lvl)
|
||||
}
|
||||
}
|
||||
const logger = require('../../utils/logger.js')
|
||||
|
||||
const commonUselessAnswerParts = [
|
||||
'A helyes válasz az ',
|
||||
|
|
@ -124,6 +122,8 @@ function answerPreProcessor(value) {
|
|||
function removeAnswerLetters(value) {
|
||||
assert(value)
|
||||
|
||||
console.log(value)
|
||||
|
||||
let val = value.split('. ')
|
||||
if (val[0].length < 2 && val.length > 1) {
|
||||
val.shift()
|
||||
|
|
@ -138,36 +138,31 @@ function simplifyQA(value, mods) {
|
|||
return
|
||||
}
|
||||
|
||||
const reducer = (res, fn) => {
|
||||
return mods.reduce((res, fn) => {
|
||||
return fn(res)
|
||||
}
|
||||
|
||||
return mods.reduce(reducer, value)
|
||||
}, value)
|
||||
}
|
||||
|
||||
function simplifyAnswer(value) {
|
||||
return simplifyQA(value, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
answerPreProcessor.bind(this),
|
||||
removeAnswerLetters.bind(this),
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
answerPreProcessor,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
|
||||
function simplifyQuestion(question) {
|
||||
if (typeof q === 'string') {
|
||||
return simplifyQA(question, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
])
|
||||
return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces])
|
||||
} else {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
])
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
])
|
||||
return question
|
||||
}
|
||||
|
|
@ -198,16 +193,20 @@ function compareData(q1, q2) {
|
|||
} else if (dataType === 'image') {
|
||||
return compareImage(q1.data, q2.data)
|
||||
} else {
|
||||
debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1)
|
||||
debugLog(q1, 'Compare question data', 2)
|
||||
logger.DebugLog(
|
||||
`Unhandled data type ${dataType}`,
|
||||
'Compare question data',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(q1, 'Compare question data', 2)
|
||||
}
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
} catch (error) {
|
||||
debugLog('Error comparing data', 'Compare question data', 1)
|
||||
debugLog(error.message, 'Compare question data', 1)
|
||||
debugLog(error, 'Compare question data', 2)
|
||||
logger.DebugLog('Error comparing data', 'Compare question data', 1)
|
||||
logger.DebugLog(error.message, 'Compare question data', 1)
|
||||
logger.DebugLog(error, 'Compare question data', 2)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -327,8 +326,8 @@ function subjectToString(subj) {
|
|||
// QuestionDB
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
function addQuestion(data, subj, question) {
|
||||
debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
debugLog(question, 'qdb add', 3)
|
||||
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
logger.DebugLog(question, 'qdb add', 3)
|
||||
assert(data)
|
||||
assert(subj)
|
||||
assert(question)
|
||||
|
|
@ -345,10 +344,10 @@ function addQuestion(data, subj, question) {
|
|||
}
|
||||
|
||||
if (i < data.length) {
|
||||
debugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
data[i].Questions.push(question)
|
||||
} else {
|
||||
debugLog('Creating new subject for question', 'qdb add', 1)
|
||||
logger.DebugLog('Creating new subject for question', 'qdb add', 1)
|
||||
data.push({
|
||||
Name: subj,
|
||||
Questions: [question],
|
||||
|
|
@ -357,72 +356,58 @@ function addQuestion(data, subj, question) {
|
|||
}
|
||||
|
||||
function searchData(data, question, subjName, questionData) {
|
||||
assert(data)
|
||||
assert(question)
|
||||
debugLog('Searching for question', 'qdb search', 1)
|
||||
debugLog('Question:', 'qdb search', 2)
|
||||
debugLog(question, 'qdb search', 2)
|
||||
debugLog(`Subject name: ${subjName}`, 'qdb search', 2)
|
||||
debugLog('Data:', 'qdb search', 2)
|
||||
debugLog(questionData || question.data, 'qdb search', 2)
|
||||
return new Promise((resolve, reject) => {
|
||||
assert(data)
|
||||
assert(question)
|
||||
logger.DebugLog('Searching for question', 'qdb search', 1)
|
||||
logger.DebugLog('Question:', 'qdb search', 2)
|
||||
logger.DebugLog(question, 'qdb search', 2)
|
||||
logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2)
|
||||
logger.DebugLog('Data:', 'qdb search', 2)
|
||||
logger.DebugLog(questionData || question.data, 'qdb search', 2)
|
||||
|
||||
if (!questionData) {
|
||||
questionData = question.data || { type: 'simple' }
|
||||
}
|
||||
if (!subjName) {
|
||||
subjName = ''
|
||||
debugLog('No subject name as param!', 'qdb search', 1)
|
||||
}
|
||||
question = simplifyQuestion(question)
|
||||
|
||||
let result = []
|
||||
data.forEach((subj) => {
|
||||
if (
|
||||
subjName
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
||||
) {
|
||||
debugLog(`Searching in ${subj.Name} `, 2)
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
if (!questionData) {
|
||||
questionData = question.data || { type: 'simple' }
|
||||
}
|
||||
})
|
||||
if (!subjName) {
|
||||
subjName = ''
|
||||
logger.DebugLog('No subject name as param!', 'qdb search', 1)
|
||||
}
|
||||
question = simplifyQuestion(question)
|
||||
|
||||
// FIXME: try to remove this? but this is also a good backup plan so idk
|
||||
if (result.length === 0) {
|
||||
debugLog(
|
||||
'Reqults length is zero when comparing names, trying all subjects',
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
data.forEach((subj) => {
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
const worker = new Worker(searchDataWorkerFile, {
|
||||
workerData: { data, subjName, question, questionData },
|
||||
})
|
||||
if (result.length > 0) {
|
||||
debugLog(
|
||||
`FIXME: '${subjName}' gave no result but '' did!`,
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
|
||||
if (code !== 0) {
|
||||
logger.Log(
|
||||
'Search Data Worker error! Exit code is not 0',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
reject(new Error('Search Data Worker error! Exit code is not 0'))
|
||||
}
|
||||
})
|
||||
|
||||
worker.on('message', (result) => {
|
||||
logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
|
||||
logger.DebugLog(result, 'ask', 2)
|
||||
|
||||
logger.DebugLog(
|
||||
`QDB search result length: ${result.length}`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
console.error(`FIXME: '${subjName}' gave no result but '' did!`)
|
||||
}
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
resolve(result)
|
||||
})
|
||||
})
|
||||
|
||||
debugLog(`QDB search result length: ${result.length}`, 'qdb search', 1)
|
||||
return result
|
||||
}
|
||||
|
||||
function addSubject(data, subj) {
|
||||
|
|
@ -458,9 +443,60 @@ function dataToString(data) {
|
|||
return result.join('\n\n')
|
||||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
const { data, subjName, question, questionData } = workerData
|
||||
let result = []
|
||||
|
||||
data.forEach((subj) => {
|
||||
if (
|
||||
subjName
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
||||
) {
|
||||
logger.DebugLog(`Searching in ${subj.Name} `, 2)
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
// FIXME: try to remove this? but this is also a good backup plan so idk
|
||||
if (result.length === 0) {
|
||||
logger.DebugLog(
|
||||
'Reqults length is zero when comparing names, trying all subjects',
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
data.forEach((subj) => {
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
})
|
||||
if (result.length > 0) {
|
||||
logger.DebugLog(
|
||||
`FIXME: '${subjName}' gave no result but '' did!`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
console.error(`FIXME: '${subjName}' gave no result but '' did!`)
|
||||
}
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
parentPort.postMessage(result)
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
minMatchAmmount,
|
||||
initLogger,
|
||||
getSubjNameWithoutYear,
|
||||
createQuestion,
|
||||
addQuestion,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue