mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Problems converting process incoming request to use new searchData worker
This commit is contained in:
mrfry
parent
c846d2591e
commit
63b08244dc
4 changed files with 233 additions and 228 deletions
|
|
@ -22,7 +22,6 @@
|
|||
const express = require('express')
|
||||
const bodyParser = require('body-parser')
|
||||
const busboy = require('connect-busboy')
|
||||
const { Worker } = require('worker_threads')
|
||||
const { v4: uuidv4 } = require('uuid')
|
||||
const fs = require('fs')
|
||||
const app = express()
|
||||
|
|
@ -33,10 +32,9 @@ const utils = require('../../utils/utils.js')
|
|||
const actions = require('../../utils/actions.js')
|
||||
const dbtools = require('../../utils/dbtools.js')
|
||||
const auth = require('../../middlewares/auth.middleware.js')
|
||||
const { dataToString } = require('../../utils/classes.js')
|
||||
const { dataToString, searchData } = require('../../utils/classes.js')
|
||||
|
||||
// files
|
||||
const searchDataWorkerFile = './src/utils/searchData.js'
|
||||
const msgFile = 'stats/msgs'
|
||||
const passwordFile = 'data/dataEditorPasswords.json'
|
||||
const dataEditsLog = 'stats/dataEdits'
|
||||
|
|
@ -763,17 +761,21 @@ function GetApp() {
|
|||
)
|
||||
}
|
||||
|
||||
const worker = new Worker(searchDataWorkerFile, {
|
||||
workerData: {
|
||||
data,
|
||||
question,
|
||||
subj,
|
||||
recData,
|
||||
},
|
||||
searchData(data, question, subj, recData)
|
||||
.then((result) => {
|
||||
res.json({
|
||||
result: result,
|
||||
success: true,
|
||||
})
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
|
||||
logger.DebugLog(
|
||||
`Question result length: ${result.length}`,
|
||||
'ask',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(result, 'ask', 2)
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.Log('Search Data error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
res.json({
|
||||
message: `There was an error processing the question: ${err.message}`,
|
||||
|
|
@ -782,34 +784,6 @@ function GetApp() {
|
|||
success: false,
|
||||
})
|
||||
})
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
|
||||
if (code !== 0) {
|
||||
logger.Log(
|
||||
'Search Data Worker error! Exit code is not 0',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
worker.on('message', (workerMsg) => {
|
||||
const result = workerMsg
|
||||
res.json({
|
||||
result: result,
|
||||
success: true,
|
||||
})
|
||||
logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
|
||||
logger.DebugLog(result, 'ask', 2)
|
||||
})
|
||||
|
||||
// let result = data.Search(question, subj, recData)
|
||||
// res.json({
|
||||
// result: result,
|
||||
// success: true,
|
||||
// })
|
||||
// logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
|
||||
// logger.DebugLog(result, 'ask', 2)
|
||||
} else {
|
||||
logger.DebugLog(`Invalid question`, 'ask', 1)
|
||||
res.json({
|
||||
|
|
|
|||
|
|
@ -24,10 +24,9 @@ module.exports = {
|
|||
|
||||
const dataFile = './publicDirs/qminingPublic/data.json'
|
||||
const recDataFile = './stats/recdata'
|
||||
const processDataWorkerFile = './src/utils/processData.js'
|
||||
|
||||
const { Worker } = require('worker_threads')
|
||||
const logger = require('../utils/logger.js')
|
||||
const { searchData } = require('../utils/classes.js')
|
||||
const idStats = require('../utils/ids.js')
|
||||
const utils = require('../utils/utils.js')
|
||||
const { addQuestion, getSubjNameWithoutYear } = require('./classes.js')
|
||||
|
|
@ -74,35 +73,24 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
|
|||
logger.DebugLog(data, 'actions', 3)
|
||||
let allQLength = data.quiz.length
|
||||
|
||||
const worker = new Worker(processDataWorkerFile, {
|
||||
workerData: {
|
||||
data: data,
|
||||
qdb: qdb,
|
||||
},
|
||||
})
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
logger.DebugLog('Process Data exit, code: ' + code, 'actions', 1)
|
||||
if (code !== 0) {
|
||||
logger.Log(
|
||||
'Process Data Worker error! Exit code is not 0',
|
||||
logger.GetColor('redbg')
|
||||
logger.DebugLog('Question:', 'actions', 2)
|
||||
logger.DebugLog(question, 'actions', 2)
|
||||
let currentQuestion = createQuestion(
|
||||
question.Q,
|
||||
question.A,
|
||||
question.data
|
||||
)
|
||||
reject(new Error('Process Data Worker exit code is not 0!'))
|
||||
}
|
||||
})
|
||||
logger.DebugLog(
|
||||
'Searching for question in subj ' + data.subj,
|
||||
'actions',
|
||||
3
|
||||
)
|
||||
logger.DebugLog(currentQuestion, 'actions', 3)
|
||||
|
||||
worker.on('message', (workerMsg) => {
|
||||
searchData(qdb, currentQuestion, data.subj)
|
||||
.then((allQuestions) => {
|
||||
logger.DebugLog('Message from processData', 'actions', 1)
|
||||
logger.DebugLog(workerMsg, 'actions', 3)
|
||||
|
||||
const allQuestions = workerMsg
|
||||
logger.DebugLog(allQuestions, 'actions', 3)
|
||||
|
||||
try {
|
||||
let color = logger.GetColor('green')
|
||||
|
|
@ -164,9 +152,16 @@ function ProcessIncomingRequest(recievedData, qdb, infos, dryRun, user) {
|
|||
'Error while processing processData worker result!',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
reject(new Error('Error while processing processData worker result!'))
|
||||
reject(
|
||||
new Error('Error while processing processData worker result!')
|
||||
)
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.Log('Process Data Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
reject(err)
|
||||
})
|
||||
} catch (err) {
|
||||
console.log(err)
|
||||
logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
var debugLogger = null
|
||||
const {
|
||||
Worker,
|
||||
isMainThread,
|
||||
parentPort,
|
||||
workerData,
|
||||
} = require('worker_threads')
|
||||
const searchDataWorkerFile = './src/utils/searchData.js'
|
||||
|
||||
function initLogger(logger) {
|
||||
debugLogger = logger
|
||||
}
|
||||
|
||||
function debugLog(msg, name, lvl) {
|
||||
if (debugLogger) {
|
||||
debugLogger(msg, name, lvl)
|
||||
}
|
||||
}
|
||||
const logger = require('../../utils/logger.js')
|
||||
|
||||
const commonUselessAnswerParts = [
|
||||
'A helyes válasz az ',
|
||||
|
|
@ -124,6 +122,8 @@ function answerPreProcessor(value) {
|
|||
function removeAnswerLetters(value) {
|
||||
assert(value)
|
||||
|
||||
console.log(value)
|
||||
|
||||
let val = value.split('. ')
|
||||
if (val[0].length < 2 && val.length > 1) {
|
||||
val.shift()
|
||||
|
|
@ -138,36 +138,31 @@ function simplifyQA(value, mods) {
|
|||
return
|
||||
}
|
||||
|
||||
const reducer = (res, fn) => {
|
||||
return mods.reduce((res, fn) => {
|
||||
return fn(res)
|
||||
}
|
||||
|
||||
return mods.reduce(reducer, value)
|
||||
}, value)
|
||||
}
|
||||
|
||||
function simplifyAnswer(value) {
|
||||
return simplifyQA(value, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
answerPreProcessor.bind(this),
|
||||
removeAnswerLetters.bind(this),
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
answerPreProcessor,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
|
||||
function simplifyQuestion(question) {
|
||||
if (typeof q === 'string') {
|
||||
return simplifyQA(question, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
])
|
||||
return simplifyQA(question, [removeSpecialChars, removeUnnecesarySpaces])
|
||||
} else {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
])
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeSpecialChars.bind(this),
|
||||
removeUnnecesarySpaces.bind(this),
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
])
|
||||
return question
|
||||
}
|
||||
|
|
@ -198,16 +193,20 @@ function compareData(q1, q2) {
|
|||
} else if (dataType === 'image') {
|
||||
return compareImage(q1.data, q2.data)
|
||||
} else {
|
||||
debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1)
|
||||
debugLog(q1, 'Compare question data', 2)
|
||||
logger.DebugLog(
|
||||
`Unhandled data type ${dataType}`,
|
||||
'Compare question data',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(q1, 'Compare question data', 2)
|
||||
}
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
} catch (error) {
|
||||
debugLog('Error comparing data', 'Compare question data', 1)
|
||||
debugLog(error.message, 'Compare question data', 1)
|
||||
debugLog(error, 'Compare question data', 2)
|
||||
logger.DebugLog('Error comparing data', 'Compare question data', 1)
|
||||
logger.DebugLog(error.message, 'Compare question data', 1)
|
||||
logger.DebugLog(error, 'Compare question data', 2)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
@ -327,8 +326,8 @@ function subjectToString(subj) {
|
|||
// QuestionDB
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
function addQuestion(data, subj, question) {
|
||||
debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
debugLog(question, 'qdb add', 3)
|
||||
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
logger.DebugLog(question, 'qdb add', 3)
|
||||
assert(data)
|
||||
assert(subj)
|
||||
assert(question)
|
||||
|
|
@ -345,10 +344,10 @@ function addQuestion(data, subj, question) {
|
|||
}
|
||||
|
||||
if (i < data.length) {
|
||||
debugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
data[i].Questions.push(question)
|
||||
} else {
|
||||
debugLog('Creating new subject for question', 'qdb add', 1)
|
||||
logger.DebugLog('Creating new subject for question', 'qdb add', 1)
|
||||
data.push({
|
||||
Name: subj,
|
||||
Questions: [question],
|
||||
|
|
@ -357,72 +356,58 @@ function addQuestion(data, subj, question) {
|
|||
}
|
||||
|
||||
function searchData(data, question, subjName, questionData) {
|
||||
return new Promise((resolve, reject) => {
|
||||
assert(data)
|
||||
assert(question)
|
||||
debugLog('Searching for question', 'qdb search', 1)
|
||||
debugLog('Question:', 'qdb search', 2)
|
||||
debugLog(question, 'qdb search', 2)
|
||||
debugLog(`Subject name: ${subjName}`, 'qdb search', 2)
|
||||
debugLog('Data:', 'qdb search', 2)
|
||||
debugLog(questionData || question.data, 'qdb search', 2)
|
||||
logger.DebugLog('Searching for question', 'qdb search', 1)
|
||||
logger.DebugLog('Question:', 'qdb search', 2)
|
||||
logger.DebugLog(question, 'qdb search', 2)
|
||||
logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2)
|
||||
logger.DebugLog('Data:', 'qdb search', 2)
|
||||
logger.DebugLog(questionData || question.data, 'qdb search', 2)
|
||||
|
||||
if (!questionData) {
|
||||
questionData = question.data || { type: 'simple' }
|
||||
}
|
||||
if (!subjName) {
|
||||
subjName = ''
|
||||
debugLog('No subject name as param!', 'qdb search', 1)
|
||||
logger.DebugLog('No subject name as param!', 'qdb search', 1)
|
||||
}
|
||||
question = simplifyQuestion(question)
|
||||
|
||||
let result = []
|
||||
data.forEach((subj) => {
|
||||
if (
|
||||
subjName
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
||||
) {
|
||||
debugLog(`Searching in ${subj.Name} `, 2)
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
const worker = new Worker(searchDataWorkerFile, {
|
||||
workerData: { data, subjName, question, questionData },
|
||||
})
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
|
||||
if (code !== 0) {
|
||||
logger.Log(
|
||||
'Search Data Worker error! Exit code is not 0',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
reject(new Error('Search Data Worker error! Exit code is not 0'))
|
||||
}
|
||||
})
|
||||
|
||||
// FIXME: try to remove this? but this is also a good backup plan so idk
|
||||
if (result.length === 0) {
|
||||
debugLog(
|
||||
'Reqults length is zero when comparing names, trying all subjects',
|
||||
worker.on('message', (result) => {
|
||||
logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
|
||||
logger.DebugLog(result, 'ask', 2)
|
||||
|
||||
logger.DebugLog(
|
||||
`QDB search result length: ${result.length}`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
data.forEach((subj) => {
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
resolve(result)
|
||||
})
|
||||
if (result.length > 0) {
|
||||
debugLog(
|
||||
`FIXME: '${subjName}' gave no result but '' did!`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
console.error(`FIXME: '${subjName}' gave no result but '' did!`)
|
||||
}
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
debugLog(`QDB search result length: ${result.length}`, 'qdb search', 1)
|
||||
return result
|
||||
}
|
||||
|
||||
function addSubject(data, subj) {
|
||||
|
|
@ -458,9 +443,60 @@ function dataToString(data) {
|
|||
return result.join('\n\n')
|
||||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
const { data, subjName, question, questionData } = workerData
|
||||
let result = []
|
||||
|
||||
data.forEach((subj) => {
|
||||
if (
|
||||
subjName
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
||||
) {
|
||||
logger.DebugLog(`Searching in ${subj.Name} `, 2)
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
// FIXME: try to remove this? but this is also a good backup plan so idk
|
||||
if (result.length === 0) {
|
||||
logger.DebugLog(
|
||||
'Reqults length is zero when comparing names, trying all subjects',
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
data.forEach((subj) => {
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
})
|
||||
if (result.length > 0) {
|
||||
logger.DebugLog(
|
||||
`FIXME: '${subjName}' gave no result but '' did!`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
console.error(`FIXME: '${subjName}' gave no result but '' did!`)
|
||||
}
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
parentPort.postMessage(result)
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
minMatchAmmount,
|
||||
initLogger,
|
||||
getSubjNameWithoutYear,
|
||||
createQuestion,
|
||||
addQuestion,
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 443203848c09e20a812f0e1f664ba9519cf28708
|
||||
Subproject commit f8d4bf2a414d2973582c08bfa8b8b8f19389b2e9
|
||||
Loading…
Add table
Add a link
Reference in a new issue