const { Worker, isMainThread, parentPort, workerData, } = require('worker_threads') const logger = require('./logger.js') const searchDataWorkerFile = './src/utils/classes.js' const assert = (val) => { if (!val) { throw new Error('Assertion failed') } } const commonUselessAnswerParts = [ 'A helyes válasz az ', 'A helyes válasz a ', 'A helyes válaszok: ', 'A helyes válaszok:', 'A helyes válasz: ', 'A helyes válasz:', 'The correct answer is:', "'", ] const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.'] const specialChars = ['&', '\\+'] const lengthDiffMultiplier = 10 /* Percent minus for length difference */ const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */ // --------------------------------------------------------------------------------------------------------- // String Utils // --------------------------------------------------------------------------------------------------------- // Exported // --------------------------------------------------------------------------------------------------------- function getSubjNameWithoutYear(subjName) { let t = subjName.split(' - ') if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { return t[1] || subjName } else { return subjName } } // Not exported // --------------------------------------------------------------------------------------------------------- function removeStuff(value, removableStrings, toReplace) { removableStrings.forEach((removableString) => { var regex = new RegExp(removableString, 'g') value = value.replace(regex, toReplace || '') }) return value } // removes whitespace from begining and and, and replaces multiple spaces with one space function removeUnnecesarySpaces(toremove) { assert(toremove) toremove = normalizeSpaces(toremove) while (toremove.includes(' ')) { toremove = toremove.replace(/ {2}/g, ' ') } return toremove.trim() } // simplifies a string for easier comparison function simplifyStringForComparison(value) { assert(value) value = removeUnnecesarySpaces(value).toLowerCase() return removeStuff(value, commonUselessStringParts) } function removeSpecialChars(value) { assert(value) return removeStuff(value, specialChars, ' ') } // damn nonbreaking space function normalizeSpaces(input) { assert(input) return input.replace(/\s/g, ' ') } function compareString(s1, s2) { if (!s1 || !s2) { if (!s1 && !s2) { return 100 } else { return 0 } } s1 = simplifyStringForComparison(s1).split(' ') s2 = simplifyStringForComparison(s2).split(' ') var match = 0 for (var i = 0; i < s1.length; i++) { if (s2.includes(s1[i])) { match++ } } var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent var lengthDifference = Math.abs(s2.length - s1.length) percent -= lengthDifference * lengthDiffMultiplier if (percent < 0) { percent = 0 } return percent } function answerPreProcessor(value) { assert(value) return removeStuff(value, commonUselessAnswerParts) } // 'a. pécsi sör' -> 'pécsi sör' function removeAnswerLetters(value) { if (!value) { return } let val = value.split('. ') if (val[0].length < 2 && val.length > 1) { val.shift() return val.join(' ') } else { return value } } function simplifyQA(value, mods) { if (!value) { return } return mods.reduce((res, fn) => { return fn(res) }, value) } function simplifyAnswer(value) { return simplifyQA(value, [ removeSpecialChars, removeUnnecesarySpaces, answerPreProcessor, removeAnswerLetters, ]) } function simplifyQuestion(question) { if (!question) { return } if (typeof question === 'string') { return simplifyQA(question, [ removeSpecialChars, removeUnnecesarySpaces, removeAnswerLetters, ]) } else { if (question.Q) { question.Q = simplifyQA(question.Q, [ removeSpecialChars, removeUnnecesarySpaces, removeAnswerLetters, ]) } if (question.A) { question.A = simplifyQA(question.A, [ removeSpecialChars, removeUnnecesarySpaces, removeAnswerLetters, ]) } return question } } // --------------------------------------------------------------------------------------------------------- // Question // --------------------------------------------------------------------------------------------------------- function createQuestion(question, answer, data) { return { Q: simplifyQuestion(question), A: simplifyAnswer(answer), data, } } function compareImage(data, data2) { return compareString(data.images.join(' '), data2.images.join(' ')) } function compareData(q1, q2) { try { if (q1.data.type === q2.data.type) { let dataType = q1.data.type if (dataType === 'simple') { return -1 } else if (dataType === 'image') { return compareImage(q1.data, q2.data) } else { logger.DebugLog( `Unhandled data type ${dataType}`, 'Compare question data', 1 ) logger.DebugLog(q1, 'Compare question data', 2) } } else { return 0 } } catch (error) { logger.DebugLog('Error comparing data', 'Compare question data', 1) logger.DebugLog(error.message, 'Compare question data', 1) logger.DebugLog(error, 'Compare question data', 2) } return 0 } function compareQuestion(q1, q2) { return compareString(q1.Q, q2.Q) } function compareAnswer(q1, q2) { return compareString(q1.A, q2.A) } function compareQuestionObj(q1, q1subjName, q2, q2subjName, data) { assert(data) assert(q1) assert(typeof q1 === 'object') assert(q2) let qObj if (typeof q2 === 'string') { qObj = { Q: q2, data: data, } } else { qObj = q2 } const qMatch = compareQuestion(q1, qObj) const aMatch = compareAnswer(q1, qObj) // -1 if botth questions are simple const dMatch = compareData(q1, qObj) let avg = -1 if (qObj.A) { if (dMatch === -1) { avg = (qMatch + aMatch) / 2 } else { avg = (qMatch + aMatch + dMatch) / 3 } } else { if (dMatch === -1) { avg = qMatch } else { avg = (qMatch + dMatch) / 2 } } return { qMatch: qMatch, aMatch: aMatch, dMatch: dMatch, matchedSubjName: q2subjName, avg: avg, } } function questionToString(question) { const { Q, A, data } = question if (data.type !== 'simple') { return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data) } else { return '?' + Q + '\n!' + A } } // --------------------------------------------------------------------------------------------------------- // Subject // --------------------------------------------------------------------------------------------------------- function searchQuestion(subj, question, questionData, subjName) { assert(question) var result = [] subj.Questions.forEach((currentQuestion) => { let percent = compareQuestionObj( currentQuestion, subjName, question, subj.Name, questionData ) if (percent.avg > minMatchAmmount) { result.push({ q: currentQuestion, match: percent.avg, detailedMatch: percent, }) } }) result = result.sort((q1, q2) => { if (q1.match < q2.match) { return 1 } else if (q1.match > q2.match) { return -1 } else { return 0 } }) return result } function subjectToString(subj) { const { Questions, Name } = subj var result = [] Questions.forEach((question) => { result.push(questionToString(question)) }) return '+' + Name + '\n' + result.join('\n') } // --------------------------------------------------------------------------------------------------------- // QuestionDB // --------------------------------------------------------------------------------------------------------- function addQuestion(data, subj, question) { logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1) logger.DebugLog(question, 'qdb add', 3) assert(data) assert(subj) assert(question) assert(typeof question === 'object') var i = 0 while ( i < data.length && !subj .toLowerCase() .includes(getSubjNameWithoutYear(data[i].Name).toLowerCase()) ) { i++ } if (i < data.length) { logger.DebugLog('Adding new question to existing subject', 'qdb add', 1) data[i].Questions.push(question) } else { logger.DebugLog('Creating new subject for question', 'qdb add', 1) data.push({ Name: subj, Questions: [question], }) } } function searchData(data, question, subjName, questionData) { return new Promise((resolve, reject) => { assert(data) assert(question) logger.DebugLog('Searching for question', 'qdb search', 1) logger.DebugLog('Question:', 'qdb search', 2) logger.DebugLog(question, 'qdb search', 2) logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2) logger.DebugLog('Data:', 'qdb search', 2) logger.DebugLog(questionData || question.data, 'qdb search', 2) if (!questionData) { questionData = question.data || { type: 'simple' } } if (!subjName) { subjName = '' logger.DebugLog('No subject name as param!', 'qdb search', 1) } question = simplifyQuestion(question) const worker = new Worker(searchDataWorkerFile, { workerData: { data, subjName, question, questionData }, }) worker.on('error', (err) => { logger.Log('Search Data Worker error!', logger.GetColor('redbg')) console.error(err) reject(err) }) worker.on('exit', (code) => { logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1) if (code !== 0) { logger.Log( 'Search Data Worker error! Exit code is not 0', logger.GetColor('redbg') ) reject(new Error('Search Data Worker error! Exit code is not 0')) } }) worker.on('message', (result) => { logger.DebugLog(`Worker message arrived`, 'worker', 2) logger.DebugLog(result, 'worker', 3) logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1) logger.DebugLog(result, 'ask', 2) logger.DebugLog( `QDB search result length: ${result.length}`, 'qdb search', 1 ) resolve(result) }) }) } function addSubject(data, subj) { assert(data) assert(subj) var i = 0 while (i < length && subj.Name !== data[i].Name) { i++ } if (i < length) { return data.map((currSubj, j) => { if (j === i) { return { ...currSubj, Questions: [...currSubj.Questions, ...subj.Questions], } } else { return currSubj } }) } else { return [...data, subj] } } function dataToString(data) { var result = [] data.forEach((subj) => { result.push(subjectToString(subj)) }) return result.join('\n\n') } // ------------------------------------------------------------------------ if (!isMainThread) { logger.DebugLog(`Starting search worker ...`, 'worker', 1) const { data, subjName, question, questionData } = workerData let result = [] data.forEach((subj) => { if ( subjName .toLowerCase() .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) ) { logger.DebugLog(`Searching in ${subj.Name} `, 2) result = result.concat( searchQuestion(subj, question, questionData, subjName) ) } }) // FIXME: try to remove this? but this is also a good backup plan so idk if (result.length === 0) { logger.DebugLog( 'Reqults length is zero when comparing names, trying all subjects', 'qdb search', 1 ) data.forEach((subj) => { result = result.concat( searchQuestion(subj, question, questionData, subjName) ) }) if (result.length > 0) { logger.DebugLog( `FIXME: '${subjName}' gave no result but '' did!`, 'qdb search', 1 ) console.error(`FIXME: '${subjName}' gave no result but '' did!`) } } result = result.sort((q1, q2) => { if (q1.match < q2.match) { return 1 } else if (q1.match > q2.match) { return -1 } else { return 0 } }) parentPort.postMessage(result) process.exit(0) } // ------------------------------------------------------------------------ module.exports = { minMatchAmmount, getSubjNameWithoutYear, createQuestion, addQuestion, addSubject, searchData, dataToString, }