mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
Renamed js files to ts
This commit is contained in:
mrfry
parent
0bddef2b78
commit
7fcb15da88
54 changed files with 8521 additions and 60 deletions
|
|
@ -1,541 +0,0 @@
|
|||
const {
|
||||
Worker,
|
||||
isMainThread,
|
||||
parentPort,
|
||||
workerData,
|
||||
} = require('worker_threads')
|
||||
|
||||
const logger = require('./logger.js')
|
||||
|
||||
const searchDataWorkerFile = './src/utils/classes.js'
|
||||
|
||||
const assert = (val) => {
|
||||
if (!val) {
|
||||
throw new Error('Assertion failed')
|
||||
}
|
||||
}
|
||||
|
||||
const commonUselessAnswerParts = [
|
||||
'A helyes válasz az ',
|
||||
'A helyes válasz a ',
|
||||
'A helyes válaszok: ',
|
||||
'A helyes válaszok:',
|
||||
'A helyes válasz: ',
|
||||
'A helyes válasz:',
|
||||
'The correct answer is:',
|
||||
"'",
|
||||
]
|
||||
|
||||
const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
|
||||
const specialChars = ['&', '\\+']
|
||||
/* Percent minus for length difference */
|
||||
const lengthDiffMultiplier = 10
|
||||
/* Minimum ammount to consider that two questions match during answering */
|
||||
const minMatchAmmount = 60
|
||||
/* If all of the results are below this match percent (when only one subject is searched due to
|
||||
* subject name matching) then all subjects are searched for answer */
|
||||
const minMatchToNotSearchOtherSubjects = 90
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// String Utils
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
// Exported
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
function getSubjNameWithoutYear(subjName) {
|
||||
let t = subjName.split(' - ')
|
||||
if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
|
||||
return t[1] || subjName
|
||||
} else {
|
||||
return subjName
|
||||
}
|
||||
}
|
||||
|
||||
// Not exported
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
function removeStuff(value, removableStrings, toReplace) {
|
||||
removableStrings.forEach((removableString) => {
|
||||
var regex = new RegExp(removableString, 'g')
|
||||
value = value.replace(regex, toReplace || '')
|
||||
})
|
||||
return value
|
||||
}
|
||||
|
||||
// removes whitespace from begining and and, and replaces multiple spaces with one space
|
||||
function removeUnnecesarySpaces(toremove) {
|
||||
assert(toremove)
|
||||
|
||||
toremove = normalizeSpaces(toremove)
|
||||
while (toremove.includes(' ')) {
|
||||
toremove = toremove.replace(/ {2}/g, ' ')
|
||||
}
|
||||
return toremove.trim()
|
||||
}
|
||||
|
||||
// simplifies a string for easier comparison
|
||||
function simplifyStringForComparison(value) {
|
||||
assert(value)
|
||||
|
||||
value = removeUnnecesarySpaces(value).toLowerCase()
|
||||
return removeStuff(value, commonUselessStringParts)
|
||||
}
|
||||
|
||||
function removeSpecialChars(value) {
|
||||
assert(value)
|
||||
|
||||
return removeStuff(value, specialChars, ' ')
|
||||
}
|
||||
|
||||
// damn nonbreaking space
|
||||
function normalizeSpaces(input) {
|
||||
assert(input)
|
||||
|
||||
return input.replace(/\s/g, ' ')
|
||||
}
|
||||
|
||||
function compareString(s1, s2) {
|
||||
if (!s1 || !s2) {
|
||||
if (!s1 && !s2) {
|
||||
return 100
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
s1 = simplifyStringForComparison(s1).split(' ')
|
||||
s2 = simplifyStringForComparison(s2).split(' ')
|
||||
var match = 0
|
||||
for (var i = 0; i < s1.length; i++) {
|
||||
if (s2.includes(s1[i])) {
|
||||
match++
|
||||
}
|
||||
}
|
||||
var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent
|
||||
var lengthDifference = Math.abs(s2.length - s1.length)
|
||||
percent -= lengthDifference * lengthDiffMultiplier
|
||||
if (percent < 0) {
|
||||
percent = 0
|
||||
}
|
||||
return percent
|
||||
}
|
||||
|
||||
function answerPreProcessor(value) {
|
||||
assert(value)
|
||||
|
||||
return removeStuff(value, commonUselessAnswerParts)
|
||||
}
|
||||
|
||||
// 'a. pécsi sör' -> 'pécsi sör'
|
||||
function removeAnswerLetters(value) {
|
||||
if (!value) {
|
||||
return
|
||||
}
|
||||
|
||||
let val = value.split('. ')
|
||||
if (val[0].length < 2 && val.length > 1) {
|
||||
val.shift()
|
||||
return val.join(' ')
|
||||
} else {
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
function simplifyQA(value, mods) {
|
||||
if (!value) {
|
||||
return
|
||||
}
|
||||
|
||||
return mods.reduce((res, fn) => {
|
||||
return fn(res)
|
||||
}, value)
|
||||
}
|
||||
|
||||
function simplifyAnswer(value) {
|
||||
if (!value) {
|
||||
return value
|
||||
}
|
||||
return simplifyQA(value, [
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
answerPreProcessor,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
|
||||
function simplifyQuestion(question) {
|
||||
if (!question) {
|
||||
return
|
||||
}
|
||||
if (typeof question === 'string') {
|
||||
return simplifyQA(question, [
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
} else {
|
||||
if (question.Q) {
|
||||
question.Q = simplifyQA(question.Q, [
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
if (question.A) {
|
||||
question.A = simplifyQA(question.A, [
|
||||
removeSpecialChars,
|
||||
removeUnnecesarySpaces,
|
||||
removeAnswerLetters,
|
||||
])
|
||||
}
|
||||
return question
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Question
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
|
||||
function createQuestion(question, answer, data) {
|
||||
return {
|
||||
Q: simplifyQuestion(question),
|
||||
A: simplifyAnswer(answer),
|
||||
data,
|
||||
}
|
||||
}
|
||||
|
||||
function compareImage(data, data2) {
|
||||
return compareString(data.images.join(' '), data2.images.join(' '))
|
||||
}
|
||||
|
||||
function compareData(q1, q2) {
|
||||
try {
|
||||
if (q1.data.type === q2.data.type) {
|
||||
let dataType = q1.data.type
|
||||
if (dataType === 'simple') {
|
||||
return -1
|
||||
} else if (dataType === 'image') {
|
||||
return compareImage(q1.data, q2.data)
|
||||
} else {
|
||||
logger.DebugLog(
|
||||
`Unhandled data type ${dataType}`,
|
||||
'Compare question data',
|
||||
1
|
||||
)
|
||||
logger.DebugLog(q1, 'Compare question data', 2)
|
||||
}
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
} catch (error) {
|
||||
logger.DebugLog('Error comparing data', 'Compare question data', 1)
|
||||
logger.DebugLog(error.message, 'Compare question data', 1)
|
||||
logger.DebugLog(error, 'Compare question data', 2)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function compareQuestion(q1, q2) {
|
||||
return compareString(q1.Q, q2.Q)
|
||||
}
|
||||
|
||||
function compareAnswer(q1, q2) {
|
||||
return compareString(q1.A, q2.A)
|
||||
}
|
||||
|
||||
function compareQuestionObj(q1, q1subjName, q2, q2subjName, data) {
|
||||
assert(data)
|
||||
assert(q1)
|
||||
assert(typeof q1 === 'object')
|
||||
assert(q2)
|
||||
let qObj
|
||||
|
||||
if (typeof q2 === 'string') {
|
||||
qObj = {
|
||||
Q: q2,
|
||||
data: data,
|
||||
}
|
||||
} else {
|
||||
qObj = q2
|
||||
}
|
||||
|
||||
const qMatch = compareQuestion(q1, qObj)
|
||||
const aMatch = compareAnswer(q1, qObj)
|
||||
// -1 if botth questions are simple
|
||||
const dMatch = compareData(q1, qObj)
|
||||
|
||||
let avg = -1
|
||||
if (qObj.A) {
|
||||
if (dMatch === -1) {
|
||||
avg = (qMatch + aMatch) / 2
|
||||
} else {
|
||||
avg = (qMatch + aMatch + dMatch) / 3
|
||||
}
|
||||
} else {
|
||||
if (dMatch === -1) {
|
||||
avg = qMatch
|
||||
} else {
|
||||
avg = (qMatch + dMatch) / 2
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
qMatch: qMatch,
|
||||
aMatch: aMatch,
|
||||
dMatch: dMatch,
|
||||
matchedSubjName: q2subjName,
|
||||
avg: avg,
|
||||
}
|
||||
}
|
||||
|
||||
function questionToString(question) {
|
||||
const { Q, A, data } = question
|
||||
|
||||
if (data.type !== 'simple') {
|
||||
return '?' + Q + '\n!' + A + '\n>' + JSON.stringify(data)
|
||||
} else {
|
||||
return '?' + Q + '\n!' + A
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// Subject
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
function searchQuestion(subj, question, questionData, subjName) {
|
||||
assert(question)
|
||||
|
||||
var result = []
|
||||
subj.Questions.forEach((currentQuestion) => {
|
||||
let percent = compareQuestionObj(
|
||||
currentQuestion,
|
||||
subjName,
|
||||
question,
|
||||
subj.Name,
|
||||
questionData
|
||||
)
|
||||
|
||||
if (percent.avg > minMatchAmmount) {
|
||||
result.push({
|
||||
q: currentQuestion,
|
||||
match: percent.avg,
|
||||
detailedMatch: percent,
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
function subjectToString(subj) {
|
||||
const { Questions, Name } = subj
|
||||
|
||||
var result = []
|
||||
Questions.forEach((question) => {
|
||||
result.push(questionToString(question))
|
||||
})
|
||||
|
||||
return '+' + Name + '\n' + result.join('\n')
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
// QuestionDB
|
||||
// ---------------------------------------------------------------------------------------------------------
|
||||
function addQuestion(data, subj, question) {
|
||||
logger.DebugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
|
||||
logger.DebugLog(question, 'qdb add', 3)
|
||||
assert(data)
|
||||
assert(subj)
|
||||
assert(question)
|
||||
assert(typeof question === 'object')
|
||||
|
||||
var i = 0
|
||||
while (
|
||||
i < data.length &&
|
||||
!subj
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(data[i].Name).toLowerCase())
|
||||
) {
|
||||
i++
|
||||
}
|
||||
|
||||
if (i < data.length) {
|
||||
logger.DebugLog('Adding new question to existing subject', 'qdb add', 1)
|
||||
data[i].Questions.push(question)
|
||||
} else {
|
||||
logger.DebugLog('Creating new subject for question', 'qdb add', 1)
|
||||
data.push({
|
||||
Name: subj,
|
||||
Questions: [question],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function searchData(data, question, subjName, questionData) {
|
||||
return new Promise((resolve, reject) => {
|
||||
assert(data)
|
||||
assert(question)
|
||||
logger.DebugLog('Searching for question', 'qdb search', 1)
|
||||
logger.DebugLog('Question:', 'qdb search', 2)
|
||||
logger.DebugLog(question, 'qdb search', 2)
|
||||
logger.DebugLog(`Subject name: ${subjName}`, 'qdb search', 2)
|
||||
logger.DebugLog('Data:', 'qdb search', 2)
|
||||
logger.DebugLog(questionData || question.data, 'qdb search', 2)
|
||||
|
||||
if (!questionData) {
|
||||
questionData = question.data || { type: 'simple' }
|
||||
}
|
||||
if (!subjName) {
|
||||
subjName = ''
|
||||
logger.DebugLog('No subject name as param!', 'qdb search', 1)
|
||||
}
|
||||
question = simplifyQuestion(question)
|
||||
|
||||
const worker = new Worker(searchDataWorkerFile, {
|
||||
workerData: { data, subjName, question, questionData },
|
||||
})
|
||||
|
||||
worker.on('error', (err) => {
|
||||
logger.Log('Search Data Worker error!', logger.GetColor('redbg'))
|
||||
console.error(err)
|
||||
reject(err)
|
||||
})
|
||||
|
||||
worker.on('exit', (code) => {
|
||||
logger.DebugLog('Search Data exit, code: ' + code, 'actions', 1)
|
||||
if (code !== 0) {
|
||||
logger.Log(
|
||||
'Search Data Worker error! Exit code is not 0',
|
||||
logger.GetColor('redbg')
|
||||
)
|
||||
reject(new Error('Search Data Worker error! Exit code is not 0'))
|
||||
}
|
||||
})
|
||||
|
||||
worker.on('message', (result) => {
|
||||
logger.DebugLog(`Worker message arrived`, 'worker', 2)
|
||||
logger.DebugLog(result, 'worker', 3)
|
||||
logger.DebugLog(`Question result length: ${result.length}`, 'ask', 1)
|
||||
logger.DebugLog(result, 'ask', 2)
|
||||
|
||||
logger.DebugLog(
|
||||
`QDB search result length: ${result.length}`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
resolve(result)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function addSubject(data, subj) {
|
||||
assert(data)
|
||||
assert(subj)
|
||||
|
||||
var i = 0
|
||||
while (i < length && subj.Name !== data[i].Name) {
|
||||
i++
|
||||
}
|
||||
|
||||
if (i < length) {
|
||||
return data.map((currSubj, j) => {
|
||||
if (j === i) {
|
||||
return {
|
||||
...currSubj,
|
||||
Questions: [...currSubj.Questions, ...subj.Questions],
|
||||
}
|
||||
} else {
|
||||
return currSubj
|
||||
}
|
||||
})
|
||||
} else {
|
||||
return [...data, subj]
|
||||
}
|
||||
}
|
||||
|
||||
function dataToString(data) {
|
||||
var result = []
|
||||
data.forEach((subj) => {
|
||||
result.push(subjectToString(subj))
|
||||
})
|
||||
return result.join('\n\n')
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------
|
||||
if (!isMainThread) {
|
||||
logger.DebugLog(`Starting search worker ...`, 'worker', 1)
|
||||
const { data, subjName, question, questionData } = workerData
|
||||
let result = []
|
||||
|
||||
data.forEach((subj) => {
|
||||
if (
|
||||
subjName
|
||||
.toLowerCase()
|
||||
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
||||
) {
|
||||
logger.DebugLog(`Searching in ${subj.Name} `, 2)
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
// FIXME: try to remove this? but this is also a good backup plan so idk
|
||||
// its sufficent to check only result[0].match, since its sorted, and the first one should have
|
||||
// the highest match
|
||||
if (
|
||||
result.length === 0 ||
|
||||
result[0].match < minMatchToNotSearchOtherSubjects
|
||||
) {
|
||||
logger.DebugLog(
|
||||
'Reqults length is zero when comparing names, trying all subjects',
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
data.forEach((subj) => {
|
||||
result = result.concat(
|
||||
searchQuestion(subj, question, questionData, subjName)
|
||||
)
|
||||
})
|
||||
if (result.length > 0) {
|
||||
logger.DebugLog(
|
||||
`FIXME: '${subjName}' gave no result but '' did!`,
|
||||
'qdb search',
|
||||
1
|
||||
)
|
||||
console.error(`FIXME: '${subjName}' gave no result but '' did!`)
|
||||
}
|
||||
}
|
||||
|
||||
result = result.sort((q1, q2) => {
|
||||
if (q1.match < q2.match) {
|
||||
return 1
|
||||
} else if (q1.match > q2.match) {
|
||||
return -1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
parentPort.postMessage(result)
|
||||
process.exit(0)
|
||||
}
|
||||
// ------------------------------------------------------------------------
|
||||
|
||||
module.exports = {
|
||||
minMatchAmmount,
|
||||
getSubjNameWithoutYear,
|
||||
createQuestion,
|
||||
addQuestion,
|
||||
addSubject,
|
||||
searchData,
|
||||
dataToString,
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue