mirror of
https://gitlab.com/MrFry/mrfrys-node-server
synced 2025-04-01 20:24:18 +02:00
539 lines
18 KiB
TypeScript
Executable file
539 lines
18 KiB
TypeScript
Executable file
/* ----------------------------------------------------------------------------
|
|
|
|
Question Server
|
|
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
|
|
|
|
This program is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
------------------------------------------------------------------------- */
|
|
// FIXME: this should be renamed to worker.ts or something
|
|
|
|
import { isMainThread, parentPort, workerData } from 'worker_threads'
|
|
|
|
import { recognizeTextFromBase64, tesseractLoaded } from './tesseract'
|
|
import logger from './logger'
|
|
import {
|
|
Question,
|
|
QuestionData,
|
|
QuestionDb,
|
|
Subject,
|
|
} from '../types/basicTypes'
|
|
import { editDb, Edits, updateQuestionsInArray } from './actions'
|
|
import {
|
|
cleanDb,
|
|
countOfQdbs,
|
|
createQuestion,
|
|
getSubjectDifference,
|
|
getSubjNameWithoutYear,
|
|
minMatchToNotSearchOtherSubjects,
|
|
noPossibleAnswerMatchPenalty,
|
|
prepareQuestion,
|
|
SearchResultQuestion,
|
|
searchSubject,
|
|
} from './qdbUtils'
|
|
// import { TaskObject } from './workerPool'
|
|
|
|
export interface WorkerResult {
|
|
msg: string
|
|
workerIndex: number
|
|
result?: SearchResultQuestion[] | number[][]
|
|
error?: boolean
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
// String Utils
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
// Exported
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
// Not exported
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
// Question
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
async function recognizeQuestionImage(question: Question): Promise<Question> {
|
|
const base64Data = question.data.base64
|
|
if (Array.isArray(base64Data) && base64Data.length) {
|
|
const res: string[] = []
|
|
for (let i = 0; i < base64Data.length; i++) {
|
|
const base64 = base64Data[i]
|
|
const text = await recognizeTextFromBase64(base64)
|
|
if (text && text.trim()) {
|
|
res.push(text)
|
|
}
|
|
}
|
|
|
|
if (res.length) {
|
|
return {
|
|
...question,
|
|
Q: res.join(' '),
|
|
data: {
|
|
...question.data,
|
|
type: 'simple',
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
return question
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
// Subject
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
// QuestionDB
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
function doSearch(
|
|
data: Array<Subject>,
|
|
subjName: string,
|
|
question: Question,
|
|
searchTillMatchPercent?: number,
|
|
searchInAllIfNoResult?: Boolean
|
|
): SearchResultQuestion[] {
|
|
let result: SearchResultQuestion[] = []
|
|
|
|
const questionToSearch = prepareQuestion(question)
|
|
|
|
data.every((subj) => {
|
|
if (
|
|
subjName
|
|
.toLowerCase()
|
|
.includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
|
|
) {
|
|
logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2)
|
|
const subjRes = searchSubject(
|
|
subj,
|
|
questionToSearch,
|
|
subjName,
|
|
searchTillMatchPercent
|
|
)
|
|
result = result.concat(subjRes)
|
|
if (searchTillMatchPercent) {
|
|
return !subjRes.some((sr) => {
|
|
return sr.match >= searchTillMatchPercent
|
|
})
|
|
}
|
|
return true
|
|
}
|
|
return true
|
|
})
|
|
|
|
if (searchInAllIfNoResult) {
|
|
// FIXME: dont research subject searched above
|
|
if (
|
|
result.length === 0 ||
|
|
result[0].match < minMatchToNotSearchOtherSubjects
|
|
) {
|
|
logger.DebugLog(
|
|
'Reqults length is zero when comparing names, trying all subjects',
|
|
'searchworker',
|
|
1
|
|
)
|
|
data.every((subj) => {
|
|
const subjRes = searchSubject(
|
|
subj,
|
|
questionToSearch,
|
|
subjName,
|
|
searchTillMatchPercent
|
|
)
|
|
result = result.concat(subjRes)
|
|
|
|
if (searchTillMatchPercent) {
|
|
const continueSearching = !subjRes.some((sr) => {
|
|
return sr.match >= searchTillMatchPercent
|
|
})
|
|
return continueSearching
|
|
}
|
|
return true
|
|
})
|
|
}
|
|
}
|
|
|
|
result = setNoPossibleAnswersPenalties(
|
|
questionToSearch.data.possibleAnswers,
|
|
result
|
|
)
|
|
|
|
result = result.sort((q1, q2) => {
|
|
if (q1.match < q2.match) {
|
|
return 1
|
|
} else if (q1.match > q2.match) {
|
|
return -1
|
|
} else {
|
|
return 0
|
|
}
|
|
})
|
|
|
|
return result
|
|
}
|
|
|
|
function setNoPossibleAnswersPenalties(
|
|
questionPossibleAnswers: QuestionData['possibleAnswers'],
|
|
results: SearchResultQuestion[]
|
|
): SearchResultQuestion[] {
|
|
if (!Array.isArray(questionPossibleAnswers)) {
|
|
return results
|
|
}
|
|
const noneHasPossibleAnswers = results.every((x) => {
|
|
return !Array.isArray(x.q.data.possibleAnswers)
|
|
})
|
|
if (noneHasPossibleAnswers) return results
|
|
|
|
let possibleAnswerMatch = false
|
|
const updated = results.map((result) => {
|
|
const matchCount = Array.isArray(result.q.data.possibleAnswers)
|
|
? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => {
|
|
return questionPossibleAnswers.some(
|
|
(questionPossibleAnswer) => {
|
|
if (
|
|
questionPossibleAnswer.val &&
|
|
resultPossibleAnswer.val
|
|
) {
|
|
return questionPossibleAnswer.val.includes(
|
|
resultPossibleAnswer.val
|
|
)
|
|
} else {
|
|
return false
|
|
}
|
|
}
|
|
)
|
|
}).length
|
|
: 0
|
|
|
|
if (matchCount === questionPossibleAnswers.length) {
|
|
possibleAnswerMatch = true
|
|
return result
|
|
} else {
|
|
return {
|
|
...result,
|
|
match: result.match - noPossibleAnswerMatchPenalty,
|
|
detailedMatch: {
|
|
...result.detailedMatch,
|
|
qMatch:
|
|
result.detailedMatch.qMatch -
|
|
noPossibleAnswerMatchPenalty,
|
|
},
|
|
}
|
|
}
|
|
})
|
|
|
|
if (possibleAnswerMatch) {
|
|
return updated
|
|
} else {
|
|
return results
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
// Multi threaded stuff
|
|
// ---------------------------------------------------------------------------------------------------------
|
|
|
|
interface WorkData {
|
|
subjName: string
|
|
question: Question
|
|
searchTillMatchPercent: number
|
|
searchInAllIfNoResult: boolean
|
|
searchIn: number[]
|
|
index: number
|
|
}
|
|
|
|
if (!isMainThread) {
|
|
handleWorkerData()
|
|
}
|
|
|
|
function handleWorkerData() {
|
|
const {
|
|
workerIndex,
|
|
initData,
|
|
}: { workerIndex: number; initData: Array<QuestionDb> } = workerData
|
|
let qdbs: Array<QuestionDb> = initData
|
|
|
|
const qdbCount = initData.length
|
|
const { subjCount, questionCount } = countOfQdbs(initData)
|
|
|
|
logger.Log(
|
|
`[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty! qdbs: ${qdbCount}, subjects: ${subjCount.toLocaleString()}, questions: ${questionCount.toLocaleString()}`
|
|
)
|
|
|
|
parentPort.on('message', async (msg /*: TaskObject */) => {
|
|
try {
|
|
await tesseractLoaded
|
|
|
|
if (msg.type === 'work') {
|
|
const {
|
|
subjName,
|
|
question: originalQuestion,
|
|
searchTillMatchPercent,
|
|
searchInAllIfNoResult,
|
|
searchIn,
|
|
index,
|
|
}: WorkData = msg.data
|
|
|
|
let searchResult: SearchResultQuestion[] = []
|
|
let error = false
|
|
|
|
const question = await recognizeQuestionImage(originalQuestion)
|
|
|
|
try {
|
|
qdbs.forEach((qdb) => {
|
|
if (searchIn.includes(qdb.index)) {
|
|
const res = doSearch(
|
|
qdb.data,
|
|
subjName,
|
|
question,
|
|
searchTillMatchPercent,
|
|
searchInAllIfNoResult
|
|
)
|
|
searchResult = [
|
|
...searchResult,
|
|
...res.map((x) => {
|
|
return {
|
|
...x,
|
|
detailedMatch: {
|
|
...x.detailedMatch,
|
|
qdb: qdb.name,
|
|
},
|
|
}
|
|
}),
|
|
]
|
|
}
|
|
})
|
|
} catch (err) {
|
|
logger.Log(
|
|
'Error in worker thread!',
|
|
logger.GetColor('redbg')
|
|
)
|
|
console.error(err)
|
|
console.error(
|
|
JSON.stringify(
|
|
{
|
|
subjName: subjName,
|
|
question: question,
|
|
searchTillMatchPercent: searchTillMatchPercent,
|
|
searchInAllIfNoResult: searchInAllIfNoResult,
|
|
searchIn: searchIn,
|
|
index: index,
|
|
},
|
|
null,
|
|
2
|
|
)
|
|
)
|
|
error = true
|
|
}
|
|
|
|
// sorting
|
|
const sortedResult: SearchResultQuestion[] = searchResult.sort(
|
|
(q1, q2) => {
|
|
if (q1.match < q2.match) {
|
|
return 1
|
|
} else if (q1.match > q2.match) {
|
|
return -1
|
|
} else {
|
|
return 0
|
|
}
|
|
}
|
|
)
|
|
|
|
const workerResult: WorkerResult = {
|
|
msg: `From thread #${workerIndex}: job ${
|
|
!isNaN(index) ? `#${index}` : ''
|
|
}done`,
|
|
workerIndex: workerIndex,
|
|
result: sortedResult,
|
|
error: error,
|
|
}
|
|
|
|
// ONDONE:
|
|
parentPort.postMessage(workerResult)
|
|
|
|
// console.log(
|
|
// `[THREAD #${workerIndex}]: Work ${
|
|
// !isNaN(index) ? `#${index}` : ''
|
|
// }done!`
|
|
// )
|
|
} else if (msg.type === 'merge') {
|
|
const {
|
|
localQdbIndex,
|
|
remoteQdb,
|
|
}: { localQdbIndex: number; remoteQdb: QuestionDb } = msg.data
|
|
const localQdb = qdbs.find((qdb) => qdb.index === localQdbIndex)
|
|
|
|
const { newData, newSubjects } = getSubjectDifference(
|
|
localQdb.data,
|
|
remoteQdb.data
|
|
)
|
|
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: merge done`,
|
|
workerIndex: workerIndex,
|
|
newData: newData,
|
|
newSubjects: newSubjects,
|
|
localQdbIndex: localQdbIndex,
|
|
})
|
|
} else if (msg.type === 'dbEdit') {
|
|
const { dbIndex, edits }: { dbIndex: number; edits: Edits } =
|
|
msg.data
|
|
const { resultDb } = editDb(qdbs[dbIndex], edits)
|
|
qdbs[dbIndex] = resultDb
|
|
logger.DebugLog(
|
|
`Worker db edit ${workerIndex}`,
|
|
'worker update',
|
|
1
|
|
)
|
|
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: db edit`,
|
|
workerIndex: workerIndex,
|
|
})
|
|
} else if (msg.type === 'newQuestions') {
|
|
const {
|
|
subjName,
|
|
qdbIndex,
|
|
newQuestions,
|
|
}: {
|
|
subjName: string
|
|
qdbIndex: number
|
|
newQuestions: Question[]
|
|
} = msg.data
|
|
|
|
const newQuestionsWithCache = newQuestions.map((question) => {
|
|
if (!question.cache) {
|
|
return createQuestion(question)
|
|
} else {
|
|
return question
|
|
}
|
|
})
|
|
|
|
let added = false
|
|
qdbs = qdbs.map((qdb) => {
|
|
if (qdb.index === qdbIndex) {
|
|
return {
|
|
...qdb,
|
|
data: qdb.data.map((subj) => {
|
|
if (subj.Name === subjName) {
|
|
added = true
|
|
return {
|
|
Name: subj.Name,
|
|
Questions: [
|
|
...subj.Questions,
|
|
...newQuestionsWithCache,
|
|
],
|
|
}
|
|
} else {
|
|
return subj
|
|
}
|
|
}),
|
|
}
|
|
} else {
|
|
return qdb
|
|
}
|
|
})
|
|
|
|
if (!added) {
|
|
qdbs = qdbs.map((qdb) => {
|
|
if (qdb.index === qdbIndex) {
|
|
return {
|
|
...qdb,
|
|
data: [
|
|
...qdb.data,
|
|
{
|
|
Name: subjName,
|
|
Questions: [...newQuestionsWithCache],
|
|
},
|
|
],
|
|
}
|
|
} else {
|
|
return qdb
|
|
}
|
|
})
|
|
}
|
|
logger.DebugLog(
|
|
`Worker new question ${workerIndex}`,
|
|
'worker update',
|
|
1
|
|
)
|
|
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: update done`,
|
|
workerIndex: workerIndex,
|
|
})
|
|
|
|
// console.log(`[THREAD #${workerIndex}]: update`)
|
|
} else if (msg.type === 'newdb') {
|
|
const { data }: { data: QuestionDb } = msg
|
|
qdbs.push(data)
|
|
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: new db add done`,
|
|
workerIndex: workerIndex,
|
|
})
|
|
// console.log(`[THREAD #${workerIndex}]: newdb`)
|
|
} else if (msg.type === 'dbClean') {
|
|
const removedIndexes = cleanDb(msg.data, qdbs)
|
|
|
|
const workerResult: WorkerResult = {
|
|
msg: `From thread #${workerIndex}: db clean done`,
|
|
workerIndex: workerIndex,
|
|
result: removedIndexes,
|
|
}
|
|
|
|
parentPort.postMessage(workerResult)
|
|
} else if (msg.type === 'rmQuestions') {
|
|
const {
|
|
questionIndexesToRemove,
|
|
subjIndex,
|
|
qdbIndex,
|
|
recievedQuestions,
|
|
} = msg.data
|
|
|
|
qdbs[qdbIndex].data[subjIndex].Questions =
|
|
updateQuestionsInArray(
|
|
questionIndexesToRemove,
|
|
qdbs[qdbIndex].data[subjIndex].Questions,
|
|
recievedQuestions
|
|
)
|
|
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: rm question done`,
|
|
workerIndex: workerIndex,
|
|
})
|
|
} else {
|
|
logger.Log(`Invalid msg type!`, logger.GetColor('redbg'))
|
|
console.error(msg)
|
|
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`,
|
|
workerIndex: workerIndex,
|
|
})
|
|
}
|
|
} catch (e) {
|
|
console.error(e)
|
|
parentPort.postMessage({
|
|
msg: `From thread #${workerIndex}: unhandled error occured!`,
|
|
workerIndex: workerIndex,
|
|
e: e,
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
// ------------------------------------------------------------------------
|
|
|
|
export { doSearch, setNoPossibleAnswersPenalties }
|