/* ---------------------------------------------------------------------------- Question Server GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ // FIXME: this should be renamed to worker.ts or something import { isMainThread, parentPort, workerData } from 'worker_threads' import { recognizeTextFromBase64, tesseractLoaded } from './tesseract' import logger from './logger' import { Question, QuestionData, QuestionDb, Subject, } from '../types/basicTypes' import { editDb, Edits, updateQuestionsInArray } from './actions' import { cleanDb, countOfQdbs, createQuestion, getSubjectDifference, getSubjNameWithoutYear, minMatchToNotSearchOtherSubjects, noPossibleAnswerMatchPenalty, prepareQuestion, SearchResultQuestion, searchSubject, } from './qdbUtils' // import { TaskObject } from './workerPool' export interface WorkerResult { msg: string workerIndex: number result?: SearchResultQuestion[] | number[][] error?: boolean } // --------------------------------------------------------------------------------------------------------- // String Utils // --------------------------------------------------------------------------------------------------------- // Exported // --------------------------------------------------------------------------------------------------------- // Not exported // --------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------- // Question // --------------------------------------------------------------------------------------------------------- async function recognizeQuestionImage(question: Question): Promise { const base64Data = question.data.base64 if (Array.isArray(base64Data) && base64Data.length) { const res: string[] = [] for (let i = 0; i < base64Data.length; i++) { const base64 = base64Data[i] const text = await recognizeTextFromBase64(base64) if (text && text.trim()) { res.push(text) } } if (res.length) { return { ...question, Q: res.join(' '), data: { ...question.data, type: 'simple', }, } } } return question } // --------------------------------------------------------------------------------------------------------- // Subject // --------------------------------------------------------------------------------------------------------- // --------------------------------------------------------------------------------------------------------- // QuestionDB // --------------------------------------------------------------------------------------------------------- function doSearch( data: Array, subjName: string, question: Question, searchTillMatchPercent?: number, searchInAllIfNoResult?: Boolean ): SearchResultQuestion[] { let result: SearchResultQuestion[] = [] const questionToSearch = prepareQuestion(question) data.every((subj) => { if ( subjName .toLowerCase() .includes(getSubjNameWithoutYear(subj.Name).toLowerCase()) ) { logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2) const subjRes = searchSubject( subj, questionToSearch, subjName, searchTillMatchPercent ) result = result.concat(subjRes) if (searchTillMatchPercent) { return !subjRes.some((sr) => { return sr.match >= searchTillMatchPercent }) } return true } return true }) if (searchInAllIfNoResult) { // FIXME: dont research subject searched above if ( result.length === 0 || result[0].match < minMatchToNotSearchOtherSubjects ) { logger.DebugLog( 'Reqults length is zero when comparing names, trying all subjects', 'searchworker', 1 ) data.every((subj) => { const subjRes = searchSubject( subj, questionToSearch, subjName, searchTillMatchPercent ) result = result.concat(subjRes) if (searchTillMatchPercent) { const continueSearching = !subjRes.some((sr) => { return sr.match >= searchTillMatchPercent }) return continueSearching } return true }) } } result = setNoPossibleAnswersPenalties( questionToSearch.data.possibleAnswers, result ) result = result.sort((q1, q2) => { if (q1.match < q2.match) { return 1 } else if (q1.match > q2.match) { return -1 } else { return 0 } }) return result } function setNoPossibleAnswersPenalties( questionPossibleAnswers: QuestionData['possibleAnswers'], results: SearchResultQuestion[] ): SearchResultQuestion[] { if (!Array.isArray(questionPossibleAnswers)) { return results } const noneHasPossibleAnswers = results.every((x) => { return !Array.isArray(x.q.data.possibleAnswers) }) if (noneHasPossibleAnswers) return results let possibleAnswerMatch = false const updated = results.map((result) => { const matchCount = Array.isArray(result.q.data.possibleAnswers) ? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => { return questionPossibleAnswers.some( (questionPossibleAnswer) => { if ( questionPossibleAnswer.val && resultPossibleAnswer.val ) { return questionPossibleAnswer.val.includes( resultPossibleAnswer.val ) } else { return false } } ) }).length : 0 if (matchCount === questionPossibleAnswers.length) { possibleAnswerMatch = true return result } else { return { ...result, match: result.match - noPossibleAnswerMatchPenalty, detailedMatch: { ...result.detailedMatch, qMatch: result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty, }, } } }) if (possibleAnswerMatch) { return updated } else { return results } } // --------------------------------------------------------------------------------------------------------- // Multi threaded stuff // --------------------------------------------------------------------------------------------------------- interface WorkData { subjName: string question: Question searchTillMatchPercent: number searchInAllIfNoResult: boolean searchIn: number[] index: number } if (!isMainThread) { handleWorkerData() } function handleWorkerData() { const { workerIndex, initData, }: { workerIndex: number; initData: Array } = workerData let qdbs: Array = initData const qdbCount = initData.length const { subjCount, questionCount } = countOfQdbs(initData) logger.Log( `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty! qdbs: ${qdbCount}, subjects: ${subjCount.toLocaleString()}, questions: ${questionCount.toLocaleString()}` ) parentPort.on('message', async (msg /*: TaskObject */) => { try { await tesseractLoaded if (msg.type === 'work') { const { subjName, question: originalQuestion, searchTillMatchPercent, searchInAllIfNoResult, searchIn, index, }: WorkData = msg.data let searchResult: SearchResultQuestion[] = [] let error = false const question = await recognizeQuestionImage(originalQuestion) try { qdbs.forEach((qdb) => { if (searchIn.includes(qdb.index)) { const res = doSearch( qdb.data, subjName, question, searchTillMatchPercent, searchInAllIfNoResult ) searchResult = [ ...searchResult, ...res.map((x) => { return { ...x, detailedMatch: { ...x.detailedMatch, qdb: qdb.name, }, } }), ] } }) } catch (err) { logger.Log( 'Error in worker thread!', logger.GetColor('redbg') ) console.error(err) console.error( JSON.stringify( { subjName: subjName, question: question, searchTillMatchPercent: searchTillMatchPercent, searchInAllIfNoResult: searchInAllIfNoResult, searchIn: searchIn, index: index, }, null, 2 ) ) error = true } // sorting const sortedResult: SearchResultQuestion[] = searchResult.sort( (q1, q2) => { if (q1.match < q2.match) { return 1 } else if (q1.match > q2.match) { return -1 } else { return 0 } } ) const workerResult: WorkerResult = { msg: `From thread #${workerIndex}: job ${ !isNaN(index) ? `#${index}` : '' }done`, workerIndex: workerIndex, result: sortedResult, error: error, } // ONDONE: parentPort.postMessage(workerResult) // console.log( // `[THREAD #${workerIndex}]: Work ${ // !isNaN(index) ? `#${index}` : '' // }done!` // ) } else if (msg.type === 'merge') { const { localQdbIndex, remoteQdb, }: { localQdbIndex: number; remoteQdb: QuestionDb } = msg.data const localQdb = qdbs.find((qdb) => qdb.index === localQdbIndex) const { newData, newSubjects } = getSubjectDifference( localQdb.data, remoteQdb.data ) parentPort.postMessage({ msg: `From thread #${workerIndex}: merge done`, workerIndex: workerIndex, newData: newData, newSubjects: newSubjects, localQdbIndex: localQdbIndex, }) } else if (msg.type === 'dbEdit') { const { dbIndex, edits }: { dbIndex: number; edits: Edits } = msg.data const { resultDb } = editDb(qdbs[dbIndex], edits) qdbs[dbIndex] = resultDb logger.DebugLog( `Worker db edit ${workerIndex}`, 'worker update', 1 ) parentPort.postMessage({ msg: `From thread #${workerIndex}: db edit`, workerIndex: workerIndex, }) } else if (msg.type === 'newQuestions') { const { subjName, qdbIndex, newQuestions, }: { subjName: string qdbIndex: number newQuestions: Question[] } = msg.data const newQuestionsWithCache = newQuestions.map((question) => { if (!question.cache) { return createQuestion(question) } else { return question } }) let added = false qdbs = qdbs.map((qdb) => { if (qdb.index === qdbIndex) { return { ...qdb, data: qdb.data.map((subj) => { if (subj.Name === subjName) { added = true return { Name: subj.Name, Questions: [ ...subj.Questions, ...newQuestionsWithCache, ], } } else { return subj } }), } } else { return qdb } }) if (!added) { qdbs = qdbs.map((qdb) => { if (qdb.index === qdbIndex) { return { ...qdb, data: [ ...qdb.data, { Name: subjName, Questions: [...newQuestionsWithCache], }, ], } } else { return qdb } }) } logger.DebugLog( `Worker new question ${workerIndex}`, 'worker update', 1 ) parentPort.postMessage({ msg: `From thread #${workerIndex}: update done`, workerIndex: workerIndex, }) // console.log(`[THREAD #${workerIndex}]: update`) } else if (msg.type === 'newdb') { const { data }: { data: QuestionDb } = msg qdbs.push(data) parentPort.postMessage({ msg: `From thread #${workerIndex}: new db add done`, workerIndex: workerIndex, }) // console.log(`[THREAD #${workerIndex}]: newdb`) } else if (msg.type === 'dbClean') { const removedIndexes = cleanDb(msg.data, qdbs) const workerResult: WorkerResult = { msg: `From thread #${workerIndex}: db clean done`, workerIndex: workerIndex, result: removedIndexes, } parentPort.postMessage(workerResult) } else if (msg.type === 'rmQuestions') { const { questionIndexesToRemove, subjIndex, qdbIndex, recievedQuestions, } = msg.data qdbs[qdbIndex].data[subjIndex].Questions = updateQuestionsInArray( questionIndexesToRemove, qdbs[qdbIndex].data[subjIndex].Questions, recievedQuestions ) parentPort.postMessage({ msg: `From thread #${workerIndex}: rm question done`, workerIndex: workerIndex, }) } else { logger.Log(`Invalid msg type!`, logger.GetColor('redbg')) console.error(msg) parentPort.postMessage({ msg: `From thread #${workerIndex}: Invalid message type (${msg.type})!`, workerIndex: workerIndex, }) } } catch (e) { console.error(e) parentPort.postMessage({ msg: `From thread #${workerIndex}: unhandled error occured!`, workerIndex: workerIndex, e: e, }) } }) } // ------------------------------------------------------------------------ export { doSearch, setNoPossibleAnswersPenalties }