worker file split, sending new questions to peers instantly

src/worker/handlers/handleDbClean.ts

@@ -0,0 +1,30 @@
+import { parentPort } from 'node:worker_threads'
+import { cleanDb } from '../../utils/qdbUtils'
+import { Question, QuestionDb } from '../../types/basicTypes'
+import { WorkerResult } from '../worker'
+
+export type DbCleanTaskObject = {
+    type: 'dbClean'
+    data: {
+        questions: Question[]
+        subjToClean: string
+        overwriteBeforeDate: number
+        qdbIndex: number
+    }
+}
+
+export const handleDbClean = async (
+    qdbs: QuestionDb[],
+    msg: DbCleanTaskObject,
+    workerIndex: number
+): Promise<void> => {
+    const removedIndexes = cleanDb(msg.data, qdbs)
+
+    const workerResult: WorkerResult = {
+        msg: `From thread #${workerIndex}: db clean done`,
+        workerIndex: workerIndex,
+        result: removedIndexes,
+    }
+
+    parentPort.postMessage(workerResult)
+}

src/worker/handlers/handleDbEdit.ts

@@ -0,0 +1,34 @@
+import { parentPort } from 'node:worker_threads'
+import { QuestionDb } from '../../types/basicTypes'
+import { Edits, editDb } from '../../utils/actions'
+import logger from '../../utils/logger'
+
+export type DbEditTaskObject = {
+    type: 'dbEdit'
+    data: { dbIndex: number; edits: Edits }
+}
+
+export const handleDbEdit = async (
+    qdbs: QuestionDb[],
+    msg: DbEditTaskObject,
+    workerIndex: number,
+    setQdbs: (newVal: Array<QuestionDb>) => void
+): Promise<void> => {
+    const { dbIndex, edits }: { dbIndex: number; edits: Edits } = msg.data
+    const { resultDb } = editDb(qdbs[dbIndex], edits)
+    setQdbs(
+        qdbs.map((qdb, i) => {
+            if (i === dbIndex) {
+                return resultDb
+            } else {
+                return qdb
+            }
+        })
+    )
+    logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1)
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: db edit`,
+        workerIndex: workerIndex,
+    })
+}

src/worker/handlers/handleMerge.ts

@@ -0,0 +1,36 @@
+import { parentPort } from 'node:worker_threads'
+import { QuestionDb } from '../../types/basicTypes'
+import { getSubjectDifference } from '../../utils/qdbUtils'
+
+export type MergeTaskObject = {
+    type: 'merge'
+    data: {
+        localQdbIndex: number
+        remoteQdb: QuestionDb
+    }
+}
+
+export const handleMerge = async (
+    qdbs: QuestionDb[],
+    msg: MergeTaskObject,
+    workerIndex: number
+): Promise<void> => {
+    const {
+        localQdbIndex,
+        remoteQdb,
+    }: { localQdbIndex: number; remoteQdb: QuestionDb } = msg.data
+    const localQdb = qdbs.find((qdb) => qdb.index === localQdbIndex)
+
+    const { newData, newSubjects } = getSubjectDifference(
+        localQdb.data,
+        remoteQdb.data
+    )
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: merge done`,
+        workerIndex: workerIndex,
+        newData: newData,
+        newSubjects: newSubjects,
+        localQdbIndex: localQdbIndex,
+    })
+}

src/worker/handlers/handleNewDb.ts

@@ -0,0 +1,22 @@
+import { parentPort } from 'node:worker_threads'
+import { QuestionDb } from '../../types/basicTypes'
+
+export type NewDbTaskObject = {
+    type: 'newdb'
+    data: QuestionDb
+}
+
+export const handleNewDb = async (
+    qdbs: QuestionDb[],
+    msg: NewDbTaskObject,
+    workerIndex: number,
+    setQdbs: (newVal: Array<QuestionDb>) => void
+): Promise<void> => {
+    const { data }: { data: QuestionDb } = msg
+    setQdbs([...qdbs, data])
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: new db add done`,
+        workerIndex: workerIndex,
+    })
+}

src/worker/handlers/handleNewQuestion.ts

@@ -0,0 +1,81 @@
+import { parentPort } from 'node:worker_threads'
+import { QuestionDb } from '../../types/basicTypes'
+import logger from '../../utils/logger'
+import { createQuestion } from '../../utils/qdbUtils'
+import { Result } from '../../utils/actions'
+
+export type NewQuestionTaskObject = {
+    type: 'newQuestions'
+    data: Omit<Result, 'qdbName'>
+}
+
+export const handleNewQuestions = async (
+    qdbs: QuestionDb[],
+    msg: NewQuestionTaskObject,
+    workerIndex: number,
+    setQdbs: (newVal: Array<QuestionDb>) => void
+): Promise<void> => {
+    const { subjName, qdbIndex, newQuestions } = msg.data
+
+    const newQuestionsWithCache = newQuestions.map((question) => {
+        if (!question.cache) {
+            return createQuestion(question)
+        } else {
+            return question
+        }
+    })
+
+    let added = false
+    setQdbs(
+        qdbs.map((qdb) => {
+            if (qdb.index === qdbIndex) {
+                return {
+                    ...qdb,
+                    data: qdb.data.map((subj) => {
+                        if (subj.Name === subjName) {
+                            added = true
+                            return {
+                                Name: subj.Name,
+                                Questions: [
+                                    ...subj.Questions,
+                                    ...newQuestionsWithCache,
+                                ],
+                            }
+                        } else {
+                            return subj
+                        }
+                    }),
+                }
+            } else {
+                return qdb
+            }
+        })
+    )
+
+    if (!added) {
+        setQdbs(
+            qdbs.map((qdb) => {
+                if (qdb.index === qdbIndex) {
+                    return {
+                        ...qdb,
+                        data: [
+                            ...qdb.data,
+                            {
+                                Name: subjName,
+                                Questions: [...newQuestionsWithCache],
+                            },
+                        ],
+                    }
+                } else {
+                    return qdb
+                }
+            })
+        )
+    }
+    logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1)
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: update done`,
+        workerIndex: workerIndex,
+    })
+}

src/worker/handlers/handleQuestionsToPeers.ts

@@ -0,0 +1,153 @@
+import { parentPort } from 'node:worker_threads'
+import { PeerInfo, Question, QuestionDb } from '../../types/basicTypes'
+import { files, paths, readAndValidateFile } from '../../utils/files'
+import utils from '../../utils/utils'
+import { RecievedData } from '../../utils/actions'
+import { removeCacheFromQuestion } from '../../utils/qdbUtils'
+import { QuestionAddResponse } from '../../modules/api/submodules/qminingapi'
+import logger from '../../utils/logger'
+import {
+    loginToPeer,
+    peerToString,
+    updatePeersFile,
+} from '../../utils/p2putils'
+import { post } from '../../utils/networkUtils'
+
+const login = async (peer: PeerInfo): Promise<string> => {
+    const loginResult = await loginToPeer(peer)
+    if (typeof loginResult === 'string') {
+        return loginResult
+    } else {
+        return null
+    }
+}
+
+export type QuestionsToPeersTaskObject = {
+    type: 'sendQuestionsToPeers'
+    data: {
+        newQuestions: Question[]
+        location: string
+        subj: string
+    }
+}
+
+export const handleQuestionsToPeers = async (
+    _qdbs: QuestionDb[],
+    msg: QuestionsToPeersTaskObject,
+    workerIndex: number
+): Promise<void> => {
+    const { newQuestions, location, subj } = msg.data
+
+    const domain = utils.ReadFile(paths.domainFile).trim()
+    const peers = readAndValidateFile<PeerInfo[]>(files.peersFile)
+
+    if (!peers || peers.length === 0 || newQuestions.length === 0) {
+        parentPort.postMessage({
+            msg: `From thread #${workerIndex}: sendQuestionsToPeers done`,
+            workerIndex: workerIndex,
+        })
+        return
+    }
+
+    const dataToSend: RecievedData = {
+        fromPeer: true,
+        subj: subj,
+        location: location,
+        id: domain, // client ID
+        version: 'P2P',
+        quiz: newQuestions.map((question) => {
+            return removeCacheFromQuestion({
+                ...question,
+                data: {
+                    ...question.data,
+                    source: domain,
+                },
+            })
+        }),
+    }
+
+    const results: {
+        errors: PeerInfo[]
+        hasNew: PeerInfo[]
+        sent: PeerInfo[]
+        loginErrors: PeerInfo[]
+    } = {
+        errors: [],
+        hasNew: [],
+        sent: [],
+        loginErrors: [],
+    }
+
+    const postData = (peer: PeerInfo, sessionCookie: string) => {
+        return post<QuestionAddResponse>({
+            hostname: peer.host,
+            port: peer.port,
+            http: peer.http,
+            path: '/api/isAdding',
+            bodyObject: dataToSend,
+            cookies: `sessionID=${sessionCookie}`,
+        })
+    }
+
+    for (const peer of peers) {
+        let sessionCookie = peer.sessionCookie
+
+        if (!sessionCookie) {
+            sessionCookie = await login(peer)
+            if (!sessionCookie) {
+                results.loginErrors.push(peer)
+                continue
+            }
+            updatePeersFile(peers, { ...peer, sessionCookie: sessionCookie })
+        }
+
+        let res = await postData(peer, sessionCookie)
+
+        if (res.data?.result === 'nouser' && sessionCookie) {
+            sessionCookie = await login(peer)
+            if (!sessionCookie) {
+                results.loginErrors.push(peer)
+                continue
+            }
+            updatePeersFile(peers, { ...peer, sessionCookie: sessionCookie })
+            res = await postData(peer, sessionCookie)
+        }
+
+        if (res.error || !res.data?.success) {
+            results.errors.push(peer)
+        } else if (res.data?.totalNewQuestions > 0) {
+            results.hasNew.push(peer)
+        } else {
+            results.sent.push(peer)
+        }
+    }
+
+    const logMsg: string[] = []
+    const addToLogMsg = (
+        peerResult: PeerInfo[],
+        prefix: string,
+        color: string
+    ) => {
+        if (peerResult.length > 0) {
+            logMsg.push(
+                `${logger.C(color)}${prefix}:${logger.C()} ` +
+                    peerResult.map((x) => peerToString(x)).join(', ')
+            )
+        }
+    }
+    addToLogMsg(results.loginErrors, 'Login error', 'red')
+    addToLogMsg(results.errors, 'Error', 'red')
+    addToLogMsg(results.hasNew, 'Had new questions', 'blue')
+    addToLogMsg(results.sent, 'Sent', 'green')
+
+    logger.Log(
+        `\t${logger.C(
+            'green'
+        )}Sent new questions to peers${logger.C()}; ${logMsg.join(', ')}`
+    )
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: sendQuestionsToPeers done`,
+        workerIndex: workerIndex,
+    })
+}

src/worker/handlers/handleRmQuestions.ts

@@ -0,0 +1,53 @@
+import { parentPort } from 'node:worker_threads'
+import { Question, QuestionDb } from '../../types/basicTypes'
+import { updateQuestionsInArray } from '../../utils/actions'
+
+export type RmQuestionsTaskObject = {
+    type: 'rmQuestions'
+    data: {
+        questionIndexesToRemove: number[][]
+        subjIndex: number
+        qdbIndex: number
+        recievedQuestions: Question[]
+    }
+}
+
+export const handleRmQuestions = async (
+    qdbs: QuestionDb[],
+    msg: RmQuestionsTaskObject,
+    workerIndex: number,
+    setQdbs: (newVal: QuestionDb[]) => void
+): Promise<void> => {
+    const { questionIndexesToRemove, subjIndex, qdbIndex, recievedQuestions } =
+        msg.data
+
+    const newQdbs = qdbs.map((qdb, i) => {
+        if (i === qdbIndex) {
+            return {
+                ...qdb,
+                data: qdb.data.map((subj, j) => {
+                    if (j === subjIndex) {
+                        return {
+                            ...subj,
+                            Questions: updateQuestionsInArray(
+                                questionIndexesToRemove,
+                                qdbs[qdbIndex].data[subjIndex].Questions,
+                                recievedQuestions
+                            ),
+                        }
+                    } else {
+                        return subj
+                    }
+                }),
+            }
+        } else {
+            return qdb
+        }
+    })
+    setQdbs(newQdbs)
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: rm question done`,
+        workerIndex: workerIndex,
+    })
+}

src/worker/handlers/handleSearch.ts

@@ -0,0 +1,286 @@
+import { parentPort } from 'worker_threads'
+import {
+    Question,
+    QuestionData,
+    QuestionDb,
+    Subject,
+} from '../../types/basicTypes'
+import logger from '../../utils/logger'
+import {
+    SearchResultQuestion,
+    getSubjNameWithoutYear,
+    minMatchToNotSearchOtherSubjects,
+    noPossibleAnswerMatchPenalty,
+    prepareQuestion,
+    searchSubject,
+} from '../../utils/qdbUtils'
+import { recognizeTextFromBase64 } from '../../utils/tesseract'
+import { WorkerResult } from '../worker'
+
+export type SearchTaskObject = {
+    type: 'search'
+    data: {
+        searchIn: number[]
+        question: Question
+        subjName: string
+        testUrl?: string
+        questionData?: QuestionData
+        searchInAllIfNoResult?: boolean
+        searchTillMatchPercent?: number
+        [key: string]: any
+    }
+}
+
+export function doSearch(
+    data: Array<Subject>,
+    subjName: string,
+    question: Question,
+    searchTillMatchPercent?: number,
+    searchInAllIfNoResult?: Boolean
+): SearchResultQuestion[] {
+    let result: SearchResultQuestion[] = []
+
+    const questionToSearch = prepareQuestion(question)
+
+    data.every((subj) => {
+        if (
+            subjName
+                .toLowerCase()
+                .includes(getSubjNameWithoutYear(subj.Name).toLowerCase())
+        ) {
+            logger.DebugLog(`Searching in ${subj.Name} `, 'searchworker', 2)
+            const subjRes = searchSubject(
+                subj,
+                questionToSearch,
+                subjName,
+                searchTillMatchPercent
+            )
+            result = result.concat(subjRes)
+            if (searchTillMatchPercent) {
+                return !subjRes.some((sr) => {
+                    return sr.match >= searchTillMatchPercent
+                })
+            }
+            return true
+        }
+        return true
+    })
+
+    if (searchInAllIfNoResult) {
+        // FIXME: dont research subject searched above
+        if (
+            result.length === 0 ||
+            result[0].match < minMatchToNotSearchOtherSubjects
+        ) {
+            logger.DebugLog(
+                'Reqults length is zero when comparing names, trying all subjects',
+                'searchworker',
+                1
+            )
+            data.every((subj) => {
+                const subjRes = searchSubject(
+                    subj,
+                    questionToSearch,
+                    subjName,
+                    searchTillMatchPercent
+                )
+                result = result.concat(subjRes)
+
+                if (searchTillMatchPercent) {
+                    const continueSearching = !subjRes.some((sr) => {
+                        return sr.match >= searchTillMatchPercent
+                    })
+                    return continueSearching
+                }
+                return true
+            })
+        }
+    }
+
+    result = setNoPossibleAnswersPenalties(
+        questionToSearch.data.possibleAnswers,
+        result
+    )
+
+    result = result.sort((q1, q2) => {
+        if (q1.match < q2.match) {
+            return 1
+        } else if (q1.match > q2.match) {
+            return -1
+        } else {
+            return 0
+        }
+    })
+
+    return result
+}
+
+export function setNoPossibleAnswersPenalties(
+    questionPossibleAnswers: QuestionData['possibleAnswers'],
+    results: SearchResultQuestion[]
+): SearchResultQuestion[] {
+    if (!Array.isArray(questionPossibleAnswers)) {
+        return results
+    }
+    const noneHasPossibleAnswers = results.every((x) => {
+        return !Array.isArray(x.q.data.possibleAnswers)
+    })
+    if (noneHasPossibleAnswers) return results
+
+    let possibleAnswerMatch = false
+    const updated = results.map((result) => {
+        const matchCount = Array.isArray(result.q.data.possibleAnswers)
+            ? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => {
+                  return questionPossibleAnswers.some(
+                      (questionPossibleAnswer) => {
+                          if (
+                              questionPossibleAnswer.val &&
+                              resultPossibleAnswer.val
+                          ) {
+                              return questionPossibleAnswer.val.includes(
+                                  resultPossibleAnswer.val
+                              )
+                          } else {
+                              return false
+                          }
+                      }
+                  )
+              }).length
+            : 0
+
+        if (matchCount === questionPossibleAnswers.length) {
+            possibleAnswerMatch = true
+            return result
+        } else {
+            return {
+                ...result,
+                match: result.match - noPossibleAnswerMatchPenalty,
+                detailedMatch: {
+                    ...result.detailedMatch,
+                    qMatch:
+                        result.detailedMatch.qMatch -
+                        noPossibleAnswerMatchPenalty,
+                },
+            }
+        }
+    })
+
+    if (possibleAnswerMatch) {
+        return updated
+    } else {
+        return results
+    }
+}
+
+async function recognizeQuestionImage(question: Question): Promise<Question> {
+    const base64Data = question.data.base64
+    if (Array.isArray(base64Data) && base64Data.length) {
+        const res: string[] = []
+        for (let i = 0; i < base64Data.length; i++) {
+            const base64 = base64Data[i]
+            const text = await recognizeTextFromBase64(base64)
+            if (text && text.trim()) {
+                res.push(text)
+            }
+        }
+
+        if (res.length) {
+            return {
+                ...question,
+                Q: res.join(' '),
+                data: {
+                    ...question.data,
+                    type: 'simple',
+                },
+            }
+        }
+    }
+
+    return question
+}
+
+export const handleSearch = async (
+    qdbs: QuestionDb[],
+    msg: SearchTaskObject,
+    workerIndex: number
+): Promise<void> => {
+    const {
+        subjName,
+        question: originalQuestion,
+        searchTillMatchPercent,
+        searchInAllIfNoResult,
+        searchIn,
+        index,
+    } = msg.data
+
+    let searchResult: SearchResultQuestion[] = []
+    let error = false
+
+    const question = await recognizeQuestionImage(originalQuestion)
+
+    try {
+        qdbs.forEach((qdb) => {
+            if (searchIn.includes(qdb.index)) {
+                const res = doSearch(
+                    qdb.data,
+                    subjName,
+                    question,
+                    searchTillMatchPercent,
+                    searchInAllIfNoResult
+                )
+                searchResult = [
+                    ...searchResult,
+                    ...res.map((x) => {
+                        return {
+                            ...x,
+                            detailedMatch: {
+                                ...x.detailedMatch,
+                                qdb: qdb.name,
+                            },
+                        }
+                    }),
+                ]
+            }
+        })
+    } catch (err) {
+        logger.Log('Error in worker thread!', logger.GetColor('redbg'))
+        console.error(err)
+        console.error(
+            JSON.stringify(
+                {
+                    subjName: subjName,
+                    question: question,
+                    searchTillMatchPercent: searchTillMatchPercent,
+                    searchInAllIfNoResult: searchInAllIfNoResult,
+                    searchIn: searchIn,
+                    index: index,
+                },
+                null,
+                2
+            )
+        )
+        error = true
+    }
+
+    // sorting
+    const sortedResult: SearchResultQuestion[] = searchResult.sort((q1, q2) => {
+        if (q1.match < q2.match) {
+            return 1
+        } else if (q1.match > q2.match) {
+            return -1
+        } else {
+            return 0
+        }
+    })
+
+    const workerResult: WorkerResult = {
+        msg: `From thread #${workerIndex}: job ${
+            !isNaN(index) ? `#${index}` : ''
+        }done`,
+        workerIndex: workerIndex,
+        result: sortedResult,
+        error: error,
+    }
+
+    parentPort.postMessage(workerResult)
+}