diff --git a/package.json b/package.json index 5995dea..20b601b 100755 --- a/package.json +++ b/package.json @@ -20,7 +20,8 @@ "dev": "npm run build && NS_THREAD_COUNT=2 NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node --inspect ./dist/server.js", "build": "tsc && bash -c './scripts/postBuild.sh'", "export": "tsc && bash -c './scripts/postBuild.sh'", - "test": "jest --detectOpenHandles" + "test": "NS_NOLOG=1 NS_THREAD_COUNT=1 jest --detectOpenHandles", + "test-debug": "NS_NOLOG=1 NS_THREAD_COUNT=1 node --inspect node_modules/.bin/jest --watch --runInBand src/tests/*.test.ts" }, "devDependencies": { "@types/better-sqlite3": "^7.5.0", diff --git a/src/tests/possibleAnswerPenalty.test.ts b/src/tests/possibleAnswerPenalty.test.ts new file mode 100644 index 0000000..bd1defe --- /dev/null +++ b/src/tests/possibleAnswerPenalty.test.ts @@ -0,0 +1,308 @@ +import { + setNoPossibleAnswersPenalties, + SearchResultQuestion, + noPossibleAnswerMatchPenalty, +} from '../utils/classes' +import { Question } from '../types/basicTypes' + +const matchPercent = 100 + +const questionWithNormalPossibleAnswers: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck' }, + { type: 'txt', val: 'super laptop' }, + { type: 'txt', val: 'nothing in particular' }, + { type: 'txt', val: 'something giberish' }, + ], + }, +} + +const questionWithNormalPossibleAnswersWithLabels: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'a) nothing in particular' }, + { type: 'txt', val: 'b) super laptop' }, + { type: 'txt', val: 'c) something giberish' }, + { type: 'txt', val: 'd) rubber duck' }, + ], + }, +} + +const questionWithNormalPossibleAnswers2: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck' }, + { type: 'txt', val: 'cat' }, + { type: 'txt', val: 'nothing in particular' }, + { type: 'txt', val: 'dog' }, + ], + }, +} + +const questionWithNormalPossibleAnswers3: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck 2' }, + { type: 'txt', val: 'whale' }, + { type: 'txt', val: 'nothing in particular 2' }, + { type: 'txt', val: 'sea lion' }, + ], + }, +} + +const questionWithNormalPossibleAnswers4: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'rubber duck' }, + { type: 'txt', val: 'super laptop' }, + ], + }, +} + +const questionWithSimilarPossibleAnswers: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'asd' }, + { type: 'txt', val: 'basd' }, + { type: 'txt', val: 'aaa' }, + { type: 'txt', val: 'bbb' }, + ], + }, +} + +const questionWithTrueFalsePossibleAnser: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + possibleAnswers: [ + { type: 'txt', val: 'true' }, + { type: 'txt', val: 'false' }, + ], + }, +} + +const questionWithNoPossibleAnswer: Question = { + Q: 'asd', + A: 'asd', + data: { + type: 'simple', + }, +} + +const resNormal: SearchResultQuestion = { + q: questionWithNormalPossibleAnswers, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const resNormal2: SearchResultQuestion = { + q: questionWithNormalPossibleAnswers2, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const resNormal3: SearchResultQuestion = { + q: questionWithNormalPossibleAnswers3, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const resNormal4: SearchResultQuestion = { + q: questionWithNormalPossibleAnswers4, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const resSimilar: SearchResultQuestion = { + q: questionWithSimilarPossibleAnswers, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const resTrueFalse: SearchResultQuestion = { + q: questionWithTrueFalsePossibleAnser, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const resNoPossibleAnswer: SearchResultQuestion = { + q: questionWithNoPossibleAnswer, + match: matchPercent, + detailedMatch: { + qMatch: matchPercent, + aMatch: matchPercent, + dMatch: matchPercent, + matchedSubjName: 'testSubj', + avg: matchPercent, + }, +} + +const testFunction = ( + question: Question, + searchResult: SearchResultQuestion[], + index: number +) => { + const updated = setNoPossibleAnswersPenalties( + question.data.possibleAnswers, + searchResult + ) + + updated.forEach((x, i) => { + if (i !== index) { + expect(x.match).toBe(matchPercent - noPossibleAnswerMatchPenalty) + expect(x.detailedMatch.qMatch).toBe( + matchPercent - noPossibleAnswerMatchPenalty + ) + } else { + expect(x.match).toBe(100) + expect(x.detailedMatch.qMatch).toBe(100) + } + }) + + return updated +} + +test('Possible answer penalty applies correctly (normal possible answers)', () => { + testFunction( + questionWithNormalPossibleAnswers, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 0 + ) +}) + +test('Possible answer penalty applies correctly (normal possible answers, with labels)', () => { + testFunction( + questionWithNormalPossibleAnswersWithLabels, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 0 + ) +}) + +test('Possible answer penalty applies correctly (similar possible answers)', () => { + testFunction( + questionWithSimilarPossibleAnswers, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 4 + ) +}) + +test('Possible answer penalty applies correctly (true false possible answers)', () => { + testFunction( + questionWithTrueFalsePossibleAnser, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ], + 5 + ) +}) + +test('Possible answer penalty applies correctly (no possible answers)', () => { + const updated = setNoPossibleAnswersPenalties( + questionWithNoPossibleAnswer.data.possibleAnswers, + [ + resNormal, + resNormal2, + resNormal3, + resNormal4, + resSimilar, + resTrueFalse, + resNoPossibleAnswer, + ] + ) + + updated.forEach((x) => { + expect(x.match).toBe(100) + expect(x.detailedMatch.qMatch).toBe(100) + }) +}) + +test('Possible answer penalty applies correctly (empty searchResult)', () => { + const updated = testFunction(questionWithTrueFalsePossibleAnser, [], 0) + expect(updated.length).toBe(0) +}) diff --git a/src/types/basicTypes.ts b/src/types/basicTypes.ts index 5854629..fb170af 100644 --- a/src/types/basicTypes.ts +++ b/src/types/basicTypes.ts @@ -31,8 +31,9 @@ export interface QuestionData { images?: Array hashedImages?: Array possibleAnswers?: Array<{ - text: string - selectedByUser: boolean + type: string + val: string + selectedByUser?: boolean }> } @@ -96,7 +97,11 @@ export interface User { export interface Request extends express.Request { body: T cookies: any - session: any + session: { + user?: User + sessionID?: string + isException?: boolean + } files: any query: { [key: string]: string } } diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 9d8042b..01fb822 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -204,7 +204,7 @@ function processIncomingRequestUsingDb( const add = result.result.every((res: SearchResultQuestion) => { return res.match < minMatchAmmountToAdd }) - if (add) { + if (add && !result.error) { allQuestions.push(recievedQuestions[i]) } }) diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 33d5c8c..5477a2e 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -33,6 +33,7 @@ export interface WorkerResult { msg: string workerIndex: number result?: SearchResultQuestion[] + error?: boolean } interface DetailedMatch { @@ -63,7 +64,7 @@ const commonUselessAnswerParts = [ // const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.'] /* Percent minus for length difference */ const lengthDiffMultiplier = 10 -const noPossibleAnswerMatchPenalty = 5 +export const noPossibleAnswerMatchPenalty = 5 /* Minimum ammount to consider that two questions match during answering */ const minMatchAmmount = 75 const magicNumber = 0.7 // same as minMatchAmmount, but /100 @@ -574,39 +575,52 @@ function doSearch( } function setNoPossibleAnswersPenalties( - possibleAnswers: QuestionData['possibleAnswers'], - result: SearchResultQuestion[] + questionPossibleAnswers: QuestionData['possibleAnswers'], + results: SearchResultQuestion[] ): SearchResultQuestion[] { - if (!Array.isArray(possibleAnswers)) { - return result + if (!Array.isArray(questionPossibleAnswers)) { + return results } - const noneHasPossibleAnswers = result.every((x) => { + const noneHasPossibleAnswers = results.every((x) => { return !Array.isArray(x.q.data.possibleAnswers) }) - if (noneHasPossibleAnswers) return result + if (noneHasPossibleAnswers) return results let possibleAnswerMatch = false - const updated = result.map((result) => { - const hasMatch = result.q.data.possibleAnswers.some((possibleAnswer) => { - return possibleAnswers.some((questionPossibleAnswer) => { - // FIXME: this could be object: questionPossibleAnswer - return questionPossibleAnswer.text.includes(possibleAnswer.text) - }) - }) - if (hasMatch) { + const updated = results.map((result) => { + const matchCount = Array.isArray(result.q.data.possibleAnswers) + ? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => { + return questionPossibleAnswers.some((questionPossibleAnswer) => { + if (questionPossibleAnswer.val && resultPossibleAnswer.val) { + return questionPossibleAnswer.val.includes( + resultPossibleAnswer.val + ) + } else { + return false + } + }) + }).length + : 0 + + if (matchCount === questionPossibleAnswers.length) { possibleAnswerMatch = true + return result } else { - result.match = result.match - noPossibleAnswerMatchPenalty - result.detailedMatch.qMatch = - result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty + return { + ...result, + match: result.match - noPossibleAnswerMatchPenalty, + detailedMatch: { + ...result.detailedMatch, + qMatch: result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty, + }, + } } - return result }) if (possibleAnswerMatch) { return updated } else { - return result + return results } } @@ -646,6 +660,7 @@ if (!isMainThread) { }: WorkData = msg.data let searchResult: SearchResultQuestion[] = [] + let error = false try { qdbs.forEach((qdb) => { @@ -674,14 +689,21 @@ if (!isMainThread) { } catch (err) { logger.Log('Error in worker thread!', logger.GetColor('redbg')) console.error(err) - console.error({ - subjName: subjName, - question: question, - searchTillMatchPercent: searchTillMatchPercent, - searchInAllIfNoResult: searchInAllIfNoResult, - searchIn: searchIn, - index: index, - }) + console.error( + JSON.stringify( + { + subjName: subjName, + question: question, + searchTillMatchPercent: searchTillMatchPercent, + searchInAllIfNoResult: searchInAllIfNoResult, + searchIn: searchIn, + index: index, + }, + null, + 2 + ) + ) + error = true } // sorting @@ -703,6 +725,7 @@ if (!isMainThread) { }done`, workerIndex: workerIndex, result: sortedResult, + error: error, } // ONDONE: @@ -810,4 +833,5 @@ export { addQuestion, dataToString, doSearch, + setNoPossibleAnswersPenalties, } diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 6134c10..0aaa387 100755 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -52,7 +52,7 @@ let uvData = {} // visit data, but per user let udvData = {} // visit data, but per user and daily let writes = 0 -let noLogips: string[] = [] +let noLogIds: string[] = [] function getColoredDateString(): string { const date = new Date() @@ -86,7 +86,9 @@ function Log(msg: string | object, color?: string): void { log = getColoredDateString() + delimiter + C(color) + msg + C() } - console.log(log) + if (!process.env.NS_NOLOG) { + console.log(log) + } utils.AppendToFile( typeof log === 'string' ? log : JSON.stringify(log), logDir + logFileName @@ -160,20 +162,20 @@ function LogReq( utils.AppendToFile(defLogs, vlogDir + logFileName) } } catch (err) { - console.log(err) + console.error(err) Log('Error at logging lol', GetColor('redbg')) } } function parseNoLogFile(newData: string) { - noLogips = newData.split('\n') - if (noLogips[noLogips.length - 1] === '') { - noLogips.pop() + noLogIds = newData.split('\n') + if (noLogIds[noLogIds.length - 1] === '') { + noLogIds.pop() } - noLogips = noLogips.filter((noLogip) => { - return noLogip !== '' + noLogIds = noLogIds.filter((noLogId) => { + return noLogId !== '' }) - Log('\tNo Log IP-s changed: ' + noLogips.join(', ')) + Log('\tNo Log user ID-s changed: ' + noLogIds.join(', ')) } function setNoLogReadInterval() { @@ -219,16 +221,23 @@ function Load(): void { setNoLogReadInterval() } -function LogStat(url: string, hostname: string, userId: number): void { +function LogStat(url: string, hostname: string, userId: number | string): void { + const nolog = noLogIds.some((noLogId) => { + return noLogId === userId.toString() + }) + if (nolog) { + return + } + url = hostname + url.split('?')[0] Inc(url) - AddUserIdStat(userId) - IncUserStat(userId) + AddUserIdStat(userId.toString()) + IncUserStat(userId.toString()) AddVisitStat(url) Save() } -function IncUserStat(userId: number) { +function IncUserStat(userId: string) { try { if (uvData[userId] === undefined) { uvData[userId] = 0 @@ -240,7 +249,7 @@ function IncUserStat(userId: number) { } } -function AddUserIdStat(userId: number) { +function AddUserIdStat(userId: string) { try { const date = new Date() const now = diff --git a/submodules/qmining-page b/submodules/qmining-page index 4f82b9d..4748c23 160000 --- a/submodules/qmining-page +++ b/submodules/qmining-page @@ -1 +1 @@ -Subproject commit 4f82b9d8c9c32ebbd9d73b951fc1db26ad1ceb22 +Subproject commit 4748c23769d16450731562bb9ffd1553ce2eb704 diff --git a/tsconfig.json b/tsconfig.json index 52fb9f7..5f0d376 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,6 +26,7 @@ "src/**/*" ], "exclude": [ + "src/tests/", "node_modules", "submodules", "devel",