Possible answers penalty fixes, logging imporvements, added tests

This commit is contained in:
mrfry 2022-03-23 16:10:16 +01:00
parent 39cb92308d
commit f5ad460e24
8 changed files with 396 additions and 48 deletions

View file

@ -204,7 +204,7 @@ function processIncomingRequestUsingDb(
const add = result.result.every((res: SearchResultQuestion) => {
return res.match < minMatchAmmountToAdd
})
if (add) {
if (add && !result.error) {
allQuestions.push(recievedQuestions[i])
}
})

View file

@ -33,6 +33,7 @@ export interface WorkerResult {
msg: string
workerIndex: number
result?: SearchResultQuestion[]
error?: boolean
}
interface DetailedMatch {
@ -63,7 +64,7 @@ const commonUselessAnswerParts = [
// const commonUselessStringParts = [',', '\\.', ':', '!', '\\+', '\\s*\\.']
/* Percent minus for length difference */
const lengthDiffMultiplier = 10
const noPossibleAnswerMatchPenalty = 5
export const noPossibleAnswerMatchPenalty = 5
/* Minimum ammount to consider that two questions match during answering */
const minMatchAmmount = 75
const magicNumber = 0.7 // same as minMatchAmmount, but /100
@ -574,39 +575,52 @@ function doSearch(
}
function setNoPossibleAnswersPenalties(
possibleAnswers: QuestionData['possibleAnswers'],
result: SearchResultQuestion[]
questionPossibleAnswers: QuestionData['possibleAnswers'],
results: SearchResultQuestion[]
): SearchResultQuestion[] {
if (!Array.isArray(possibleAnswers)) {
return result
if (!Array.isArray(questionPossibleAnswers)) {
return results
}
const noneHasPossibleAnswers = result.every((x) => {
const noneHasPossibleAnswers = results.every((x) => {
return !Array.isArray(x.q.data.possibleAnswers)
})
if (noneHasPossibleAnswers) return result
if (noneHasPossibleAnswers) return results
let possibleAnswerMatch = false
const updated = result.map((result) => {
const hasMatch = result.q.data.possibleAnswers.some((possibleAnswer) => {
return possibleAnswers.some((questionPossibleAnswer) => {
// FIXME: this could be object: questionPossibleAnswer
return questionPossibleAnswer.text.includes(possibleAnswer.text)
})
})
if (hasMatch) {
const updated = results.map((result) => {
const matchCount = Array.isArray(result.q.data.possibleAnswers)
? result.q.data.possibleAnswers.filter((resultPossibleAnswer) => {
return questionPossibleAnswers.some((questionPossibleAnswer) => {
if (questionPossibleAnswer.val && resultPossibleAnswer.val) {
return questionPossibleAnswer.val.includes(
resultPossibleAnswer.val
)
} else {
return false
}
})
}).length
: 0
if (matchCount === questionPossibleAnswers.length) {
possibleAnswerMatch = true
return result
} else {
result.match = result.match - noPossibleAnswerMatchPenalty
result.detailedMatch.qMatch =
result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty
return {
...result,
match: result.match - noPossibleAnswerMatchPenalty,
detailedMatch: {
...result.detailedMatch,
qMatch: result.detailedMatch.qMatch - noPossibleAnswerMatchPenalty,
},
}
}
return result
})
if (possibleAnswerMatch) {
return updated
} else {
return result
return results
}
}
@ -646,6 +660,7 @@ if (!isMainThread) {
}: WorkData = msg.data
let searchResult: SearchResultQuestion[] = []
let error = false
try {
qdbs.forEach((qdb) => {
@ -674,14 +689,21 @@ if (!isMainThread) {
} catch (err) {
logger.Log('Error in worker thread!', logger.GetColor('redbg'))
console.error(err)
console.error({
subjName: subjName,
question: question,
searchTillMatchPercent: searchTillMatchPercent,
searchInAllIfNoResult: searchInAllIfNoResult,
searchIn: searchIn,
index: index,
})
console.error(
JSON.stringify(
{
subjName: subjName,
question: question,
searchTillMatchPercent: searchTillMatchPercent,
searchInAllIfNoResult: searchInAllIfNoResult,
searchIn: searchIn,
index: index,
},
null,
2
)
)
error = true
}
// sorting
@ -703,6 +725,7 @@ if (!isMainThread) {
}done`,
workerIndex: workerIndex,
result: sortedResult,
error: error,
}
// ONDONE:
@ -810,4 +833,5 @@ export {
addQuestion,
dataToString,
doSearch,
setNoPossibleAnswersPenalties,
}

View file

@ -52,7 +52,7 @@ let uvData = {} // visit data, but per user
let udvData = {} // visit data, but per user and daily
let writes = 0
let noLogips: string[] = []
let noLogIds: string[] = []
function getColoredDateString(): string {
const date = new Date()
@ -86,7 +86,9 @@ function Log(msg: string | object, color?: string): void {
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
console.log(log)
if (!process.env.NS_NOLOG) {
console.log(log)
}
utils.AppendToFile(
typeof log === 'string' ? log : JSON.stringify(log),
logDir + logFileName
@ -160,20 +162,20 @@ function LogReq(
utils.AppendToFile(defLogs, vlogDir + logFileName)
}
} catch (err) {
console.log(err)
console.error(err)
Log('Error at logging lol', GetColor('redbg'))
}
}
function parseNoLogFile(newData: string) {
noLogips = newData.split('\n')
if (noLogips[noLogips.length - 1] === '') {
noLogips.pop()
noLogIds = newData.split('\n')
if (noLogIds[noLogIds.length - 1] === '') {
noLogIds.pop()
}
noLogips = noLogips.filter((noLogip) => {
return noLogip !== ''
noLogIds = noLogIds.filter((noLogId) => {
return noLogId !== ''
})
Log('\tNo Log IP-s changed: ' + noLogips.join(', '))
Log('\tNo Log user ID-s changed: ' + noLogIds.join(', '))
}
function setNoLogReadInterval() {
@ -219,16 +221,23 @@ function Load(): void {
setNoLogReadInterval()
}
function LogStat(url: string, hostname: string, userId: number): void {
function LogStat(url: string, hostname: string, userId: number | string): void {
const nolog = noLogIds.some((noLogId) => {
return noLogId === userId.toString()
})
if (nolog) {
return
}
url = hostname + url.split('?')[0]
Inc(url)
AddUserIdStat(userId)
IncUserStat(userId)
AddUserIdStat(userId.toString())
IncUserStat(userId.toString())
AddVisitStat(url)
Save()
}
function IncUserStat(userId: number) {
function IncUserStat(userId: string) {
try {
if (uvData[userId] === undefined) {
uvData[userId] = 0
@ -240,7 +249,7 @@ function IncUserStat(userId: number) {
}
}
function AddUserIdStat(userId: number) {
function AddUserIdStat(userId: string) {
try {
const date = new Date()
const now =