From 64f41d67481e64d1b491a6b1aa370f86a4c48e73 Mon Sep 17 00:00:00 2001 From: mrfry Date: Fri, 8 Jan 2021 13:44:27 +0100 Subject: [PATCH] Reqlogger exceptions, searchig: search till match percent and search in other subjs --- src/middlewares/reqlogger.middleware.ts | 9 ++- src/modules/api/api.ts | 2 +- src/server.ts | 2 + src/utils/actions.ts | 6 +- src/utils/classes.ts | 88 +++++++++++++------------ 5 files changed, 61 insertions(+), 46 deletions(-) diff --git a/src/middlewares/reqlogger.middleware.ts b/src/middlewares/reqlogger.middleware.ts index dee31d8..3f32289 100644 --- a/src/middlewares/reqlogger.middleware.ts +++ b/src/middlewares/reqlogger.middleware.ts @@ -8,10 +8,16 @@ interface Options { export default function(options: Options): any { const loggableKeywords = options ? options.loggableKeywords : undefined const loggableModules = options ? options.loggableModules : undefined + const exceptions = options ? options.exceptions : [] return function(req, res, next) { res.on('finish', function() { - if (req.url.includes('_next/static')) { + // TODO: test this + const isException = exceptions.some((ex) => { + return req.url.includes(ex) + }) + + if (isException) { return } @@ -26,7 +32,6 @@ export default function(options: Options): any { console.log(req.headers) } - // fixme: regexp includes checking const hasLoggableKeyword = loggableKeywords && loggableKeywords.some((keyword) => { diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts index ec573f5..cb30773 100644 --- a/src/modules/api/api.ts +++ b/src/modules/api/api.ts @@ -1166,7 +1166,7 @@ function GetApp(): ModuleType { question: question, subjName: subj, questionData: recData, - firstMatchOnly: true, // search till the first match only + searchInAllIfNoResult: true, }, }) .then((taskResult) => { diff --git a/src/server.ts b/src/server.ts index 132d536..31a0e71 100755 --- a/src/server.ts +++ b/src/server.ts @@ -148,12 +148,14 @@ app.use( // origin: [ /\.frylabs\.net$/ ] }) ) + const cookieSecret = uuidv4() app.use(cookieParser(cookieSecret)) app.use( reqlogger({ loggableKeywords: ['news.json'], loggableModules: ['dataeditor'], + exceptions: ['stable.user.js?up', '_next/static'], }) ) diff --git a/src/utils/actions.ts b/src/utils/actions.ts index 46cf8dc..08370b8 100755 --- a/src/utils/actions.ts +++ b/src/utils/actions.ts @@ -32,7 +32,7 @@ import { QuestionDb, Question, User, DataFile } from '../types/basicTypes' // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db -const minMatchToAmmountToAdd = 90 +const minMatchAmmountToAdd = 90 const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate let currWrites = 0 @@ -166,6 +166,7 @@ function processIncomingRequestUsingDb( ) logger.DebugLog(currentQuestion, 'actions', 3) recievedQuestions.push(currentQuestion) + // This here searches only in relevant subjects, and not all subjects questionSearchPromises.push( doALongTask({ type: 'work', @@ -174,6 +175,7 @@ function processIncomingRequestUsingDb( qdb: qdb.data, question: currentQuestion, subjName: recievedData.subj, + searchTillMatchPercent: minMatchAmmountToAdd, }, }) ) @@ -184,7 +186,7 @@ function processIncomingRequestUsingDb( const allQuestions = [] // all new questions here that do not have result results.forEach((result, i) => { const add = result.result.every((res) => { - return res.match < minMatchToAmmountToAdd + return res.match < minMatchAmmountToAdd }) if (add) { allQuestions.push(recievedQuestions[i]) diff --git a/src/utils/classes.ts b/src/utils/classes.ts index 8df86e7..60abe67 100755 --- a/src/utils/classes.ts +++ b/src/utils/classes.ts @@ -319,7 +319,7 @@ function searchQuestion( subj: Subject, question: Question, subjName: string, - firstMatchOnly?: Boolean + searchTillMatchPercent?: number ) { assert(question) @@ -341,7 +341,7 @@ function searchQuestion( }) } - if (firstMatchOnly && percent.avg === 100) { + if (searchTillMatchPercent && percent.avg >= searchTillMatchPercent) { return false } @@ -388,6 +388,7 @@ function addQuestion( assert(typeof question === 'object') let i = 0 + // FIXME: this only adds to the first matched subject name. Check if this doesnt cause any bugs while ( i < data.length && !subj @@ -443,14 +444,13 @@ function dataToString(data: Array): string { return result.join('\n\n') } -// ------------------------------------------------------------------------ - function doSearch( data: Array, subjName: string, question: Question | string, questionData?: QuestionData, - firstMatchOnly?: Boolean + searchTillMatchPercent?: number, + searchInAllIfNoResult?: Boolean ): any { let result = [] @@ -469,53 +469,47 @@ function doSearch( subj, questionToSearch, subjName, - firstMatchOnly + searchTillMatchPercent ) result = result.concat(subjRes) - if (firstMatchOnly) { + if (searchTillMatchPercent) { + // TODO: this should return true to continue searching? return subjRes.every((sr) => { - return sr.match < 100 + return sr.match <= searchTillMatchPercent }) } return true } + // TODO: data.every should always return something! true or false? }) - // FIXME: try to remove this? but this is also a good backup plan so idk - // its sufficent to check only result[0].match, since its sorted, and the first one should have - // the highest match - if ( - result.length === 0 || - result[0].match < minMatchToNotSearchOtherSubjects - ) { - logger.DebugLog( - 'Reqults length is zero when comparing names, trying all subjects', - 'searchworker', - 1 - ) - data.every((subj) => { - const subjRes = searchQuestion( - subj, - questionToSearch, - subjName, - firstMatchOnly - ) - result = result.concat(subjRes) - if (firstMatchOnly) { - return subjRes.every((sr) => { - return sr.match < 100 - }) - } - return true - }) - - if (result.length > 0) { + if (searchInAllIfNoResult) { + // FIXME: dont research subject searched above + if ( + result.length === 0 || + result[0].match < minMatchToNotSearchOtherSubjects + ) { logger.DebugLog( - `FIXME: '${subjName}' gave no result but '' did!`, + 'Reqults length is zero when comparing names, trying all subjects', 'searchworker', 1 ) - console.error(`FIXME: '${subjName}' gave no result but '' did!`) + data.every((subj) => { + const subjRes = searchQuestion( + subj, + questionToSearch, + subjName, + searchTillMatchPercent + ) + result = result.concat(subjRes) + if (searchTillMatchPercent) { + // TODO: this should return true to continue searching? + return subjRes.every((sr) => { + return sr.match < searchTillMatchPercent + }) + } + return true + }) } } @@ -532,6 +526,10 @@ function doSearch( return result } +// --------------------------------------------------------------------------------------------------------- +// Multi threaded stuff +// --------------------------------------------------------------------------------------------------------- + if (!isMainThread) { const { workerIndex } = workerData let qdbs: Array = workerData.initData @@ -542,7 +540,14 @@ if (!isMainThread) { parentPort.on('message', (msg) => { if (msg.type === 'work') { - const { subjName, question, questionData, firstMatchOnly } = msg.data + const { + subjName, + question, + questionData, + searchTillMatchPercent, + searchInAllIfNoResult, + } = msg.data + const index = msg.index const searchIn = msg.data.searchIn @@ -562,7 +567,8 @@ if (!isMainThread) { subjName, question, questionData, - firstMatchOnly + searchTillMatchPercent, + searchInAllIfNoResult ) searchResult = [...searchResult, ...res] }