Excluding things from request stats, script registering endpoint

src/middlewares/reqlogger.middleware.ts

@@ -4,12 +4,14 @@ interface Options {
   loggableKeywords: Array<string>
   loggableModules: Array<string>
   exceptions: Array<string>
+  excludeFromStats: Array<string>
 }
 
 export default function(options: Options): any {
   const loggableKeywords = options ? options.loggableKeywords : undefined
   const loggableModules = options ? options.loggableModules : undefined
-  const exceptions = options ? options.exceptions : []
+  const exceptions = options.exceptions || []
+  const excludeFromStats = options.excludeFromStats || []
 
   return function(req, res, next) {
     res.on('finish', function() {
@@ -49,7 +51,12 @@ export default function(options: Options): any {
       if (toLog) {
         logger.LogReq(req)
       }
-      if (res.statusCode !== 404) {
+
+      const shouldLogStat = !excludeFromStats.some((ex) => {
+        return req.url.includes(ex)
+      })
+
+      if (res.statusCode !== 404 && shouldLogStat) {
         logger.LogStat(
           req.url,
           ip,

src/modules/api/api.ts

@@ -71,6 +71,7 @@ const todosFile = 'data/todos.json'
 const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js'
 const rootRedirectToFile = 'data/apiRootRedirectTo'
 const recievedQuestionFile = 'stats/recievedQuestions'
+const registeredScriptsFile = 'stats/registeredScripts.json'
 const savedQuestionsFileName = 'savedQuestions.json'
 
 // other constants
@@ -126,6 +127,7 @@ function GetApp(): ModuleType {
       userDB: userDB,
       jsonResponse: true,
       exceptions: [
+        '/register',
         '/favicon.ico',
         '/login',
         '/postfeedbackfile',
@@ -789,6 +791,8 @@ function GetApp(): ModuleType {
     const db: any = req.query.db
     let stringifiedData = ''
 
+    res.setHeader('content-type', 'text/plain; charset=utf-8')
+
     if (db) {
       const requestedDb = questionDbs.find((qdb) => {
         return qdb.name === db
@@ -805,7 +809,6 @@ function GetApp(): ModuleType {
       stringifiedData += dataToString(requestedDb.data)
       stringifiedData += '\n' + line + line + '\n'
     } else {
-      res.set('Content-Type', 'text/plain')
       stringifiedData = questionDbs
         .map((qdb) => {
           let result = ''
@@ -986,8 +989,16 @@ function GetApp(): ModuleType {
     logger.LogReq(req)
     const user: User = req.session.user
     const dryRun = testUsers.includes(user.id)
+    if (!req.body.location) {
+      res.json({ msg: 'invalid location parameter!' })
+      return
+    }
+
     const location = req.body.location.split('/')[2]
 
+    // TODO: handle undefined location
+    //  * if location is undefined still try to get suited question dbs
+    //      * if there isnt one dont create one, just resp.json(gebasz)
     try {
       let maxIndex = -1
       const suitedQuestionDbs = questionDbs.filter((qdb) => {
@@ -1105,6 +1116,7 @@ function GetApp(): ModuleType {
       return
     }
     const subj: any = req.body.subj || ''
+    // TODO: test if testUrl is undefined
     const testUrl = req.body.testUrl
       ? req.body.testUrl.split('/')[2]
       : undefined
@@ -1125,7 +1137,7 @@ function GetApp(): ModuleType {
       res.json(response)
 
       const saveableQuestions = response.reduce((acc, res) => {
-        // TODO
+        // TODO: only save if there isnt an answer for it
         // if (res.answers.length === 0) {
         //   acc.push(res.question)
         // }
@@ -1133,7 +1145,9 @@ function GetApp(): ModuleType {
         return acc
       }, [])
 
+      if (saveableQuestions.length > 0) {
         saveQuestion(saveableQuestions, subj, testUrl, user.id)
+      }
     })
   })
 
@@ -1194,7 +1208,6 @@ function GetApp(): ModuleType {
           subjName: subj,
           questionData: recData,
           searchInAllIfNoResult: true,
-          searchTillMatchPercent: 30,
         },
       })
         .then((taskResult) => {
@@ -1306,6 +1319,63 @@ function GetApp(): ModuleType {
     res.json(result)
   })
 
+  app.post('/registerscript', function(req: Request, res) {
+    logger.LogReq(req)
+
+    if (!utils.FileExists(registeredScriptsFile)) {
+      utils.WriteFile('[]', registeredScriptsFile)
+    }
+
+    const ip: any =
+      req.headers['cf-connecting-ip'] || req.connection.remoteAddress
+    const ua: any = req.headers['user-agent']
+    const registeredScripts = utils.ReadJSON(registeredScriptsFile)
+    const { cid, uid, version, date } = req.body
+
+    const index = registeredScripts.findIndex((registration) => {
+      return registration.cid === cid
+    })
+
+    if (index === -1) {
+      const x: any = {
+        cid: cid,
+        version: version,
+        date: date,
+        ip: ip,
+        userAgent: ua,
+      }
+
+      if (uid) {
+        x.uid = uid
+        x.loginDate = date
+      }
+      registeredScripts.push(x)
+    } else {
+      const currRegistration = registeredScripts[index]
+
+      if (!currRegistration.uid && uid) {
+        registeredScripts[index] = {
+          ...registeredScripts[index],
+          uid: uid,
+          loginDate: date,
+        }
+      } else {
+        logger.DebugLog(
+          `cid: ${cid}, uid: ${uid} tried to register multiple times`,
+          'register',
+          1
+        )
+      }
+    }
+
+    utils.WriteFile(
+      JSON.stringify(registeredScripts, null, 2),
+      registeredScriptsFile
+    )
+
+    res.json({ msg: 'done' })
+  })
+
   // -------------------------------------------------------------------------------------------
 
   app.get('*', function(req: Request, res: any) {

src/server.ts

@@ -155,7 +155,8 @@ app.use(
   reqlogger({
     loggableKeywords: ['news.json'],
     loggableModules: ['dataeditor'],
-    exceptions: ['stable.user.js?up', '_next/static'],
+    exceptions: ['_next/static'],
+    excludeFromStats: ['stable.user.js?up'],
   })
 )
 

submodules/qmining-data-editor

@@ -1 +1 @@
-Subproject commit a0d51266a018f53d983bb6acacb4c59214d26284
+Subproject commit 07b34762c3265abde099297f1794349022f2e12d

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 5a1bd258bcdeba3f268c802fe1a81632ce57f035
+Subproject commit 21bd0c01771cce8f693a32a0ae5d2f5fda4d55fb