Merge

package-lock.json

@@ -8,7 +8,6 @@
       "dependencies": {
         "@types/express": "^4.17.9",
         "@types/node": "^15.0.1",
-        "@types/socket.io": "^3.0.2",
         "better-sqlite3": "^7.1.5",
         "connect-busboy": "0.0.2",
         "cookie-parser": "^1.4.5",
@@ -1144,15 +1143,6 @@
         "@types/node": "*"
       }
     },
-    "node_modules/@types/socket.io": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-3.0.2.tgz",
-      "integrity": "sha512-pu0sN9m5VjCxBZVK8hW37ZcMe8rjn4HHggBN5CbaRTvFwv5jOmuIRZEuddsBPa9Th0ts0SIo3Niukq+95cMBbQ==",
-      "deprecated": "This is a stub types definition. socket.io provides its own type definitions, so you do not need this installed.",
-      "dependencies": {
-        "socket.io": "*"
-      }
-    },
     "node_modules/@types/stack-utils": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.0.tgz",
@@ -9461,14 +9451,6 @@
         "@types/node": "*"
       }
     },
-    "@types/socket.io": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@types/socket.io/-/socket.io-3.0.2.tgz",
-      "integrity": "sha512-pu0sN9m5VjCxBZVK8hW37ZcMe8rjn4HHggBN5CbaRTvFwv5jOmuIRZEuddsBPa9Th0ts0SIo3Niukq+95cMBbQ==",
-      "requires": {
-        "socket.io": "*"
-      }
-    },
     "@types/stack-utils": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.0.tgz",

package.json

@@ -4,7 +4,6 @@
   "dependencies": {
     "@types/express": "^4.17.9",
     "@types/node": "^15.0.1",
-    "@types/socket.io": "^3.0.2",
     "better-sqlite3": "^7.1.5",
     "connect-busboy": "0.0.2",
     "cookie-parser": "^1.4.5",
@@ -23,7 +22,7 @@
     "dev": "npm run build && NS_DEVEL=1 NS_NOUSER=1 NS_LOGLEVEL=1 node ./dist/server.js",
     "build": "tsc && bash -c './scripts/postBuild.sh'",
     "export": "tsc && bash -c './scripts/postBuild.sh'",
-    "test": "jest"
+    "test": "jest --detectOpenHandles"
   },
   "devDependencies": {
     "@types/jest": "^26.0.23",

scripts/make.sh

@@ -1,79 +1,85 @@
 #!/bin/bash
 
+hr() {
+  printf '\033[0;32m%*s\033[0m\n' "$(tput cols)" '' | tr ' ' '='
+}
+
+log() {
+  hr
+  echo -e "\033[0;32m${@}\033[0m"
+  hr
+}
+
 checkFile() {
   if [ ! -f "$@" ]; then
-    echo "$@ does not exists, exiting"
+    log "$@ does not exists, exiting"
     exit
   fi
 }
 
-checkFile "$PWD/src/server.js"
+makeNextSubmodule() {
+  log "Making ${1}"
+  pushd "submodules/${1}/"
+  npm install
+  npm audit fix
+  npm run export
+  popd
+  ln -sf "$PWD/submodules/${1}/out" "$PWD/nextStatic/${2}"
+}
+
+checkFile "$PWD/src/server.ts"
 checkFile "$PWD/package.json"
 
 git pull
 git submodule update --init --recursive
 
-echo "Installing node modules for server"
+log "Installing node modules for server"
 npm install
+npm audit fix
+npm run export
 
-echo "Seting up next.js static stuff..."
+log "Seting up next.js static stuff..."
 mkdir "$PWD/nextStatic"
 
-echo "Making qmining page"
+# ------------------------------------------------------------------------------------
-pushd submodules/qmining-page/
+makeNextSubmodule "qmining-page" "qminingPagePublic"
-npm install
+makeNextSubmodule "qmining-data-editor" "dataEditorPublic"
-npm run export
+# ------------------------------------------------------------------------------------
-popd
-checkFile "$PWD/submodules/qmining-page/out"
-ln -sf "$PWD/submodules/qmining-page/out" "$PWD/nextStatic/qminingPagePublic"
 
-echo "Making data editor page"
+log "Making moodle test userscript"
-pushd submodules/qmining-data-editor/
+mkdir -pv "$PWD/publicDirs/qminingPublic/moodle-test-userscript"
-npm install
-npm run export
-popd
-checkFile "$PWD/submodules/qmining-data-editor/out"
-ln -sf "$PWD/submodules/qmining-data-editor/out" "$PWD/nextStatic/dataEditorPublic"
-
-echo "Making moodle test userscript"
-mkdir -p "$PWD/publicDirs/qminingPublic/moodle-test-userscript"
 checkFile "$PWD/submodules/moodle-test-userscript/stable.user.js"
-ln -sf "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/"
+ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic/moodle-test-userscript/"
-ln -sf "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic"
+ln -sfv "$PWD/submodules/moodle-test-userscript/stable.user.js" "$PWD/publicDirs/qminingPublic"
 
-echo "mkdir-ing/touching :3"
+log "mkdir-ing/touching :3"
 
 # TODO: make server create these itself
 mkdir -p stats
 mkdir -p stats/logs
 mkdir -p stats/vlogs
-touch nolog
+mkdir -p data
+mkdir -p data/dbs
+touch data/nolog
 
 #JSONS
 echo '{}' >stats/stats
 echo '{}' >stats/vstats
 echo '{}' >stats/idstats
 echo '{}' >stats/idvstats
-echo '{}' >qminingPublic/userSpecificMotd.json
+echo '{}' >publicDirs/qminingPublic/userSpecificMotd.json
-echo '{}' >qminingPublic/news.json
+echo '{}' >publicDirs/qminingPublic/news.json
 
-touch qminingPublic/version
+touch publicDirs/qminingPublic/version
-touch qminingPublic/motd
+touch publicDirs/qminingPublic/motd
 
-echo "wgetting data.json from frylabs..."
+pushd src/standaloneUtils
-wget "http://qmining.frylabs.net/data.json" -O ./qminingPublic/data.json
+NS_LOGLEVEL=2 node dbSetup.js
-
+mv *db ../../data/dbs
-if [ "$?" -ne "0" ]; then
-  echo "Failed to wget data.json, please create it yourself!"
-  echo "Now starting with empty data!"
-  echo '{"Subjects":[],"version":"TESET","motd":"hai"}' >./qminingPublic/data.json
-fi
-
-pushd standaloneUtils
-rm -v ../data/dbs/users.db
-NS_SQL_DEBUG_LOG=true NS_LOGLEVEL=2 node dbSetup.js
 popd
 
+hr
 echo "Done!"
 echo "npm start {loglevel}"
 echo "To start server"
+hr

scripts/makeQminingPage.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-pushd modules/qmining/qmining-page/
-./make.sh
-popd

scripts/serverStats.js

@@ -8,12 +8,10 @@ const colWidth = 40
 const maxStatLength = colWidth // Math.floor(cols / 4)
 const statNameSpacing = 4
 const beforeRowSpace = 13
-const rowsToPrint = Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || 1
+const colsToPrint =
+  Math.floor(cols / (colWidth + statNameSpacing + beforeRowSpace / 3 + 5)) || 1
 const coloredWords = {
-  red: [
+  red: ['lred', 'thanks'],
-    'lred',
-    'thanks'
-  ],
   cyan: [
     'getveteranpw',
     'pwrequest',
@@ -53,12 +51,7 @@ const coloredWords = {
     'registerscript',
     'install',
   ],
-  magenta: [
+  magenta: ['addPost', 'comment', 'postfeedback', 'quickvote'],
-    'addPost',
-    'comment',
-    'postfeedback',
-    'quickvote',
-  ],
 }
 const filterFromDailyStats = [
   'savedQuestions',
@@ -164,10 +157,7 @@ function tail(text, number) {
 }
 
 function head(text, number) {
-  return text
+  return text.split('\n').slice(0, number).join('\n')
-    .split('\n')
-    .slice(0, number)
-    .join('\n')
 }
 
 function countLinesMatching(text, toMatch) {
@@ -180,6 +170,20 @@ function countLinesMatching(text, toMatch) {
   return count
 }
 
+function getDayName(day) {
+  switch (day) {
+    case 0:
+    case undefined:
+      return 'Today'
+    case -1:
+      return 'Yesterday'
+    case -2:
+      return 'Before yesterday'
+    default:
+      return `Day ${day.toString()}`
+  }
+}
+
 function readFile(name) {
   if (fs.existsSync(name)) {
     return fs.readFileSync(name, 'utf8')
@@ -198,7 +202,7 @@ function pCols(cols, rowTitles, colorNames, firstRowColor) {
   // console.log(cols)
   let maxLength = 0
   cols.reverse().forEach((col, i) => {
-    if (i >= rowsToPrint) {
+    if (i >= colsToPrint) {
       return
     }
     if (col.length > maxLength) {
@@ -211,54 +215,53 @@ function pCols(cols, rowTitles, colorNames, firstRowColor) {
   for (let i = 0; i < maxLength; i++) {
     const row = []
 
-    cols
+    cols.forEach((val, colIndex) => {
-      .forEach((val, colIndex) => {
+      if (colIndex >= colsToPrint) {
-    if (colIndex >= rowsToPrint) {
+        return
-      return
+      }
-    }
+      if (!val[i]) {
-        if (!val[i]) {
+        row.push(getLetterNTimes(' ', maxStatLength + statNameSpacing + 2))
-          row.push(getLetterNTimes(' ', maxStatLength + statNameSpacing + 2))
+        return
-          return
+      }
-        }
 
-        const keyName = val[i].name || val[i]
+      const keyName = val[i].name || val[i]
 
-        let slicedName = keyName.slice(0, maxStatLength)
+      let slicedName = keyName.slice(0, maxStatLength)
-        const toColor = colorNames
+      const toColor = colorNames
-          ? Object.keys(coloredWords).reduce((acc, key) => {
+        ? Object.keys(coloredWords).reduce((acc, key) => {
-              const colorArray = coloredWords[key]
+            const colorArray = coloredWords[key]
 
-              const includes = colorArray.some((colorableIdName) => {
+            const includes = colorArray.some((colorableIdName) => {
-                return keyName
+              return keyName
-                  .toLowerCase()
+                .toLowerCase()
-                  .includes(colorableIdName.toLowerCase())
+                .includes(colorableIdName.toLowerCase())
-              })
+            })
 
-              if (includes) {
+            if (includes) {
-                return key
+              return key
-              }
+            }
 
-              return acc
+            return acc
-            }, '')
+          }, '')
-          : false
+        : false
 
-        const sep = (i + 1) % 5 === 0 ? '.' : ' '
+      const sep = (i + 1) % 5 === 0 ? '.' : ' '
 
-        while (slicedName.length < maxStatLength) {
+      while (slicedName.length < maxStatLength) {
-          slicedName = slicedName + sep
+        slicedName = slicedName + sep
-        }
+      }
 
-        let ammount = val[i].val ? val[i].val.toString() : ''
+      let ammount = val[i].val ? val[i].val.toString() : ''
-        while (ammount.length < 5) {
+      while (ammount.length < 5) {
-          ammount = ammount + ' '
+        ammount = ammount + ' '
-        }
+      }
 
-        if (toColor) {
+      if (toColor) {
-          row.push(C(toColor) + slicedName + ' ' + ammount + C())
+        row.push(C(toColor) + slicedName + ' ' + ammount + C())
-        } else {
+      } else {
-          row.push(slicedName + ' ' + ammount)
+        row.push(slicedName + ' ' + ammount)
-        }
+      }
-      })
+    })
 
     // ROW TITLE ---------------------------------------------------
     let currRowTitle =
@@ -311,7 +314,7 @@ function preProcessDailyStats(obj) {
         if (i === index) {
           return {
             ...x,
-            val: z.val + x.val
+            val: z.val + x.val,
           }
         }
         return z
@@ -333,12 +336,15 @@ function preProcessDailyStats(obj) {
     }
   })
 }
+function getDailyStat(day) {
+  return preProcessDailyStats(dailyStats[getDayIndex(day)])
+}
 
 pCols(
   [
-    preProcessDailyStats(dailyStats[getDayIndex()]),
+    ...[...Array(colsToPrint).keys()].map((x) => {
-    preProcessDailyStats(dailyStats[getDayIndex(-1)]),
+      return getDailyStat(-x)
-    preProcessDailyStats(dailyStats[getDayIndex(-2)]),
+    }),
   ],
   null,
   true
@@ -361,15 +367,18 @@ function preProcessUIdTestSolving(obj, minLength) {
     return Object.keys(obj).length.toString()
   }
 }
+function getUserIdTestSolving(day) {
+  return [
+    getDayName(day),
+    preProcessUIdTestSolving(userIdTestSolving[getDayIndex(day)]),
+  ]
+}
 
 pCols(
   [
-    ['Today', preProcessUIdTestSolving(userIdTestSolving[getDayIndex()])],
+    ...[...Array(colsToPrint).keys()].map((x) => {
-    ['Yesterday', preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-1)])],
+      return getUserIdTestSolving(-x)
-    [
+    }),
-      'Before yesterday',
-      preProcessUIdTestSolving(userIdTestSolving[getDayIndex(-2)]),
-    ],
   ],
   null,
   false,
@@ -379,24 +388,19 @@ pCols(
 // ------------------------------------------------------------------------------
 printHeader('User id requests')
 const clientIdTestSolving = readJSON(`${dir}stats/uvstats`)
+function getUserIdRequests(day) {
+  return [
+    getDayName(day),
+    preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)]),
+    preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(day)], 5),
+  ]
+}
 
 pCols(
   [
-    [
+    ...[...Array(colsToPrint).keys()].map((x) => {
-      'Today',
+      return getUserIdRequests(-x)
-      preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()]),
+    }),
-      preProcessUIdTestSolving(clientIdTestSolving[getDayIndex()], 5),
-    ],
-    [
-      'Yesterday',
-      preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)]),
-      preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-1)], 5),
-    ],
-    [
-      'Before Yesterday',
-      preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)]),
-      preProcessUIdTestSolving(clientIdTestSolving[getDayIndex(-2)], 5),
-    ],
   ],
   ['', 'All', 'More than 5'],
   false,
@@ -406,15 +410,18 @@ pCols(
 // ------------------------------------------------------------------------------
 printHeader('Daily data count')
 const dailyDataCount = readFile(`${dir}stats/dailyDataCount`)
+function getDailyDataCount(count) {
+  return [...Array(count).keys()].map((x) => {
+    return JSON.parse(head(tail(dailyDataCount, x + 1), 1))
+  })
+}
 
-printLastDataCount([
+printLastDataCount(getDailyDataCount(colsToPrint))
-  JSON.parse(head(tail(dailyDataCount, 1), 1)),
-  JSON.parse(head(tail(dailyDataCount, 2), 1)),
-  JSON.parse(head(tail(dailyDataCount, 3), 1)),
-])
 
 function printLastDataCount(data) {
-  const res = [['Today'], ['Yesterday'], ['Before yesterday']]
+  const res = [...Array(colsToPrint).keys()].map((x) => {
+    return [getDayName(-x)]
+  })
   data.forEach((dataCount, i) => {
     res[i].push(dataCount.userCount.toString())
     res[i].push(dataCount.subjectCount.toString())
@@ -425,26 +432,24 @@ function printLastDataCount(data) {
 }
 // ------------------------------------------------------------------------------
 printHeader('Daily script install / update check count')
-const todaysLogs = readFile(`${dir}stats/vlogs/log`)
+function getDailyScriptStat(day) {
-const yesterdaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex(-1)}`) || ''
+  const log =
-const beforeYesterdaysLogs = readFile(`${dir}stats/vlogs/${getDayIndex(-2)}`) || ''
+    !day || day === 0
+      ? readFile(`${dir}stats/vlogs/log`)
+      : readFile(`${dir}stats/vlogs/${getDayIndex(day)}`)
 
-const installs = [
+  if (!log) {
-  [
+    return [getDayName(day), 0, 0]
-    'Today',
+  }
-    countLinesMatching(todaysLogs, '?install').toString(),
+  return [
-    countLinesMatching(todaysLogs, '?up').toString(),
+    getDayName(day),
-  ],
+    countLinesMatching(log, '?install').toString(),
-  [
+    countLinesMatching(log, '?up').toString(),
-    'Yesterday',
+  ]
-    countLinesMatching(yesterdaysLogs, '?install').toString(),
+}
-    countLinesMatching(yesterdaysLogs, '?up').toString(),
+
-  ],
+const installs = [...Array(colsToPrint).keys()].map((x) => {
-  [
+  return getDailyScriptStat(-x)
-    'Before yesterday',
+})
-    countLinesMatching(beforeYesterdaysLogs, '?install').toString(),
-    countLinesMatching(beforeYesterdaysLogs, '?up').toString(),
-  ],
-]
 
 pCols(installs, ['', 'Installs', 'Updates'], false, 'green')

src/modules/api/submodules/chat.ts

@@ -46,19 +46,9 @@ function setup(data: SubmoduleData): void {
 
   io.on('connection', (socket: ExtendedSocket) => {
     const userid = socket.user.id
-    logger.Log(`Chat connect: ${userid}`, logger.GetColor('green'))
 
     socket.on('join', function (/*data*/) {
       socket.join(userid.toString())
-      let currUser: any = dbtools.Select(userDB, 'users', {
-        id: userid,
-      })
-      if (!currUser || currUser.length === 0) {
-        // TODO: handle somehow?
-        socket.disconnect()
-      } else {
-        currUser = currUser[0]
-      }
 
       const groups = dbtools
         .runStatement(

src/server.ts

@@ -261,7 +261,7 @@ function setLogTimer() {
     1
   )
   logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
-  const msToMidnight = night.getTime() - now.getTime() + 1000
+  const msToMidnight = night.getTime() - now.getTime() + 10000
   logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
   logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)