merge

.eslintrc.js

@@ -28,7 +28,7 @@ module.exports = {
     '@typescript-eslint/ban-types': 'off',
     'id-length': [
       'warn',
-      { exceptions: ['i', 'j', 't', 'Q', 'A', 'C', 'q', 'a', 'b', 'x'] },
+      { exceptions: ['i', 'j', 't', 'Q', 'A', 'C', 'q', 'a', 'b', 'x', 'e'] },
     ],
     'object-shorthand': ['warn', 'never'],
     'prefer-const': 'warn',

.gitignore

@@ -5,6 +5,5 @@ dist/
 nextStatic/
 publicDirs/
 extraModules/
-nolog
 src/extraModules
 

package.json

@@ -13,7 +13,8 @@
     "eslint-plugin-typescript": "^0.14.0",
     "express": "^4.6.1",
     "express-ejs-layouts": "^1.1.0",
-    "generic-pool": "^3.7.1",
+    "express-fileupload": "^1.2.1",
+    "queue-microtask": "^1.2.3",
     "sqlite3": "^4.1.1",
     "ts-node": "^9.0.0",
     "typescript": "^4.1.2",
@@ -30,7 +31,7 @@
   },
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^4.8.1",
-    "@typescript-eslint/parser": "^4.8.1",
+    "@typescript-eslint/parser": "^4.22.0",
     "eslint": "^7.14.0"
   }
 }

src/middlewares/auth.middleware.ts

@@ -10,7 +10,7 @@ interface Options {
 
 const testUser = {
   id: 19,
-  avaiblePWRequests: 999,
+  avaiblePWRequests: 645,
   pwRequestCount: 19,
   created: new Date(),
 }

src/modules.json

@@ -31,7 +31,8 @@
     "name": "api",
     "urls": [
       "api.frylabs.net",
-      "localhost"
+      "localhost",
+      "superhedghog.frylabs.net"
     ]
   },
   "main": {

src/modules/api/submodules/feedback.ts

@@ -0,0 +1,63 @@
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+
+const msgFile = 'stats/msgs'
+const uloadFiles = 'data/f'
+
+function setup(data: SubmoduleData): void {
+  const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
+
+  app.post('/postfeedbackfile', function(req: Request, res: any) {
+    utils.uploadFile(req, uloadFiles).then(() => {
+      res.json({ success: true })
+    })
+
+    logger.LogReq(req)
+    logger.Log('New feedback file', logger.GetColor('bluebg'))
+  })
+
+  app.post('/postfeedback', function(req: Request, res: any) {
+    logger.LogReq(req)
+    if (req.body.fromLogin) {
+      logger.Log(
+        'New feedback message from Login page',
+        logger.GetColor('bluebg')
+      )
+    } else {
+      logger.Log(
+        'New feedback message from feedback page',
+        logger.GetColor('bluebg')
+      )
+    }
+
+    const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
+    const user: User = req.session.user
+
+    utils.AppendToFile(
+      utils.GetDateString() +
+        ':\n' +
+        JSON.stringify(
+          {
+            ...req.body,
+            userID: user ? user.id : 'no user',
+            ip: ip,
+          },
+          null,
+          2
+        ),
+      msgFile
+    )
+    res.json({ success: true })
+  })
+
+  app.route('/fosuploader').post(function(req: Request, res: any) {
+    utils.uploadFile(req, uloadFiles).then(({ fileName }) => {
+      res.redirect('/f/' + fileName)
+    })
+  })
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/forum.ts

@@ -0,0 +1,394 @@
+import { v4 as uuidv4 } from 'uuid'
+
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+
+const adminUsersFile = 'data/admins.json'
+const forumContentsFileName = '.contents.json'
+
+function countComments(comments) {
+  return comments.reduce((acc, comment) => {
+    if (comment.subComments) {
+      acc += countComments(comment.subComments) + 1
+    } else {
+      acc += 1
+    }
+    return acc
+  }, 0)
+}
+
+function addComment(obj, path, comment) {
+  if (path.length === 0) {
+    obj.push(comment)
+  } else {
+    const i = path.pop()
+    if (!obj[i].subComments) {
+      obj[i].subComments = []
+    }
+    addComment(obj[i].subComments, path, comment)
+  }
+}
+
+function deleteComment(obj: any, path: Array<number>, userid: number): boolean {
+  if (path.length === 1) {
+    if (obj[path[0]].user === userid) {
+      obj.splice(path[0], 1)
+      return true
+    } else {
+      return false
+    }
+  } else {
+    const i = path.pop()
+    deleteComment(obj[i].subComments, path, userid)
+    return true
+  }
+}
+
+function addReaction(obj, path, { reaction, isDelete, uid }) {
+  if (path.length === 1) {
+    const index = path[0]
+    if (!obj[index].reacts) {
+      obj[index].reacts = {}
+    }
+    if (isDelete) {
+      if (obj[index].reacts[reaction]) {
+        obj[index].reacts[reaction] = obj[index].reacts[reaction].filter(
+          (uid) => {
+            return uid !== uid
+          }
+        )
+        if (obj[index].reacts[reaction].length === 0) {
+          delete obj[index].reacts[reaction]
+        }
+      }
+    } else {
+      if (!obj[index].reacts[reaction]) {
+        obj[index].reacts[reaction] = [uid]
+      } else {
+        if (!obj[index].reacts[reaction].includes(uid)) {
+          obj[index].reacts[reaction].push(uid)
+        }
+      }
+    }
+  } else {
+    const i = path.pop()
+    addReaction(obj[i].subComments, path, {
+      reaction: reaction,
+      isDelete: isDelete,
+      uid: uid,
+    })
+  }
+}
+
+function getForumData(
+  forumName: string,
+  forumDir: string
+): { forumPath: string; contentFilePath: string; contents: any } {
+  const safeForumName = forumName.replace(/\./g, '').replace(/\/+/g, '')
+  const forumPath = forumDir + '/' + safeForumName
+  const contentFilePath = forumPath + '/' + forumContentsFileName
+
+  if (!utils.FileExists(forumPath)) {
+    utils.CreatePath(forumPath, true)
+  }
+
+  if (!utils.FileExists(contentFilePath)) {
+    utils.WriteFile('[]', contentFilePath)
+  }
+  const contents = utils.ReadJSON(contentFilePath)
+  return {
+    forumPath: forumPath,
+    contentFilePath: contentFilePath,
+    contents: contents,
+  }
+}
+
+function getPostData(postKey, contents, forumPath) {
+  const safePostKey = postKey.replace(/\./g, '').replace(/\/+/g, '')
+  const postPath = forumPath + '/' + safePostKey
+  if (!contents[postKey] || !utils.FileExists(postPath)) {
+    return
+  }
+  return { postData: utils.ReadJSON(postPath), postPath: postPath }
+}
+
+function setup(data: SubmoduleData): void {
+  const { app, /* userDB, url, */ publicdirs /*, moduleSpecificData */ } = data
+
+  const publicDir = publicdirs[0]
+
+  const forumDir = publicDir + 'forum'
+
+  if (!utils.FileExists(forumDir)) {
+    utils.CreatePath(forumDir, true)
+  }
+
+  app.get('/forumEntries', (req: Request, res) => {
+    logger.LogReq(req)
+    const forumName: any = req.query.forumName
+    if (!forumName) {
+      res.json({
+        success: false,
+        msg: `Forum name is not specified!`,
+      })
+      return
+    }
+
+    const { forumPath, contents } = getForumData(forumName, forumDir)
+
+    const from = req.query.from || Object.keys(contents).reverse()[0]
+    const count = parseInt(req.query.count ? req.query.count.toString() : '5')
+    const getContent = req.query.getContent
+
+    let entries = {}
+    let i = 0
+    let passed = false
+    let lastKey = undefined
+    Object.keys(contents)
+      .reverse()
+      .some((key) => {
+        const entry = getContent
+          ? utils.ReadJSON(forumPath + '/' + key)
+          : contents[key]
+        if (key === from) {
+          passed = true
+        }
+        if (i < count && passed) {
+          entries = {
+            ...entries,
+            [key]: entry,
+          }
+        }
+        if (i === count) {
+          lastKey = key
+          return true
+        }
+        if (passed) {
+          i++
+        }
+      })
+
+    res.json({
+      entries: entries,
+      nextKey: lastKey,
+    })
+  })
+
+  app.post('/addPost', (req: Request, res) => {
+    logger.LogReq(req)
+
+    const { title, content } = req.body
+    const forumName: any = req.body.forumName
+    if (!forumName) {
+      res.json({
+        success: false,
+        msg: `Forum name is not specified!`,
+      })
+      return
+    }
+    const { forumPath, contents, contentFilePath } = getForumData(
+      forumName,
+      forumDir
+    )
+    const user: User = req.session.user
+    const admins: any = utils.FileExists(adminUsersFile)
+      ? utils.ReadJSON(adminUsersFile)
+      : []
+
+    const newPostKey = uuidv4()
+    const postData = {
+      date: utils.GetDateString(),
+      user: user.id,
+      title: title,
+      admin: admins.includes(user.id),
+      commentCount: 0,
+    }
+
+    contents[newPostKey] = postData
+    utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath)
+    utils.WriteFile(
+      JSON.stringify({ ...postData, content: content }),
+      forumPath + '/' + newPostKey
+    )
+
+    res.json({
+      success: true,
+      newPostKey: newPostKey,
+      newEntry: { ...postData, content: content },
+    })
+  })
+
+  app.post('/rmPost', (req: Request, res) => {
+    logger.LogReq(req)
+    const { postKey } = req.body
+    const forumName: any = req.body.forumName
+    if (!forumName) {
+      res.json({
+        success: false,
+        msg: `Forum name is not specified!`,
+      })
+      return
+    }
+    const { forumPath, contentFilePath, contents } = getForumData(
+      forumName,
+      forumDir
+    )
+    const user: User = req.session.user
+
+    if (contents[postKey] && contents[postKey].user === user.id) {
+      utils.deleteFile(forumPath + '/' + postKey)
+      delete contents[postKey]
+      utils.WriteFile(JSON.stringify(contents), contentFilePath)
+
+      res.json({ success: true, deletedId: postKey })
+    } else {
+      res.json({
+        success: false,
+        msg:
+          'Entry does not exists, or you are not authorized to delete this post',
+      })
+      return
+    }
+  })
+
+  app.post('/comment', (req: Request, res) => {
+    logger.LogReq(req)
+
+    const forumName: any = req.body.forumName
+    if (!forumName) {
+      res.json({
+        success: false,
+        msg: `Forum name is not specified!`,
+      })
+      return
+    }
+    const { forumPath, contentFilePath, contents } = getForumData(
+      forumName,
+      forumDir
+    )
+    const user: User = req.session.user
+    const admins: any = utils.FileExists(adminUsersFile)
+      ? utils.ReadJSON(adminUsersFile)
+      : []
+    const { type, path, postKey } = req.body
+    if (!postKey || !type || !path) {
+      res.json({ success: false, msg: 'type or path or postKey is undefined' })
+      return
+    }
+
+    const { postPath, postData } = getPostData(postKey, contents, forumPath)
+    if (!postData) {
+      res.json({
+        success: false,
+        msg: `Post entry '${postKey}' from forum '${forumName}' does not exits!`,
+      })
+    }
+
+    if (type === 'add') {
+      const { content } = req.body
+      const comment = {
+        date: utils.GetDateString(),
+        user: user.id,
+        content: content,
+        admin: admins.includes(user.id),
+      }
+      if (!postData.comments) {
+        postData.comments = []
+      }
+      addComment(postData.comments, path, comment)
+    } else if (type === 'delete') {
+      if (postData.comments) {
+        const success = deleteComment(postData.comments, path, user.id)
+        if (!success) {
+          res.json({
+            success: false,
+            msg: 'you cant delete other users comments',
+            postData: postData,
+          })
+          return
+        }
+      }
+    } else {
+      res.json({ success: false, msg: 'no such type' })
+      return
+    }
+
+    contents[postKey].commentCount = countComments(postData.comments)
+
+    utils.WriteFile(JSON.stringify(postData, null, 2), postPath)
+    utils.WriteFile(JSON.stringify(contents, null, 2), contentFilePath)
+    res.json({ success: true, postKey: postKey, postData: postData })
+  })
+
+  app.post('/react', (req: Request, res) => {
+    logger.LogReq(req)
+
+    const forumName: any = req.body.forumName
+    if (!forumName) {
+      res.json({
+        success: false,
+        msg: `Forum name is not specified!`,
+      })
+      return
+    }
+    const { forumPath, contents } = getForumData(forumName, forumDir)
+    const user: User = req.session.user
+
+    const { postKey, path, reaction, isDelete } = req.body
+    if (!postKey || !reaction) {
+      res.json({ success: false, msg: 'no postkey or reaction' })
+      return
+    }
+
+    const { postPath, postData } = getPostData(postKey, contents, forumPath)
+    if (!postData) {
+      res.json({
+        success: false,
+        msg: `Post entry '${postKey}' from forum '${forumName}' does not exits!`,
+      })
+    }
+
+    if (!path || path.length === 0) {
+      if (postData) {
+        if (isDelete) {
+          if (postData.reacts) {
+            postData.reacts[reaction] = postData.reacts[reaction].filter(
+              (uid) => {
+                return uid !== user.id
+              }
+            )
+            if (postData.reacts[reaction].length === 0) {
+              delete postData.reacts[reaction]
+            }
+          }
+        } else {
+          if (!postData.reacts) {
+            postData.reacts = { [reaction]: [user.id] }
+          } else {
+            if (Array.isArray(postData.reacts[reaction])) {
+              if (!postData.reacts[reaction].includes(user.id)) {
+                postData.reacts[reaction].push(user.id)
+              }
+            } else {
+              postData.reacts[reaction] = [user.id]
+            }
+          }
+        }
+      }
+    } else {
+      addReaction(postData.comments, path, {
+        reaction: reaction,
+        isDelete: isDelete,
+        uid: user.id,
+      })
+    }
+
+    utils.WriteFile(JSON.stringify(postData, null, 2), postPath)
+    res.json({ success: true, postData: postData })
+  })
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/qminingapi.ts

@@ -0,0 +1,995 @@
+import fs from 'fs'
+
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import {
+  User,
+  DataFile,
+  Request,
+  QuestionDb,
+  SubmoduleData,
+} from '../../../types/basicTypes'
+import {
+  processIncomingRequest,
+  logResult,
+  shouldSaveDataFile,
+  Result,
+  backupData,
+  shouldSearchDataFile,
+  loadJSON,
+  writeData,
+  editDb,
+} from '../../../utils/actions'
+import {
+  dataToString,
+  getSubjNameWithoutYear,
+  // compareQuestionObj,
+} from '../../../utils/classes'
+import {
+  doALongTask,
+  msgAllWorker,
+  initWorkerPool,
+} from '../../../utils/workerPool'
+import dbtools from '../../../utils/dbtools'
+
+const line = '====================================================' // lol
+const registeredScriptsFile = 'stats/registeredScripts.json'
+const testUsersFile = 'data/testUsers.json'
+const userScriptFile = 'submodules/moodle-test-userscript/stable.user.js'
+const recievedQuestionFile = 'stats/recievedQuestions'
+const savedQuestionsFileName = 'savedQuestions.json'
+const oldMotdFile = 'publicDirs/qminingPublic/oldMotd'
+const dailyDataCountFile = 'stats/dailyDataCount'
+const dataEditsLog = 'stats/dataEdits'
+
+function getSubjCount(qdbs) {
+  return qdbs.reduce((acc, qdb) => {
+    return acc + qdb.data.length
+  }, 0)
+}
+
+function getQuestionCount(qdbs) {
+  return qdbs.reduce((acc, qdb) => {
+    return (
+      acc +
+      qdb.data.reduce((qacc, subject) => {
+        return qacc + subject.Questions.length
+      }, 0)
+    )
+  }, 0)
+}
+
+function ExportDailyDataCount(questionDbs, userDB) {
+  logger.Log('Saving daily data count ...')
+  utils.AppendToFile(
+    JSON.stringify({
+      date: utils.GetDateString(),
+      subjectCount: getSubjCount(questionDbs),
+      questionCount: getQuestionCount(questionDbs),
+      userCount: dbtools.TableInfo(userDB, 'users').dataCount,
+    }),
+    dailyDataCountFile
+  )
+}
+
+function getDbIndexesToSearchIn(
+  testUrl: string,
+  questionDbs,
+  trueIfAlways?: boolean
+) {
+  return testUrl
+    ? questionDbs.reduce((acc, qdb, i) => {
+        if (shouldSearchDataFile(qdb, testUrl, trueIfAlways)) {
+          acc.push(i)
+        }
+        return acc
+      }, [])
+    : []
+}
+
+function getSimplreRes(questionDbs) {
+  return {
+    subjects: getSubjCount(questionDbs),
+    questions: getQuestionCount(questionDbs),
+  }
+}
+
+function getDetailedRes(questionDbs) {
+  return questionDbs.map((qdb) => {
+    return {
+      dbName: qdb.name,
+      subjs: qdb.data.map((subj) => {
+        return {
+          name: subj.Name,
+          count: subj.Questions.length,
+        }
+      }),
+    }
+  })
+}
+
+function getMotd(version, motd) {
+  if (version) {
+    if (version.startsWith('2.0.')) {
+      if (utils.FileExists(oldMotdFile)) {
+        return utils.ReadFile(oldMotdFile)
+      }
+    }
+  }
+  return motd
+}
+
+function searchInDbs(
+  question,
+  subj,
+  recData,
+  recievedData,
+  searchIn,
+  testUrl?
+) {
+  // searchIn could be [0], [1], ... to search every db in different thread. Put this into a
+  // forEach(qdbs) to achieve this
+  return new Promise((resolve) => {
+    doALongTask({
+      type: 'work',
+      data: {
+        searchIn: searchIn,
+        testUrl: testUrl,
+        question: question,
+        subjName: subj,
+        questionData: recData,
+        searchInAllIfNoResult: true,
+      },
+    })
+      .then((taskResult) => {
+        try {
+          logger.DebugLog(
+            `Question result length: ${taskResult.length}`,
+            'ask',
+            1
+          )
+          logger.DebugLog(taskResult, 'ask', 2)
+          resolve({
+            question: question,
+            result: taskResult.result,
+            success: true,
+          })
+        } catch (err) {
+          console.error(err)
+          logger.Log(
+            'Error while sending ask results',
+            logger.GetColor('redbg')
+          )
+        }
+      })
+      .catch((err) => {
+        logger.Log('Search Data error!', logger.GetColor('redbg'))
+        console.error(err)
+        resolve({
+          mesage: `There was an error processing the question: ${err.message}`,
+          result: [],
+          recievedData: JSON.stringify(recievedData),
+          success: false,
+        })
+      })
+  })
+}
+
+function getResult(data) {
+  const { question, subj, recData, recievedData, questionDbs, testUrl } = data
+  return new Promise((resolve) => {
+    const searchIn = getDbIndexesToSearchIn(testUrl, questionDbs, false)
+
+    searchInDbs(question, subj, recData, recievedData, searchIn, testUrl).then(
+      (res: any) => {
+        if (res.result.length === 0) {
+          logger.DebugLog(
+            `No result while searching specific question db ${testUrl}`,
+            'ask',
+            1
+          )
+          const searchInMore = getDbIndexesToSearchIn(
+            testUrl,
+            questionDbs,
+            true
+          ).filter((x) => {
+            return !searchIn.includes(x)
+          })
+          searchInDbs(
+            question,
+            subj,
+            recData,
+            recievedData,
+            searchInMore,
+            testUrl
+          ).then((res) => {
+            resolve(res)
+          })
+        } else {
+          resolve(res)
+        }
+      }
+    )
+  })
+}
+
+function dbExists(location, qdbs: Array<QuestionDb>) {
+  return qdbs.some((qdb) => {
+    return qdb.name === location
+  })
+}
+
+function writeAskData(body) {
+  try {
+    let towrite = utils.GetDateString() + '\n'
+    towrite +=
+      '------------------------------------------------------------------------------\n'
+    towrite += JSON.stringify(body)
+    towrite +=
+      '\n------------------------------------------------------------------------------\n'
+    utils.AppendToFile(towrite, recievedQuestionFile)
+  } catch (err) {
+    logger.Log('Error writing revieved /ask POST data')
+    console.error(err)
+  }
+}
+
+function saveQuestion(questions, subj, testUrl, userid, savedQuestionsDir) {
+  // TODO: clear folder every now and then, check if saved questions exist
+  if (!subj) {
+    logger.Log('No subj name to save test question')
+    return
+  }
+  const questionsToSave = {
+    questions: questions,
+    subj: subj,
+    userid: userid,
+    testUrl: testUrl,
+    date: new Date(),
+  }
+  const fname = `${utils.GetDateString()}_${userid}_${testUrl}.json`
+  const subject = getSubjNameWithoutYear(subj).replace(/\//g, '-')
+  const subjPath = `${savedQuestionsDir}/${subject}`
+  const savedSubjQuestionsFilePath = `${subjPath}/${savedQuestionsFileName}`
+
+  utils.CreatePath(subjPath, true)
+  if (!utils.FileExists(savedSubjQuestionsFilePath)) {
+    utils.WriteFile('[]', savedSubjQuestionsFilePath)
+  }
+
+  const savedQuestions = utils.ReadJSON(savedSubjQuestionsFilePath)
+
+  const testExists = false
+  // TODO: do this on another thread?
+  // const testExists = savedQuestions.some((savedQuestion) => {
+  //   const data = utils.ReadJSON(`${subjPath}/${savedQuestion.fname}`)
+
+  //   return data.questions.some((dQuestion) => {
+  //     return questions.some((question) => {
+  //       const percent = compareQuestionObj(
+  //         createQuestion(question),
+  //         '',
+  //         createQuestion(dQuestion),
+  //         ''
+  //       )
+
+  //       return percent.avg === 100
+  //     })
+  //   })
+  // })
+
+  if (testExists) {
+    return
+  }
+
+  savedQuestions.push({
+    fname: fname,
+    subj: subj,
+    userid: userid,
+    testUrl: testUrl,
+    date: new Date(),
+  })
+  utils.WriteFile(JSON.stringify(savedQuestions), savedSubjQuestionsFilePath)
+  utils.WriteFile(JSON.stringify(questionsToSave), `${subjPath}/${fname}`)
+}
+
+function LoadVersion() {
+  const scriptContent = utils.ReadFile(userScriptFile)
+
+  let temp: any = scriptContent.split('\n').find((x) => {
+    return x.includes('@version')
+  })
+  temp = temp.split(' ')
+  temp = temp[temp.length - 1]
+
+  return temp
+}
+
+function LoadMOTD(motdFile) {
+  return utils.ReadFile(motdFile)
+}
+
+function LoadUserSpecificMOTD(userSpecificMotdFile) {
+  try {
+    return utils.ReadJSON(userSpecificMotdFile)
+  } catch (err) {
+    logger.Log('Couldnt parse user specific motd!', logger.GetColor('redbg'))
+    console.error(err)
+  }
+}
+
+function LoadTestUsers() {
+  let testUsers = utils.ReadJSON(testUsersFile)
+  if (testUsers) {
+    testUsers = testUsers.userIds
+  }
+  return testUsers
+}
+
+function getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs) {
+  logger.Log(
+    `No suitable questiondbs found for ${location}, creating a new one...`
+  )
+  const newDb: DataFile = {
+    path: `questionDbs/${location}.json`,
+    name: location,
+    shouldSearch: {
+      location: {
+        val: location,
+      },
+    },
+    shouldSave: {
+      location: {
+        val: location,
+      },
+    },
+  }
+
+  utils.WriteFile(
+    JSON.stringify(
+      [
+        ...utils.ReadJSON(dbsFile),
+        newDb, // stored as 'data.json', but is './publicDirs/.../data.json' runtime
+      ],
+      null,
+      2
+    ),
+    dbsFile
+  )
+
+  // "loading" new db
+  const loadedNewDb: QuestionDb = {
+    ...newDb,
+    data: [],
+    path: publicDir + newDb.path,
+    index: maxIndex,
+  }
+  utils.WriteFile('[]', loadedNewDb.path)
+
+  questionDbs.push(loadedNewDb)
+  msgAllWorker({
+    newdb: loadedNewDb,
+    type: 'newdb',
+  })
+
+  return loadedNewDb
+}
+
+function setup(data: SubmoduleData): any {
+  const { app, userDB, /* url */ publicdirs /* moduleSpecificData */ } = data
+
+  const publicDir = publicdirs[0]
+  const motdFile = publicDir + 'motd'
+  const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
+  const dbsFile = publicDir + 'questionDbs.json'
+  const savedQuestionsDir = publicDir + 'savedQuestions'
+
+  let version = LoadVersion()
+  let motd = LoadMOTD(motdFile)
+  let userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
+  let testUsers: any = LoadTestUsers()
+
+  const dataFiles: Array<DataFile> = utils.ReadJSON(dbsFile)
+  const questionDbs: Array<QuestionDb> = loadJSON(dataFiles, publicDir)
+  initWorkerPool(questionDbs)
+
+  const filesToWatch = [
+    {
+      fname: userSpecificMotdFile,
+      logMsg: 'User Specific Motd updated',
+      action: () => {
+        userSpecificMotd = LoadUserSpecificMOTD(userSpecificMotdFile)
+      },
+    },
+    {
+      fname: motdFile,
+      logMsg: 'Motd updated',
+      action: () => {
+        motd = LoadMOTD(motdFile)
+      },
+    },
+    {
+      fname: testUsersFile,
+      logMsg: 'Test Users file changed',
+      action: () => {
+        testUsers = LoadTestUsers()
+      },
+    },
+    {
+      fname: userScriptFile,
+      logMsg: 'User script file changed',
+      action: () => {
+        version = LoadVersion()
+      },
+    },
+  ]
+
+  app.get('/getDbs', (req: Request, res: any) => {
+    logger.LogReq(req)
+    res.json(
+      questionDbs.map((qdb) => {
+        return {
+          path: qdb.path.replace(publicDir, ''),
+          name: qdb.name,
+          locked: qdb.locked,
+        }
+      })
+    )
+  })
+
+  app.get('/allqr.txt', function(req: Request, res: any) {
+    logger.LogReq(req)
+    const db: any = req.query.db
+    let stringifiedData = ''
+
+    res.setHeader('content-type', 'text/plain; charset=utf-8')
+
+    if (db) {
+      const requestedDb = questionDbs.find((qdb) => {
+        return qdb.name === db
+      })
+
+      if (!requestedDb) {
+        res.end(`No such db ${db}`)
+        return
+      }
+
+      stringifiedData = '\n' + line
+      stringifiedData += `  Questions in ${requestedDb.name}:  `
+      stringifiedData += line + '\n'
+      stringifiedData += dataToString(requestedDb.data)
+      stringifiedData += '\n' + line + line + '\n'
+    } else {
+      stringifiedData = questionDbs
+        .map((qdb) => {
+          let result = ''
+          result += '\n' + line
+          result += `  Questions in ${qdb.name}:  `
+          result += line + '\n'
+          result += dataToString(qdb.data)
+          result += '\n' + line + line + '\n'
+          return result
+        })
+        .join('\n\n')
+    }
+    res.end(stringifiedData)
+  })
+
+  app.post('/isAdding', function(req: Request, res: any) {
+    logger.LogReq(req)
+    const user: User = req.session.user
+    const dryRun = testUsers.includes(user.id)
+    if (!req.body.location) {
+      logger.Log(
+        '\tbody.location is missing, client version:' + req.body.version
+      )
+      res.json({ msg: 'body.location is missing' })
+      return
+    }
+
+    const location = req.body.location.split('/')[2]
+
+    try {
+      let maxIndex = -1
+      const suitedQuestionDbs = questionDbs.filter((qdb) => {
+        if (maxIndex < qdb.index) {
+          maxIndex = qdb.index
+        }
+        return shouldSaveDataFile(qdb, req.body)
+      }, [])
+
+      if (suitedQuestionDbs.length === 0) {
+        if (!dbExists(location, questionDbs)) {
+          suitedQuestionDbs.push(
+            getNewQdb(location, maxIndex, dbsFile, publicDir, questionDbs)
+          )
+        } else {
+          logger.Log(
+            `Tried to add existing db named ${location}!`,
+            logger.GetColor('red')
+          )
+        }
+      }
+      if (suitedQuestionDbs.length === 0) {
+        res.json({
+          status: 'fail',
+          msg: 'No suitable dbs to add questions to',
+        })
+        return
+      }
+
+      processIncomingRequest(req.body, suitedQuestionDbs, dryRun, user)
+        .then((resultArray: Array<Result>) => {
+          logResult(req.body, resultArray, user.id, dryRun)
+
+          const totalNewQuestions = resultArray.reduce((acc, sres) => {
+            return acc + sres.newQuestions.length
+          }, 0)
+
+          res.json({
+            success: resultArray.length > 0,
+            newQuestions: resultArray,
+            totalNewQuestions: totalNewQuestions,
+          })
+
+          if (totalNewQuestions > 0) {
+            resultArray.forEach((result) => {
+              msgAllWorker({
+                type: 'newQuestions',
+                data: result,
+              })
+            })
+          }
+        })
+        .catch((err) => {
+          logger.Log(
+            'Error during processing incoming request',
+            logger.GetColor('redbg')
+          )
+          console.error(err)
+          res.json({
+            success: false,
+          })
+        })
+    } catch (err) {
+      logger.Log(
+        'Error during getting incoming request processor promises ',
+        logger.GetColor('redbg')
+      )
+      console.error(err)
+      res.json({
+        success: false,
+      })
+    }
+  })
+
+  app.post('/ask', function(req: Request, res) {
+    const user: User = req.session.user
+
+    if (!req.body.questions) {
+      res.json({
+        message: `ask something! { question:'' ,subject:'', location:'' }`,
+        result: [],
+        recievedData: JSON.stringify(req.body),
+        success: false,
+      })
+      return
+    }
+    const subj: any = req.body.subj || ''
+    // TODO: test if testUrl is undefined (it shouldnt)
+    const testUrl = req.body.testUrl
+      ? req.body.testUrl.split('/')[2]
+      : undefined
+
+    writeAskData(req.body)
+
+    // every question in a different thread
+    const resultPromises = req.body.questions.map((question) => {
+      return getResult({
+        question: question,
+        subj: subj,
+        recData: req.body,
+        testUrl: testUrl,
+        questionDbs: questionDbs,
+      })
+    })
+
+    Promise.all(resultPromises).then((results) => {
+      const response = results.map((result: any) => {
+        return {
+          answers: result.result,
+          question: result.question,
+        }
+      })
+      res.json(response)
+
+      const saveableQuestions = response.reduce((acc, res) => {
+        if (res.answers.length === 0) {
+          acc.push(res.question)
+        }
+        return acc
+      }, [])
+
+      if (saveableQuestions.length > 0) {
+        saveQuestion(
+          saveableQuestions,
+          subj,
+          testUrl,
+          user.id,
+          savedQuestionsDir
+        )
+      }
+    })
+  })
+  app.get('/ask', function(req: Request, res) {
+    if (Object.keys(req.query).length === 0) {
+      logger.DebugLog(`No query params /ask GET`, 'ask', 1)
+      res.json({
+        message:
+          'ask something! ?q=[question]&subj=[subject]&data=[question data]. "subj" is optimal for faster result',
+        result: [],
+        recievedData: JSON.stringify(req.query),
+        success: false,
+      })
+      return
+    }
+
+    if (req.query.q && req.query.data) {
+      const subj: any = req.query.subj || ''
+      const question = req.query.q
+      const recData: any = req.query.data
+      getResult({
+        question: question,
+        subj: subj,
+        recData: recData,
+        recievedData: req.query,
+        questionDbs: questionDbs,
+      }).then((result) => {
+        res.json(result)
+      })
+    } else {
+      logger.DebugLog(`Invalid question`, 'ask', 1)
+      res.json({
+        message: `Invalid question :(`,
+        result: [],
+        recievedData: JSON.stringify(req.query),
+        success: false,
+      })
+    }
+  })
+  app.get('/datacount', function(req: Request, res: any) {
+    logger.LogReq(req)
+    if (req.query.detailed === 'all') {
+      res.json({
+        detailed: getDetailedRes(questionDbs),
+        simple: getSimplreRes(questionDbs),
+      })
+    } else if (req.query.detailed) {
+      res.json(getDetailedRes(questionDbs))
+    } else {
+      res.json(getSimplreRes(questionDbs))
+    }
+  })
+  app.get('/infos', function(req: Request, res) {
+    const user: User = req.session.user
+
+    const result: any = {
+      result: 'success',
+      uid: user.id,
+    }
+
+    if (req.query.subjinfo) {
+      result.subjinfo = getSimplreRes(questionDbs)
+    }
+    if (req.query.version) {
+      result.version = version
+    }
+    if (req.query.motd) {
+      result.motd = getMotd(req.query.cversion, motd)
+      if (userSpecificMotd[user.id]) {
+        result.userSpecificMotd = {
+          msg: userSpecificMotd[user.id].msg,
+          seen: userSpecificMotd[user.id].seen,
+        }
+      }
+    }
+    res.json(result)
+  })
+  app.post('/infos', (req: Request, res) => {
+    const user: User = req.session.user
+
+    if (req.body.userSpecificMotdSeen && !userSpecificMotd[user.id].seen) {
+      userSpecificMotd[user.id].seen = true
+
+      logger.Log(
+        `User #${user.id}'s user specific motd is now seen:`,
+        logger.GetColor('bluebg')
+      )
+      logger.Log(userSpecificMotd[user.id].msg)
+      utils.WriteFile(
+        JSON.stringify(userSpecificMotd, null, 2),
+        userSpecificMotdFile
+      )
+    }
+
+    res.json({ msg: 'done' })
+  })
+
+  app.post('/registerscript', function(req: Request, res) {
+    logger.LogReq(req)
+
+    if (!utils.FileExists(registeredScriptsFile)) {
+      utils.WriteFile('[]', registeredScriptsFile)
+    }
+
+    const ip: any =
+      req.headers['cf-connecting-ip'] || req.connection.remoteAddress
+    const ua: any = req.headers['user-agent']
+    const registeredScripts = utils.ReadJSON(registeredScriptsFile)
+    const { cid, uid, version, installSource, date } = req.body
+
+    const index = registeredScripts.findIndex((registration) => {
+      return registration.cid === cid
+    })
+
+    if (index === -1) {
+      const x: any = {
+        cid: cid,
+        version: version,
+        installSource: installSource,
+        date: date,
+        ip: ip,
+        userAgent: ua,
+      }
+
+      if (uid) {
+        x.uid = uid
+        x.loginDate = date
+      }
+      registeredScripts.push(x)
+    } else {
+      const currRegistration = registeredScripts[index]
+
+      if (!currRegistration.uid && uid) {
+        registeredScripts[index] = {
+          ...registeredScripts[index],
+          uid: uid,
+          loginDate: date,
+        }
+      } else {
+        logger.DebugLog(
+          `cid: ${cid}, uid: ${uid} tried to register multiple times`,
+          'register',
+          1
+        )
+      }
+    }
+
+    utils.WriteFile(
+      JSON.stringify(registeredScripts, null, 2),
+      registeredScriptsFile
+    )
+
+    res.json({ msg: 'done' })
+  })
+
+  app.get('/possibleAnswers', (req: Request, res: any) => {
+    logger.LogReq(req)
+    const files = utils.ReadDir(savedQuestionsDir)
+
+    files.sort(function(a, b) {
+      return (
+        fs.statSync(savedQuestionsDir + '/' + b).mtime.getTime() -
+        fs.statSync(savedQuestionsDir + '/' + a).mtime.getTime()
+      )
+    })
+
+    res.json({
+      savedQuestionsFileName: savedQuestionsFileName,
+      subjects: files.map((subj) => {
+        return {
+          name: subj,
+          path: `savedQuestions/${subj}/`,
+        }
+      }),
+    })
+  })
+
+  app.post('/rmPossibleAnswer', (req: Request, res: any) => {
+    logger.LogReq(req)
+    const user: User = req.session.user
+
+    const subj = req.body.subj
+    const file = req.body.file
+    const savedQuestionsPath = `${savedQuestionsDir}/${subj}/${savedQuestionsFileName}`
+    const savedQuestions = utils.ReadJSON(savedQuestionsPath)
+    let path = `${savedQuestionsDir}/${subj}/${file}`
+    // to prevent deleting ../../../ ... /etc/shadow
+    while (path.includes('..')) {
+      path = path.replace(/\.\./g, '.')
+    }
+
+    if (utils.FileExists(path)) {
+      utils.deleteFile(path)
+
+      utils.WriteFile(
+        JSON.stringify(
+          savedQuestions.filter((sq) => {
+            return sq.fname !== file
+          })
+        ),
+        savedQuestionsPath
+      )
+
+      logger.Log(
+        `User #${user.id} deleted '${file}' from subject '${subj}'`,
+        logger.GetColor('cyan')
+      )
+      res.json({
+        res: 'ok',
+      })
+    } else {
+      logger.Log(
+        `User #${user.id} tried to delete '${file}' from subject '${subj}', but failed`,
+        logger.GetColor('red')
+      )
+      res.json({
+        res: 'fail',
+      })
+    }
+  })
+
+  app.post('/updateQuestion', (req: Request, res) => {
+    logger.LogReq(req)
+    const user: User = req.session.user
+    const date = utils.GetDateString()
+
+    const editType = req.body.type
+    const selectedDb = req.body.selectedDb
+    if (!editType || !selectedDb) {
+      res.json({
+        status: 'fail',
+        msg: 'No .editType or .selectedDb !',
+      })
+      return
+    }
+
+    const dbIndex = questionDbs.findIndex((qdb) => {
+      return qdb.name === selectedDb.name
+    })
+    const currDb = questionDbs[dbIndex]
+
+    if (dbIndex === -1) {
+      res.json({
+        status: 'fail',
+        msg: `No question db named like ${selectedDb.name}!`,
+      })
+      return
+    }
+
+    // -----------------
+    const {
+      success,
+      msg,
+      resultDb,
+      deletedQuestion,
+      newVal,
+      oldVal,
+      deletedQuestions,
+      changedQuestions,
+    } = editDb(currDb, req.body)
+
+    if (!success) {
+      res.json({ success: success, msg: msg })
+      return
+    }
+    if (resultDb) {
+      questionDbs[dbIndex] = resultDb
+    }
+
+    if (editType === 'delete') {
+      const { index, subjName } = req.body
+      logger.Log(
+        `User #${user.id} deleted a question from '${subjName}'`,
+        logger.GetColor('cyan')
+      )
+      utils.AppendToFile(
+        `${date}: User ${user.id} deleted a question from '${subjName}' (index: ${index})`,
+        dataEditsLog
+      )
+      utils.AppendToFile(JSON.stringify(deletedQuestion, null, 2), dataEditsLog)
+    }
+
+    if (editType === 'edit') {
+      const { index, subjName } = req.body
+      logger.Log(
+        `User #${user.id} edited a question in '${subjName}'`,
+        logger.GetColor('cyan')
+      )
+      utils.AppendToFile(
+        `${date}: User ${user.id} edited a question in '${subjName}' (index: ${index})`,
+        dataEditsLog
+      )
+      utils.AppendToFile(
+        JSON.stringify(
+          {
+            newVal: newVal,
+            oldVal: oldVal,
+          },
+          null,
+          2
+        ),
+        dataEditsLog
+      )
+    }
+
+    if (editType === 'subjEdit') {
+      const { subjName } = req.body
+      logger.Log(
+        `User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
+        logger.GetColor('cyan')
+      )
+      utils.AppendToFile(
+        `${date} User #${user.id} modified '${subjName}'. Edited: ${deletedQuestions.length}, deleted: ${deletedQuestions.length}`,
+        dataEditsLog
+      )
+      utils.AppendToFile(
+        JSON.stringify(
+          {
+            deletedQuestions: deletedQuestions,
+            changedQuestions: changedQuestions,
+          },
+          null,
+          2
+        ),
+        dataEditsLog
+      )
+    }
+    // ------------------
+
+    if (success) {
+      writeData(currDb.data, currDb.path)
+      msgAllWorker({
+        type: 'dbEdit',
+        data: {
+          dbIndex: dbIndex,
+          edits: req.body,
+        },
+      })
+    }
+
+    res.json({
+      success: true,
+      msg: 'OK',
+    })
+  })
+
+  return {
+    dailyAction: () => {
+      backupData(questionDbs)
+      ExportDailyDataCount(questionDbs, userDB)
+    },
+    load: () => {
+      backupData(questionDbs)
+
+      filesToWatch.forEach((ftw) => {
+        if (utils.FileExists(ftw.fname)) {
+          utils.WatchFile(ftw.fname, () => {
+            logger.Log(ftw.logMsg)
+            ftw.action()
+          })
+          ftw.action()
+        } else {
+          logger.Log(
+            `File ${ftw.fname} does not exists to watch!`,
+            logger.GetColor('redbg')
+          )
+        }
+      })
+    },
+  }
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/quickvote.ts

@@ -0,0 +1,90 @@
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+
+const quickVoteResultsDir = 'stats/qvote'
+const quickVotes = 'stats/qvote/votes.json'
+
+function setup(data: SubmoduleData): void {
+  const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
+
+  app.get('/quickvote', (req: Request, res: any) => {
+    const key = req.query.key
+    const val: any = req.query.val
+    const user: User = req.session.user
+
+    if (!key || !val) {
+      res.render('votethank', {
+        results: 'error',
+        msg: 'no key or val query param!',
+      })
+      return
+    }
+
+    // FIXME: check vote type in file
+    let votes: any = {}
+    if (utils.FileExists(quickVotes)) {
+      votes = utils.ReadJSON(quickVotes)
+    } else {
+      logger.Log(
+        `No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`,
+        logger.GetColor('blue')
+      )
+      res.render('votethank', {
+        result: 'no such pool',
+      })
+      return
+    }
+
+    if (!votes.voteNames.includes(key)) {
+      logger.Log(
+        `No such vote "${key}" ( #${user.id}: ${key}-${val} )`,
+        logger.GetColor('blue')
+      )
+      res.render('votethank', {
+        result: 'no such pool',
+      })
+      return
+    }
+
+    const voteFile = quickVoteResultsDir + '/' + key + '.json'
+
+    let voteData = {
+      votes: {},
+      sum: {},
+    }
+
+    if (utils.FileExists(voteFile)) {
+      voteData = utils.ReadJSON(voteFile)
+    } else {
+      utils.CreatePath(quickVoteResultsDir)
+    }
+
+    const prevVote = voteData.votes[user.id]
+
+    voteData.votes[user.id] = val
+    if (voteData.sum[val]) {
+      voteData.sum[val]++
+    } else {
+      voteData.sum[val] = 1
+    }
+    if (prevVote) {
+      if (voteData.sum[prevVote]) {
+        voteData.sum[prevVote] -= 1
+      }
+    }
+
+    logger.Log(`Vote from #${user.id}: ${key}: ${val}`, logger.GetColor('blue'))
+    res.render('votethank', {
+      result: prevVote ? 'already voted' : 'success',
+      prevVote: prevVote,
+      msg: 'vote added',
+    })
+
+    utils.WriteFile(JSON.stringify(voteData), voteFile)
+  })
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/ranklist.ts

@@ -0,0 +1,104 @@
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+
+const idStatFile = 'stats/idstats'
+const idvStatFile = 'stats/idvstats'
+
+function mergeObjSum(a, b) {
+  const res = { ...b }
+  Object.keys(a).forEach((key) => {
+    if (res[key]) {
+      res[key] += a[key]
+    } else {
+      res[key] = a[key]
+    }
+  })
+
+  return res
+}
+
+function setup(data: SubmoduleData): void {
+  const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
+
+  app.get('/ranklist', (req: Request, res: any) => {
+    logger.LogReq(req)
+    let result
+    const querySince: any = req.query.since
+    const user: User = req.session.user
+
+    if (!querySince) {
+      result = utils.ReadJSON(idStatFile)
+    } else {
+      try {
+        const since = new Date(querySince)
+        if (!(since instanceof Date) || isNaN(since.getTime())) {
+          throw new Error('Not a date')
+        }
+        const data = utils.ReadJSON(idvStatFile)
+        result = {}
+
+        Object.keys(data).forEach((key) => {
+          const dailyStat = data[key]
+
+          if (new Date(key) > since) {
+            Object.keys(dailyStat).forEach((userId) => {
+              const userStat = dailyStat[userId]
+              const uidRes = result[userId]
+
+              if (!uidRes) {
+                result[userId] = userStat
+              } else {
+                result[userId] = {
+                  count: uidRes.count + userStat.count,
+                  newQuestions: uidRes.newQuestions + userStat.newQuestions,
+                  allQuestions: uidRes.allQuestions + userStat.allQuestions,
+                  subjs: mergeObjSum(uidRes.subjs, userStat.subjs),
+                }
+              }
+            })
+          }
+        })
+      } catch (err) {
+        res.json({
+          msg: 'invalid date format, or other error occured',
+        })
+      }
+    }
+
+    const list = []
+    const sum = {
+      count: 0,
+      newQuestions: 0,
+      allQuestions: 0,
+    }
+    Object.keys(result).forEach((key) => {
+      list.push({
+        userId: parseInt(key),
+        ...result[key],
+      })
+
+      sum.count = sum.count + result[key].count
+      sum.newQuestions = sum.newQuestions + result[key].newQuestions
+      sum.allQuestions = sum.allQuestions + result[key].allQuestions
+    })
+
+    if (list.length === 0) {
+      res.json({
+        msg: 'There are no users in the stats db :c',
+      })
+      return
+    }
+
+    res.json({
+      since: querySince,
+      sum: sum,
+      list: list,
+      selfuserId: user.id,
+    })
+  })
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/todos.ts

@@ -0,0 +1,65 @@
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData } from '../../../types/basicTypes'
+
+const todosFile = 'data/todos.json'
+
+function setup(data: SubmoduleData): void {
+  const { app /* userDB, url, publicdirs, moduleSpecificData */ } = data
+
+  app.get('/voteTodo', (req: Request, res: any) => {
+    logger.LogReq(req)
+    const userId = req.session.user.id
+    const id: any = req.query.id
+    const todos = utils.ReadJSON(todosFile)
+
+    if (!id) {
+      res.json({
+        msg: 'id query undefined',
+        result: 'not ok',
+      })
+    }
+
+    const cardIndex = todos.cards.findIndex((currcard) => {
+      return currcard.id === parseInt(id)
+    })
+    if (cardIndex === -1) {
+      res.json({
+        msg: 'card not found',
+        result: 'not ok',
+      })
+      return
+    }
+
+    const ind = todos.cards[cardIndex].votes.indexOf(userId)
+    if (ind === -1) {
+      todos.cards[cardIndex].votes.push(userId)
+    } else {
+      todos.cards[cardIndex].votes.splice(ind, 1)
+    }
+
+    utils.WriteFile(JSON.stringify(todos, null, 2), todosFile)
+    res.json({
+      todos: todos,
+      userId: userId,
+      msg: 'updated',
+      result: 'ok',
+    })
+  })
+
+  app.get('/todos', (req: Request, res: any) => {
+    logger.LogReq(req)
+    const userId = req.session.user.id
+    const todos = utils.ReadJSON(todosFile)
+
+    res.json({
+      todos: todos,
+      userId: userId,
+      result: 'ok',
+    })
+  })
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/userFiles.ts

@@ -0,0 +1,195 @@
+import fs from 'fs'
+
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+
+const usersFileName = '.users.json'
+
+function setup(data: SubmoduleData): void {
+  const { app, /* userDB, url, */ publicdirs /* moduleSpecificData */ } = data
+
+  const publicDir = publicdirs[0]
+
+  const userFilesDir = publicDir + 'userFiles'
+
+  if (!utils.FileExists(userFilesDir)) {
+    utils.CreatePath(userFilesDir, true)
+  }
+
+  app.get('/listUserDir', (req: Request, res) => {
+    logger.LogReq(req)
+
+    const subdir: any = req.query.subdir
+
+    if (subdir) {
+      const safeSubdir = subdir.replace(/\.+/g, '').replace(/\/+/g, '')
+      const dir = userFilesDir + '/' + safeSubdir
+      const usersFile = dir + '/' + usersFileName
+
+      if (!utils.FileExists(dir)) {
+        res.json({
+          success: false,
+          msg: `Directory ${subdir} does not exists`,
+        })
+        return
+      }
+      if (!utils.FileExists(usersFile)) {
+        utils.WriteFile('{}', usersFile)
+      }
+      const users = utils.ReadJSON(usersFile)
+
+      if (!utils.FileExists(dir)) {
+        res.json({
+          success: false,
+          msg: `Path '${safeSubdir}' does not exists`,
+        })
+        return
+      }
+
+      res.json({
+        success: true,
+        files: utils.ReadDir(dir).reduce((acc, file) => {
+          const stat = fs.lstatSync(dir + '/' + file)
+
+          if (stat.isDirectory()) {
+            return acc
+          }
+
+          acc.push({
+            name: file,
+            path: dir.replace(publicDir, '') + '/' + file,
+            size: stat.size,
+            date: stat.mtime.getTime(),
+            user: users[file] || -1,
+          })
+          return acc
+        }, []),
+      })
+    } else {
+      res.json({
+        success: true,
+        dirs: utils.ReadDir(userFilesDir).reduce((acc, file) => {
+          const stat = fs.lstatSync(userFilesDir + '/' + file)
+
+          if (!stat.isDirectory()) {
+            return acc
+          }
+
+          acc.push({
+            name: file,
+            date: stat.mtime.getTime(),
+            size: utils.ReadDir(userFilesDir + '/' + file).length,
+          })
+          return acc
+        }, []),
+      })
+    }
+  })
+
+  app.post('/deleteUserFile', (req: Request, res) => {
+    const dir: any = req.body.dir
+    const fname: any = req.body.fname
+    if (!dir || !fname) {
+      res.json({
+        success: false,
+        msg: `'dir' or 'fname' is undefined!`,
+      })
+      return
+    }
+    const safeDir = dir.replace(/\.+/g, '').replace(/\/+/g, '')
+    const safeFname = fname.replace(/\.+/g, '.').replace(/\/+/g, '')
+    const filePath = userFilesDir + '/' + safeDir + '/' + safeFname
+
+    if (!utils.FileExists(filePath)) {
+      res.json({
+        success: false,
+        msg: `path does not exists!`,
+      })
+      return
+    }
+    utils.deleteFile(filePath)
+    const usersFile = userFilesDir + '/' + safeDir + '/' + usersFileName
+    const users = utils.ReadJSON(usersFile)
+    delete users[safeFname]
+    utils.WriteFile(JSON.stringify(users), usersFile)
+
+    res.json({
+      success: true,
+    })
+  })
+
+  app.post('/newUserDir', (req: Request, res) => {
+    logger.LogReq(req)
+
+    const name: any = req.body.name
+    if (!name) {
+      res.json({
+        success: false,
+        msg: `name is undefined!`,
+      })
+      return
+    }
+    const safeName = name.replace(/\.+/g, '').replace(/\/+/g, '')
+
+    if (utils.FileExists(userFilesDir + '/' + safeName)) {
+      res.json({
+        success: false,
+        msg: `Dir ${name} already exists`,
+      })
+      return
+    }
+    utils.CreatePath(userFilesDir + '/' + safeName, true)
+
+    res.json({
+      success: true,
+    })
+  })
+
+  app.post('/uploadUserFile', (req: Request, res) => {
+    logger.LogReq(req)
+
+    const user: User = req.session.user
+    const dir = req.body.dir
+    if (!dir) {
+      res.json({
+        success: false,
+        msg: `dir '${dir}' is undefined!`,
+      })
+      return
+    }
+    const safeDir = dir.replace(/\.+/g, '.').replace(/\/+/g, '/')
+    if (!utils.FileExists(userFilesDir + '/' + safeDir)) {
+      res.json({
+        success: false,
+        msg: `dir '${dir}' does not exists!`,
+      })
+      return
+    }
+
+    utils
+      .uploadFile(req, userFilesDir + '/' + safeDir)
+      .then((body) => {
+        logger.Log(
+          `Successfull upload ${body.filePath}`,
+          logger.GetColor('blue')
+        )
+
+        const usersFile = userFilesDir + '/' + safeDir + '/' + usersFileName
+        const users = utils.ReadJSON(usersFile)
+        users[body.fileName] = user.id
+        utils.WriteFile(JSON.stringify(users), usersFile)
+
+        res.json({
+          success: true,
+        })
+      })
+      .catch(() => {
+        res.end('something bad happened :s')
+      })
+  })
+}
+
+export default {
+  setup: setup,
+}

src/modules/api/submodules/userManagement.ts

@@ -0,0 +1,305 @@
+import { v4 as uuidv4 } from 'uuid'
+
+import logger from '../../../utils/logger'
+import utils from '../../../utils/utils'
+import { Request, SubmoduleData, User } from '../../../types/basicTypes'
+import dbtools from '../../../utils/dbtools'
+
+const minimumAlowwedSessions = 2 // how many sessions are allowed for a user
+const addPWPerDay = 3 // every x day a user can give a pw
+const maxPWCount = 6 // maximum pw give opportunities a user can have at once
+const addPWCount = 1 // how many pw gen opportunities to add each time
+const daysAfterUserGetsPWs = 5 // days after user gets pw-s
+const usersDbBackupPath = 'data/dbs/backup'
+
+function BackupDB(usersDbBackupPath, userDB) {
+  logger.Log('Backing up auth DB ...')
+  utils.CreatePath(usersDbBackupPath, true)
+  userDB
+    .backup(
+      `${usersDbBackupPath}/users.${utils
+        .GetDateString()
+        .replace(/ /g, '_')}.db`
+    )
+    .then(() => {
+      logger.Log('Auth DB backup complete!')
+    })
+    .catch((err) => {
+      logger.Log('Auth DB backup failed!', logger.GetColor('redbg'))
+      console.error(err)
+    })
+}
+
+function setup(data: SubmoduleData): any {
+  const { app, userDB, url /* publicdirs, moduleSpecificData */ } = data
+  let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
+  domain.shift() // [ "frylabs", "net" ]
+  domain = domain.join('.') // "frylabs.net"
+  logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
+
+  app.get('/avaiblePWS', (req: Request, res: any) => {
+    logger.LogReq(req)
+
+    const user: User = req.session.user
+
+    res.json({
+      success: true,
+      userCreated: user.created,
+      availablePWS: user.avaiblePWRequests,
+      requestedPWS: user.pwRequestCount,
+      maxPWCount: maxPWCount,
+      daysAfterUserGetsPWs: daysAfterUserGetsPWs,
+      addPWPerDay: addPWPerDay,
+      addPWCount: addPWCount,
+      dayDiff: getDayDiff(user.created),
+      userCount: dbtools.TableInfo(userDB, 'users').dataCount,
+    })
+  })
+
+  app.post('/getpw', function(req: Request, res: any) {
+    logger.LogReq(req)
+
+    const requestingUser = req.session.user
+
+    if (requestingUser.avaiblePWRequests <= 0) {
+      res.json({
+        result: 'error',
+        msg:
+          'Too many passwords requested or cant request password yet, try later',
+      })
+      logger.Log(
+        `User #${requestingUser.id} requested too much passwords`,
+        logger.GetColor('cyan')
+      )
+      return
+    }
+
+    dbtools.Update(
+      userDB,
+      'users',
+      {
+        avaiblePWRequests: requestingUser.avaiblePWRequests - 1,
+        pwRequestCount: requestingUser.pwRequestCount + 1,
+      },
+      {
+        id: requestingUser.id,
+      }
+    )
+
+    const pw = uuidv4()
+    const insertRes = dbtools.Insert(userDB, 'users', {
+      pw: pw,
+      avaiblePWRequests: 0,
+      created: utils.GetDateString(),
+    })
+
+    logger.Log(
+      `User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`,
+      logger.GetColor('cyan')
+    )
+
+    res.json({
+      pw: pw,
+      success: true,
+      userCreated: requestingUser.created,
+      availablePWS: requestingUser.avaiblePWRequests,
+      requestedPWS: requestingUser.pwRequestCount,
+      maxPWCount: maxPWCount,
+      daysAfterUserGetsPWs: daysAfterUserGetsPWs,
+      addPWPerDay: addPWPerDay,
+      addPWCount: addPWCount,
+      dayDiff: getDayDiff(requestingUser.created),
+      userCount: dbtools.TableInfo(userDB, 'users').dataCount,
+    })
+  })
+
+  app.post('/login', (req: Request, res: any) => {
+    logger.LogReq(req)
+    const pw = req.body.pw
+      ? req.body.pw
+          .replace(/'/g, '')
+          .replace(/"/g, '')
+          .replace(/;/g, '')
+      : false
+    const isScript = req.body.script
+    const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
+    const user: User = dbtools.Select(userDB, 'users', {
+      pw: pw,
+    })[0]
+
+    if (user) {
+      const sessionID = uuidv4()
+
+      const existingSessions = dbtools
+        .Select(userDB, 'sessions', {
+          userID: user.id,
+          isScript: isScript ? 1 : 0,
+        })
+        .sort((a, b) => {
+          return new Date(a).getTime() - new Date(b).getTime()
+        })
+
+      const diff = existingSessions.length - minimumAlowwedSessions
+      if (diff > 0) {
+        logger.Log(
+          `Multiple ${isScript ? 'script' : 'website'} sessions ( ${
+            existingSessions.length
+          } ) for #${user.id}, deleting olds`,
+          logger.GetColor('cyan')
+        )
+        for (let i = 0; i < diff; i++) {
+          const id = existingSessions[i].id
+          dbtools.Delete(userDB, 'sessions', {
+            id: id,
+            isScript: isScript ? 1 : 0,
+          })
+        }
+      }
+
+      dbtools.Update(
+        userDB,
+        'users',
+        {
+          loginCount: user.loginCount + 1,
+          lastIP: ip,
+          lastLogin: utils.GetDateString(),
+        },
+        {
+          id: user.id,
+        }
+      )
+
+      dbtools.Insert(userDB, 'sessions', {
+        id: sessionID,
+        ip: ip,
+        userID: user.id,
+        isScript: isScript ? 1 : 0,
+        createDate: utils.GetDateString(),
+      })
+
+      // https://www.npmjs.com/package/cookie
+      // FIXME: cookies are not configured coorectly
+      res.cookie('sessionID', sessionID, {
+        domain: domain,
+        expires: new Date(
+          new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
+        ),
+        sameSite: 'none',
+        secure: true,
+      })
+      res.cookie('sessionID', sessionID, {
+        expires: new Date(
+          new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
+        ),
+        sameSite: 'none',
+        secure: true,
+      })
+
+      res.json({
+        result: 'success',
+        msg: 'you are now logged in',
+      })
+      logger.Log(
+        `Successfull login to ${
+          isScript ? 'script' : 'website'
+        } with user ID: #${user.id}`,
+        logger.GetColor('cyan')
+      )
+    } else {
+      logger.Log(
+        `Login attempt with invalid pw: ${pw} to ${
+          isScript ? 'script' : 'website'
+        }`,
+        logger.GetColor('cyan')
+      )
+      res.json({
+        result: 'error',
+        msg: 'Invalid password',
+      })
+    }
+  })
+
+  app.get('/logout', (req: Request, res: any) => {
+    logger.LogReq(req)
+    const sessionID = req.cookies.sessionID
+    const user: User = req.session.user
+
+    if (!user) {
+      res.json({
+        msg: 'You are not logged in',
+        success: false,
+      })
+      return
+    }
+
+    logger.Log(
+      `Successfull logout with user ID: #${user.id}`,
+      logger.GetColor('cyan')
+    )
+
+    // removing session from db
+    dbtools.Delete(userDB, 'sessions', {
+      id: sessionID,
+    })
+    res.clearCookie('sessionID').json({
+      msg: 'Successfull logout',
+      result: 'success',
+    })
+  })
+
+  function getDayDiff(dateString) {
+    const msdiff = new Date().getTime() - new Date(dateString).getTime()
+    return Math.floor(msdiff / (1000 * 3600 * 24))
+  }
+
+  function IncrementAvaiblePWs() {
+    // FIXME: check this if this is legit and works
+    logger.Log('Incrementing avaible PW-s ...')
+    const users: Array<User> = dbtools.SelectAll(userDB, 'users')
+
+    users.forEach((user) => {
+      if (user.avaiblePWRequests >= maxPWCount) {
+        return
+      }
+
+      const dayDiff = getDayDiff(user.created)
+      if (dayDiff < daysAfterUserGetsPWs) {
+        logger.Log(
+          `User #${user.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`,
+          logger.GetColor('cyan')
+        )
+        return
+      }
+
+      if (dayDiff % addPWPerDay === 0) {
+        logger.Log(
+          `Incrementing avaible PW-s for user #${user.id}: ${
+            user.avaiblePWRequests
+          } -> ${user.avaiblePWRequests + addPWCount}`,
+          logger.GetColor('cyan')
+        )
+        dbtools.Update(
+          userDB,
+          'users',
+          {
+            avaiblePWRequests: user.avaiblePWRequests + addPWCount,
+          },
+          {
+            id: user.id,
+          }
+        )
+      }
+    })
+  }
+
+  return {
+    dailyAction: () => {
+      BackupDB(usersDbBackupPath, userDB)
+      IncrementAvaiblePWs()
+    },
+  }
+}
+
+export default {
+  setup: setup,
+}

src/server.ts

@@ -281,8 +281,8 @@ function rotateLog() {
     utils.CopyFile(vlogFile, logger.vlogDir + fname)
   }
 
-  utils.WriteFile(fname, logger.logDir + fname)
+  utils.WriteFile(fname, logFile)
-  utils.WriteFile(fname, logger.vlogDir + fname)
+  utils.WriteFile(fname, vlogFile)
 }
 
 function LogTimerAction() {

src/standaloneUtils/rmDuplicates.js

@@ -94,6 +94,8 @@ if (stat.isDirectory()) {
 // possible answers duplicate removing
 // ---------------------------------------------------------------------------------
 
+// TODO: dont check every file, only check per directorires
+// only compare questions of same subjects
 function removePossibleAnswersDuplicates(path) {
   let count = 0
   let currIndex = 1

src/types/basicTypes.ts

@@ -72,4 +72,14 @@ export interface Request extends express.Request {
   cookies: any
   session: any
   busboy: any
+  files: any
+}
+
+export interface SubmoduleData {
+  app: express.Application
+  url: string
+  publicdirs: Array<string>
+  userDB?: any
+  nextdir?: string
+  moduleSpecificData?: any
 }

src/utils/actions.ts

@@ -53,8 +53,10 @@ export interface RecievedData {
 }
 
 export interface Result {
+  qdbIndex: number
   qdbName: string
-  newQuestions: number
+  subjName: string
+  newQuestions: Array<Question>
 }
 
 export function logResult(
@@ -84,9 +86,9 @@ export function logResult(
       const allQLength = recievedData.quiz.length
       let msg = `${res.qdbName}: `
       let color = logger.GetColor('green')
-      if (res.newQuestions > 0) {
+      if (res.newQuestions.length > 0) {
         color = logger.GetColor('blue')
-        msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
+        msg += `New questions: ${res.newQuestions.length} ( All: ${allQLength} )`
       } else {
         msg += `No new data ( ${allQLength} )`
       }
@@ -241,7 +243,9 @@ function processIncomingRequestUsingDb(
 
             logger.DebugLog('ProcessIncomingRequest done', 'isadding', 1)
             resolve({
-              newQuestions: allQuestions.length,
+              newQuestions: allQuestions,
+              subjName: recievedData.subj,
+              qdbIndex: qdb.index,
               qdbName: qdb.name,
             })
           } catch (error) {
@@ -439,3 +443,288 @@ export function backupData(questionDbs: Array<QuestionDb>): void {
     }
   })
 }
+
+function deleteFromDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    type: string
+    selectedDb: { path: string; name: string }
+  }
+): {
+  success: Boolean
+  msg: string
+  deletedQuestion?: Question
+  resultDb?: QuestionDb
+} {
+  // {
+  //   "index": 0,
+  //   "subjName": "VHDL programozás",
+  //   "type": "delete",
+  //   "selectedDb": {
+  //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+  //     "name": "elearning.uni-obuda.hu"
+  //   }
+  // }
+  const { index, subjName } = edits
+  let deletedQuestion
+  if (isNaN(index) || !subjName) {
+    return {
+      success: false,
+      msg: 'No .index or .subjName !',
+    }
+  }
+
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.filter((question, i) => {
+          if (index === i) {
+            deletedQuestion = question
+            return false
+          } else {
+            return true
+          }
+        }),
+      }
+    }
+  })
+
+  return {
+    success: true,
+    msg: 'Delete successfull',
+    deletedQuestion: deletedQuestion,
+    resultDb: questionDb,
+  }
+}
+
+function editQuestionInDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    type: string
+    selectedDb: { path: string; name: string }
+    newVal?: Question
+  }
+): {
+  success: Boolean
+  msg: string
+  newVal?: Question
+  oldVal?: Question
+  resultDb?: QuestionDb
+} {
+  // {
+  //   "index": 0,
+  //   "subjName": "Elektronika",
+  //   "type": "edit",
+  //   "newVal": {
+  //     "Q": "Analóg műszer esetén az érzékenység az a legkisebb mennyiség, amely a műszer kijelzőjén meghatározott mértékű változást okoz.",
+  //     "A": "Igaz",
+  //     "data": {
+  //       "type": "simple",
+  //       "possibleAnswers": [
+  //         "Igaz"
+  //       ]
+  //     },
+  //     "possibleAnswers": [
+  //       "Igaz"
+  //     ]
+  //   },
+  //   "selectedDb": {
+  //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+  //     "name": "elearning.uni-obuda.hu"
+  //   }
+  // }
+  const { index, subjName, newVal } = edits
+  let oldVal
+  if (isNaN(index) || !subjName) {
+    return {
+      success: false,
+      msg: 'No .index or .subjName !',
+    }
+  }
+  if (!isQuestionValid(newVal)) {
+    return {
+      success: false,
+      msg: 'edited question is not valid',
+    }
+  }
+
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.map((question, i) => {
+          if (index === i) {
+            oldVal = question
+            return createQuestion(newVal)
+          } else {
+            return question
+          }
+        }),
+      }
+    }
+  })
+
+  return {
+    success: true,
+    msg: 'Edit successfull',
+    oldVal: oldVal,
+    newVal: newVal,
+    resultDb: questionDb,
+  }
+}
+
+function editSubjInDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    type: string
+    selectedDb: { path: string; name: string }
+    deletedQuestions?: Array<number>
+    changedQuestions?: Array<{
+      index: number
+      value: Question
+    }>
+  }
+): {
+  success: Boolean
+  msg: string
+  deletedQuestions?: Array<Question>
+  changedQuestions?: Array<Question>
+  resultDb?: QuestionDb
+} {
+  // {
+  //   "subjName": "Elektronika",
+  //   "changedQuestions": [
+  //     {
+  //       "index": 1,
+  //       "value": {
+  //         "Q": "A műszer pontosságát a hibájával fejezzük ki, melyet az osztályjel (osztálypontosság ) mutat meg.",
+  //         "A": "Hamis",
+  //         "data": {
+  //           "type": "simple",
+  //           "possibleAnswers": [
+  //             "Igaz",
+  //             "Hamis"
+  //           ]
+  //         }
+  //       }
+  //     }
+  //   ],
+  //   "deletedQuestions": [
+  //     0
+  //   ],
+  //   "type": "subjEdit",
+  //   "selectedDb": {
+  //     "path": "questionDbs/elearning.uni-obuda.hu.json",
+  //     "name": "elearning.uni-obuda.hu"
+  //   }
+  // }
+  const { subjName, changedQuestions, deletedQuestions } = edits
+  const deletedQuestionsToWrite = []
+  const changedQuestionsToWrite = []
+  if (!Array.isArray(changedQuestions) || !Array.isArray(deletedQuestions)) {
+    return {
+      success: false,
+      msg: 'no changedQuestions or deletedQuestions!',
+    }
+  }
+
+  // processing changed questions
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.map((question, i) => {
+          const changedTo = changedQuestions.find((cq) => {
+            return cq.index === i
+          })
+          if (changedTo) {
+            changedQuestionsToWrite.push({
+              oldVal: question,
+              newVal: changedTo.value,
+            })
+            return createQuestion(changedTo.value)
+          } else {
+            return question
+          }
+        }),
+      }
+    }
+  })
+
+  // processing deletedQuestions
+  questionDb.data = questionDb.data.map((subj) => {
+    if (subj.Name !== subjName) {
+      return subj
+    } else {
+      return {
+        ...subj,
+        Questions: subj.Questions.filter((question, i) => {
+          const isDeleted = deletedQuestions.includes(i)
+          if (isDeleted) {
+            deletedQuestionsToWrite.push(question)
+            return false
+          } else {
+            return true
+          }
+        }),
+      }
+    }
+  })
+
+  return {
+    success: true,
+    msg: 'subj edit successfull',
+    deletedQuestions: deletedQuestionsToWrite,
+    changedQuestions: changedQuestionsToWrite,
+    resultDb: questionDb,
+  }
+}
+
+export function editDb(
+  questionDb: QuestionDb,
+  edits: {
+    index: number
+    subjName: string
+    selectedDb: { path: string; name: string }
+    type: string
+    newVal?: Question
+    deletedQuestion?: Array<number>
+    changedQuestions?: Array<{
+      index: number
+      value: Question
+    }>
+  }
+): {
+  success: Boolean
+  msg: string
+  resultDb?: QuestionDb
+  deletedQuestion?: Question
+  newVal?: Question
+  oldVal?: Question
+  deletedQuestions?: Array<Question>
+  changedQuestions?: Array<Question>
+} {
+  if (edits.type === 'delete') {
+    return deleteFromDb(questionDb, edits)
+  }
+  if (edits.type === 'edit') {
+    return editQuestionInDb(questionDb, edits)
+  }
+
+  if (edits.type === 'subjEdit') {
+    return editSubjInDb(questionDb, edits)
+  }
+}

src/utils/classes.ts

@@ -1,6 +1,12 @@
 import { isMainThread, parentPort, workerData } from 'worker_threads'
 import logger from './logger'
-import { Question, QuestionData, Subject } from '../types/basicTypes'
+import {
+  Question,
+  QuestionData,
+  QuestionDb,
+  Subject,
+} from '../types/basicTypes'
+import { editDb } from './actions'
 
 interface SearchResultQuestion extends Question {
   match: number
@@ -32,7 +38,7 @@ const commonUselessAnswerParts = [
 /* Percent minus for length difference */
 const lengthDiffMultiplier = 10
 /* Minimum ammount to consider that two questions match during answering */
-const minMatchAmmount = 70
+const minMatchAmmount = 75
 const magicNumber = 0.7 // same as minMatchAmmount, but /100
 /* If all of the results are below this match percent (when only one subject is searched due to
  * subject name matching) then all subjects are searched for answer */
@@ -290,10 +296,22 @@ function compareData(q1: Question, q2: Question) {
 
 function compareQuestion(q1: Question, q2: Question) {
   return compareString(q1.Q, q1.cache.Q, q2.Q, q2.cache.Q)
+  // return compareString(
+  //   q1.Q,
+  //   q1.Q ? q1.Q.split(' ') : [],
+  //   q2.Q,
+  //   q2.Q ? q2.Q.split(' ') : []
+  // )
 }
 
 function compareAnswer(q1: Question, q2: Question) {
   return compareString(q1.A, q1.cache.A, q2.A, q2.cache.A)
+  // return compareString(
+  //   q1.A,
+  //   q1.A ? q1.A.split(' ') : [],
+  //   q2.A,
+  //   q2.A ? q2.A.split(' ') : []
+  // )
 }
 
 function compareQuestionObj(
@@ -569,7 +587,7 @@ function doSearch(
 
 if (!isMainThread) {
   const { workerIndex } = workerData
-  let qdbs: Array<any> = workerData.initData
+  let qdbs: Array<QuestionDb> = workerData.initData
 
   logger.Log(
     `[THREAD #${workerIndex}]: Worker ${workerIndex} reporting for duty`
@@ -658,14 +676,72 @@ if (!isMainThread) {
       //     !isNaN(index) ? `#${index}` : ''
       //   }done!`
       // )
-    } else if (msg.type === 'update') {
+    } else if (msg.type === 'dbEdit') {
-      qdbs = msg.qdbs
+      const { dbIndex, edits } = msg.data
-      logger.DebugLog(`Worker update ${workerIndex}`, 'worker update', 1)
+      const { resultDb } = editDb(qdbs[dbIndex], edits)
+      qdbs[dbIndex] = resultDb
+      logger.DebugLog(`Worker db edit ${workerIndex}`, 'worker update', 1)
+
+      parentPort.postMessage({
+        msg: `From thread #${workerIndex}: db edit`,
+        workerIndex: workerIndex,
+      })
+    } else if (msg.type === 'newQuestions') {
+      const { subjName, qdbIndex, newQuestions } = msg.data
+      let added = false
+      qdbs = qdbs.map((qdb) => {
+        if (qdb.index === qdbIndex) {
+          return {
+            ...qdb,
+            data: qdb.data.map((subj) => {
+              if (subj.Name === subjName) {
+                added = true
+                return {
+                  Name: subj.Name,
+                  Questions: [...subj.Questions, ...newQuestions],
+                }
+              } else {
+                return subj
+              }
+            }),
+          }
+        } else {
+          return qdb
+        }
+      })
+
+      if (!added) {
+        qdbs = qdbs.map((qdb) => {
+          if (qdb.index === qdbIndex) {
+            return {
+              ...qdb,
+              data: [
+                ...qdb.data,
+                {
+                  Name: subjName,
+                  Questions: [...newQuestions],
+                },
+              ],
+            }
+          } else {
+            return qdb
+          }
+        })
+      }
+      logger.DebugLog(`Worker new question ${workerIndex}`, 'worker update', 1)
+
+      // parentPort.postMessage({
+      //   msg: `From thread #${workerIndex}: update done`,
+      //   workerIndex: workerIndex,
+      // })
 
       // console.log(`[THREAD #${workerIndex}]: update`)
     } else if (msg.type === 'newdb') {
       qdbs.push(msg.newdb)
       // console.log(`[THREAD #${workerIndex}]: newdb`)
+    } else {
+      logger.Log(`Invalid msg type!`, logger.GetColor('redbg'))
+      console.error(msg)
     }
   })
 } else {

src/utils/logger.ts

@@ -33,7 +33,7 @@ const statFile = 'stats/stats'
 const vStatFile = 'stats/vstats'
 const uStatsFile = 'stats/ustats'
 const uvStatsFile = 'stats/uvstats'
-const nologFile = './nolog'
+const nologFile = './data/nolog'
 
 const colors = ['green', 'red', 'yellow', 'blue', 'magenta', 'cyan']
 const logFileName = 'log'

src/utils/utils.ts

@@ -12,11 +12,14 @@ export default {
   GetDateString: GetDateString,
   formatUrl: formatUrl,
   deleteFile: deleteFile,
+  uploadFile: uploadFile,
 }
 
 import fs from 'fs'
 import logger from '../utils/logger'
 
+import { Request } from '../types/basicTypes'
+
 interface URLFormatOptions {
   pathname?: string
   query?: any
@@ -70,8 +73,14 @@ function CopyFile(from: string, to: string): void {
   fs.copyFileSync(from, to)
 }
 
-function ReadDir(path: string): Array<string> {
+function ReadDir(path: string, listHidden?: boolean): Array<string> {
+  if (listHidden) {
     return fs.readdirSync(path)
+  } else {
+    return fs.readdirSync(path).filter((file) => {
+      return !file.startsWith('.')
+    })
+  }
 }
 
 function ReadJSON(name: string): any {
@@ -171,3 +180,46 @@ function deleteFile(fname: string): Boolean {
   }
   return false
 }
+
+function uploadFile(req: Request, path: string): Promise<any> {
+  return new Promise((resolve, reject) => {
+    try {
+      const file = req.files.file
+      // FIXME: Object.keys(req.files).forEach((file) => { saveFile() })
+      logger.Log('Uploading: ' + file.name, logger.GetColor('blue'))
+
+      CreatePath(path, true)
+
+      let fileName = file.name.replace(/\.+/g, '.').replace(/\/+/g, '/')
+      let fileDestination = path + '/' + fileName
+      if (FileExists(fileDestination)) {
+        const date = new Date()
+        const id = date.getHours() + date.getMinutes() + date.getSeconds()
+
+        const temp = file.name.split('.')
+        const extension = temp.pop()
+        fileName = temp.join('.') + '_' + id + '.' + extension
+        console.log(fileName)
+        fileDestination = path + '/' + fileName
+      }
+
+      file.mv(fileDestination, (err) => {
+        if (err) {
+          logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
+          console.error(err)
+          reject(err)
+        } else {
+          resolve({
+            body: req.body,
+            fileName: fileName,
+            filePath: fileDestination,
+          })
+        }
+      })
+    } catch (err) {
+      logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
+      console.error(err)
+      reject(err)
+    }
+  })
+}

src/utils/workerPool.ts

@@ -1,84 +1,173 @@
 import { Worker } from 'worker_threads'
-import genericPool from 'generic-pool'
+import { v4 as uuidv4 } from 'uuid'
+import { EventEmitter } from 'events'
 import os from 'os'
 
 import logger from './logger'
-// import { QuestionDb } from '../types/basicTypes'
 
-// ---------------------------------------------------------------------------
+interface WorkerObj {
+  worker: any
+  index: number
+  free: Boolean
+}
 
+interface PendingJob {
+  workData: any
+  doneEvent: any
+  targetWorkerIndex?: number
+}
+
+const alertOnPendingCount = 10
 const workerFile = './src/utils/classes.ts'
-let pool: any = null
+let workers: Array<WorkerObj>
-let workers: any = null
+const pendingJobs: {
+  [id: string]: PendingJob
+} = {}
+
+const jobEvents = new EventEmitter()
+
+jobEvents.on('jobDone', () => {
+  processJob()
+})
+
+jobEvents.on('newJob', () => {
+  processJob()
+})
 
 // ---------------------------------------------------------------------------
 
-// doALongTask(i, taskObj).then((res) => {
+function handleWorkerError(worker: WorkerObj, err) {
-//   msgAll(res)
+  // TODO: restart worker if exited or things like that
-//   console.log('[MSGFROMCLIENT]', res)
-// })
-
-export function doALongTask(obj: any): Promise<any> {
-  return new Promise((resolve) => {
-    pool
-      .acquire()
-      .then(function(client) {
-        doSomething(client, obj).then((res) => {
-          resolve(res)
-          // TODO: check if result is really a result, and want to release port
-          pool.release(client)
-          // console.log('[RELEASE]: #' + client.index)
-        })
-      })
-      .catch(function(err) {
   logger.Log('resourcePromise error', logger.GetColor('redbg'))
   console.error(err)
-        // handle error - this is generally a timeout or maxWaitingClients
+}
-        // error
+
+// TODO: accuire all workers here, and handle errors so they can be removed if threads exit
+export function msgAllWorker(data: any): Promise<any> {
+  console.log('MSGING ALL WORKER')
+  return new Promise((resolve) => {
+    const promises = []
+    workers.forEach((worker) => {
+      promises.push(doALongTask(data, worker.index))
+    })
+    Promise.all(promises).then((res) => {
+      resolve(res)
+    })
+  })
+}
+
+export function doALongTask(
+  obj: any,
+  targetWorkerIndex?: number
+): Promise<any> {
+  if (Object.keys(pendingJobs).length > alertOnPendingCount) {
+    logger.Log(
+      `More than ${alertOnPendingCount} callers waiting for free resource! (${
+        Object.keys(pendingJobs).length
+      })`,
+      logger.GetColor('redbg')
+    )
+  }
+
+  const jobId = uuidv4()
+  // FIXME: delete doneEvent?
+  const doneEvent = new EventEmitter()
+  pendingJobs[jobId] = {
+    workData: obj,
+    targetWorkerIndex: targetWorkerIndex,
+    doneEvent: doneEvent,
+  }
+  jobEvents.emit('newJob')
+  return new Promise((resolve) => {
+    doneEvent.once('done', (result) => {
+      jobEvents.emit('jobDone')
+      resolve(result)
     })
   })
 }
 
 export function initWorkerPool(initData: any): void {
-  if (workers && pool) {
+  if (workers) {
-    logger.Log('WORKER AND POOL ALREADY EXISTS', logger.GetColor('redbg'))
+    logger.Log('WORKERS ALREADY EXISTS', logger.GetColor('redbg'))
     return
   }
   workers = []
   const factory = {
-    create: function() {
+    create: function(index) {
-      const currInd = workers.length
+      return getAWorker(index, initData)
-      const worker = getAWorker(currInd, initData)
-      workers.push(worker)
-      return {
-        worker: worker,
-        index: currInd,
-      }
     },
     destroy: function(client) {
-      // console.log('[DESTROY]')
       client.worker.terminate()
-      // console.log('[DESTROYED] #' + client.index)
     },
   }
 
-  const opts = {
+  const threadCount = process.env.NS_THREAD_COUNT || os.cpus().length
-    min: os.cpus().length - 1, // minimum size of the pool
+  if (process.env.NS_THREAD_COUNT) {
-    max: os.cpus().length - 1, // maximum size of the pool
+    logger.Log(
-    maxWaitingClients: 999,
+      `Setting thread count from enviroment variable NS_WORKER_COUNT: '${threadCount}'`,
+      logger.GetColor('red')
+    )
   }
 
-  pool = genericPool.createPool(factory, opts)
+  for (let i = 0; i < threadCount; i++) {
-}
+    workers.push({
-
+      worker: factory.create(i),
-export function msgAllWorker(data: any): void {
+      index: i,
-  workers.forEach((worker) => {
+      free: true,
-    worker.postMessage(data)
     })
   }
+}
 
 // ---------------------------------------------------------------------------
 
+function processJob() {
+  if (Object.keys(pendingJobs).length > 0) {
+    // FIXME: FIFO OR ANYTHING ELSE (JOB PROCESSING ORDER)
+    const keys = Object.keys(pendingJobs)
+    let jobKey, freeWorker
+    let i = 0
+    while (!freeWorker && i < keys.length) {
+      jobKey = keys[i]
+      if (!isNaN(pendingJobs[jobKey].targetWorkerIndex)) {
+        if (workers[pendingJobs[jobKey].targetWorkerIndex].free) {
+          freeWorker = workers[pendingJobs[jobKey].targetWorkerIndex]
+          console.log(
+            `RESERVING WORKER ${pendingJobs[jobKey].targetWorkerIndex}`
+          )
+        }
+      } else {
+        freeWorker = workers.find((worker) => {
+          return worker.free
+        })
+        if (freeWorker) {
+          console.log(`RESERVING FIRST AVAILABLE WORKER ${freeWorker.index}`)
+        }
+      }
+      i++
+    }
+
+    if (!freeWorker) {
+      console.log('NO FREE WORKER')
+      return
+    }
+
+    if (freeWorker.free) {
+      freeWorker.free = false
+    }
+    const job = pendingJobs[jobKey]
+    delete pendingJobs[jobKey]
+
+    doSomething(freeWorker, job.workData)
+      .then((res) => {
+        freeWorker.free = true
+        job.doneEvent.emit('done', res)
+      })
+      .catch(function(err) {
+        handleWorkerError(freeWorker, err)
+      })
+  }
+}
+
 function getAWorker(i, initData) {
   const worker = workerTs(workerFile, {
     workerData: {
@@ -89,14 +178,6 @@ function getAWorker(i, initData) {
 
   worker.setMaxListeners(50)
 
-  // worker.on('message', (msg) => {
-  //   logger.Log(`[MAIN]: Msg from worker #${i}`, msg)
-  // })
-
-  // worker.on('online', () => {
-  //   logger.Log(`[THREAD #${i}]: Worker ${i} online`)
-  // })
-
   worker.on('error', (err) => {
     logger.Log('Worker error!', logger.GetColor('redbg'))
     console.error(err)
@@ -114,10 +195,9 @@ function getAWorker(i, initData) {
 
 // ---------------------------------------------------------------------------
 
-function doSomething(client, obj) {
+function doSomething(currWorker, obj) {
-  const { /* index, */ worker } = client
+  const { /* index, */ worker } = currWorker
   return new Promise((resolve) => {
-    // console.log('[ACCUIRE]: #' + index)
     worker.postMessage(obj)
     worker.once('message', (msg) => {
       resolve(msg)