Seperate dbs

src/modules/api/api.ts

@@ -57,11 +57,13 @@ const idvStatFile = 'stats/idvstats'
 const todosFile = 'data/todos.json'
 
 // other constants
+const line = '====================================================' // lol
 const maxVeteranPwGetCount = 10
 const addPWPerDay = 3 // every x day a user can give a pw
 const maxPWCount = 6 // maximum pw give opportunities a user can have at once
 const addPWCount = 1 // how many pw gen opportunities to add each time
 const daysAfterUserGetsPWs = 5 // days after user gets pw-s
+const minimumAlowwedSessions = 1 // how many sessions are allowed for a user
 
 // stuff gotten from server.js
 let userDB
@@ -79,19 +81,27 @@ function GetApp(): ModuleType {
   // files in public dirs
   const recivedFiles = publicDir + 'recivedfiles'
   const uloadFiles = publicDir + 'f'
+  // FIXME: this to seperate file?
   const dataFiles: Array<DataFile> = [
     {
       path: `${publicDir}oldData.json`,
       name: 'oldData',
       shouldSave: (recData: RecievedData): boolean => {
-        return recData.version.includes('2.0.')
+        return recData.version.startsWith('2.0.')
       },
     },
     {
       path: `${publicDir}data.json`,
       name: 'newData',
       shouldSave: (recData: RecievedData): boolean => {
-        return recData.version.includes('2.1.')
+        return recData.version.startsWith('2.1.')
+      },
+    },
+    {
+      path: `${publicDir}fromwebsiteData.json`,
+      name: 'fromwebsiteData',
+      shouldSave: (recData: RecievedData): boolean => {
+        return recData.version === 'WEBSITE'
       },
     },
   ]
@@ -99,6 +109,11 @@ function GetApp(): ModuleType {
   const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
   const versionFile = publicDir + 'version'
 
+  let domain = url.split('.') // [ "https://api", "frylabs", "net" ]
+  domain.shift() // [ "frylabs", "net" ]
+  domain = domain.join('.') // "frylabs.net"
+  logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
+
   app.use(
     bodyParser.urlencoded({
       limit: '10mb',
@@ -246,7 +261,21 @@ function GetApp(): ModuleType {
 
   // -------------------------------------------------------------
 
+  app.get('/getDbs', (req: any, res: any) => {
+    logger.LogReq(req)
+
+    res.json(
+      dataFiles.map((df) => {
+        return {
+          path: df.path.split('/').pop(),
+          name: df.name,
+        }
+      })
+    )
+  })
+
   app.get('/updateTodo', (req: any, res: any) => {
+    logger.LogReq(req)
     const userId = req.session.user.id
     const id = req.query.id
     const todos = utils.ReadJSON(todosFile)
@@ -632,13 +661,12 @@ function GetApp(): ModuleType {
     if (user) {
       const sessionID = uuidv4()
 
-      // FIXME: Users now can only log in in one session, this might be too strict.
       const existingSessions = dbtools.Select(userDB, 'sessions', {
         userID: user.id,
         isScript: isScript ? 1 : 0,
       })
 
-      if (existingSessions.length > 0) {
+      if (existingSessions.length >= minimumAlowwedSessions) {
         logger.Log(
           `Multiple ${isScript ? 'script' : 'website'} sessions ( ${
             existingSessions.length
@@ -675,9 +703,9 @@ function GetApp(): ModuleType {
       })
 
       // https://www.npmjs.com/package/cookie
-      // TODO: cookie age
+      // FIXME: cookies are not configured coorectly
       res.cookie('sessionID', sessionID, {
-        domain: '.frylabs.net', // TODO: use url. url: "https://api.frylabs.net"
+        domain: domain,
         expires: new Date(
           new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
         ),
@@ -724,7 +752,6 @@ function GetApp(): ModuleType {
     dbtools.Delete(userDB, 'sessions', {
       id: sessionID,
     })
-    // TODO: remove old sessions every once in a while
     res.clearCookie('sessionID').json({
       result: 'success',
     })
@@ -829,9 +856,14 @@ function GetApp(): ModuleType {
   app.get('/allqr.txt', function(req: any, res: any) {
     res.set('Content-Type', 'text/plain')
     const stringifiedData = questionDbs.map((qdb) => {
-      return dataToString(qdb.data)
+      let result = ''
+      result += '\n' + line
+      result += `  Questions in ${qdb.name}:  `
+      result += line + '\n'
+      result += dataToString(qdb.data)
+      result += '\n' + line + line + '\n'
+      return result
     })
-    // TODO: test this
     res.send(stringifiedData.join('\n\n'))
     res.end()
     logger.LogReq(req)
@@ -949,26 +981,38 @@ function GetApp(): ModuleType {
 
   app.post('/isAdding', function(req: any, res: any) {
     logger.LogReq(req)
-
     const user: User = req.session.user
-
     const dryRun = testUsers.includes(user.id)
 
-    processIncomingRequest(req.body, questionDbs, dryRun, user)
+    try {
-      .then((resultArray) => {
+      processIncomingRequest(req.body, questionDbs, dryRun, user)
-        logResult(req.body, resultArray)
+        .then((resultArray) => {
-        res.json({
+          logResult(req.body, resultArray, user.id, dryRun)
-          success: resultArray.length > 0, // TODO check for -1s
+          res.json({
-          newQuestions: resultArray,
+            success: resultArray.length > 0,
+            newQuestions: resultArray,
+          })
         })
+        .catch((err) => {
+          logger.Log(
+            'Error during processing incoming request',
+            logger.GetColor('redbg')
+          )
+          console.error(err)
+          res.json({
+            success: false,
+          })
+        })
+    } catch (err) {
+      logger.Log(
+        'Error during getting incoming request processor promises ',
+        logger.GetColor('redbg')
+      )
+      console.error(err)
+      res.json({
+        success: false,
       })
-      .catch((err) => {
+    }
-        logger.Log(
-          'Couldnt process incoming request!',
-          logger.GetColor('redbg')
-        )
-        console.error(err)
-      })
   })
 
   app.get('/ask', function(req: any, res) {
@@ -996,8 +1040,6 @@ function GetApp(): ModuleType {
 
         searchDatas(questionDbs, question, subj, recData)
           .then((result) => {
-            // TODO: test
-            console.log(result)
             res.json({
               result: result,
               success: true,

src/modules/stuff/stuff.ts

@@ -74,7 +74,7 @@ function GetApp(): ModuleType {
   // app, '/*.mp4', 'video/mp4', 'stuff/video'
   function appGetFileType(app, wildcard, contentType, pageToRender) {
     app.get(wildcard, function(req, res) {
-      const path = decodeURI(req.url)
+      const path = decodeURIComponent(req.url)
       let fp: any = path
       if (path.includes('?')) {
         fp = path.split('?')
@@ -151,7 +151,7 @@ function GetApp(): ModuleType {
   })
 
   app.get('/*', function(req, res) {
-    const parsedUrl = decodeURI(req.url)
+    const parsedUrl = decodeURIComponent(req.url)
     let curr =
       listedFiles +
       '/' +

src/types/basicTypes.ts

@@ -38,7 +38,6 @@ export interface ModuleType {
   dailyAction?: Function
 }
 
-// TODO: add more props based on db
 export interface User {
   id: number
   pw: string

src/utils/actions.ts

@@ -24,7 +24,7 @@ import logger from '../utils/logger'
 import { searchData, createQuestion } from '../utils/classes'
 import idStats from '../utils/ids'
 import utils from '../utils/utils'
-import { addQuestion, getSubjNameWithoutYear } from './classes'
+import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
 
 // types
 import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
@@ -51,29 +51,42 @@ interface Result {
 
 export function logResult(
   recievedData: RecievedData,
-  result: Array<Result>
+  result: Array<Result>,
+  userId: number,
+  dryRun?: boolean
 ): void {
-  let subjRow = '\t' + recievedData.subj
+  logger.Log('\t' + recievedData.subj)
-  if (recievedData.id) {
-    subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
-  }
-  logger.Log(subjRow)
 
-  result.forEach((res: Result) => {
+  if (dryRun) {
-    const allQLength = recievedData.quiz.length
+    logger.Log('\tDry run')
-    let msg = `${res.qdbName}: `
+  }
-    let color = logger.GetColor('green')
+
-    if (res.newQuestions > 0) {
+  let idRow = '\t'
-      color = logger.GetColor('blue')
+  if (recievedData.version) {
-      msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
+    idRow += 'Version: ' + logger.logHashed(recievedData.version)
-    } else {
+  }
-      msg += `No new data ( ${allQLength} )`
+  if (recievedData.id) {
-    }
+    idRow += ', CID: ' + logger.logHashed(recievedData.id)
-    if (recievedData.version !== undefined) {
+  }
-      msg += '. Version: ' + recievedData.version
+  idRow += `, User #${logger.logHashed(userId.toString())}`
-    }
+  logger.Log(idRow)
-    logger.Log('\t' + msg, color)
+
-  })
+  if (result.length > 0) {
+    result.forEach((res: Result) => {
+      const allQLength = recievedData.quiz.length
+      let msg = `${res.qdbName}: `
+      let color = logger.GetColor('green')
+      if (res.newQuestions > 0) {
+        color = logger.GetColor('blue')
+        msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
+      } else {
+        msg += `No new data ( ${allQLength} )`
+      }
+      logger.Log('\t' + msg, color)
+    })
+  } else {
+    logger.Log('\tNo db-s passed shouldSave!', logger.GetColor('red'))
+  }
 }
 
 export function processIncomingRequest(
@@ -90,7 +103,7 @@ export function processIncomingRequest(
   }
 
   try {
-    let towrite = logger.GetDateString() + '\n'
+    let towrite = utils.GetDateString() + '\n'
     towrite +=
       '------------------------------------------------------------------------------\n'
     if (typeof recievedData === 'object') {
@@ -153,15 +166,15 @@ function processIncomingRequestUsingDb(
         logger.DebugLog(currentQuestion, 'actions', 3)
         recievedQuestions.push(currentQuestion)
         questionSearchPromises.push(
-          searchData(qdb.data, currentQuestion, recievedData.subj)
+          searchData(qdb, currentQuestion, recievedData.subj)
         )
       })
 
       Promise.all(questionSearchPromises)
-        .then((results) => {
+        .then((results: Array<SearchResult>) => {
           const allQuestions = [] // all new questions here that do not have result
           results.forEach((result, i) => {
-            const add = result.every((res) => {
+            const add = result.result.every((res) => {
               return res.match < minMatchToAmmountToAdd
             })
             if (add) {
@@ -192,8 +205,6 @@ function processIncomingRequestUsingDb(
                 currWrites = 0
                 logger.DebugLog('Writing data.json', 'actions', 1)
                 utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
-              } else if (dryRun) {
-                logger.Log('\tDry run')
               }
             }
 

src/utils/classes.ts

@@ -7,11 +7,15 @@ import {
   Subject,
 } from '../types/basicTypes'
 
-// TODO
 interface SearchResultQuestion extends Question {
   match: number
 }
 
+export interface SearchResult {
+  result: Array<SearchResultQuestion>
+  dbName: string
+}
+
 const searchDataWorkerFile = './src/utils/classes.ts'
 
 const assert = (val) => {
@@ -217,7 +221,10 @@ function createQuestion(
 }
 
 function compareImage(data: QuestionData, data2: QuestionData) {
-  return compareString(data.images.join(' '), data2.images.join(' '))
+  // TODO: img comparing (hashed images vs images)
+  const imgs1 = data.hashedImages ? data.hashedImages : data.images
+  const imgs2 = data2.hashedImages ? data2.hashedImages : data2.images
+  return compareString(imgs1.join(' '), imgs2.join(' '))
 }
 
 function compareData(q1: Question, q2: Question) {
@@ -412,23 +419,22 @@ function searchDatas(
   question: any,
   subjName: string,
   questionData?: QuestionData
-): Promise<Array<Array<SearchResultQuestion>>> {
+): Promise<Array<SearchResult>> {
   return Promise.all(
-    data.map((db) => {
+    data.map((db: QuestionDb) => {
-      return searchData(db.data, question, subjName, questionData)
+      return searchData(db, question, subjName, questionData)
     })
   )
 }
 
-// TODO: remove questionData, make question only Question type
+// FIXME: remove questionData, make question only Question type
 function searchData(
-  data: Array<Subject>,
+  qdb: QuestionDb,
   question: any,
   subjName: string,
   questionData?: QuestionData
-): Promise<Array<SearchResultQuestion>> {
+): Promise<SearchResult> {
   return new Promise((resolve, reject) => {
-    assert(data)
     assert(question)
     logger.DebugLog('Searching for question', 'qdb search', 1)
     logger.DebugLog('Question:', 'qdb search', 2)
@@ -447,7 +453,7 @@ function searchData(
     question = simplifyQuestion(question)
 
     const worker = workerTs(searchDataWorkerFile, {
-      workerData: { data, subjName, question, questionData },
+      workerData: { data: qdb.data, subjName, question, questionData },
     })
 
     worker.on('error', (err) => {
@@ -478,7 +484,10 @@ function searchData(
         'qdb search',
         1
       )
-      resolve(result)
+      resolve({
+        result: result,
+        dbName: qdb.name,
+      })
     })
   })
 }
@@ -494,10 +503,10 @@ function dataToString(data: Array<Subject>): string {
 // ------------------------------------------------------------------------
 
 function searchWorker(
-  data: any,
+  data: Array<Subject>,
-  subjName: any,
+  subjName: string,
-  question: any,
+  question: Question,
-  questionData: any
+  questionData?: QuestionData
 ): any {
   let result = []
 

src/utils/dbtools.ts

@@ -126,7 +126,6 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
     const cols = Object.keys(columns)
       .reduce((acc, key) => {
         const item = columns[key]
-        // FIXME: array, and push stuff, then join()
         const flags = []
         const toCheck = {
           primary: 'PRIMARY KEY',

src/utils/logger.ts

@@ -22,7 +22,7 @@ const hr =
   '---------------------------------------------------------------------------------'
 
 export default {
-  GetDateString: GetDateString,
+  getColoredDateString: getColoredDateString,
   Log: Log,
   DebugLog: DebugLog,
   GetColor: GetColor,
@@ -70,7 +70,7 @@ function setNewLogfileName(): void {
   logFileName = utils.GetDateString(true)
 }
 
-function GetDateString(): string {
+function getColoredDateString(): string {
   const date = new Date()
   const dateString = utils.GetDateString()
   return GetRandomColor(date.getHours().toString()) + dateString + C()
@@ -99,7 +99,7 @@ function Log(msg: string | object, color?: string): void {
   let log = msg
   if (typeof msg !== 'object') {
     const delimiter = DELIM + C(color)
-    log = C(color) + GetDateString() + delimiter + msg + C()
+    log = getColoredDateString() + delimiter + C(color) + msg + C()
   }
 
   console.log(log)
@@ -109,7 +109,6 @@ function Log(msg: string | object, color?: string): void {
   )
 }
 
-// FIXME: express.Request type, but with .cookies and .session
 function LogReq(
   req: any /*express.Request*/,
   toFile?: boolean,
@@ -162,7 +161,7 @@ function LogReq(
     if (!toFile) {
       Log(logEntry)
     } else {
-      const defLogs = GetDateString() + dl + logEntry
+      const defLogs = getColoredDateString() + dl + logEntry
 
       utils.AppendToFile(defLogs, vlogDir + logFileName)
     }

src/utils/utils.ts

@@ -17,14 +17,22 @@ import fs from 'fs'
 import logger from '../utils/logger'
 
 interface URLFormatOptions {
-  query: any
   pathname?: string
+  query?: any
 }
 
-// TODO
 function formatUrl(options: URLFormatOptions): string {
   console.log(options.pathname, options.query)
-  return options.pathname + JSON.stringify(options.query)
+  const queryString = options.query
+    ? '?' +
+      Object.keys(options.query)
+        .map((key) => {
+          return `${key}=${encodeURIComponent(options.query[key])}`
+        })
+        .join('&')
+    : ''
+  const path = options.pathname || ''
+  return path + queryString
 }
 
 function GetDateString(noTime?: boolean): string {

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3
+Subproject commit 309436b2270ebf2398945b2cd707d1f133fe8ec1