readme update, updated some queries to use numbers instead of strings

README.md

@@ -33,31 +33,41 @@ threads, and serve more requests at once. The used cores can be limited with env
  | Peer to peer functionality | The ability to share question databases and users with other instances of this server |
  | Peer | Another instance of this server, with peer to peer functionality set up |
  | User \#1 | The first user created, admin of the server |
+ | Submodule | A server submodule, dynamically loaded, and stored in `./src/modules/` |
+ | Git module | Git modules stored in `./submodules` |
 
 ## Setup
 
 Run `./scripts/setup.sh`, then `npm run dev` for development, or `npm run start` for prod.
 
-On the first run there will be a number of errors, that some files weren't found. Please create them
+On the first run there will be a number of errors, that some files weren't found, or some
-according to the messages, these are necessary for the server to function.
+environment variables needed to be specified. Please create them according to the messages, these
+are necessary for the server to function.
 
 There will be also a lot of information about files and other necessary things being created. **Please
 read them very carefully, you should know about what was created!**
 
-### Setup notes
+### Setup notes and tips
 
- * **Gitlab specific notes**: You need a gitlab account with SSH keys set up in order to be able to use
+ * **Gitlab specific notes**: You need a gitlab account with SSH keys set up in order to be able to
-   SSH for cloning this (and any) project. This includes git modules of this project. See
+   use SSH for cloning this (and any) project. This includes git modules of this project. To use HTTPS
-   `CLONE_WITH_HTTPS` in environment variables below.
+   set the `CLONE_WITH_HTTPS` environment variable before running `setup.sh`. To set up SSH keys:
- * The `DOMAIN` env var can be replaced by making the file `./data/domain`. This is necesarry, to make
+   https://docs.gitlab.com/ee/user/ssh.html
-   xmlhttp requests and some redirects work in statically built HTML files (by next.js), in the
+ * You either need to specify a `DOMAIN` env var, or write the domain to `./data/domain`. This is
-   userscript, and in the server itself.
+   necesarry to make xmlhttp requests and some redirects work in: statically built HTML files (by
- * The setup script can be also used to update and rebuild all git modules.
+   next.js), userscript, and the server itself. Domain is for example: 'qmining.com', without
+   'http://' and '/'-s
+ * By default the server redirects all HTTP traffic to HTTPS. To disable this use
+   `NS_NO_HTTPS_FORCE`
+ * The server launches a thread for each CPU core. This could be an overkill on 4+ cored CPU-s. Use
+   `NS_THREAD_COUNT` to restrict the number of threads
+ * The setup script can be also used to update and rebuild all git modules if ran after the initial
+   setup
 
 ## Web server paths
 
-There are different routes assigned to different modules. You can see these in detail in the files
+There are different routes assigned to different submodules. You can see these in detail in the
-`./src/modules.json`.
+files `./src/modules.json`.
 
 ## Peer to peer
 
@@ -71,7 +81,7 @@ To setup P2P functionality you have to create a few files in `./data/p2p`:
     {
         "name": "any name you choose",
         "contact": "contact to server administrator (irc server, e-mail, anything)",
-        "host": "server host (like somesite.com, without 'http(s):// and /-s')",
+        "host": "server host (like 'qmining.com', without 'http(s):// and /-s')",
         "pw": "password to the host, so the server can log in there. Please use a dedicated password, that only the server uses!",
         "port": "server port, number"
     }
@@ -148,23 +158,27 @@ result can be found in the directory it was ran.
 
 ### For the server
 
+Set these before launching the server (`VAR="" npm run start`)
+
  | Name | Type | Description |
  | --- | --- | --- |
  | PORT | number | The port the http server should run on |
- | NS_THREAD_COUNT | number | Nubmer of CPU cores to use |
- | NS_NOUSER | boolean | If the authorization should be skipped (for testing) |
- | NS_NO_HTTPS_FORCE | boolean | Disables automatic redirects from http to https |
- | NS_LOGLEVEL | number | Debug log level, 0 is the least verbose |
- | NS_NOLOG | boolean | If logging should be skipped |
- | NS_SQL_DEBUG_LOG | boolean | If the SQL queries should be logged |
  | DOMAIN | string | The domain that the server should use for redirects, cookies, etc. ex.: `qmining.com`, without 'https://', and '/'-s. If not specified `./data/domain` will be used |
+ | NS_NO_HTTPS_FORCE | boolean | Disables automatic redirects from http to https |
+ | NS_THREAD_COUNT | number | Nubmer of CPU cores to use |
+ | NS_LOGLEVEL | number | Debug log level, 0 is the least verbose |
+ | NS_SQL_DEBUG_LOG | boolean | If the SQL queries should be logged |
+ | NS_NOUSER | boolean | If the authorization should be skipped (for testing) |
+ | NS_NOLOG | boolean | If logging should be skipped |
 
 ### For the setup script (`./scripts/setup.sh`)
 
+Set these before starting the setup (`VAR="" ./scripts/setup.sh`)
+
  | Name | Type | Description |
  | --- | --- | --- |
- | CLONE_WITH_HTTPS | boolean | By default git clones with ssh, which can cause troubles. Set this
  | DOMAIN | string | The domain that the server should use for redirects, cookies, etc. ex.: `qmining.com`, without 'https://', and '/'-s. If not specified `./data/domain` will be used |
+ | CLONE_WITH_HTTPS | boolean | By default git clones with ssh, which can cause troubles. Set this to use https |
 
 ## npm scripts
 
@@ -179,7 +193,7 @@ result can be found in the directory it was ran.
 
 ## Git modules
 
-Details about the specific git module should be found in its directory.
+More details about the specific git module should be found in its directory: `./submodules/`
 
 # qmining-page
 

scripts/setup.sh

@@ -49,7 +49,7 @@ if [ -z "${domain}" ]; then
     error "DOMAIN is not set, and ${domainPath} does not exist!"
     echo "Set DOMAIN to the preferred domain"
     echo "Or please create the file ${domainPath}, and re-run this script"
-    echo "Expected content: domain, ex.:'frylabs.net' (without 'http://' and '/')"
+    echo "Expected content: domain, ex.:'qmining.com' (without 'http://' and '/')"
     exit 1
 fi
 

src/middlewares/auth.middleware.ts

@@ -52,7 +52,6 @@ function renderLogin(req: Request, res: Response) {
         })
     } else {
         res.render('login', {
-            useHttp: process.env.NS_NO_HTTPS_FORCE,
             domain: constants.domain,
         })
     }

src/modules/api/api.ts

@@ -60,9 +60,9 @@ function GetApp(): ModuleType {
         throw new Error(`No public dir! ( API )`)
     }
 
-    let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
+    let domain: any = url.split('.')
-    domain.shift() // [ "frylabs", "net" ]
+    domain.shift()
-    domain = domain.join('.') // "frylabs.net"
+    domain = domain.join('.') // "qmining.com"
     logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
 
     // -------------------------------------------------------------------------------------------

src/modules/api/submodules/userManagement.ts

@@ -40,8 +40,8 @@ const daysAfterUserGetsPWs = 7 // days after user gets pw-s
 interface Session {
     id: string
     userId: number
-    createDate: string
+    createDate: number
-    lastAccess: string
+    lastAccess: number
     isScript: number
 }
 
@@ -69,7 +69,7 @@ function createDefaultUser(userDb: Database) {
     const insertRes = dbtools.Insert(userDb, 'users', {
         pw: pw,
         avaiblePWRequests: 50,
-        created: utils.GetDateString(),
+        created: new Date().getTime(),
     })
     logger.Log('ID and PW for user #1: ', 'yellowbg')
     console.log(`ID: #${insertRes.lastInsertRowid}, PW: "${pw}"`)
@@ -153,7 +153,7 @@ function setup(data: SubmoduleData): Submodule {
         const insertRes = dbtools.Insert(userDB, 'users', {
             pw: pw,
             avaiblePWRequests: 0,
-            created: utils.GetDateString(),
+            created: new Date().getTime(),
             createdBy: requestingUser.id,
         })
 
@@ -194,10 +194,7 @@ function setup(data: SubmoduleData): Submodule {
                     isScript: isScript ? 1 : 0,
                 })
                 .sort((a: Session, b: Session) => {
-                    return (
+                    return a.lastAccess - b.lastAccess
-                        new Date(a.lastAccess).getTime() -
-                        new Date(b.lastAccess).getTime()
-                    )
                 })
 
             const diff = existingSessions.length - minimumAlowwedSessions
@@ -222,7 +219,7 @@ function setup(data: SubmoduleData): Submodule {
                 'users',
                 {
                     loginCount: user.loginCount + 1,
-                    lastLogin: utils.GetDateString(),
+                    lastLogin: new Date().getTime(),
                 },
                 {
                     id: user.id,
@@ -233,7 +230,7 @@ function setup(data: SubmoduleData): Submodule {
                 id: sessionID,
                 userID: user.id,
                 isScript: isScript ? 1 : 0,
-                createDate: utils.GetDateString(),
+                createDate: new Date().getTime(),
             })
 
             // https://www.npmjs.com/package/cookie

src/sharedViews/login.ejs

@@ -123,7 +123,7 @@
       button.classList.add('disabledButton')
       button.disabled = true
       // TODO: get url from controller
-        const rawResponse = await fetch('<%= useHttp ? `http://${domain}/api/login` : `https://${domain}/api/login` %>', {
+        const rawResponse = await fetch('/api/login', {
         method: 'POST',
         credentials: 'include',
         headers: {

src/standaloneUtils/createUserDataList.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-scp -P 19192 qminer@megacarrot.frylabs.net:/home/qminer/mrfrys-node-server/stats/idstats "./idstats.new"
-scp -P 19192 qminer@megacarrot.frylabs.net:/home/qminer/old.nodeServ/stats/idstats "./idstats.old"
-
-cat idstats.new | jq 'keys' > idstats.new.keys
-cat idstats.old | jq 'keys' > idstats.old.keys
-
-cat idstats.new.keys idstats.old.keys | sort | uniq > keys.tmp
-
-cat keys.tmp | grep -Eo '[0-9]{13}' > ids
-
-rm idstats.new
-rm idstats.new.keys
-rm idstats.old
-rm idstats.old.keys
-rm keys.tmp

src/tests/addQuestion.test.ts

@@ -56,7 +56,7 @@ test('Adds new subjects, multiple new questions', () => {
 test("New subject names shouldn't be empty", () => {
     const data: RecievedData = JSON.parse(
         fs.readFileSync(
-            __dirname + '/../../devel/tests/testData/ROS.json',
+            __dirname + '/../../testingTools/tests/testData/ROS.json',
             'utf8'
         )
     )
@@ -80,7 +80,7 @@ test("New subject names shouldn't be empty", () => {
 test('New questions shouldnt be added to accidental empty named subjects', () => {
     const data: RecievedData = JSON.parse(
         fs.readFileSync(
-            __dirname + '/../../devel/tests/testData/ROS.json',
+            __dirname + '/../../testingTools/tests/testData/ROS.json',
             'utf8'
         )
     )
@@ -103,7 +103,7 @@ test('New questions shouldnt be added to accidental empty named subjects', () =>
 test('Question gets added to the correct subject', () => {
     const data: RecievedData = JSON.parse(
         fs.readFileSync(
-            __dirname + '/../../devel/tests/testData/ROS.json',
+            __dirname + '/../../testingTools/tests/testData/ROS.json',
             'utf8'
         )
     )

submodules/moodle-test-userscript

@@ -1 +1 @@
-Subproject commit d9e5c8bdb0bd7b66148623ac8585e6cfe9a752b2
+Subproject commit 8e91b93d298f0bfcd8b3ace9cd35ab0565109491

submodules/qmining-data-editor

@@ -1 +1 @@
-Subproject commit 89edf8609922d6bb4054fca4f9712a9efbbce9f0
+Subproject commit f8d423f162268ed7734cb768ad40df9924b46bb2

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 509aff7cdd5b619568e7c433b85a56fd4ef3399b
+Subproject commit a1dab7a3ca2b5006d6eda00d1aea7e787e4de532

testingTools/tests/testScripts/ask.get.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 url=$(head -n 1 ../serverAddress)
-# url='https://api.frylabs.net'
 echo "Server url: $url"
 
 simpleType='\{"type":"simple"\}'

testingTools/tests/testScripts/ask.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 url=$(head -n 1 ../serverAddress)
-# url='https://api.frylabs.net'
 echo "Server url: $url"
 
 simpleType="{\"type\":\"simple\"}"

testingTools/tests/testScripts/postTestData.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 url=$(head -n 1 ../serverAddress)
-# url='https://api.frylabs.net'
 echo "Server url: $url"
 
 if [ "$#" -ne 1 ]; then