From f8a7c6a4e508558b0fa0b76be876eb6b1861fd3e Mon Sep 17 00:00:00 2001
From: mrfry <mr.fry@tutanota.com>
Date: Tue, 11 Apr 2023 14:33:47 +0200
Subject: [PATCH] readme update, updated some queries to use numbers instead of
strings
---
README.md | 56 ++++++++++++-------
scripts/setup.sh | 2 +-
src/middlewares/auth.middleware.ts | 1 -
src/modules/api/api.ts | 6 +-
src/modules/api/submodules/userManagement.ts | 17 +++---
src/sharedViews/login.ejs | 2 +-
src/standaloneUtils/createUserDataList.sh | 16 ------
src/tests/addQuestion.test.ts | 6 +-
submodules/moodle-test-userscript | 2 +-
submodules/qmining-data-editor | 2 +-
submodules/qmining-page | 2 +-
testingTools/tests/testScripts/ask.get.sh | 1 -
testingTools/tests/testScripts/ask.sh | 1 -
.../tests/testScripts/postTestData.sh | 1 -
14 files changed, 53 insertions(+), 62 deletions(-)
delete mode 100755 src/standaloneUtils/createUserDataList.sh
diff --git a/README.md b/README.md
index 2f609b2..5a0b697 100755
--- a/README.md
+++ b/README.md
@@ -33,31 +33,41 @@ threads, and serve more requests at once. The used cores can be limited with env
| Peer to peer functionality | The ability to share question databases and users with other instances of this server |
| Peer | Another instance of this server, with peer to peer functionality set up |
| User \#1 | The first user created, admin of the server |
+ | Submodule | A server submodule, dynamically loaded, and stored in `./src/modules/` |
+ | Git module | Git modules stored in `./submodules` |
## Setup
Run `./scripts/setup.sh`, then `npm run dev` for development, or `npm run start` for prod.
-On the first run there will be a number of errors, that some files weren't found. Please create them
-according to the messages, these are necessary for the server to function.
+On the first run there will be a number of errors, that some files weren't found, or some
+environment variables needed to be specified. Please create them according to the messages, these
+are necessary for the server to function.
There will be also a lot of information about files and other necessary things being created. **Please
read them very carefully, you should know about what was created!**
-### Setup notes
+### Setup notes and tips
- * **Gitlab specific notes**: You need a gitlab account with SSH keys set up in order to be able to use
- SSH for cloning this (and any) project. This includes git modules of this project. See
- `CLONE_WITH_HTTPS` in environment variables below.
- * The `DOMAIN` env var can be replaced by making the file `./data/domain`. This is necesarry, to make
- xmlhttp requests and some redirects work in statically built HTML files (by next.js), in the
- userscript, and in the server itself.
- * The setup script can be also used to update and rebuild all git modules.
+ * **Gitlab specific notes**: You need a gitlab account with SSH keys set up in order to be able to
+ use SSH for cloning this (and any) project. This includes git modules of this project. To use HTTPS
+ set the `CLONE_WITH_HTTPS` environment variable before running `setup.sh`. To set up SSH keys:
+ https://docs.gitlab.com/ee/user/ssh.html
+ * You either need to specify a `DOMAIN` env var, or write the domain to `./data/domain`. This is
+ necesarry to make xmlhttp requests and some redirects work in: statically built HTML files (by
+ next.js), userscript, and the server itself. Domain is for example: 'qmining.com', without
+ 'http://' and '/'-s
+ * By default the server redirects all HTTP traffic to HTTPS. To disable this use
+ `NS_NO_HTTPS_FORCE`
+ * The server launches a thread for each CPU core. This could be an overkill on 4+ cored CPU-s. Use
+ `NS_THREAD_COUNT` to restrict the number of threads
+ * The setup script can be also used to update and rebuild all git modules if ran after the initial
+ setup
## Web server paths
-There are different routes assigned to different modules. You can see these in detail in the files
-`./src/modules.json`.
+There are different routes assigned to different submodules. You can see these in detail in the
+files `./src/modules.json`.
## Peer to peer
@@ -71,7 +81,7 @@ To setup P2P functionality you have to create a few files in `./data/p2p`:
{
"name": "any name you choose",
"contact": "contact to server administrator (irc server, e-mail, anything)",
- "host": "server host (like somesite.com, without 'http(s):// and /-s')",
+ "host": "server host (like 'qmining.com', without 'http(s):// and /-s')",
"pw": "password to the host, so the server can log in there. Please use a dedicated password, that only the server uses!",
"port": "server port, number"
}
@@ -148,23 +158,27 @@ result can be found in the directory it was ran.
### For the server
+Set these before launching the server (`VAR="" npm run start`)
+
| Name | Type | Description |
| --- | --- | --- |
| PORT | number | The port the http server should run on |
- | NS_THREAD_COUNT | number | Nubmer of CPU cores to use |
- | NS_NOUSER | boolean | If the authorization should be skipped (for testing) |
- | NS_NO_HTTPS_FORCE | boolean | Disables automatic redirects from http to https |
- | NS_LOGLEVEL | number | Debug log level, 0 is the least verbose |
- | NS_NOLOG | boolean | If logging should be skipped |
- | NS_SQL_DEBUG_LOG | boolean | If the SQL queries should be logged |
| DOMAIN | string | The domain that the server should use for redirects, cookies, etc. ex.: `qmining.com`, without 'https://', and '/'-s. If not specified `./data/domain` will be used |
+ | NS_NO_HTTPS_FORCE | boolean | Disables automatic redirects from http to https |
+ | NS_THREAD_COUNT | number | Nubmer of CPU cores to use |
+ | NS_LOGLEVEL | number | Debug log level, 0 is the least verbose |
+ | NS_SQL_DEBUG_LOG | boolean | If the SQL queries should be logged |
+ | NS_NOUSER | boolean | If the authorization should be skipped (for testing) |
+ | NS_NOLOG | boolean | If logging should be skipped |
### For the setup script (`./scripts/setup.sh`)
+Set these before starting the setup (`VAR="" ./scripts/setup.sh`)
+
| Name | Type | Description |
| --- | --- | --- |
- | CLONE_WITH_HTTPS | boolean | By default git clones with ssh, which can cause troubles. Set this
| DOMAIN | string | The domain that the server should use for redirects, cookies, etc. ex.: `qmining.com`, without 'https://', and '/'-s. If not specified `./data/domain` will be used |
+ | CLONE_WITH_HTTPS | boolean | By default git clones with ssh, which can cause troubles. Set this to use https |
## npm scripts
@@ -179,7 +193,7 @@ result can be found in the directory it was ran.
## Git modules
-Details about the specific git module should be found in its directory.
+More details about the specific git module should be found in its directory: `./submodules/`
# qmining-page
diff --git a/scripts/setup.sh b/scripts/setup.sh
index 939689b..fa92694 100755
--- a/scripts/setup.sh
+++ b/scripts/setup.sh
@@ -49,7 +49,7 @@ if [ -z "${domain}" ]; then
error "DOMAIN is not set, and ${domainPath} does not exist!"
echo "Set DOMAIN to the preferred domain"
echo "Or please create the file ${domainPath}, and re-run this script"
- echo "Expected content: domain, ex.:'frylabs.net' (without 'http://' and '/')"
+ echo "Expected content: domain, ex.:'qmining.com' (without 'http://' and '/')"
exit 1
fi
diff --git a/src/middlewares/auth.middleware.ts b/src/middlewares/auth.middleware.ts
index e9a559a..fc2a16b 100644
--- a/src/middlewares/auth.middleware.ts
+++ b/src/middlewares/auth.middleware.ts
@@ -52,7 +52,6 @@ function renderLogin(req: Request, res: Response) {
})
} else {
res.render('login', {
- useHttp: process.env.NS_NO_HTTPS_FORCE,
domain: constants.domain,
})
}
diff --git a/src/modules/api/api.ts b/src/modules/api/api.ts
index c4fece5..b97bd4c 100644
--- a/src/modules/api/api.ts
+++ b/src/modules/api/api.ts
@@ -60,9 +60,9 @@ function GetApp(): ModuleType {
throw new Error(`No public dir! ( API )`)
}
- let domain: any = url.split('.') // [ "https://api", "frylabs", "net" ]
- domain.shift() // [ "frylabs", "net" ]
- domain = domain.join('.') // "frylabs.net"
+ let domain: any = url.split('.')
+ domain.shift()
+ domain = domain.join('.') // "qmining.com"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
// -------------------------------------------------------------------------------------------
diff --git a/src/modules/api/submodules/userManagement.ts b/src/modules/api/submodules/userManagement.ts
index dbe25c0..9c6f31e 100644
--- a/src/modules/api/submodules/userManagement.ts
+++ b/src/modules/api/submodules/userManagement.ts
@@ -40,8 +40,8 @@ const daysAfterUserGetsPWs = 7 // days after user gets pw-s
interface Session {
id: string
userId: number
- createDate: string
- lastAccess: string
+ createDate: number
+ lastAccess: number
isScript: number
}
@@ -69,7 +69,7 @@ function createDefaultUser(userDb: Database) {
const insertRes = dbtools.Insert(userDb, 'users', {
pw: pw,
avaiblePWRequests: 50,
- created: utils.GetDateString(),
+ created: new Date().getTime(),
})
logger.Log('ID and PW for user #1: ', 'yellowbg')
console.log(`ID: #${insertRes.lastInsertRowid}, PW: "${pw}"`)
@@ -153,7 +153,7 @@ function setup(data: SubmoduleData): Submodule {
const insertRes = dbtools.Insert(userDB, 'users', {
pw: pw,
avaiblePWRequests: 0,
- created: utils.GetDateString(),
+ created: new Date().getTime(),
createdBy: requestingUser.id,
})
@@ -194,10 +194,7 @@ function setup(data: SubmoduleData): Submodule {
isScript: isScript ? 1 : 0,
})
.sort((a: Session, b: Session) => {
- return (
- new Date(a.lastAccess).getTime() -
- new Date(b.lastAccess).getTime()
- )
+ return a.lastAccess - b.lastAccess
})
const diff = existingSessions.length - minimumAlowwedSessions
@@ -222,7 +219,7 @@ function setup(data: SubmoduleData): Submodule {
'users',
{
loginCount: user.loginCount + 1,
- lastLogin: utils.GetDateString(),
+ lastLogin: new Date().getTime(),
},
{
id: user.id,
@@ -233,7 +230,7 @@ function setup(data: SubmoduleData): Submodule {
id: sessionID,
userID: user.id,
isScript: isScript ? 1 : 0,
- createDate: utils.GetDateString(),
+ createDate: new Date().getTime(),
})
// https://www.npmjs.com/package/cookie
diff --git a/src/sharedViews/login.ejs b/src/sharedViews/login.ejs
index fb62f83..12151b8 100644
--- a/src/sharedViews/login.ejs
+++ b/src/sharedViews/login.ejs
@@ -123,7 +123,7 @@
button.classList.add('disabledButton')
button.disabled = true
// TODO: get url from controller
- const rawResponse = await fetch('<%= useHttp ? `http://${domain}/api/login` : `https://${domain}/api/login` %>', {
+ const rawResponse = await fetch('/api/login', {
method: 'POST',
credentials: 'include',
headers: {
diff --git a/src/standaloneUtils/createUserDataList.sh b/src/standaloneUtils/createUserDataList.sh
deleted file mode 100755
index 3d3a7da..0000000
--- a/src/standaloneUtils/createUserDataList.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-scp -P 19192 qminer@megacarrot.frylabs.net:/home/qminer/mrfrys-node-server/stats/idstats "./idstats.new"
-scp -P 19192 qminer@megacarrot.frylabs.net:/home/qminer/old.nodeServ/stats/idstats "./idstats.old"
-
-cat idstats.new | jq 'keys' > idstats.new.keys
-cat idstats.old | jq 'keys' > idstats.old.keys
-
-cat idstats.new.keys idstats.old.keys | sort | uniq > keys.tmp
-
-cat keys.tmp | grep -Eo '[0-9]{13}' > ids
-
-rm idstats.new
-rm idstats.new.keys
-rm idstats.old
-rm idstats.old.keys
-rm keys.tmp
diff --git a/src/tests/addQuestion.test.ts b/src/tests/addQuestion.test.ts
index a009589..9511b87 100644
--- a/src/tests/addQuestion.test.ts
+++ b/src/tests/addQuestion.test.ts
@@ -56,7 +56,7 @@ test('Adds new subjects, multiple new questions', () => {
test("New subject names shouldn't be empty", () => {
const data: RecievedData = JSON.parse(
fs.readFileSync(
- __dirname + '/../../devel/tests/testData/ROS.json',
+ __dirname + '/../../testingTools/tests/testData/ROS.json',
'utf8'
)
)
@@ -80,7 +80,7 @@ test("New subject names shouldn't be empty", () => {
test('New questions shouldnt be added to accidental empty named subjects', () => {
const data: RecievedData = JSON.parse(
fs.readFileSync(
- __dirname + '/../../devel/tests/testData/ROS.json',
+ __dirname + '/../../testingTools/tests/testData/ROS.json',
'utf8'
)
)
@@ -103,7 +103,7 @@ test('New questions shouldnt be added to accidental empty named subjects', () =>
test('Question gets added to the correct subject', () => {
const data: RecievedData = JSON.parse(
fs.readFileSync(
- __dirname + '/../../devel/tests/testData/ROS.json',
+ __dirname + '/../../testingTools/tests/testData/ROS.json',
'utf8'
)
)
diff --git a/submodules/moodle-test-userscript b/submodules/moodle-test-userscript
index d9e5c8b..8e91b93 160000
--- a/submodules/moodle-test-userscript
+++ b/submodules/moodle-test-userscript
@@ -1 +1 @@
-Subproject commit d9e5c8bdb0bd7b66148623ac8585e6cfe9a752b2
+Subproject commit 8e91b93d298f0bfcd8b3ace9cd35ab0565109491
diff --git a/submodules/qmining-data-editor b/submodules/qmining-data-editor
index 89edf86..f8d423f 160000
--- a/submodules/qmining-data-editor
+++ b/submodules/qmining-data-editor
@@ -1 +1 @@
-Subproject commit 89edf8609922d6bb4054fca4f9712a9efbbce9f0
+Subproject commit f8d423f162268ed7734cb768ad40df9924b46bb2
diff --git a/submodules/qmining-page b/submodules/qmining-page
index 509aff7..a1dab7a 160000
--- a/submodules/qmining-page
+++ b/submodules/qmining-page
@@ -1 +1 @@
-Subproject commit 509aff7cdd5b619568e7c433b85a56fd4ef3399b
+Subproject commit a1dab7a3ca2b5006d6eda00d1aea7e787e4de532
diff --git a/testingTools/tests/testScripts/ask.get.sh b/testingTools/tests/testScripts/ask.get.sh
index 7fe2ea2..6f29117 100755
--- a/testingTools/tests/testScripts/ask.get.sh
+++ b/testingTools/tests/testScripts/ask.get.sh
@@ -1,6 +1,5 @@
#!/bin/bash
url=$(head -n 1 ../serverAddress)
-# url='https://api.frylabs.net'
echo "Server url: $url"
simpleType='\{"type":"simple"\}'
diff --git a/testingTools/tests/testScripts/ask.sh b/testingTools/tests/testScripts/ask.sh
index 0c6b8fa..f25504d 100755
--- a/testingTools/tests/testScripts/ask.sh
+++ b/testingTools/tests/testScripts/ask.sh
@@ -1,6 +1,5 @@
#!/bin/bash
url=$(head -n 1 ../serverAddress)
-# url='https://api.frylabs.net'
echo "Server url: $url"
simpleType="{\"type\":\"simple\"}"
diff --git a/testingTools/tests/testScripts/postTestData.sh b/testingTools/tests/testScripts/postTestData.sh
index 80aef73..3d13228 100755
--- a/testingTools/tests/testScripts/postTestData.sh
+++ b/testingTools/tests/testScripts/postTestData.sh
@@ -1,6 +1,5 @@
#!/bin/bash
url=$(head -n 1 ../serverAddress)
-# url='https://api.frylabs.net'
echo "Server url: $url"
if [ "$#" -ne 1 ]; then