albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Seperate dbs

Browse source
This commit is contained in:
mrfry 2020-11-26 09:16:12 +01:00
parent 728931d56e
commit 4e34267d44
9 changed files with 149 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

94
src/modules/api/api.ts
View file

@ -57,11 +57,13 @@ const idvStatFile = 'stats/idvstats'
const todosFile = 'data/todos.json'
// other constants
const line = '====================================================' // lol
const maxVeteranPwGetCount = 10
const addPWPerDay = 3 // every x day a user can give a pw
const maxPWCount = 6 // maximum pw give opportunities a user can have at once
const addPWCount = 1 // how many pw gen opportunities to add each time
const daysAfterUserGetsPWs = 5 // days after user gets pw-s
const minimumAlowwedSessions = 1 // how many sessions are allowed for a user
// stuff gotten from server.js
let userDB
@ -79,19 +81,27 @@ function GetApp(): ModuleType {
// files in public dirs
const recivedFiles = publicDir + 'recivedfiles'
const uloadFiles = publicDir + 'f'
// FIXME: this to seperate file?
const dataFiles: Array<DataFile> = [
{
path: `${publicDir}oldData.json`,
name: 'oldData',
shouldSave: (recData: RecievedData): boolean => {
return recData.version.includes('2.0.')
return recData.version.startsWith('2.0.')
},
},
{
path: `${publicDir}data.json`,
name: 'newData',
shouldSave: (recData: RecievedData): boolean => {
return recData.version.includes('2.1.')
return recData.version.startsWith('2.1.')
},
},
{
path: `${publicDir}fromwebsiteData.json`,
name: 'fromwebsiteData',
shouldSave: (recData: RecievedData): boolean => {
return recData.version === 'WEBSITE'
},
},
]
@ -99,6 +109,11 @@ function GetApp(): ModuleType {
const userSpecificMotdFile = publicDir + 'userSpecificMotd.json'
const versionFile = publicDir + 'version'
let domain = url.split('.') // [ "https://api", "frylabs", "net" ]
domain.shift() // [ "frylabs", "net" ]
domain = domain.join('.') // "frylabs.net"
logger.DebugLog(`Cookie domain: ${domain}`, 'cookie', 1)
app.use(
bodyParser.urlencoded({
limit: '10mb',
@ -246,7 +261,21 @@ function GetApp(): ModuleType {
// -------------------------------------------------------------
app.get('/getDbs', (req: any, res: any) => {
logger.LogReq(req)
res.json(
dataFiles.map((df) => {
return {
path: df.path.split('/').pop(),
name: df.name,
}
})
)
})
app.get('/updateTodo', (req: any, res: any) => {
logger.LogReq(req)
const userId = req.session.user.id
const id = req.query.id
const todos = utils.ReadJSON(todosFile)
@ -632,13 +661,12 @@ function GetApp(): ModuleType {
if (user) {
const sessionID = uuidv4()
// FIXME: Users now can only log in in one session, this might be too strict.
const existingSessions = dbtools.Select(userDB, 'sessions', {
userID: user.id,
isScript: isScript ? 1 : 0,
})
if (existingSessions.length > 0) {
if (existingSessions.length >= minimumAlowwedSessions) {
logger.Log(
`Multiple ${isScript ? 'script' : 'website'} sessions ( ${
existingSessions.length
@ -675,9 +703,9 @@ function GetApp(): ModuleType {
})
// https://www.npmjs.com/package/cookie
// TODO: cookie age
// FIXME: cookies are not configured coorectly
res.cookie('sessionID', sessionID, {
domain: '.frylabs.net', // TODO: use url. url: "https://api.frylabs.net"
domain: domain,
expires: new Date(
new Date().getTime() + 10 * 365 * 24 * 60 * 60 * 1000
),
@ -724,7 +752,6 @@ function GetApp(): ModuleType {
dbtools.Delete(userDB, 'sessions', {
id: sessionID,
})
// TODO: remove old sessions every once in a while
res.clearCookie('sessionID').json({
result: 'success',
})
@ -829,9 +856,14 @@ function GetApp(): ModuleType {
app.get('/allqr.txt', function(req: any, res: any) {
res.set('Content-Type', 'text/plain')
const stringifiedData = questionDbs.map((qdb) => {
return dataToString(qdb.data)
let result = ''
result += '\n' + line
result += ` Questions in ${qdb.name}: `
result += line + '\n'
result += dataToString(qdb.data)
result += '\n' + line + line + '\n'
return result
})
// TODO: test this
res.send(stringifiedData.join('\n\n'))
res.end()
logger.LogReq(req)
@ -949,26 +981,38 @@ function GetApp(): ModuleType {
app.post('/isAdding', function(req: any, res: any) {
logger.LogReq(req)
const user: User = req.session.user
const dryRun = testUsers.includes(user.id)
processIncomingRequest(req.body, questionDbs, dryRun, user)
.then((resultArray) => {
logResult(req.body, resultArray)
res.json({
success: resultArray.length > 0, // TODO check for -1s
newQuestions: resultArray,
try {
processIncomingRequest(req.body, questionDbs, dryRun, user)
.then((resultArray) => {
logResult(req.body, resultArray, user.id, dryRun)
res.json({
success: resultArray.length > 0,
newQuestions: resultArray,
})
})
.catch((err) => {
logger.Log(
'Error during processing incoming request',
logger.GetColor('redbg')
)
console.error(err)
res.json({
success: false,
})
})
} catch (err) {
logger.Log(
'Error during getting incoming request processor promises ',
logger.GetColor('redbg')
)
console.error(err)
res.json({
success: false,
})
.catch((err) => {
logger.Log(
'Couldnt process incoming request!',
logger.GetColor('redbg')
)
console.error(err)
})
}
})
app.get('/ask', function(req: any, res) {
@ -996,8 +1040,6 @@ function GetApp(): ModuleType {
searchDatas(questionDbs, question, subj, recData)
.then((result) => {
// TODO: test
console.log(result)
res.json({
result: result,
success: true,

4
src/modules/stuff/stuff.ts
View file

@ -74,7 +74,7 @@ function GetApp(): ModuleType {
// app, '/*.mp4', 'video/mp4', 'stuff/video'
function appGetFileType(app, wildcard, contentType, pageToRender) {
app.get(wildcard, function(req, res) {
const path = decodeURI(req.url)
const path = decodeURIComponent(req.url)
let fp: any = path
if (path.includes('?')) {
fp = path.split('?')
@ -151,7 +151,7 @@ function GetApp(): ModuleType {
})
app.get('/*', function(req, res) {
const parsedUrl = decodeURI(req.url)
const parsedUrl = decodeURIComponent(req.url)
let curr =
listedFiles +
'/' +

1
src/types/basicTypes.ts
View file

@ -38,7 +38,6 @@ export interface ModuleType {
dailyAction?: Function
}
// TODO: add more props based on db
export interface User {
id: number
pw: string

67
src/utils/actions.ts
View file

@ -24,7 +24,7 @@ import logger from '../utils/logger'
import { searchData, createQuestion } from '../utils/classes'
import idStats from '../utils/ids'
import utils from '../utils/utils'
import { addQuestion, getSubjNameWithoutYear } from './classes'
import { SearchResult, addQuestion, getSubjNameWithoutYear } from './classes'
// types
import { QuestionDb, Question, User, DataFile } from '../types/basicTypes'
@ -51,29 +51,42 @@ interface Result {
export function logResult(
recievedData: RecievedData,
result: Array<Result>
result: Array<Result>,
userId: number,
dryRun?: boolean
): void {
let subjRow = '\t' + recievedData.subj
if (recievedData.id) {
subjRow += ' ( CID: ' + logger.logHashed(recievedData.id) + ')'
}
logger.Log(subjRow)
logger.Log('\t' + recievedData.subj)
result.forEach((res: Result) => {
const allQLength = recievedData.quiz.length
let msg = `${res.qdbName}: `
let color = logger.GetColor('green')
if (res.newQuestions > 0) {
color = logger.GetColor('blue')
msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
} else {
msg += `No new data ( ${allQLength} )`
}
if (recievedData.version !== undefined) {
msg += '. Version: ' + recievedData.version
}
logger.Log('\t' + msg, color)
})
if (dryRun) {
logger.Log('\tDry run')
}
let idRow = '\t'
if (recievedData.version) {
idRow += 'Version: ' + logger.logHashed(recievedData.version)
}
if (recievedData.id) {
idRow += ', CID: ' + logger.logHashed(recievedData.id)
}
idRow += `, User #${logger.logHashed(userId.toString())}`
logger.Log(idRow)
if (result.length > 0) {
result.forEach((res: Result) => {
const allQLength = recievedData.quiz.length
let msg = `${res.qdbName}: `
let color = logger.GetColor('green')
if (res.newQuestions > 0) {
color = logger.GetColor('blue')
msg += `New questions: ${res.newQuestions} ( All: ${allQLength} )`
} else {
msg += `No new data ( ${allQLength} )`
}
logger.Log('\t' + msg, color)
})
} else {
logger.Log('\tNo db-s passed shouldSave!', logger.GetColor('red'))
}
}
export function processIncomingRequest(
@ -90,7 +103,7 @@ export function processIncomingRequest(
}
try {
let towrite = logger.GetDateString() + '\n'
let towrite = utils.GetDateString() + '\n'
towrite +=
'------------------------------------------------------------------------------\n'
if (typeof recievedData === 'object') {
@ -153,15 +166,15 @@ function processIncomingRequestUsingDb(
logger.DebugLog(currentQuestion, 'actions', 3)
recievedQuestions.push(currentQuestion)
questionSearchPromises.push(
searchData(qdb.data, currentQuestion, recievedData.subj)
searchData(qdb, currentQuestion, recievedData.subj)
)
})
Promise.all(questionSearchPromises)
.then((results) => {
.then((results: Array<SearchResult>) => {
const allQuestions = [] // all new questions here that do not have result
results.forEach((result, i) => {
const add = result.every((res) => {
const add = result.result.every((res) => {
return res.match < minMatchToAmmountToAdd
})
if (add) {
@ -192,8 +205,6 @@ function processIncomingRequestUsingDb(
currWrites = 0
logger.DebugLog('Writing data.json', 'actions', 1)
utils.WriteFile(JSON.stringify(qdb.data), qdb.path)
} else if (dryRun) {
logger.Log('\tDry run')
}
}

39
src/utils/classes.ts
View file

@ -7,11 +7,15 @@ import {
Subject,
} from '../types/basicTypes'
// TODO
interface SearchResultQuestion extends Question {
match: number
}
export interface SearchResult {
result: Array<SearchResultQuestion>
dbName: string
}
const searchDataWorkerFile = './src/utils/classes.ts'
const assert = (val) => {
@ -217,7 +221,10 @@ function createQuestion(
}
function compareImage(data: QuestionData, data2: QuestionData) {
return compareString(data.images.join(' '), data2.images.join(' '))
// TODO: img comparing (hashed images vs images)
const imgs1 = data.hashedImages ? data.hashedImages : data.images
const imgs2 = data2.hashedImages ? data2.hashedImages : data2.images
return compareString(imgs1.join(' '), imgs2.join(' '))
}
function compareData(q1: Question, q2: Question) {
@ -412,23 +419,22 @@ function searchDatas(
question: any,
subjName: string,
questionData?: QuestionData
): Promise<Array<Array<SearchResultQuestion>>> {
): Promise<Array<SearchResult>> {
return Promise.all(
data.map((db) => {
return searchData(db.data, question, subjName, questionData)
data.map((db: QuestionDb) => {
return searchData(db, question, subjName, questionData)
})
)
}
// TODO: remove questionData, make question only Question type
// FIXME: remove questionData, make question only Question type
function searchData(
data: Array<Subject>,
qdb: QuestionDb,
question: any,
subjName: string,
questionData?: QuestionData
): Promise<Array<SearchResultQuestion>> {
): Promise<SearchResult> {
return new Promise((resolve, reject) => {
assert(data)
assert(question)
logger.DebugLog('Searching for question', 'qdb search', 1)
logger.DebugLog('Question:', 'qdb search', 2)
@ -447,7 +453,7 @@ function searchData(
question = simplifyQuestion(question)
const worker = workerTs(searchDataWorkerFile, {
workerData: { data, subjName, question, questionData },
workerData: { data: qdb.data, subjName, question, questionData },
})
worker.on('error', (err) => {
@ -478,7 +484,10 @@ function searchData(
'qdb search',
1
)
resolve(result)
resolve({
result: result,
dbName: qdb.name,
})
})
})
}
@ -494,10 +503,10 @@ function dataToString(data: Array<Subject>): string {
// ------------------------------------------------------------------------
function searchWorker(
data: any,
subjName: any,
question: any,
questionData: any
data: Array<Subject>,
subjName: string,
question: Question,
questionData?: QuestionData
): any {
let result = []

1
src/utils/dbtools.ts
View file

@ -126,7 +126,6 @@ function CreateTable(db: any, name: any, columns: any, foreignKeys: any): any {
const cols = Object.keys(columns)
.reduce((acc, key) => {
const item = columns[key]
// FIXME: array, and push stuff, then join()
const flags = []
const toCheck = {
primary: 'PRIMARY KEY',

9
src/utils/logger.ts
View file

@ -22,7 +22,7 @@ const hr =
'---------------------------------------------------------------------------------'
export default {
GetDateString: GetDateString,
getColoredDateString: getColoredDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
@ -70,7 +70,7 @@ function setNewLogfileName(): void {
logFileName = utils.GetDateString(true)
}
function GetDateString(): string {
function getColoredDateString(): string {
const date = new Date()
const dateString = utils.GetDateString()
return GetRandomColor(date.getHours().toString()) + dateString + C()
@ -99,7 +99,7 @@ function Log(msg: string | object, color?: string): void {
let log = msg
if (typeof msg !== 'object') {
const delimiter = DELIM + C(color)
log = C(color) + GetDateString() + delimiter + msg + C()
log = getColoredDateString() + delimiter + C(color) + msg + C()
}
console.log(log)
@ -109,7 +109,6 @@ function Log(msg: string | object, color?: string): void {
)
}
// FIXME: express.Request type, but with .cookies and .session
function LogReq(
req: any /*express.Request*/,
toFile?: boolean,
@ -162,7 +161,7 @@ function LogReq(
if (!toFile) {
Log(logEntry)
} else {
const defLogs = GetDateString() + dl + logEntry
const defLogs = getColoredDateString() + dl + logEntry
utils.AppendToFile(defLogs, vlogDir + logFileName)
}

14
src/utils/utils.ts
View file

@ -17,14 +17,22 @@ import fs from 'fs'
import logger from '../utils/logger'
interface URLFormatOptions {
query: any
pathname?: string
query?: any
}
// TODO
function formatUrl(options: URLFormatOptions): string {
console.log(options.pathname, options.query)
return options.pathname + JSON.stringify(options.query)
const queryString = options.query
? '?' +
Object.keys(options.query)
.map((key) => {
return `${key}=${encodeURIComponent(options.query[key])}`
})
.join('&')
: ''
const path = options.pathname || ''
return path + queryString
}
function GetDateString(noTime?: boolean): string {

2
submodules/qmining-page

@ -1 +1 @@
Subproject commit 4a7ad2d6c8e0651bdc8edbdaa99e4069411b98d3
Subproject commit 309436b2270ebf2398945b2cd707d1f133fe8ec1