albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Moved submodules and every stuff into seperate folders neatly #4

Browse source
This commit is contained in:
mrfry 2020-10-01 14:16:19 +02:00
parent 7e44ca30f1
commit ae91801fbd
51 changed files with 0 additions and 799 deletions
Download patch file Download diff file Expand all files Collapse all files

135
src/middlewares/auth.middleware.js Normal file
View file

@ -0,0 +1,135 @@
const logger = require('../utils/logger.js')
const utils = require('../utils/utils.js')
const dbtools = require('../utils/dbtools.js')
module.exports = function (options) {
const { userDB, jsonResponse, exceptions } = options
const renderLogin = (req, res) => {
res.status('401') // Unauthorized
if (jsonResponse) {
res.json({
result: 'nouser',
msg: 'You are not logged in'
})
} else {
res.render('login', {
devel: process.env.NS_DEVEL
})
}
}
return function (req, res, next) {
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
const sessionID = req.cookies.sessionID
const isException = exceptions.some((exc) => {
return req.url.split('?')[0] === exc
})
if (process.env.NS_NOUSER) {
req.session = {
user: {
id: 21323
},
sessionID: sessionID || 111111111111111111,
isException: false
}
next()
return
}
// FIXME Allowing all urls with _next in it, but not in params
if (req.url.split('?')[0].includes('_next') || req.url.split('?')[0].includes('well-known/acme-challenge')) {
req.session = { isException: true }
next()
return
}
if (!sessionID) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No session ID: ${req.url}`, 'auth', 1)
renderLogin(req, res)
return
}
const user = GetUserBySessionID(userDB, sessionID, req)
if (!user) {
if (isException) {
logger.DebugLog(`EXCEPTION: ${req.url}`, 'auth', 1)
req.session = { isException: true }
next()
return
}
logger.DebugLog(`No user:${req.url}`, 'auth', 1)
renderLogin(req, res)
return
}
req.session = {
user: user,
sessionID: sessionID,
isException: isException
}
logger.DebugLog(`ID #${user.id}: ${req.url}`, 'auth', 1)
UpdateAccess(userDB, user, ip, sessionID)
dbtools.Update(userDB, 'sessions', {
lastAccess: utils.GetDateString()
}, {
id: sessionID
})
dbtools.Update(userDB, 'users', {
lastIP: ip,
lastAccess: utils.GetDateString()
}, {
id: user.id
})
next()
}
}
function UpdateAccess (db, user, ip, sessionID) {
const accesses = dbtools.Select(db, 'accesses', {
userId: user.id,
ip: ip
})
if (accesses.length === 0) {
dbtools.Insert(db, 'accesses', {
userID: user.id,
ip: ip,
sessionID: sessionID,
date: utils.GetDateString()
})
}
}
function GetUserBySessionID (db, sessionID, req) {
logger.DebugLog(`Getting user from db`, 'auth', 2)
const session = dbtools.Select(db, 'sessions', {
id: sessionID
})[0]
if (!session) {
return
}
const user = dbtools.Select(db, 'users', {
id: session.userID
})[0]
if (user) {
return user
}
}

45
src/middlewares/reqlogger.middleware.js Normal file
View file

@ -0,0 +1,45 @@
const logger = require('../utils/logger.js')
module.exports = function (options) {
const loggableKeywords = options ? options.loggableKeywords : undefined
const loggableModules = options ? options.loggableModules : undefined
return function (req, res, next) {
res.on('finish', function () {
if (req.url.includes('_next/static')) {
return
}
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
let hostname = 'NOHOST'
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
logger.Log('Hostname is undefined!', logger.GetColor('redbg'))
console.log(req.body)
console.log(req.query)
console.log(req.headers)
}
// fixme: regexp includes checking
const hasLoggableKeyword = loggableKeywords && loggableKeywords.some((x) => {
return req.url.includes(x)
})
const hasLoggableModule = loggableModules && loggableModules.some((x) => {
return hostname.includes(x)
})
const toLog = hasLoggableModule || hasLoggableKeyword
logger.LogReq(req, true, res.statusCode)
if (toLog) { logger.LogReq(req) }
if (res.statusCode !== 404) {
logger.LogStat(req.url,
ip,
hostname,
req.session && req.session.user ? req.session.user.id : 'NOUSER'
)
}
})
next()
}
}

68
src/modules.json Normal file
View file

@ -0,0 +1,68 @@
{
"dataEditor": {
"path": "./modules/dataEditor/dataEditor.js",
"publicdirs": [
"qminingPublic/"
],
"nextdir": "modules/dataEditor/public/",
"name": "dataeditor",
"urls": [
"dataeditor.frylabs.net"
],
"isNextJs": true
},
"qmining": {
"path": "./modules/qmining/qmining.js",
"publicdirs": [
"qminingPublic/"
],
"nextdir": "modules/qmining/public/",
"name": "qmining",
"urls": [
"qmining.frylabs.net"
],
"isNextJs": true
},
"api": {
"path": "./modules/api/api.js",
"publicdirs": [
"qminingPublic/"
],
"name": "api",
"urls": [
"api.frylabs.net",
"localhost"
]
},
"main": {
"path": "./modules/main/main.js",
"publicdirs": [
"public/"
],
"name": "main",
"urls": [
"frylabs.net",
"www.frylabs.net"
]
},
"sio": {
"path": "./modules/sio/sio.js",
"publicdirs": [
"sioPublic/"
],
"name": "sio",
"urls": [
"sio.frylabs.net"
]
},
"stuff": {
"path": "./modules/stuff/stuff.js",
"publicdirs": [
"stuffPublic/"
],
"name": "stuff",
"urls": [
"stuff.frylabs.net"
]
}
}

780
src/modules/api/api.js Normal file
View file

@ -0,0 +1,780 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
const express = require('express')
const bodyParser = require('body-parser')
const busboy = require('connect-busboy')
const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported
const fs = require('fs')
const app = express()
// other requires
const logger = require('../../utils/logger.js')
const utils = require('../../utils/utils.js')
const actions = require('../../utils/actions.js')
const dbtools = require('../../utils/dbtools.js')
const auth = require('../../middlewares/auth.middleware.js')
// files
const msgFile = 'stats/msgs'
const passwordFile = 'data/dataEditorPasswords.json'
const dataEditsLog = 'stats/dataEdits'
const dailyDataCountFile = 'stats/dailyDataCount'
const usersDbBackupPath = 'data/dbs/backup'
const quickVoteResultsDir = 'stats/qvote'
const quickVotes = 'stats/qvote/votes.json'
const testUsersFile = 'data/testUsers.json'
// other constants
const maxVeteranPwGetCount = 10
const addPWPerDay = 3 // every x day a user can give a pw
const maxPWCount = 6 // maximum pw give opportunities a user can have at once
// const daysAfterUserGetsPWs = 2 // days after user gets pw-s
// stuff gotten from server.js
let userDB
let url // eslint-disable-line
let publicdirs = []
function GetApp () {
const p = publicdirs[0]
if (!p) {
throw new Error(`No public dir! ( API )`)
}
// files in public dirs
const recivedFiles = p + 'recivedfiles'
const uloadFiles = p + 'f'
const dataFile = p + 'data.json'
const motdFile = p + 'motd'
const versionFile = p + 'version'
app.use(bodyParser.urlencoded({
limit: '10mb',
extended: true
}))
app.use(bodyParser.json({
limit: '10mb'
}))
app.set('view engine', 'ejs')
app.set('views', [
'./modules/api/views',
'./sharedViews'
])
app.use(auth({
userDB: userDB,
jsonResponse: true,
exceptions: [
'/favicon.ico',
'/login',
'/getveteranpw',
'/postfeedbackfile',
'/postfeedback',
'/fosuploader',
'/badtestsender'
]
}))
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(busboy({
limits: {
fileSize: 50000 * 1024 * 1024
}
}))
var data = actions.LoadJSON(dataFile)
var version = ''
var motd = ''
var testUsers = []
function LoadVersion () {
version = utils.ReadFile(versionFile)
}
function LoadMOTD () {
motd = utils.ReadFile(motdFile)
}
function LoadTestUsers () {
testUsers = utils.ReadJSON(testUsersFile)
if (testUsers) {
testUsers = testUsers.userIds
}
}
function Load () {
utils.WatchFile(motdFile, (newData) => {
logger.Log(`Motd changed: ${newData.replace(/\/n/g, '')}`)
LoadMOTD()
})
utils.WatchFile(versionFile, (newData) => {
logger.Log(`Version changed: ${newData.replace(/\/n/g, '')}`)
LoadVersion()
})
utils.WatchFile(testUsersFile, (newData) => {
logger.Log(`Test Users file changed: ${newData.replace(/\/n/g, '')}`)
LoadTestUsers()
})
LoadTestUsers()
LoadVersion()
LoadMOTD()
}
Load()
// -------------------------------------------------------------
app.get('/quickvote', (req, res) => {
const key = req.query.key
const val = req.query.val
const user = req.session.user
if (!key || !val) {
res.render('votethank', {
results: 'error',
msg: 'no key or val query param!'
})
return
}
let votes = {}
if (utils.FileExists(quickVotes)) {
votes = utils.ReadJSON(quickVotes)
} else {
logger.Log(`No such vote "${key}", and quickVotes.json is missing ( #${user.id}: ${key}-${val} )`, logger.GetColor('blue'))
res.render('votethank', {
result: 'no such pool'
})
return
}
if (!votes.voteNames.includes(key)) {
logger.Log(`No such vote "${key}" ( #${user.id}: ${key}-${val} )`, logger.GetColor('blue'))
res.render('votethank', {
result: 'no such pool'
})
return
}
const voteFile = quickVoteResultsDir + '/' + key + '.json'
let voteData = {
votes: {},
users: []
}
if (utils.FileExists(voteFile)) {
voteData = utils.ReadJSON(voteFile)
} else {
utils.CreatePath(quickVoteResultsDir)
}
if (!voteData.users.includes(user.id)) {
if (voteData.votes[val]) {
voteData.votes[val]++
} else {
voteData.votes[val] = 1
}
voteData.users.push(user.id)
logger.Log(`Vote from #${user.id}: ${key}: ${val}`, logger.GetColor('blue'))
res.render('votethank', {
result: 'success',
msg: 'vote added'
})
} else {
logger.Log(`#${user.id} already voted for: ${key}: ${val}`, logger.GetColor('blue'))
res.render('votethank', {
result: 'already voted',
msg: 'already voted'
})
}
utils.WriteFile(JSON.stringify(voteData), voteFile)
})
app.get('/avaiblePWS', (req, res) => {
logger.LogReq(req)
const user = req.session.user
res.json({
result: 'success',
userCreated: user.created,
avaiblePWS: user.avaiblePWRequests,
requestedPWS: user.pwRequestCount,
maxPWCount: maxPWCount,
// daysAfterUserGetsPWs: daysAfterUserGetsPWs,
addPWPerDay: addPWPerDay
})
})
app.post('/getpw', function (req, res) {
logger.LogReq(req)
const requestingUser = req.session.user
if (requestingUser.avaiblePWRequests <= 0) {
res.json({
result: 'error',
msg: 'Too many passwords requested or cant request password yet, try later'
})
logger.Log(`User #${requestingUser.id} requested too much passwords`, logger.GetColor('cyan'))
return
}
dbtools.Update(userDB, 'users', {
avaiblePWRequests: requestingUser.avaiblePWRequests - 1,
pwRequestCount: requestingUser.pwRequestCount + 1
}, {
id: requestingUser.id
})
const pw = uuidv4()
const insertRes = dbtools.Insert(userDB, 'users', {
pw: pw,
avaiblePWRequests: 0,
created: utils.GetDateString()
})
logger.Log(`User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`, logger.GetColor('cyan'))
res.json({
result: 'success',
pw: pw,
requestedPWS: requestingUser.pwRequestCount + 1,
remaining: requestingUser.avaiblePWRequests - 1
})
})
app.post('/getveteranpw', function (req, res) {
logger.LogReq(req)
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
const tries = dbtools.Select(userDB, 'veteranPWRequests', {
ip: ip
})[0]
if (tries) {
if (tries.count > maxVeteranPwGetCount) {
res.json({
result: 'error',
msg: 'Too many tries from this IP'
})
logger.Log(`Too many veteran PW requests from ${ip}!`, logger.GetColor('cyan'))
return
} else {
dbtools.Update(userDB, 'veteranPWRequests', {
count: tries.count + 1,
lastDate: utils.GetDateString()
}, {
id: tries.id
})
}
} else {
dbtools.Insert(userDB, 'veteranPWRequests', {
ip: ip,
lastDate: utils.GetDateString()
})
}
const oldUserID = req.body.cid
if (!oldUserID) {
res.json({
result: 'error',
msg: 'No Client ID recieved'
})
logger.Log(`No client ID recieved`, logger.GetColor('cyan'))
return
}
const user = dbtools.Select(userDB, 'users', {
oldCID: oldUserID
})[0]
if (user) {
if (user.pwGotFromCID === 0) {
logger.Log(`Sent password to veteran user #${user.id}`, logger.GetColor('cyan'))
dbtools.Update(userDB, 'users', {
pwGotFromCID: 1
}, {
id: user.id
})
res.json({
result: 'success',
pw: user.pw
})
} else {
logger.Log(`Veteran user #${user.id} already requested password`, logger.GetColor('cyan'))
res.json({
result: 'error',
msg: 'Password already requested'
})
}
} else {
logger.Log(`Invalid password request with CID: ${oldUserID}`, logger.GetColor('cyan'))
res.json({
result: 'error',
msg: 'No such Client ID'
})
}
})
app.post('/login', (req, res) => {
logger.LogReq(req)
const pw = req.body.pw || false
const cid = req.body.cid
const isScript = req.body.script
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
const user = dbtools.Select(userDB, 'users', {
pw: pw
})[0]
if (user) {
const sessionID = uuidv4()
// FIXME: Users now can only log in in one session, this might be too strict.
const existingSessions = dbtools.Select(userDB, 'sessions', {
userID: user.id,
isScript: isScript ? 1 : 0
})
if (existingSessions.length > 0) {
logger.Log(`Multiple ${isScript ? 'script' : 'website'} sessions ( ${existingSessions.length} ) for #${user.id}, deleting olds`, logger.GetColor('cyan'))
existingSessions.forEach((sess) => {
dbtools.Delete(userDB, 'sessions', {
id: sess.id,
isScript: isScript ? 1 : 0
})
})
}
dbtools.Update(userDB, 'users', {
loginCount: user.loginCount + 1,
lastIP: ip,
lastLogin: utils.GetDateString()
}, {
id: user.id
})
dbtools.Insert(userDB, 'sessions', {
id: sessionID,
ip: ip,
userID: user.id,
isScript: isScript ? 1 : 0,
createDate: utils.GetDateString()
})
// https://www.npmjs.com/package/cookie
// TODO: cookie age
res.cookie('sessionID', sessionID, {
domain: '.frylabs.net', // TODO: use url. url: "https://api.frylabs.net"
expires: new Date(new Date().getTime() + (10 * 365 * 24 * 60 * 60 * 1000)),
sameSite: 'none',
secure: true
})
res.cookie('sessionID', sessionID, {
expires: new Date(new Date().getTime() + (10 * 365 * 24 * 60 * 60 * 1000)),
sameSite: 'none',
secure: true
})
res.json({
result: 'success',
msg: 'you are now logged in'
})
logger.Log(`Successfull login to ${isScript ? 'script' : 'website'} with user ID: #${user.id}`, logger.GetColor('cyan'))
} else {
logger.Log(`Login attempt with invalid pw: ${pw} to ${isScript ? 'script' : 'website'}${cid ? ', CID:' + cid : ''}`, logger.GetColor('cyan'))
res.json({
result: 'error',
msg: 'Invalid password'
})
}
})
app.post('/logout', (req, res) => {
logger.LogReq(req)
const sessionID = req.cookies.sessionID
// removing session from db
dbtools.Delete(userDB, 'sessions', {
id: sessionID
})
// TODO: remove old sessions every once in a while
res.clearCookie('sessionID').json({
result: 'success'
})
})
// --------------------------------------------------------------
app.get('/', function (req, res) {
logger.LogReq(req)
res.redirect('https://www.youtube.com/watch?v=ieqGJgqiXFk')
})
app.post('/postfeedbackfile', function (req, res) {
UploadFile(req, res, uloadFiles, (fn) => {
res.json({ success: true })
})
logger.LogReq(req)
logger.Log('New feedback file', logger.GetColor('bluebg'), true)
})
app.post('/postfeedback', function (req, res) {
logger.LogReq(req)
if (req.body.fromLogin) {
logger.Log('New feedback message from Login page', logger.GetColor('bluebg'), true)
} else {
logger.Log('New feedback message from feedback page', logger.GetColor('bluebg'), true)
}
const ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
const user = req.session.user
utils.AppendToFile(utils.GetDateString() + ':\n' + JSON.stringify({
...req.body,
userID: user ? user.id : 'no user',
ip: ip
}), msgFile)
res.json({ success: true })
})
function UploadFile (req, res, path, next) {
try {
var fstream
req.pipe(req.busboy)
req.busboy.on('file', function (fieldname, file, filename) {
logger.Log('Uploading: ' + filename, logger.GetColor('blue'))
utils.CreatePath(path, true)
let d = new Date()
let fn = d.getHours() + '' + d.getMinutes() + '' + d.getSeconds() + '_' + filename
fstream = fs.createWriteStream(path + '/' + fn)
file.pipe(fstream)
fstream.on('close', function () {
logger.Log('Upload Finished of ' + path + '/' + fn, logger.GetColor('blue'))
next(fn)
})
fstream.on('error', function (err) {
console.log(err)
res.end('something bad happened :s')
})
})
} catch (e) {
logger.Log(`Unable to upload file!`, logger.GetColor('redbg'))
console.log(e)
}
}
app.route('/fosuploader').post(function (req, res, next) {
UploadFile(req, res, uloadFiles, (fn) => {
res.redirect('/f/' + fn)
})
})
app.route('/badtestsender').post(function (req, res, next) {
UploadFile(req, res, recivedFiles, (fn) => {
res.redirect('back')
})
logger.LogReq(req)
})
app.get('/allqr.txt', function (req, res) {
res.set('Content-Type', 'text/plain')
res.send(data.toString())
res.end()
logger.LogReq(req)
})
// -------------------------------------------------------------------------------------------
// API
app.post('/uploaddata', (req, res) => {
// body: JSON.stringify({
// newData: data,
// count: getCount(data),
// initialCount: initialCount,
// password: password,
// editedQuestions: editedQuestions
// })
const { count, initialCount, editedQuestions, password, newData } = req.body
const respStatuses = {
invalidPass: 'invalidPass',
ok: 'ok',
error: 'error'
}
logger.LogReq(req)
try {
// finding user
const pwds = JSON.parse(utils.ReadFile(passwordFile))
let user = Object.keys(pwds).find((key) => {
const u = pwds[key]
return u.password === password
})
user = pwds[user]
// logging and stuff
logger.Log(`Data upload`, logger.GetColor('bluebg'))
logger.Log(`PWD: ${password}`, logger.GetColor('bluebg'))
// returning if user password is not ok
if (!user) {
logger.Log(`Data upload: invalid password ${password}`, logger.GetColor('red'))
utils.AppendToFile(utils.GetDateString() + '\n' + password + '(FAILED PASSWORD)\n' + JSON.stringify(editedQuestions) + '\n\n', dataEditsLog)
res.json({ status: respStatuses.invalidPass })
return
}
logger.Log(`Password accepted for ${user.name}`, logger.GetColor('bluebg'))
logger.Log(`Old Subjects/Questions: ${initialCount.subjectCount} / ${initialCount.questionCount} | New: ${count.subjectCount} / ${count.questionCount} | Edited question count: ${Object.keys(editedQuestions).length}`, logger.GetColor('bluebg'))
// saving detailed editedCount
utils.AppendToFile(utils.GetDateString() + '\n' + JSON.stringify(user) + '\n' + JSON.stringify(editedQuestions) + '\n\n', dataEditsLog)
// making backup
utils.CopyFile('./' + dataFile, `./qminingPublic/backs/data_before_${user.name}_${utils.GetDateString().replace(/ /g, '_')}`) // TODO: rewrite to dinamyc public!!!
logger.Log('Backup made')
// writing data
utils.WriteFile(JSON.stringify(newData), dataFile)
logger.Log('New data file written')
// reloading data file
data = actions.LoadJSON(dataFile)
// data = newData
logger.Log('Data set to newData')
res.json({
status: respStatuses.ok,
user: user.name
})
logger.Log('Data updating done!', logger.GetColor('bluebg'))
} catch (e) {
logger.Log(`Data upload error! `, logger.GetColor('redbg'))
console.error(e)
res.json({ status: respStatuses.error, msg: e.message })
}
})
app.post('/isAdding', function (req, res) {
logger.LogReq(req)
const user = req.session.user
const dryRun = testUsers.includes(user.id)
// automatically saves to dataFile every n write
// FIXME: req.body.datatoadd is for backwards compatibility, remove this sometime in the future
actions.ProcessIncomingRequest(
req.body.datatoadd || req.body,
data,
{ motd, version },
dryRun
).then((r) => {
res.json({
success: r !== -1,
newQuestions: r
})
})
})
app.get('/ask', function (req, res) {
if (Object.keys(req.query).length === 0) {
logger.DebugLog(`No query params`, 'ask', 1)
res.json({
message: `ask something! ?q=[question]&subj=[subject]&data=[question data]. 'subj' is optimal for faster result`,
result: [],
recievedData: JSON.stringify(req.query),
success: false
})
} else {
if (req.query.q && req.query.data) {
let subj = req.query.subj || ''
let question = req.query.q
let recData = {}
try {
recData = JSON.parse(req.query.data)
} catch (e) {
logger.Log(`Unable to parse recieved question data! '${req.query.data}'`, logger.GetColor('redbg'))
}
let r = data.Search(question, subj, recData)
res.json({
result: r,
success: true
})
logger.DebugLog(`Question result length: ${r.length}`, 'ask', 1)
logger.DebugLog(r, 'ask', 2)
} else {
logger.DebugLog(`Invalid question`, 'ask', 1)
res.json({
message: `Invalid question :(`,
result: [],
recievedData: JSON.stringify(req.query),
success: false
})
}
}
})
function getSimplreRes () {
return {
subjects: data.length,
questions: data.Subjects.reduce((acc, subj) => {
return acc + subj.length
}, 0)
}
}
function getDetailedRes () {
return data.Subjects.map((subj) => {
return {
name: subj.Name,
count: subj.length
}
})
}
app.get('/datacount', function (req, res) {
logger.LogReq(req)
if (req.query.detailed === 'all') {
res.json({
detailed: getDetailedRes(),
simple: getSimplreRes()
})
} else if (req.query.detailed) {
res.json(getDetailedRes())
} else {
res.json(getSimplreRes())
}
})
app.get('/infos', function (req, res) {
const user = req.session.user
let result = {
result: 'success',
uid: user.id
}
if (req.query.subjinfo) {
result.subjinfo = getSimplreRes()
}
if (req.query.version) {
result.version = version
}
if (req.query.motd) {
result.motd = motd
}
res.json(result)
})
// -------------------------------------------------------------------------------------------
app.get('*', function (req, res) {
res.status(404).render('404')
})
app.post('*', function (req, res) {
res.status(404).render('404')
})
function ExportDailyDataCount () {
logger.Log('Saving daily data count ...')
utils.AppendToFile(JSON.stringify({
date: utils.GetDateString(),
subjectCount: data.Subjects.length,
questionCOunt: data.Subjects.reduce((acc, subj) => {
return acc + subj.Questions.length
}, 0),
userCount: dbtools.TableInfo(userDB, 'users').dataCount
}), dailyDataCountFile)
}
function BackupDB () {
logger.Log('Backing up auth DB ...')
utils.CreatePath(usersDbBackupPath, true)
userDB.backup(`${usersDbBackupPath}/users.${utils.GetDateString().replace(/ /g, '_')}.db`)
.then(() => {
logger.Log('Auth DB backup complete!')
})
.catch((err) => {
logger.Log('Auth DB backup failed!', logger.GetColor('redbg'))
console.error(err)
})
}
function IncrementAvaiblePWs () {
// FIXME: check this if this is legit and works
logger.Log('Incrementing avaible PW-s ...')
const users = dbtools.SelectAll(userDB, 'users')
const today = new Date()
const getDayDiff = (dateString) => {
let msdiff = today - new Date(dateString)
return Math.floor(msdiff / (1000 * 3600 * 24))
}
users.forEach((u) => {
if (u.avaiblePWRequests >= maxPWCount) {
return
}
const dayDiff = getDayDiff(u.created)
// if (dayDiff < daysAfterUserGetsPWs) {
// logger.Log(`User #${u.id} is not registered long enough to get password ( ${dayDiff} days, ${daysAfterUserGetsPWs} needed)`, logger.GetColor('cyan'))
// return
// }
if (dayDiff % addPWPerDay === 0) {
logger.Log(`Incrementing avaible PW-s for user #${u.id}: ${u.avaiblePWRequests} -> ${u.avaiblePWRequests + 1}`, logger.GetColor('cyan'))
dbtools.Update(userDB, 'users', {
avaiblePWRequests: u.avaiblePWRequests + 1
}, {
id: u.id
})
}
})
}
function DailyAction () {
ExportDailyDataCount()
BackupDB()
IncrementAvaiblePWs()
}
return {
dailyAction: DailyAction,
app: app
}
}
exports.name = 'API'
exports.getApp = GetApp
exports.setup = (data) => {
userDB = data.userDB
url = data.url
publicdirs = data.publicdirs
}

148
src/modules/api/apiDBStruct.json Normal file
View file

@ -0,0 +1,148 @@
{
"users": {
"tableStruct": {
"id": {
"type": "integer",
"primary": true,
"autoIncrement": true
},
"pw": {
"type": "text",
"notNull": true,
"unique": true
},
"oldCID": {
"type": "text",
"unique": true
},
"lastIP": {
"type": "text"
},
"notes": {
"type": "text"
},
"loginCount": {
"type": "number",
"defaultZero": true
},
"created": {
"type": "text",
"notNull": true
},
"lastLogin": {
"type": "text"
},
"lastAccess": {
"type": "text"
},
"avaiblePWRequests": {
"type": "number",
"defaultZero": true
},
"pwRequestCount": {
"type": "number",
"defaultZero": true
},
"pwGotFromCID": {
"type": "number",
"defaultZero": true
}
}
},
"sessions": {
"foreignKey": [
{
"keysFrom": [
"userID"
],
"table": "users",
"keysTo": [
"id"
]
}
],
"tableStruct": {
"id": {
"type": "text",
"primary": true,
"notNull": true
},
"ip": {
"type": "text",
"notNull": true
},
"userID": {
"type": "number",
"notNull": true
},
"createDate": {
"type": "text",
"notNull": true
},
"lastAccess": {
"type": "text"
},
"isScript": {
"type": "number",
"notNull": true
}
}
},
"accesses": {
"foreignKey": [
{
"keysFrom": [
"userID"
],
"table": "users",
"keysTo": [
"id"
]
}
],
"tableStruct": {
"accessID": {
"type": "integer",
"primary": true,
"autoIncrement": true
},
"userID": {
"type": "number",
"notNull": true
},
"ip": {
"type": "text",
"notNull": true
},
"date": {
"type": "text",
"notNull": true
},
"sessionID": {
"type": "text",
"notNull": true
}
}
},
"veteranPWRequests": {
"tableStruct": {
"id": {
"type": "integer",
"primary": true,
"autoIncrement": true
},
"ip": {
"type": "text",
"notNull": true
},
"count": {
"type": "number",
"defaultZero": true
},
"lastDate": {
"type": "text",
"notNull": true
}
}
}
}

30
src/modules/api/views/votethank.ejs Executable file
View file

@ -0,0 +1,30 @@
<html>
<body bgcolor="#212127">
<head>
<title>Shit uploader</title>
<meta charset="UTF-8">
<style>
.main {
font-size: 28px;
color: #ffffff;
text-align: center;
}
</style>
</head>
<div class='main'>
<%
if (result == 'success') {
%> ty a szavazásért c: <%
} else if (result == 'no such pool') {
%> Ilyen nevű szavazás nincs :c <%
} else if (result == 'already voted') {
%> Már szavaztál, de azért ty c: <%
} else if (result == 'error') {
%> Helytelen url paraméterek :c <%
} else {
%> bit of a fuckup here <%
}
%>
</div>
</body>
</html>

116
src/modules/dataEditor/dataEditor.js Normal file
View file

@ -0,0 +1,116 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
const express = require('express')
const bodyParser = require('body-parser')
const busboy = require('connect-busboy')
const app = express()
// other requires
const utils = require('../../utils/utils.js')
const logger = require('../../utils/logger.js')
const auth = require('../../middlewares/auth.middleware.js')
// stuff gotten from server.js
let userDB
let publicdirs = []
let nextdir = ''
function GetApp () {
app.use(bodyParser.urlencoded({
limit: '5mb',
extended: true
}))
app.use(bodyParser.json({
limit: '5mb'
}))
app.set('view engine', 'ejs')
app.set('views', [
'./modules/dataEditor/views',
'./sharedViews'
])
app.use(auth({
userDB: userDB,
jsonResponse: false,
exceptions: [
'/favicon.ico',
'/getVeteranPw'
]
}))
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
app.use(busboy({
limits: {
fileSize: 10000 * 1024 * 1024
}
}))
// --------------------------------------------------------------
function AddHtmlRoutes (files) {
const routes = files.reduce((acc, f) => {
if (f.includes('html')) {
acc.push(f.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'DataEditor routes', 1)
app.get(`/${route}`, function (req, res) {
logger.LogReq(req)
res.redirect(`${route}.html`)
})
})
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
app.get('/', function (req, res) {
res.end('hai')
logger.LogReq(req)
})
app.get('*', function (req, res) {
res.status(404).render('404')
})
app.post('*', function (req, res) {
res.status(404).render('404')
})
return {
app: app
}
}
exports.name = 'Data editor'
exports.getApp = GetApp
exports.setup = (data) => {
userDB = data.userDB
publicdirs = data.publicdirs
nextdir = data.nextdir
}

84
src/modules/main/main.js Normal file
View file

@ -0,0 +1,84 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
const express = require('express')
const bodyParser = require('body-parser')
const busboy = require('connect-busboy')
const app = express()
// other requires
const logger = require('../../utils/logger.js')
// stuff gotten from server.js
let publicdirs = []
let url = '' // http(s)//asd.basd
function GetApp () {
app.set('view engine', 'ejs')
app.set('views', [
'./modules/main/views',
'./sharedViews'
])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(busboy({
limits: {
fileSize: 10000 * 1024 * 1024
}
}))
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({
limit: '5mb',
extended: true
}))
app.use(bodyParser.json({
limit: '5mb'
}))
// --------------------------------------------------------------
app.get('/', function (req, res) {
res.render('main', {
siteurl: url
})
})
app.get('*', function (req, res) {
res.status(404).render('404')
})
app.post('*', function (req, res) {
res.status(404).render('404')
})
return {
app: app
}
}
exports.name = 'Main'
exports.getApp = GetApp
exports.setup = (data) => {
url = data.url
publicdirs = data.publicdirs
}

41
src/modules/main/views/main.ejs Executable file
View file

@ -0,0 +1,41 @@
<html>
<body bgcolor="#212127">
<head>
<title>FryLabs.net</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.8" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top
}
a {
color: #9999ff;
text-decoration: none;
}
</style>
</head>
</p>
<h2>
<a>
<pre>
____ __ __
/ __/_____ __ / /__ _/ / ___
/ _// __/ // / / / _ `/ _ \(_-<
/_/ /_/ \_, / /_/\_,_/_.__/___/
/___/
</pre>
</a>
</h2>
</body>
</html>

218
src/modules/qmining/qmining.js Normal file
View file

@ -0,0 +1,218 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
const express = require('express')
const bodyParser = require('body-parser')
const busboy = require('connect-busboy')
const app = express()
// other requires
const utils = require('../../utils/utils.js')
const logger = require('../../utils/logger.js')
const auth = require('../../middlewares/auth.middleware.js')
// stuff gotten from server.js
let donateURL = ''
let publicdirs = []
let userDB
let nextdir = ''
try {
donateURL = utils.ReadFile('./data/donateURL')
} catch (e) {
logger.Log('Couldnt read donate URL file!', logger.GetColor('red'))
}
function GetApp () {
app.use(bodyParser.urlencoded({
limit: '5mb',
extended: true
}))
app.use(bodyParser.json({
limit: '5mb'
}))
app.set('view engine', 'ejs')
app.set('views', [
'./modules/qmining/views',
'./sharedViews'
])
app.use(auth({
userDB: userDB,
jsonResponse: false,
exceptions: [
'/thanks',
'/thanks.html',
'/img/thanks.gif',
'/install',
'/favicon.ico',
'/getVeteranPw',
'/moodle-test-userscript/stable.user.js',
'/donate',
'/irc'
]
}))
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(express.static(nextdir))
app.use(busboy({
limits: {
fileSize: 10000 * 1024 * 1024
}
}))
// --------------------------------------------------------------
// REDIRECTS
// --------------------------------------------------------------
// to be backwards compatible
app.get('/ask', function (req, res) {
logger.DebugLog(`Qmining module ask redirect`, 'ask', 1)
res.redirect(`http://api.frylabs.net/ask?q=${req.query.q}&subj=${req.query.subj}&data=${req.query.data}`)
})
const simpleRedirects = [
{
from: '/dataeditor',
to: 'https://dataeditor.frylabs.net'
},
{
from: '/install',
to: 'https://qmining.frylabs.net/moodle-test-userscript/stable.user.js'
},
{
from: '/servergit',
to: 'https://gitlab.com/MrFry/mrfrys-node-server'
},
{
from: '/scriptgit',
to: 'https://gitlab.com/MrFry/moodle-test-userscript'
},
{
from: '/qminingSite',
to: 'https://gitlab.com/MrFry/qmining-page'
},
{
from: '/classesgit',
to: 'https://gitlab.com/MrFry/question-classes'
},
{
from: '/menuClick',
to: '/'
},
{
from: '/lred',
to: '/allQuestions.html'
},
{
from: '/donate',
to: donateURL
},
{ // to be backwards compatible
from: '/legacy',
to: '/allQuestions.html'
},
{
from: '/allqr',
to: 'http://api.frylabs.net/allqr.txt'
},
{
from: '/allqr.txt',
to: 'http://api.frylabs.net/allqr.txt'
},
{
from: '/infos',
to: 'http://api.frylabs.net/infos?version=true&motd=true&subjinfo=true',
nolog: true
},
{
from: '/irc',
to: 'https://kiwiirc.com/nextclient/irc.sub.fm/#qmining'
}
]
simpleRedirects.forEach((redirect) => {
app.get(redirect.from, function (req, res) {
if (!redirect.nolog) {
logger.LogReq(req)
}
logger.DebugLog(`Qmining module ${redirect.from} redirect`, 'infos', 1)
res.redirect(`${redirect.to}`)
})
})
// --------------------------------------------------------------
function AddHtmlRoutes (files) {
const routes = files.reduce((acc, f) => {
if (f.includes('html')) {
acc.push(f.split('.')[0])
return acc
}
return acc
}, [])
routes.forEach((route) => {
logger.DebugLog(`Added route /${route}`, 'Qmining routes', 1)
app.get(`/${route}`, function (req, res) {
logger.LogReq(req)
res.redirect(`${route}.html`)
})
})
}
AddHtmlRoutes(utils.ReadDir(nextdir))
// --------------------------------------------------------------
app.get('/', function (req, res) {
res.end('hai')
logger.LogReq(req)
})
app.get('/getVeteranPw', function (req, res) {
res.render('veteranPw', {
cid: req.query.cid || '',
devel: process.env.NS_DEVEL
})
logger.LogReq(req)
})
app.get('*', function (req, res) {
res.status(404).render('404')
})
app.post('*', function (req, res) {
res.status(404).render('404')
})
return {
app: app
}
}
exports.name = 'Qmining'
exports.getApp = GetApp
exports.setup = (data) => {
userDB = data.userDB
publicdirs = data.publicdirs
nextdir = data.nextdir
}

150
src/modules/qmining/views/veteranPw.ejs Normal file
View file

@ -0,0 +1,150 @@
<html>
<body bgcolor="#222426">
<head>
<title>Frylabs</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
a {
color: lightblue;
}
.center {
width: 440px;
height: 340px;
position: absolute;
left: 0;
right: 0;
top: 0;
bottom: 0;
margin: auto;
max-width: 100%;
max-height: 100%;
overflow: auto;
text-align: center;
}
.text {
font-size: 18px;
color: white;
margin: 20px;
}
.title {
font-size: 50px;
font-weight: bold;
color: white;
}
.inputContainer {
width: 100%;
}
.showpwContainer {
color: white;
width: 40px;
cursor: pointer;
}
input[type=text], input[type=password] {
font-size: 20px;
color: #ffffff;
background-color: #181a1b;
width: 100%;
padding: 12px 20px;
margin: 8px 0;
box-sizing: border-box;
border: 2px solid #333333;
text-align: center;
}
input[type=text], input[type=password]:focus {
border: 2px solid #000;
}
button {
width: 100px;
background-color: #9999ff;
border: none;
color: white;
padding: 16px 32px;
text-decoration: none;
margin: 4px 2px;
cursor: pointer;
}
#irc {
display: none;
}
</style>
</head>
<div class='center'>
<div class='title'>
Frylabs
</div>
<div id='text' class='text'>
Másold be ide régi kliens ID-d, és az alapján jelszót kapsz. Ezt csak egyszer teheted meg,
ezért a kapott jelszót tuti helyre írd le!
</div>
<div id='irc'>
<a class='ircLink' href='<%= devel? 'http' : 'https' %>://qmining.frylabs.net/irc?vetPwReqClick'>IRC</a>
</div>
<div id='form'>
<div class='inputContainer'>
<input type='text' id='cid' name='pw' value='<%= cid %>' autocomplete="off"/>
</div>
<input type='hidden' name='redirect' value='asd' autocomplete="off"/>
<button id='sendButton' onclick="GetVeteranPW(this)">Submit</button>
</div>
</div>
</body>
<script>
function HandleResp (resp) {
const textNode = document.getElementById('text')
document.getElementById('sendButton').innerText = 'Submit'
if (resp.result === 'success') {
document.getElementById('form').style.display = 'none'
textNode.innerText = 'Password:'
const pwDiv = document.createElement('div')
pwDiv.innerText = resp.pw
pwDiv.style.fontSize = '20px'
textNode.appendChild(pwDiv)
} else {
textNode.innerText = resp.msg
}
}
function HandleZeroStart () {
document.getElementById('form').style.display = 'none'
document.getElementById('irc').style.display = 'block'
document.getElementById('text').innerText = 'Client ID-d 0-val kezdődik. Ez azt jelenti hogy a jelszavasítás után telepítetted a scriptet, ezért nem vagy jogosult itt jelszót kérni. Ennek ellenére más felhasználóktól (akiknek már van jelszavuk) kérhetsz. Ha úgy gondolod valami nem stimmel:'
}
async function GetVeteranPW(button) {
button.innerText = '...'
const cid = document.getElementById('cid').value
if (cid[0] === '0') {
HandleZeroStart()
return
}
const rawResponse = await fetch('<%= devel? 'http' : 'https' %>://api.frylabs.net/getveteranpw', {
method: 'POST',
credentials: 'include',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
cid: cid
})
})
if (!rawResponse.ok) {
document.getElementById('text').innerText = 'Internal server error'
document.getElementById('sendButton').innerText = 'Submit'
}
try {
rawResponse.json()
.then((resp) => {
HandleResp(resp)
})
} catch (e) {
document.getElementById('text').innerText = 'Invalid data recieved from server'
document.getElementById('sendButton').innerText = 'Submit'
console.log(e)
}
}
</script>
</html>

119
src/modules/sio/sio.js Normal file
View file

@ -0,0 +1,119 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
const express = require('express')
const bodyParser = require('body-parser')
const busboy = require('connect-busboy')
const fs = require('fs')
const app = express()
// other requires
const logger = require('../../utils/logger.js')
const utils = require('../../utils/utils.js')
// stuff gotten from server.js
let publicdirs = []
function GetApp () {
const p = publicdirs[0]
if (!p) {
throw new Error(`No public dir! ( SIO )`)
}
// files in public dirs
const uloadFiles = p + 'f'
app.set('view engine', 'ejs')
app.set('views', [
'./modules/sio/views',
'./sharedViews'
])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(busboy({
limits: {
fileSize: 10000 * 1024 * 1024
}
}))
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({
limit: '5mb',
extended: true
}))
app.use(bodyParser.json({
limit: '5mb'
}))
// --------------------------------------------------------------
app.get('/', function (req, res) {
res.render('uload')
res.end()
})
function UploadFile (req, res, path, next) {
var fstream
req.pipe(req.busboy)
req.busboy.on('file', function (fieldname, file, filename) {
logger.Log('Uploading: ' + filename, logger.GetColor('blue'))
utils.CreatePath(path, true)
let d = new Date()
let fn = d.getHours() + '' + d.getMinutes() + '' + d.getSeconds() + '_' + filename
fstream = fs.createWriteStream(path + '/' + fn)
file.pipe(fstream)
fstream.on('close', function () {
logger.Log('Upload Finished of ' + path + '/' + fn, logger.GetColor('blue'))
next(fn)
})
fstream.on('error', function (err) {
console.log(err)
res.end('something bad happened :s')
})
})
}
app.route('/fosuploader').post(function (req, res, next) {
UploadFile(req, res, uloadFiles, (fn) => {
res.redirect('/f/' + fn)
})
})
app.get('*', function (req, res) {
res.status(404).render('404')
})
app.post('*', function (req, res) {
res.status(404).render('404')
})
return {
app: app
}
}
exports.name = 'Sio'
exports.getApp = GetApp
exports.setup = (data) => {
publicdirs = data.publicdirs
}

39
src/modules/sio/views/uload.ejs Executable file
View file

@ -0,0 +1,39 @@
<html>
<body bgcolor="#212127">
<head>
<title>Shit uploader</title>
<meta charset="UTF-8">
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top
}
textarea {
font: normal 14px Verdana;
color: #999999;
background-color: #212127;
width: 100%;
height: 700;
}
a {
color: #9999ff;
}
</style>
</head>
<form action="/fosuploader" enctype=multipart/form-data method="post">
<input type="file" name="dasfile" />
<input type="submit" value="Upload" />
</form>
</body>
</html>

237
src/modules/stuff/stuff.js Normal file
View file

@ -0,0 +1,237 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// package requires
const express = require('express')
const bodyParser = require('body-parser')
const busboy = require('connect-busboy')
const fs = require('fs')
const app = express()
// other requires
const logger = require('../../utils/logger.js')
// stuff gotten from server.js
let publicdirs = []
let url = ''
function GetApp () {
const p = publicdirs[0]
if (!p) {
throw new Error(`No public dir! ( Stuff )`)
}
// files in public dirs
const listedFiles = './' + p + 'files'
app.set('view engine', 'ejs')
app.set('views', [
'./modules/stuff/views',
'./sharedViews'
])
publicdirs.forEach((pdir) => {
logger.Log(`Using public dir: ${pdir}`)
app.use(express.static(pdir))
})
app.use(busboy({
limits: {
fileSize: 10000 * 1024 * 1024
}
}))
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({
limit: '5mb',
extended: true
}))
app.use(bodyParser.json({
limit: '5mb'
}))
// --------------------------------------------------------------
// app, '/*.mp4', 'video/mp4', 'stuff/video'
function appGetFileType (app, wildcard, contentType, pageToRender) {
app.get(wildcard, function (req, res) {
let p = decodeURI(req.url)
let fp = p
if (p.includes('?')) {
fp = p.split('?')
fp.pop()
fp = fp.join('/')
}
const fpath = listedFiles + fp
if (!fs.existsSync(fpath)) {
res.render('nofile', {
missingFile: fpath,
url
})
return
}
if (req.query.stream || !pageToRender) {
const stat = fs.statSync(fpath)
const fileSize = stat.size
const range = req.headers.range
if (range) {
const parts = range.replace(/bytes=/, '').split('-')
const start = parseInt(parts[0], 10)
const end = parts[1]
? parseInt(parts[1], 10)
: fileSize - 1
const chunksize = (end - start) + 1
const file = fs.createReadStream(fpath, { start, end })
const head = {
'Content-Range': `bytes ${start}-${end}/${fileSize}`,
'Accept-Ranges': 'bytes',
'Content-Length': chunksize,
'Content-Type': contentType
}
res.writeHead(206, head)
file.pipe(res)
} else {
const head = {
'Content-Length': fileSize,
'Content-Type': contentType
}
res.writeHead(200, head)
fs.createReadStream(fpath).pipe(res)
}
} else {
logger.LogReq(req)
let fname = fpath.split('/')
fname = fname.pop()
res.render(pageToRender, {
path: fp,
fname,
url,
contentType,
albumArt: GetAlbumArt(p)
})
}
})
}
const fileTypes = [
['/*.mp4', 'video/mp4', 'video'],
['/*.mkv', 'audio/x-matroska', 'video'],
['/*.mp3', 'audio/mpeg', 'audio'],
['/*.pdf', 'application/pdf'],
['/*.zip', 'application/zip']
]
function GetAlbumArt (path) {
let tmp = path.split('.')
tmp.pop()
tmp = tmp.join('.').split('/')
let last = tmp.pop()
return tmp.join('/') + '/.' + last + '.png'
}
fileTypes.forEach((t) => {
appGetFileType(app, t[0], t[1], t[2])
})
app.get('/*', function (req, res) {
let parsedUrl = decodeURI(req.url)
let curr = listedFiles + '/' + parsedUrl.substring('/'.length, parsedUrl.length).split('?')[0]
let relPath = curr.substring(listedFiles.length, curr.length)
if (relPath[relPath.length - 1] !== '/') { relPath += '/' }
let t = relPath.split('/')
let prevDir = ''
for (let i = 0; i < t.length - 2; i++) { prevDir += t[i] + '/' }
// curr = curr.replace(/\//g, "/");
// relPath = relPath.replace(/\//g, "/");
logger.LogReq(req)
try {
const stat = fs.lstatSync(curr)
if (stat.isDirectory() || stat.isSymbolicLink()) {
if (curr[curr.length - 1] !== '/') { curr += '/' }
let f = []
let files = fs.readdirSync(curr)
files.sort(function (a, b) {
return fs.statSync(curr + b).mtime.getTime() -
fs.statSync(curr + a).mtime.getTime()
})
files.forEach((item) => {
if (item[0] !== '.') {
let res = { name: item }
let stat = fs.statSync(curr + '/' + item)
let fileSizeInBytes = stat['size']
res.size = Math.round(fileSizeInBytes / 1000000)
res.path = relPath
if (res.path[res.path.length - 1] !== '/') { res.path += '/' }
res.path += item
res.mtime = stat['mtime'].toLocaleString()
res.isDir = stat.isDirectory()
f.push(res)
}
})
res.render('folders', {
folders: f,
dirname: relPath,
prevDir,
url
})
} else {
let fileStream = fs.createReadStream(curr)
fileStream.pipe(res)
}
} catch (e) {
res.render('nofile', {
missingFile: curr,
url
})
}
})
// -----------------------------------------------------------------------------------------------
app.get('*', function (req, res) {
res.status(404).render('404')
})
app.post('*', function (req, res) {
res.status(404).render('404')
})
return {
app: app
}
}
exports.name = 'Stuff'
exports.getApp = GetApp
exports.setup = (data) => {
url = data.url
publicdirs = data.publicdirs
}

39
src/modules/stuff/views/audio.ejs Executable file
View file

@ -0,0 +1,39 @@
<html>
<body bgcolor="#212127">
<head>
<title><%= fname %></title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
video {
width: 100%
}
</style>
</head>
<center>
<h2>
<%= fname %>
</h2>
<img
id="coverArt"
style="width:auto; max-height: 100%;"
src="<%= url + albumArt %>"
alt=' '
/>
<audio id="audioPlayer" controls style="width:100%">
<source src="<%= url %><%= path %>?stream=true" type=<%= contentType %>>
</audio>
</center>
</body>
<script>
console.log('a')
document.getElementById('coverArt').style.height = window.innerHeight - 140
</script>
</html>

130
src/modules/stuff/views/folders.ejs Executable file
View file

@ -0,0 +1,130 @@
<html>
<body bgcolor="#212127">
<head>
<title><%=dirname%></title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top;
word-wrap: break-word;
word-break: break-all;
table-layout: fixed;
padding: 0 12;
vertical-align: middle;
}
textarea {
font: normal 14px Verdana;
color: #999999;
background-color: #212127;
width: 100%;
height: 700;
}
a {
color: #9999ff;
}
.subtable {
border-collapse: collapse;
table-layout:fixed;
width:100%
}
.maintable {
border-collapse: collapse;
table-layout:fixed;
width:100%
padding:0; margin:0;
border: none !important;
}
tr {
width:32%;
}
.butt {
background-color: #212127;
color: #999999;
cursor: pointer;
width: 100%;
border: none;
text-align: left;
outline: none;
font-size: 13px;
}
</style>
</head>
<center>
<h1>
<%=dirname%>
</h1>
</center>
<h2>
<a href="<%= url + prevDir%>" > Up one level </a>
</h2>
</p>
<table class="maintable">
<% for (var i = 0; i < folders.length; i++) { %>
<tr>
<td>
<a
href="<%= url + folders[i].path %>"
style="font-size: 0px"
>
<button
class="butt"
style='<%= i % 2 === 0 ? "background-color: #2f2f37" : "" %>'
onmouseenter='mouseEnter(this, <%= i %>)'
onmouseleave='mouseLeave(this, <%= i %>)'
>
<table class="subtable">
<tr height="62">
<td style='width:30%;'>
<%=folders[i].name %>
</td>
<td style='width:20%;'>
<%=folders[i].mtime %>
</td>
<td style='width:10%;'>
<%
if (folders[i].isDir) {
%> <%= "DIR" %> <%
} else {
if (folders[i].size === 0) {
%> <%= "~0 MB" %> <%
} else {
%> <%= folders[i].size + ' MB' %> <%
}
}
%>
</td>
</tr>
</table>
</button>
</a>
</td>
</tr>
<% } %>
</table>
</body>
<script>
console.log('hi')
function mouseEnter (e, i) {
e.style.backgroundColor = "#555"
}
function mouseLeave (e, i) {
if (i % 2 == 0) {
e.style.backgroundColor = "#2f2f37"
} else {
e.style.backgroundColor = "#212127"
}
}
</script>
</html>

94
src/modules/stuff/views/nofile.ejs Executable file
View file

@ -0,0 +1,94 @@
<html>
<body bgcolor="#212127">
<head>
<title>No such file/folder</title>
<meta charset="UTF-8">
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
td {
vertical-align: top
}
textarea {
font: normal 14px Verdana;
color: #999999;
background-color: #212127;
width: 100%;
height: 700;
}
a {
color: #9999ff;
}
.subtable {
border-collapse: collapse;
table-layout:fixed;
width:100%
}
.maintable {
border-collapse: collapse;
table-layout:fixed;
width:100%
padding:0; margin:0;
border: none !important;
}
tr {
line-height: 29px;
width:32%;
}
.butt {
background-color: #212127;
color: #999999;
cursor: pointer;
width: 100%;
border: none;
text-align: left;
outline: none;
font-size: 13px;
}
.active,
.butt:hover {
background-color: #555;
}
</style>
</head>
<center>
<h1>
No such file / folder:
</br>
<%= missingFile %>
</br>
<a href="<%= url %>" > Back to root </a>
</br>
<a
onclick='goBack("<%=missingFile%>")'
href="#"
>
Go back
</a>
</h1>
</center>
</body>
<script>
function goBack(path) {
path = path.replace('./public/files', '')
if (path[path.length - 1] == '/') {
path = path.substring(0, path.length -2)
}
let p = path.split('/')
p.pop()
if (p.length > 1) {
location.href = p.join('/')
} else {
location.href = '/'
}
}
</script>
</html>

34
src/modules/stuff/views/video.ejs Executable file
View file

@ -0,0 +1,34 @@
<html>
<body bgcolor="#212127">
<head>
<title><%= fname %></title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
video {
width: 100%
}
</style>
</head>
<center>
<h2>
<%= fname %>
</h2>
</center>
<video id="videoPlayer" controls>
<source src="<%= url %><%= path %>?stream=true" type="video/mp4">
</video>
</body>
<script>
var v = document.getElementsByTagName('video')[0]
v.style.maxHeight = window.innerHeight - 100
v.maxWidth = window.innerWidth
console.log('a')
</script>
</html>

248
src/server.js Executable file
View file

@ -0,0 +1,248 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
const startHTTPS = true
const isRoot = process.getuid && process.getuid() === 0
const port = isRoot ? 80 : 8080
const httpsport = isRoot ? 443 : 5001
const express = require('express')
const vhost = require('vhost')
const logger = require('./utils/logger.js')
const utils = require('./utils/utils.js')
const http = require('http')
const https = require('https')
const cors = require('cors')
const cookieParser = require('cookie-parser')
const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported
const dbtools = require('./utils/dbtools.js')
const reqlogger = require('./middlewares/reqlogger.middleware.js')
const extraModulesFile = './extraModules.json'
const modulesFile = './modules.json'
const usersDBPath = 'data/dbs/users.db'
if (!utils.FileExists(usersDBPath)) {
throw new Error('No user DB exists yet! please run utils/dbSetup.js first!')
}
const userDB = dbtools.GetDB(usersDBPath)
let modules = JSON.parse(utils.ReadFile(modulesFile))
logger.Load()
try {
if (utils.FileExists(extraModulesFile)) {
const extraModules = JSON.parse(utils.ReadFile(extraModulesFile))
modules = {
...extraModules,
...modules
}
}
} catch (e) {
logger.Log('Failed to read extra modules file')
console.log(e)
}
// Setting up exits
// process.on('exit', () => exit('exit'))
process.on('SIGINT', () => exit('SIGINT'))
process.on('SIGTERM', () => exit('SIGTERM'))
function exit (reason) {
console.log()
logger.Log(`Exiting, reason: ${reason}`)
Object.keys(modules).forEach((k, i) => {
const x = modules[k]
if (x.cleanup) {
try {
x.cleanup()
} catch (e) {
logger.Log(`Error in ${k} cleanup! Details in STDERR`, logger.GetColor('redbg'))
console.error(e)
}
}
})
logger.Log('Closing Auth DB')
userDB.close()
process.exit()
}
const app = express()
if (!process.env.NS_DEVEL) {
app.use(function (req, res, next) {
if (req.secure) {
next()
} else {
logger.DebugLog(`HTTPS ${req.method} redirect to: ${'https://' + req.headers.host + req.url}`, 'https', 1)
if (req.method === 'POST') {
res.redirect(307, 'https://' + req.headers.host + req.url)
} else {
res.redirect('https://' + req.headers.host + req.url)
}
}
})
}
// https://github.com/expressjs/cors#configuration-options
app.use(cors({
credentials: true,
origin: true
// origin: [ /\.frylabs\.net$/ ]
}))
const cookieSecret = uuidv4()
app.use(cookieParser(cookieSecret))
app.use(reqlogger({
loggableKeywords: [
'stable.user.js'
],
loggableModules: [
'dataeditor'
]
}))
Object.keys(modules).forEach(function (k, i) {
let x = modules[k]
try {
let mod = require(x.path)
logger.Log(`Loading ${mod.name} module`, logger.GetColor('yellow'))
x.publicdirs.forEach((pdir) => {
utils.CreatePath(pdir)
})
if (mod.setup) {
mod.setup({
url: 'https://' + x.urls[0],
userDB: userDB,
publicdirs: x.publicdirs,
nextdir: x.nextdir
})
}
const modApp = mod.getApp()
x.app = modApp.app
x.dailyAction = modApp.dailyAction
x.cleanup = modApp.cleanup
x.urls.forEach((url) => {
app.use(vhost(url, x.app))
})
} catch (e) {
console.log(e)
}
})
const locLogFile = './stats/logs'
const allLogFile = '/nlogs/log'
// https://certbot.eff.org/
const privkeyFile = '/etc/letsencrypt/live/frylabs.net/privkey.pem'
const fullchainFile = '/etc/letsencrypt/live/frylabs.net/fullchain.pem'
const chainFile = '/etc/letsencrypt/live/frylabs.net/chain.pem'
var certsLoaded = false
if (startHTTPS && utils.FileExists(privkeyFile) && utils.FileExists(fullchainFile) && utils.FileExists(
chainFile)) {
try {
const key = utils.ReadFile(privkeyFile)
const cert = utils.ReadFile(fullchainFile)
const ca = utils.ReadFile(chainFile)
var certs = {
key: key,
cert: cert,
ca: ca
}
certsLoaded = true
} catch (e) {
logger.Log('Error loading cert files!', logger.GetColor('redbg'))
console.log(e)
}
}
setLogTimer()
function setLogTimer () {
const now = new Date()
const night = new Date(
now.getFullYear(),
now.getMonth(),
now.getDate() + 1,
0,
0,
0
)
logger.DebugLog(`Next daily action: ${night}`, 'daily', 1)
const msToMidnight = night.getTime() - now.getTime()
logger.DebugLog(`msToMidnight: ${msToMidnight}`, 'daily', 1)
logger.DebugLog(`Seconds To Midnight: ${msToMidnight / 1000}`, 'daily', 1)
if (msToMidnight < 0) {
logger.Log(`Error setting up Log Timer, msToMidnight is negative! (${msToMidnight})`, logger.GetColor('redbg'))
return
}
setTimeout(function () {
LogTimerAction()
setLogTimer()
}, msToMidnight)
}
function LogTimerAction () {
logger.DebugLog(`Running Log Timer Action`, 'daily', 1)
Object.keys(modules).forEach((k, i) => {
const x = modules[k]
logger.DebugLog(`Ckecking ${k}`, 'daily', 1)
if (x.dailyAction) {
try {
logger.Log(`Running daily action of ${k}`)
x.dailyAction()
} catch (e) {
logger.Log(`Error in ${k} daily action! Details in STDERR`, logger.GetColor('redbg'))
console.err(e)
}
}
})
const line = '==================================================================================================================================================='
logger.Log(line)
utils.AppendToFile(line, locLogFile)
utils.AppendToFile(line, allLogFile)
}
logger.Log('Node version: ' + process.version)
logger.Log('Current working directory: ' + process.cwd())
logger.Log('Listening on port: ' + port)
if (isRoot) {
logger.Log('Running as root', logger.GetColor('red'))
}
const httpServer = http.createServer(app)
httpServer.listen(port)
if (certsLoaded) {
const httpsServer = https.createServer(certs, app)
httpsServer.listen(httpsport)
logger.Log('Listening on port: ' + httpsport + ' (https)')
} else {
logger.Log('Https not avaible')
}
// app.listen(port)

25
src/sharedViews/404.ejs Executable file
View file

@ -0,0 +1,25 @@
<html>
<body bgcolor="#212127">
<head>
<style>
body {
font: normal 14px Verdana;
color: #999999;
}
a {
color: #9999ff;
}
</style>
</head>
<center>
<h1>404</h1>
<iframe width="660" height="465" src="https://www.youtube-nocookie.com/embed/qLrnkK2YEcE" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
</center>
</body>
</html>

204
src/sharedViews/login.ejs Normal file
View file

@ -0,0 +1,204 @@
<html>
<body bgcolor="#222426">
<head>
<title>Frylabs</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=0.6" />
<style>
a {
color: lightblue;
}
.center {
width: 380px;
height: 340px;
position: absolute;
left: 0;
right: 0;
top: 0;
bottom: 0;
margin: auto;
max-width: 100%;
max-height: 100%;
overflow: none;
text-align: center;
}
.text {
font-size: 18px;
color: white;
margin: 20px;
}
.title {
margin: 20px;
font-size: 50px;
font-weight: bold;
color: white;
}
.inputContainer {
width: 100%;
}
.showpwContainer {
color: white;
width: 40px;
cursor: pointer;
}
input[type=text], input[type=password], textarea {
font-size: 20px;
color: #ffffff;
background-color: #181a1b;
width: 100%;
padding: 12px 20px;
margin: 8px 0;
box-sizing: border-box;
border: 2px solid #333333;
text-align: center;
}
input[type=text], input[type=password]:focus {
border: 2px solid #000;
}
button {
background-color: #9999ff;
border: none;
color: white;
padding: 16px 32px;
text-decoration: none;
margin: 4px 2px;
cursor: pointer;
}
.ircLinkContainer {
display: flex;
justify-content: flex-end
}
.ircLink {
color: #9999ff;
font-size: 12px;
text-decoration: underline;
cursor: pointer;
}
#feedback {
display: none;
}
#feedbackTextArea {
text-align: left;
font-size: 16px;
height: 160px;
resize: none;
}
</style>
</head>
<div class='center'>
<div class='title'>
Frylabs
</div>
<div id='feedback'>
<textarea placeholder='Feedback' id='feedbackTextArea'></textarea>
<div class='ircLinkContainer' >
<a class='ircLink' href='<%= devel? 'http' : 'https' %>://qmining.frylabs.net/irc?loginClick'>IRC chatszoba</a>
</div>
<button id='sendFeedbackButton' onclick="SendFeedback(this)">Submit</button>
</div>
<div id='form'>
<div class='inputContainer'>
<input onkeyup="PWKeyUp(this)" type='text' id='pw' name='pw' autocomplete="off" autofocus/>
<input type='hidden' id='cid' name='pw' autocomplete="off"/>
</div>
<div class='ircLinkContainer' >
<a class='ircLink' onclick='ShowFeedback()'>Contact</a>
</div>
<button id='sendButton' onclick="Login(this)">Login</button>
</div>
</div>
</body>
<script>
function PWKeyUp (inputField) {
if (event.keyCode === 13) {
event.preventDefault();
Login(document.getElementById('sendButton'))
}
}
function HandleFeedbackResp (resp) {
document.getElementById('sendButton').innerText = 'Submit'
const textNode = document.getElementById('text')
const feedback = document.getElementById('feedback').style.display = "none";
if (resp.success) {
textNode.innerText = 'Visszajelzés elküldve'
} else {
textNode.innerText = 'Szerver oldali hiba :c'
}
}
async function SendFeedback (button) {
const feedback = document.getElementById('feedbackTextArea').value
button.innerText = '...'
const rawResponse = await fetch('<%= devel? 'http' : 'https' %>://api.frylabs.net/postfeedback', {
method: 'POST',
credentials: 'include',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
description: feedback,
fromLogin: true
})
})
if (!rawResponse.ok) {
document.getElementById('text').innerText = 'Internal server error'
button.innerText = 'Submit'
}
try {
rawResponse.json()
.then((resp) => {
HandleFeedbackResp(resp)
})
} catch (e) {
document.getElementById('text').innerText = 'Invalid data recieved from server'
button.innerText = 'Submit'
console.log(e)
}
}
function ShowFeedback () {
const form = document.getElementById('form').style.display = "none";
const feedback = document.getElementById('feedback').style.display = "block";
document.getElementById('text').innerText = 'Ha szeretnél választ kapni akkor kérdésed mellé írd be e-mailed, vagy kattints a lenti "IRC" linkre. Jelszót meglévő felhasználóktól kérj! E-mail esetén válasz spam-be is érkezhet!'
}
function HandleResp (resp) {
document.getElementById('sendButton').innerText = 'Login'
const textNode = document.getElementById('text')
if (resp.result === 'success') {
location.reload()
textNode.innerText = resp.msg
} else {
textNode.innerText = resp.msg
}
}
async function Login(button) {
button.innerText = '...'
const rawResponse = await fetch('<%= devel? 'http' : 'https' %>://api.frylabs.net/login', {
method: 'POST',
credentials: 'include',
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({
pw: document.getElementById('pw').value,
cid: document.getElementById('cid').value
})
})
if (!rawResponse.ok) {
document.getElementById('text').innerText = 'Internal server error'
button.innerText = 'Login'
}
try {
rawResponse.json()
.then((resp) => {
HandleResp(resp)
})
} catch (e) {
document.getElementById('text').innerText = 'Invalid data recieved from server'
button.innerText = 'Login'
console.log(e)
}
}
</script>
</html>

178
src/utils/actions.js Executable file
View file

@ -0,0 +1,178 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
module.exports = {
ProcessIncomingRequest: ProcessIncomingRequest,
LoadJSON: LoadJSON
}
const dataFile = './qminingPublic/data.json'
const recDataFile = './stats/recdata'
const logger = require('../utils/logger.js')
const idStats = require('../utils/ids.js')
idStats.Load() // FIXME: dont always load when actions.js is used
const utils = require('../utils/utils.js')
const classes = require('./classes.js')
classes.initLogger(logger.DebugLog)
// if a recievend question doesnt match at least this % to any other question in the db it gets
// added to db
const minMatchAmmountToAdd = 90 // FIXME: test this value
const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate
var currWrites = 0
function ProcessIncomingRequest (recievedData, qdb, infos, dryRun) {
return new Promise((resolve, reject) => {
logger.DebugLog('Processing incoming request', 'actions', 1)
if (recievedData === undefined) {
logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg'))
reject(new Error('Recieved data is undefined!'))
}
try {
let towrite = logger.GetDateString() + '\n'
towrite += '------------------------------------------------------------------------------\n'
if (typeof recievedData === 'object') {
towrite += JSON.stringify(recievedData)
} else {
towrite += recievedData
}
towrite += '\n------------------------------------------------------------------------------\n'
utils.AppendToFile(towrite, recDataFile)
logger.DebugLog('recDataFile written', 'actions', 1)
} catch (e) {
logger.log('Error writing recieved data.')
}
try {
// recievedData: { version: "", id: "", subj: "" quiz: {} }
let d = recievedData
// FIXME: if is for backwards compatibility, remove this sometime in the future
if (typeof d !== 'object') {
d = JSON.parse(recievedData)
}
logger.DebugLog('recievedData JSON parsed', 'actions', 1)
logger.DebugLog(d, 'actions', 3)
let allQLength = d.quiz.length
let allQuestions = []
d.quiz.forEach((question) => {
logger.DebugLog('Question:', 'actions', 2)
logger.DebugLog(question, 'actions', 2)
let q = new classes.Question(question.Q, question.A, question.data)
logger.DebugLog('Searching for question in subj ' + d.subj, 'actions', 3)
logger.DebugLog(q, 'actions', 3)
let sames = qdb.Search(q, d.subj)
logger.DebugLog('Same questions:', 'actions', 2)
logger.DebugLog('Length: ' + sames.length, 'actions', 2)
logger.DebugLog(sames, 'actions', 3)
// if it didnt find any question, or every found questions match is lower thatn 80
let isNew = sames.length === 0 || sames.every(searchResItem => {
return searchResItem.match < minMatchAmmountToAdd
})
logger.DebugLog('isNew: ' + isNew, 'actions', 2)
if (isNew) {
allQuestions.push(q)
}
})
let color = logger.GetColor('green')
let msg = ''
if (allQuestions.length > 0) {
color = logger.GetColor('blue')
msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )`
allQuestions.forEach((q) => {
const sName = classes.SUtils.GetSubjNameWithoutYear(d.subj)
logger.DebugLog('Adding question with subjName: ' + sName + ' :', 'actions', 3)
logger.DebugLog(q, 'actions', 3)
qdb.AddQuestion(sName, q)
})
currWrites++
logger.DebugLog('currWrites for data.json: ' + currWrites, 'actions', 1)
if (currWrites >= writeAfter && !dryRun) {
currWrites = 0
try {
qdb.version = infos.version
qdb.motd = infos.motd
logger.DebugLog('version and motd set for data.json', 'actions', 3)
} catch (e) {
logger.Log('MOTD/Version writing/reading error!')
}
logger.DebugLog('Writing data.json', 'actions', 1)
utils.WriteFile(JSON.stringify(qdb), dataFile)
logger.Log('\tData file written', color)
} else if (dryRun) {
logger.Log('\tDry run')
}
} else {
msg += `No new data ( ${allQLength} )`
}
let subjRow = '\t' + d.subj
if (d.id) {
subjRow += ' ( CID: ' + logger.logHashed(d.id) + ')'
idStats.LogId(d.id, d.subj)
}
logger.Log(subjRow)
if (d.version !== undefined) { msg += '. Version: ' + d.version }
logger.Log('\t' + msg, color)
logger.DebugLog('New Questions:', 'actions', 2)
logger.DebugLog(allQuestions, 'actions', 2)
logger.DebugLog('ProcessIncomingRequest done', 'actions', 1)
resolve(allQLength.length)
} catch (e) {
console.log(e)
logger.Log('Couldnt parse JSON data', logger.GetColor('redbg'))
reject(new Error('Couldnt parse JSON data'))
}
})
}
// loading stuff
function LoadJSON (dataFile) {
try {
var d = JSON.parse(utils.ReadFile(dataFile))
var r = new classes.QuestionDB()
var rt = []
for (var i = 0; i < d.Subjects.length; i++) {
let s = new classes.Subject(d.Subjects[i].Name)
var j = 0
for (j = 0; j < d.Subjects[i].Questions.length; j++) {
var currQ = d.Subjects[i].Questions[j]
s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data))
}
rt.push({
name: d.Subjects[i].Name,
count: j
})
r.AddSubject(s)
}
return r
} catch (e) {
logger.Log('Error loading sutff', logger.GetColor('redbg'), true)
console.log(e)
}
}

31
src/utils/changedataversion.js Executable file
View file

@ -0,0 +1,31 @@
const utils = require('../utils/utils.js')
const dataFile = '../public/data.json'
const versionFile = '../public/version'
var p = GetParams()
if (p.length <= 0) {
console.log('no params!')
process.exit(0)
}
var param = p.join(' ')
console.log('param: ' + param)
var d = utils.ReadFile(dataFile)
var parsed = JSON.parse(d)
console.log('Old version:')
console.log(parsed.version)
parsed.version = param
console.log('New version:')
console.log(parsed.version)
utils.WriteFile(JSON.stringify(parsed), dataFile)
utils.WriteFile(parsed.version, versionFile)
function GetParams () {
return process.argv.splice(2)
}

508
src/utils/classes.js Executable file
View file

@ -0,0 +1,508 @@
var debugLogger = null
function initLogger (logger) {
debugLogger = logger
}
function debugLog (msg, name, lvl) {
if (debugLogger) {
debugLogger(msg, name, lvl)
}
}
const commonUselessAnswerParts = [
'A helyes válasz az ',
'A helyes válasz a ',
'A helyes válaszok: ',
'A helyes válaszok:',
'A helyes válasz: ',
'A helyes válasz:',
'The correct answer is:',
'\''
]
const commonUselessStringParts = [
',',
'\\.',
':',
'!',
'\\+',
'\\s*\\.'
]
const specialChars = [ '&', '\\+' ]
const lengthDiffMultiplier = 10 /* Percent minus for length difference */
const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */
const assert = (val) => {
if (!val) { throw new Error('Assertion failed') }
}
class StringUtils {
GetSubjNameWithoutYear (subjName) {
let t = subjName.split(' - ')
if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
return t[1] || subjName
} else {
return subjName
}
}
RemoveStuff (value, removableStrings, toReplace) {
removableStrings.forEach((x) => {
var regex = new RegExp(x, 'g')
value = value.replace(regex, toReplace || '')
})
return value
}
SimplifyQuery (q) {
assert(q)
var result = q.replace(/\n/g, ' ').replace(/\s/g, ' ')
return this.RemoveUnnecesarySpaces(result)
}
ShortenString (toShorten, ammount) {
assert(toShorten)
var result = ''
var i = 0
while (i < toShorten.length && i < ammount) {
result += toShorten[i]
i++
}
return result
}
ReplaceCharsWithSpace (val, char) {
assert(val)
assert(char)
var toremove = this.NormalizeSpaces(val)
var regex = new RegExp(char, 'g')
toremove = toremove.replace(regex, ' ')
return this.RemoveUnnecesarySpaces(toremove)
}
// removes whitespace from begining and and, and replaces multiple spaces with one space
RemoveUnnecesarySpaces (toremove) {
assert(toremove)
toremove = this.NormalizeSpaces(toremove)
while (toremove.includes(' ')) {
toremove = toremove.replace(/ {2}/g, ' ')
}
return toremove.trim()
}
// simplifies a string for easier comparison
SimplifyStringForComparison (value) {
assert(value)
value = this.RemoveUnnecesarySpaces(value).toLowerCase()
return this.RemoveStuff(value, commonUselessStringParts)
}
RemoveSpecialChars (value) {
assert(value)
return this.RemoveStuff(value, specialChars, ' ')
}
// if the value is empty, or whitespace
EmptyOrWhiteSpace (value) {
// replaces /n-s with "". then replaces spaces with "". if it equals "", then its empty, or only consists of white space
if (value === undefined) { return true }
return (value.replace(/\n/g, '').replace(/ /g, '').replace(/\s/g, ' ') === '')
}
// damn nonbreaking space
NormalizeSpaces (input) {
assert(input)
return input.replace(/\s/g, ' ')
}
CompareString (s1, s2) {
if (!s1 || !s2) {
if (!s1 && !s2) {
return 100
} else {
return 0
}
}
s1 = this.SimplifyStringForComparison(s1).split(' ')
s2 = this.SimplifyStringForComparison(s2).split(' ')
var match = 0
for (var i = 0; i < s1.length; i++) {
if (s2.includes(s1[i])) { match++ }
}
var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent
var lengthDifference = Math.abs(s2.length - s1.length)
percent -= lengthDifference * lengthDiffMultiplier
if (percent < 0) { percent = 0 }
return percent
}
AnswerPreProcessor (value) {
assert(value)
return this.RemoveStuff(
value, commonUselessAnswerParts)
}
// 'a. pécsi sör' -> 'pécsi sör'
RemoveAnswerLetters (value) {
assert(value)
let s = value.split('. ')
if (s[0].length < 2 && s.length > 1) {
s.shift()
return s.join(' ')
} else {
return value
}
}
SimplifyQA (value, mods) {
if (!value) { return }
const reducer = (res, fn) => {
return fn(res)
}
return mods.reduce(reducer, value)
}
SimplifyAnswer (value) {
return this.SimplifyQA(
value,
[
this.RemoveSpecialChars.bind(this),
this.RemoveUnnecesarySpaces.bind(this),
this.AnswerPreProcessor.bind(this),
this.RemoveAnswerLetters.bind(this)
])
}
SimplifyQuestion (value) {
return this.SimplifyQA(
value,
[
this.RemoveSpecialChars.bind(this),
this.RemoveUnnecesarySpaces.bind(this)
])
}
SimplifyStack (stack) {
return this.SimplifyQuery(stack)
}
}
const SUtils = new StringUtils()
class Question {
constructor (q, a, data) {
this.Q = SUtils.SimplifyQuestion(q)
this.A = SUtils.SimplifyAnswer(a)
this.data = { ...data }
}
toString () {
if (this.data.type !== 'simple') {
return '?' + this.Q + '\n!' + this.A + '\n>' + JSON.stringify(this.data)
} else {
return '?' + this.Q + '\n!' + this.A
}
}
HasQuestion () {
return this.Q !== undefined
}
HasAnswer () {
return this.A !== undefined
}
HasImage () {
return this.data.type === 'image'
}
IsComplete () {
return this.HasQuestion() && this.HasAnswer()
}
CompareImage (data2) {
return SUtils.CompareString(this.data.images.join(' '), data2.images.join(' '))
}
// returns -1 if botth is simple
CompareData (qObj) {
try {
if (qObj.data.type === this.data.type) {
let dataType = qObj.data.type
if (dataType === 'simple') {
return -1
} else if (dataType === 'image') {
return this.CompareImage(qObj.data)
} else {
debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1)
debugLog(qObj, 'Compare question data', 2)
}
} else {
return 0
}
} catch (e) {
debugLog('Error comparing data', 'Compare question data', 1)
debugLog(e.message, 'Compare question data', 1)
debugLog(e, 'Compare question data', 2)
}
return 0
}
CompareQuestion (qObj) {
return SUtils.CompareString(this.Q, qObj.Q)
}
CompareAnswer (qObj) {
return SUtils.CompareString(this.A, qObj.A)
}
Compare (q2, data) {
assert(q2)
let qObj
if (typeof q2 === 'string') {
qObj = {
Q: q2,
data: data
}
} else {
qObj = q2
}
const qMatch = this.CompareQuestion(qObj)
const aMatch = this.CompareAnswer(qObj)
// -1 if botth questions are simple
const dMatch = this.CompareData(qObj)
let avg = -1
if (qObj.A) {
if (dMatch === -1) {
avg = (qMatch + aMatch) / 2
} else {
avg = (qMatch + aMatch + dMatch) / 3
}
} else {
if (dMatch === -1) {
avg = qMatch
} else {
avg = (qMatch + dMatch) / 2
}
}
return {
qMatch: qMatch,
aMatch: aMatch,
dMatch: dMatch,
avg: avg
}
}
}
class Subject {
constructor (n) {
assert(n)
this.Name = n
this.Questions = []
}
setIndex (i) {
this.index = i
}
getIndex () {
return this.index
}
get length () {
return this.Questions.length
}
AddQuestion (q) {
assert(q)
this.Questions.push(q)
}
getSubjNameWithoutYear () {
return SUtils.GetSubjNameWithoutYear(this.Name)
}
getYear () {
let t = this.Name.split(' - ')[0]
if (t.match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) {
return t
} else {
return ''
}
}
Search (q, data) {
assert(q)
var r = []
for (let i = 0; i < this.length; i++) {
let percent = this.Questions[i].Compare(q, data)
if (percent.avg > minMatchAmmount) {
r.push({
q: this.Questions[i],
match: percent.avg,
detailedMatch: percent
})
}
}
for (let i = 0; i < r.length; i++) {
for (var j = i; j < r.length; j++) {
if (r[i].match < r[j].match) {
var tmp = r[i]
r[i] = r[j]
r[j] = tmp
}
}
}
return r
}
toString () {
var r = []
for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) }
return '+' + this.Name + '\n' + r.join('\n')
}
}
class QuestionDB {
constructor () {
this.Subjects = []
}
get length () {
return this.Subjects.length
}
AddQuestion (subj, q) {
debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1)
debugLog(q, 'qdb add', 3)
assert(subj)
var i = 0
while (i < this.Subjects.length &&
!subj.toLowerCase().includes(this.Subjects[i].getSubjNameWithoutYear().toLowerCase())) {
i++
}
if (i < this.Subjects.length) {
debugLog('Adding new question to existing subject', 'qdb add', 1)
this.Subjects[i].AddQuestion(q)
} else {
debugLog('Creating new subject for question', 'qdb add', 1)
const n = new Subject(subj)
n.AddQuestion(q)
this.Subjects.push(n)
}
}
SimplifyQuestion (q) {
if (typeof q === 'string') {
return SUtils.SimplifyQuestion(q)
} else {
q.Q = SUtils.SimplifyQuestion(q.Q)
q.A = SUtils.SimplifyQuestion(q.A)
return q
}
}
Search (q, subjName, data) {
assert(q)
debugLog('Searching for question', 'qdb search', 1)
debugLog('Question:', 'qdb search', 2)
debugLog(q, 'qdb search', 2)
debugLog(`Subject name: ${subjName}`, 'qdb search', 2)
debugLog('Data:', 'qdb search', 2)
debugLog(data || q.data, 'qdb search', 2)
if (!data) {
data = q.data || { type: 'simple' }
}
if (!subjName) {
subjName = ''
debugLog('No subject name as param!', 'qdb search', 1)
}
q = this.SimplifyQuestion(q)
var r = []
this.Subjects.forEach((subj) => {
if (subjName.toLowerCase().includes(subj.getSubjNameWithoutYear().toLowerCase())) {
debugLog(`Searching in ${subj.Name} `, 2)
r = r.concat(subj.Search(q, data))
}
})
// FIXME: try to remove this? but this is also a good backup plan so idk
if (r.length === 0) {
debugLog('Reqults length is zero when comparing names, trying all subjects', 'qdb search', 1)
this.Subjects.forEach((subj) => {
r = r.concat(subj.Search(q, data))
})
if (r.length > 0) {
debugLog(`FIXME: '${subjName}' gave no result but '' did!`, 'qdb search', 1)
console.error(`FIXME: '${subjName}' gave no result but '' did!`)
}
}
for (let i = 0; i < r.length; i++) {
for (var j = i; j < r.length; j++) {
if (r[i].match < r[j].match) {
var tmp = r[i]
r[i] = r[j]
r[j] = tmp
}
}
}
debugLog(`QDB search result length: ${r.length}`, 'qdb search', 1)
return r
}
AddSubject (subj) {
assert(subj)
var i = 0
while (i < this.length && subj.Name !== this.Subjects[i].Name) { i++ }
if (i < this.length) {
this.Subjects.concat(subj.Questions)
} else {
this.Subjects.push(subj)
}
}
toString () {
var r = []
for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) }
return r.join('\n\n')
}
}
module.exports.StringUtils = StringUtils // TODO: export singleton string utils, remove nea StringUtils from other files
module.exports.SUtils = SUtils
module.exports.Question = Question
module.exports.Subject = Subject
module.exports.QuestionDB = QuestionDB
module.exports.minMatchAmmount = minMatchAmmount
module.exports.initLogger = initLogger

62
src/utils/dbSetup.js Normal file
View file

@ -0,0 +1,62 @@
const utils = require('../utils/utils.js')
const logger = require('../utils/logger.js')
const dbtools = require('../utils/dbtools.js')
const dbStructPath = '../modules/api/apiDBStruct.json'
const usersDBPath = '../data/dbs/users.db'
const uuidv4 = require('uuid/v4') // TODO: deprecated, but imports are not supported
let authDB
console.clear()
CreateDB()
authDB.close()
function CreateDB () {
const dbStruct = utils.ReadJSON(dbStructPath)
// authDB = dbtools.GetDB(':memory:')
authDB = dbtools.GetDB(usersDBPath)
authDB.pragma('synchronous = OFF')
Object.keys(dbStruct).forEach((tableName) => {
const tableData = dbStruct[tableName]
dbtools.CreateTable(authDB, tableName, tableData.tableStruct, tableData.foreignKey)
})
try {
if (utils.FileExists('./ids')) {
const uids = utils.ReadFile('./ids').split('\n')
uids.forEach((cid, i) => {
if (!cid) { return }
logger.Log(`[ ${i} / ${uids.length} ]`)
try {
dbtools.Insert(authDB, 'users', {
pw: uuidv4(),
oldCID: cid,
avaiblePWRequests: 4,
created: utils.GetDateString()
})
} catch (e) {
logger.Log('Error during inserting', logger.GetColor('redbg'))
console.error(e)
}
})
}
} catch (e) {
console.error(e)
}
const dir = `./dbSetupResult/${utils.GetDateString().replace(/ /g, '_')}`
utils.CreatePath(dir)
Object.keys(dbStruct).forEach((key) => {
const path = `${dir}/${key}.json`
logger.Log(`Writing ${path}...`)
utils.WriteFile(JSON.stringify({
tableInfo: dbtools.TableInfo(authDB, key),
tableRows: dbtools.SelectAll(authDB, key)
}), path)
})
logger.Log('Done')
}

224
src/utils/dbtools.js Normal file
View file

@ -0,0 +1,224 @@
// https://www.sqlitetutorial.net/sqlite-nodejs/
// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
module.exports = {
GetDB,
AddColumn,
TableInfo,
Update,
Delete,
CreateTable,
SelectAll,
Select,
Insert,
CloseDB
}
const Sqlite = require('better-sqlite3')
const logger = require('../utils/logger.js')
const utils = require('../utils/utils.js')
const debugLog = process.env.NS_SQL_DEBUG_LOG
// { asd: 'asd', basd: 4 } => asd = 'asd', basd = 4
function GetSqlQuerry (conditions, type) {
const res = Object.keys(conditions).reduce((acc, key) => {
const item = conditions[key]
if (typeof item === 'string') {
acc.push(`${key} = '${conditions[key]}'`)
} else {
acc.push(`${key} = ${conditions[key]}`)
}
return acc
}, [])
if (type === 'where') {
return res.join(' AND ')
} else {
return res.join(', ')
}
}
// -------------------------------------------------------------------------
function GetDB (path) {
utils.CreatePath(path)
const res = new Sqlite(path)
res.pragma('synchronous = OFF')
return res
}
function DebugLog (msg) {
if (debugLog) {
logger.DebugLog(msg, 'sql', 0)
}
}
function AddColumn (db, table, col) {
try {
const colName = Object.keys(col)[0]
const colType = col.type
const s = `ALTER TABLE ${table} ADD COLUMN ${colName} ${colType}`
const stmt = PrepareStatement(db, s)
return stmt.run()
} catch (e) {
console.error(e)
}
}
function TableInfo (db, table) {
try {
} catch (e) {
console.error(e)
}
const s = `PRAGMA table_info(${table})`
const stmt = PrepareStatement(db, s)
const infoRes = stmt.all()
const s2 = `SELECT COUNT(*) FROM ${table}`
const stmt2 = PrepareStatement(db, s2)
const countRes = stmt2.get()
return {
columns: infoRes,
dataCount: countRes[Object.keys(countRes)[0]]
}
}
function Update (db, table, newData, conditions) {
try {
const s = `UPDATE ${table} SET ${GetSqlQuerry(newData, 'set')} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, s)
return stmt.run()
} catch (e) {
console.error(e)
}
}
function Delete (db, table, conditions) {
try {
const s = `DELETE FROM ${table} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, s)
return stmt.run()
} catch (e) {
console.error(e)
}
}
function CreateTable (db, name, columns, foreignKeys) {
// CREATE TABLE users(pw text PRIMARY KEY NOT NULL, id number, lastIP text, notes text, loginCount
// number, lastLogin text, lastAccess text
//
// FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) )
try {
const cols = Object.keys(columns).reduce((acc, key) => {
const item = columns[key]
// FIXME: array, and push stuff, then join()
const flags = []
const toCheck = {
primary: 'PRIMARY KEY',
notNull: 'NOT NULL',
unique: 'UNIQUE',
autoIncrement: 'AUTOINCREMENT',
defaultZero: 'DEFAULT 0'
}
Object.keys(toCheck).forEach((key) => {
if (item[key]) {
flags.push(toCheck[key])
}
})
acc.push(`${key} ${item.type} ${flags.join(' ')}`)
return acc
}, []).join(', ')
let fKeys = []
if (foreignKeys) {
foreignKeys.forEach((f) => {
const { keysFrom, table, keysTo } = f
fKeys.push(`, FOREIGN KEY(${keysFrom.join(', ')}) REFERENCES ${table}(${keysTo.join(', ')})`)
})
}
// IF NOT EXISTS
const s = `CREATE TABLE ${name}(${cols}${fKeys.join(', ')})`
const stmt = PrepareStatement(db, s)
return stmt.run()
} catch (e) {
console.error(e)
}
}
function SelectAll (db, from) {
try {
const s = `SELECT * from ${from}`
const stmt = PrepareStatement(db, s)
return stmt.all()
} catch (e) {
console.error(e)
}
}
function Select (db, from, conditions) {
try {
const s = `SELECT * from ${from} WHERE ${GetSqlQuerry(conditions, 'where')}`
const stmt = PrepareStatement(db, s)
return stmt.all()
} catch (e) {
console.error(e)
}
}
function Insert (db, table, data) {
try {
const cols = Object.keys(data).reduce((acc, key) => {
acc.push(`${key}`)
return acc
}, []).join(', ')
const values = Object.keys(data).reduce((acc, key) => {
const item = data[key]
if (typeof item === 'string') {
acc.push(`'${item}'`)
} else {
acc.push(`${item}`)
}
return acc
}, []).join(', ')
const s = `INSERT INTO ${table} (${cols}) VALUES (${values})`
const stmt = PrepareStatement(db, s)
return stmt.run()
} catch (e) {
console.error(e)
}
}
function CloseDB (db) {
db.close((err) => {
if (err) {
return console.error(err.message)
}
DebugLog('Close the database connection.')
})
}
// -------------------------------------------------------------------------
function PrepareStatement (db, s) {
if (!db) {
throw new Error('DB is undefined in prepare statement! DB action called with undefined db')
}
DebugLog(s)
return db.prepare(s)
}

112
src/utils/ids.js Executable file
View file

@ -0,0 +1,112 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
module.exports = {
LogId: LogId,
Load: Load
}
const utils = require('../utils/utils.js')
const logger = require('../utils/logger.js')
const idStatFile = 'stats/idstats'
const idVStatFile = 'stats/idvstats'
const writeInterval = 1
let data = {}
let vData = {}
let writes = 0
function Load () {
try {
var prevData = utils.ReadFile(idStatFile)
data = JSON.parse(prevData)
} catch (e) {
logger.Log('Error at loading id logs! (@ first run its normal)', logger.GetColor('redbg'))
console.log(e)
}
try {
var prevVData = utils.ReadFile(idVStatFile)
vData = JSON.parse(prevVData)
} catch (e) {
logger.Log('Error at loading id logs! (@ first run its normal)', logger.GetColor('redbg'))
console.log(e)
}
}
function LogId (id, subj) {
Inc(id, subj)
AddVisitStat(id, subj)
Save()
}
function AddSubjToList (list, subj) {
if (!list[subj]) {
list[subj] = 0
}
list[subj]++
}
function Inc (value, subj) {
if (data[value] === undefined) {
data[value] = {
count: 0,
subjs: {}
}
}
data[value].count++
AddSubjToList(data[value].subjs, subj)
}
function AddVisitStat (name, subj) {
var m = new Date()
const now = m.getFullYear() + '/' + ('0' + (m.getMonth() + 1)).slice(-2) + '/' + ('0' + m.getDate()).slice(-2)
if (vData[now] === undefined) { vData[now] = {} }
if (vData[now][name] === undefined) {
vData[now][name] = {
count: 0,
subjs: {}
}
}
vData[now][name].count++
AddSubjToList(vData[now][name].subjs, subj)
}
function Save () {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(data), idStatFile)
// Log("Stats wrote.");
} catch (e) {
logger.Log('Error at writing logs!', logger.GetColor('redbg'))
console.log(e)
}
try {
utils.WriteFile(JSON.stringify(vData), idVStatFile)
// Log("Stats wrote.");
} catch (e) {
logger.Log('Error at writing visit logs!', logger.GetColor('redbg'))
console.log(e)
}
writes = 0
}
}

329
src/utils/logger.js Executable file
View file

@ -0,0 +1,329 @@
/* ----------------------------------------------------------------------------
Question Server
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
const hr = '---------------------------------------------------------------------------------'
module.exports = {
GetDateString: GetDateString,
Log: Log,
DebugLog: DebugLog,
GetColor: GetColor,
LogReq: LogReq,
LogStat: LogStat,
Load: Load,
logHashed: logHashed,
hr: hr,
C: C
}
const DELIM = C('green') + '|' + C()
const utils = require('../utils/utils.js')
const locLogFile = './stats/logs'
const logFile = '/nlogs/nlogs'
const allLogFile = '/nlogs/log'
const statFile = 'stats/stats'
const vStatFile = 'stats/vstats'
const uStatsFile = 'stats/ustats'
const uvStatsFile = 'stats/uvstats'
const nologFile = './nolog'
const colors = [
'green',
'red',
'yellow',
'blue',
'magenta',
'cyan'
]
const writeInterval = 10
const debugLevel = parseInt(process.env.NS_LOGLEVEL) || 0
Log('Loglevel is: ' + debugLevel)
let data = {} // visit data
let vData = {} // visit data, but daily
let uData = {} // visit data, but per user
let uvData = {} // visit data, but per user and daily
let writes = 0
let noLogips = []
function GetDateString () {
const m = new Date()
const d = utils.GetDateString()
return GetRandomColor(m.getHours().toString()) + d + C()
}
function DebugLog (msg, name, lvl) {
if (lvl <= debugLevel) {
if (msg === 'hr') {
msg = hr
}
let s = msg
let header = `${C('red')}#DEBUG${lvl}#${C('yellow')}${name.toUpperCase()}${C('red')}#${C()}${DELIM}${C()}`
if (typeof msg !== 'object') {
s = header + msg
} else {
Log(header + 'OBJECT:', 'yellow')
s = msg
}
Log(s, 'yellow')
}
}
function Log (s, c) {
let log = s
if (typeof s !== 'object') {
let dl = DELIM + C(c)
log = C(c) + GetDateString() + dl + s + C()
}
console.log(log)
utils.AppendToFile(log, logFile)
}
function LogReq (req, toFile, sc) {
try {
let ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress
let nolog = noLogips.some((x) => {
return ip.includes(x)
})
if (nolog) {
return
}
let logEntry = GetRandomColor(ip) + ip + C()
let dl = DELIM
if (req.url.includes('lred')) {
dl += C('red')
}
let hostname
if (req.hostname) {
hostname = req.hostname.replace('www.', '').split('.')[0]
} else {
hostname = 'NOHOST'
Log('req.hostname is undefined! req.hostname: ' + req.hostname, GetColor('redbg'))
}
logEntry += dl +
hostname + dl +
req.headers['user-agent'] + dl +
req.method + dl
if (req.session && req.session.user) {
logEntry += C('cyan') + req.session.user.id + C() + dl
} else if (req.session && req.session.isException === true) {
logEntry += C('cyan') + 'EX' + C() + dl
} else {
logEntry += C('red') + 'NOUSER' + C() + dl
}
logEntry += GetRandomColor(req.url.split('?')[0]) + req.url
if (sc !== undefined) { logEntry += dl + sc }
logEntry += C()
if (!toFile) {
Log(logEntry)
} else {
let defLogs = GetDateString() + dl + logEntry
utils.AppendToFile(defLogs, locLogFile)
utils.AppendToFile(defLogs, allLogFile)
}
} catch (e) {
console.log(e)
Log('Error at logging lol', GetColor('redbg'), true)
}
}
function parseNoLogFile (newData) {
noLogips = newData.split('\n')
if (noLogips[noLogips.length - 1] === '') {
noLogips.pop()
}
noLogips = noLogips.filter((x) => {
return x !== ''
})
Log('\tNo Log IP-s changed: ' + noLogips.join(', '))
}
function setNoLogReadInterval () {
utils.WatchFile(nologFile, (newData) => {
parseNoLogFile(newData)
})
parseNoLogFile(utils.ReadFile(nologFile))
}
function Load () {
Log('Loading logger...')
try {
uData = JSON.parse(utils.ReadFile(uStatsFile))
} catch (e) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.log(e)
}
try {
uvData = JSON.parse(utils.ReadFile(uvStatsFile))
} catch (e) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.log(e)
}
try {
var prevData = utils.ReadFile(statFile)
data = JSON.parse(prevData)
} catch (e) {
Log('Error at loading logs! (@ first run its normal)', GetColor('redbg'))
console.log(e)
}
try {
var prevVData = utils.ReadFile(vStatFile)
vData = JSON.parse(prevVData)
} catch (e) {
Log('Error at loading visit logs! (@ first run its normal)', GetColor('redbg'))
console.log(e)
}
setNoLogReadInterval()
}
function LogStat (url, ip, hostname, userId) {
let nolog = noLogips.some((x) => {
return x.includes(ip)
})
if (nolog) {
return
}
url = hostname + url.split('?')[0]
Inc(url)
AddUserIdStat(userId)
IncUserStat(userId)
AddVisitStat(url)
Save()
}
function IncUserStat (userId) {
try {
if (uData[userId] === undefined) { uData[userId] = 0 }
uData[userId]++
} catch (e) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(e)
}
}
function AddUserIdStat (userId) {
try {
var m = new Date()
const now = m.getFullYear() + '/' + ('0' + (m.getMonth() + 1)).slice(-2) + '/' + ('0' + m.getDate()).slice(-2)
if (uvData[now] === undefined) { uvData[now] = {} }
if (uvData[now][userId] === undefined) { uvData[now][userId] = 0 }
uvData[now][userId]++
} catch (e) {
Log('Error at making user ID stats!', GetColor('redbg'))
console.error(e)
}
}
function Inc (value) {
if (value.startsWith('/?')) { value = '/' }
if (data[value] === undefined) { data[value] = 0 }
data[value]++
}
function AddVisitStat (name) {
var m = new Date()
const now = m.getFullYear() + '/' + ('0' + (m.getMonth() + 1)).slice(-2) + '/' + ('0' + m.getDate()).slice(-2)
if (vData[now] === undefined) { vData[now] = {} }
if (vData[now][name] === undefined) { vData[now][name] = 0 }
vData[now][name]++
}
function Save () {
writes++
if (writes === writeInterval) {
try {
utils.WriteFile(JSON.stringify(uData), uStatsFile)
} catch (e) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(e)
}
try {
utils.WriteFile(JSON.stringify(uvData), uvStatsFile)
} catch (e) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(e)
}
try {
utils.WriteFile(JSON.stringify(data), statFile)
// Log("Stats wrote.");
} catch (e) {
Log('Error at writing logs! (more in stderr)', GetColor('redbg'))
console.error(e)
}
try {
utils.WriteFile(JSON.stringify(vData), vStatFile)
// Log("Stats wrote.");
} catch (e) {
Log('Error at writing visit logs! (more in stderr)', GetColor('redbg'))
console.error(e)
}
writes = 0
}
}
function logHashed (x) {
return GetRandomColor(x.toString()) + x + C()
}
function GetRandomColor (ip) {
if (!ip) {
return 'red'
}
let res = ip.split('').reduce((res, x) => {
return res + x.charCodeAt(0)
}, 0)
return C(colors[res % colors.length])
}
function GetColor (c) {
return c
}
function C (c) {
if (c !== undefined) { c = c.toLowerCase() }
if (c === 'redbg') { return '\x1b[41m' }
if (c === 'bluebg') { return '\x1b[44m' }
if (c === 'green') { return '\x1b[32m' }
if (c === 'red') { return '\x1b[31m' }
if (c === 'yellow') { return '\x1b[33m' }
if (c === 'blue') { return '\x1b[34m' }
if (c === 'magenta') { return '\x1b[35m' }
if (c === 'cyan') { return '\x1b[36m' }
return '\x1b[0m'
}

12
src/utils/merge.sh Executable file
View file

@ -0,0 +1,12 @@
#!/bin/bash
p=$(echo $PWD | rev | cut -d'/' -f2- | rev)
cp -v $p/public/data.json /tmp/data.json
node $p/utils/rmDuplicates.js /tmp/data.json 2> /dev/null
mkdir -p "$p/public/backs/"
mv -v $p/public/data.json "$p/public/backs/data.json $(date)"
mv -v $p/utils/res.json $p/public/data.json
echo Done

22
src/utils/motd.js Executable file
View file

@ -0,0 +1,22 @@
const utils = require('../utils/utils.js')
const dataFile = '../public/data.json'
const motdFile = '../public/motd'
var p = GetParams()
if (p.length <= 0) {
console.log('no params!')
process.exit(0)
}
var param = p.join(' ')
console.log('param: ' + param)
var d = utils.ReadFile(dataFile)
var parsed = JSON.parse(d)
parsed.motd = param
utils.WriteFile(JSON.stringify(parsed), dataFile)
utils.WriteFile(parsed.motd, motdFile)
function GetParams () {
return process.argv.splice(2)
}

86
src/utils/readme.md Normal file
View file

@ -0,0 +1,86 @@
# Contents
Utils mappa tartalom
## merger.js
Törölve lesz
## merge.sh
Törölve lesz
## rmDuplicates.js
Tervezve
Kitörli a két vagy többször szereplő kérdéseket
Params:
1. Elérési út a JSON adatbázishoz
## actions.js
Át lesz nevezve: `questionProcessor.js`
JSON adatbázist betölti, és az új bejövő kérdéseket hozzáadja
Exportált funkciók:
1. ProcessIncomingRequest
2. LoadJSON
## utils.js
Általános eszközök
Exportált funkciók:
1. ReadFile
2. WriteFile
3. writeFileAsync
4. AppendToFile
5. Beep
6. WriteBackup
7. FileExists
8. CreatePath
9. WatchFile
10. ReadDir
## logger.js
Logolást kezeli
Exportált funkciók:
1. GetDateString
2. Log
3. DebugLog
4. GetColor
5. LogReq
6. LogStat
7. Load
8. logHashed
9. hr
## motd.js
Az adatbázis MOTD-ét változtatja meg. Elvileg már nem kell
`node motd.js "new motd"`
## changedataversion.js
Az adatbázis jelenlegi legfrissebb kliens verzióját változtatja meg. Elvileg már nem kell
`node motd.js "new version"`
## ids.js
`./stats/idstats` és `./stats/idvstats` ba írja a egyedi kliens ID statisztikákat
Exportált funkciók:
1. LogId
2. Load

260
src/utils/rmDuplicates.js Normal file
View file

@ -0,0 +1,260 @@
/* ----------------------------------------------------------------------------
Question Server question file merger
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
const utils = require('./utils.js')
const classes = require('./classes.js')
const actions = require('./actions.js')
const logger = require('./logger.js')
const resultFileName = 'res.json'
const minMatchAmmount = 100
const logPath = './mergeLogs/mergelog_' + GetDateString().replace(/ /g, '_')
Main()
function Main () {
const params = GetParams()
console.log(params)
if (params.length === 0) {
console.error('No params! Need a path to a question database!')
process.exit()
}
const data = actions.LoadJSON(params[0])
PrintDB(data)
console.log(hr('='))
const { res, stats } = RemoveDuplicates(data)
console.log(hr('='))
LogStats(stats, data, res)
console.log(hr('='))
console.log('Result database:')
PrintDB(res)
console.log(hr('='))
utils.WriteFile(JSON.stringify(res), resultFileName)
console.log(C('green') + resultFileName + ' written!' + C())
console.log(hr('='))
console.log(C('green') + 'Done' + C())
}
function LogStats (stats, oldData, newData) {
const maxSubjNameLength = MaxLengthOf(stats, 'name')
const maxPrevLength = MaxLengthOf(stats, 'prevQuestions')
const maxAddedLength = MaxLengthOf(stats, 'addedQuestions')
const maxRemovedLength = MaxLengthOf(stats, 'removedQuestions')
stats.forEach((currStat) => {
const { name, prevQuestions, addedQuestions, removedQuestions } = currStat
let toLog = ''
toLog += C('green')
toLog += GetExactLength(name, maxSubjNameLength)
toLog += C()
toLog += ' '
toLog += C('magenta')
toLog += GetExactLength(prevQuestions, maxPrevLength)
toLog += C()
toLog += C('cyan')
toLog += ' -> '
toLog += C()
toLog += C('green')
toLog += GetExactLength(addedQuestions, maxAddedLength)
toLog += C()
toLog += ' [ '
toLog += C('red')
toLog += GetExactLength(removedQuestions, maxRemovedLength)
toLog += C()
toLog += ' ]'
console.log(toLog)
})
console.log(hr())
console.log('Old data:')
LogDataCount(oldData)
console.log('New data:')
LogDataCount(newData)
}
function LogDataCount (data) {
const subjLength = data.Subjects.length
const qLength = data.Subjects.reduce((acc, subj) => {
return acc + subj.Questions.length
}, 0)
console.log('Subjects: ' + C('green') + subjLength + C() + ', Questions: ' + C('green') + qLength + C())
}
function PrintDB (data) {
const maxSubjNameLength = MaxLengthOf(data.Subjects, 'Name')
data.Subjects.forEach((subj) => {
let toLog = ''
toLog += C('green')
toLog += GetExactLength(subj.Name, maxSubjNameLength)
toLog += C()
toLog += ' [ '
toLog += C('cyan')
toLog += subj.Questions.length
toLog += C()
toLog += ' ]'
console.log(toLog)
})
console.log(hr())
LogDataCount(data)
console.log(hr())
}
function GetExactLength (s, length) {
let toLog = s.toString()
const lengthDiff = length - toLog.length
for (let i = 0; i < lengthDiff; i++) {
toLog += ' '
}
return toLog
}
function MaxLengthOf (prop, key) {
return prop.reduce((acc, currStat) => {
if (acc < currStat[key].toString().length) {
acc = currStat[key].toString().length
}
return acc
}, 0)
}
function RemoveDuplicates (data) {
console.log(C('yellow') + 'Removing duplicates' + C())
const res = new classes.QuestionDB()
const stats = []
data.Subjects.forEach((subj, i) => {
const logFile = logPath + '/' + subj.Name.replace(/ /g, '_').replace(/\//g, '-')
LogSubjProgress(i, subj, data.Subjects.length)
let addedQuestions = 0
let removedQuestions = 0
subj.Questions.forEach((question) => {
// Searching for same question in result database
let r = res.Search(question).reduce((acc, r) => {
if (r.match >= minMatchAmmount) {
acc.push(r)
}
return acc
}, [])
// if htere are more that one same questions in the new database
if (r.length > 0) {
utils.AppendToFile(hr('#'), logFile)
utils.AppendToFile('QUESTION', logFile)
utils.AppendToFile(JSON.stringify(question, null, 2), logFile)
utils.AppendToFile(hr(), logFile)
utils.AppendToFile('SAMES', logFile)
utils.AppendToFile(JSON.stringify(r, null, 2), logFile)
removedQuestions++
} else {
// if no same questions are fount then adding it to then new db
res.AddQuestion(subj.getSubjNameWithoutYear(), question)
addedQuestions++
}
})
LogResultProgress(subj, addedQuestions, removedQuestions)
stats.push({
name: subj.Name,
prevQuestions: subj.Questions.length,
addedQuestions: addedQuestions,
removedQuestions: removedQuestions
})
})
return { res, stats }
}
function LogSubjProgress (i, subj, subjCount) {
log(
'[ ' +
C('cyan') +
(i + 1) +
C() +
' / ' +
C('green') +
subjCount +
C() +
' ] ' +
C('yellow') +
subj.Name +
C() +
': ' +
C('green') +
subj.Questions.length
)
}
function LogResultProgress (subj, addedQuestions, removedQuestions) {
log(
' ' +
C('cyan') +
'-> ' +
C('green') +
addedQuestions +
C() +
', removed: ' +
C('red') +
removedQuestions +
C() +
'\n'
)
}
function log (msg) {
process.stdout.write(msg)
}
function hr (char) {
let h = ''
const cols = process.stdout.columns || 20
for (let i = 0; i < cols; i++) {
h += char || '-'
}
return h
}
function C (color) {
return logger.C(color)
}
function GetParams () {
return process.argv.splice(2)
}
function GetDateString () {
const m = new Date()
const d = m.getFullYear() + '-' +
('0' + (m.getMonth() + 1)).slice(-2) + '-' +
('0' + m.getDate()).slice(-2) + ' ' +
('0' + m.getHours()).slice(-2) + ':' +
('0' + m.getMinutes()).slice(-2) + ':' +
('0' + m.getSeconds()).slice(-2)
return d
}

22
src/utils/runSqliteCmds.sh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/bash
if [ "$#" -lt "2" ]; then
echo "No params! 2 file required: db, commands file"
echo "usage: ./runSqliteCmds db.db commands"
exit 1
fi
echo "Executing:"
cat $2
echo
cmd=''
while read p; do
cmd="$cmd -cmd \"${p}\" -cmd \".shell echo\""
done <"$2"
echo "sqlite3 -bail $1 $cmd"
eval "sqlite3 -bail $1 $cmd" > cmdRes 2> /dev/null
echo "Done, result written to cmdRes file!"

9
src/utils/sqliteBatchCommands/showAll Normal file
View file

@ -0,0 +1,9 @@
.mode column
.headers ON
select * from users
select * from sessions
select * from veteranPWRequests
select * from accesses
.tables
.bail
select * from EXIT

138
src/utils/utils.js Executable file
View file

@ -0,0 +1,138 @@
module.exports = {
ReadFile: ReadFile,
ReadJSON: ReadJSON,
WriteFile: WriteFile,
writeFileAsync: WriteFileAsync,
AppendToFile: AppendToFile,
Beep: Beep,
WriteBackup: WriteBackup,
FileExists: FileExists,
CreatePath: CreatePath,
WatchFile: WatchFile,
ReadDir: ReadDir,
CopyFile: CopyFile,
GetDateString: GetDateString
}
var fs = require('fs')
var logger = require('../utils/logger.js')
const dataFile = './qminingPublic/data.json'
function GetDateString () {
const m = new Date()
return m.getFullYear() + '-' +
('0' + (m.getMonth() + 1)).slice(-2) + '-' +
('0' + m.getDate()).slice(-2) + ' ' +
('0' + m.getHours()).slice(-2) + ':' +
('0' + m.getMinutes()).slice(-2) + ':' +
('0' + m.getSeconds()).slice(-2)
}
function CopyFile (from, to) {
CreatePath(to)
fs.copyFileSync(from, to)
}
function ReadDir (path) {
return fs.readdirSync(path)
}
function ReadJSON (name) {
try {
return JSON.parse(ReadFile(name))
} catch (e) {
console.log(e)
throw new Error('Coulndt parse JSON in "ReadJSON", file: ' + name)
}
}
function ReadFile (name) {
if (!FileExists(name)) { throw new Error('No such file: ' + name) }
return fs.readFileSync(name, 'utf8')
}
function FileExists (path) {
return fs.existsSync(path)
}
function WatchFile (file, callback) {
if (FileExists(file)) {
fs.watchFile(file, (curr, prev) => {
fs.readFile(file, 'utf8', (err, data) => {
if (err) {
// console.log(err)
} else {
callback(data)
}
})
})
} else {
console.log(file + ' does not eadjsalék')
setTimeout(() => {
WatchFile(file)
}, 1000)
}
}
function CreatePath (path, onlyPath) {
if (FileExists(path)) { return }
var p = path.split('/')
var currDir = p[0]
for (var i = 1; i < p.length; i++) {
if (currDir !== '' && !fs.existsSync(currDir)) {
try {
fs.mkdirSync(currDir)
} catch (e) { console.log('Failed to make ' + currDir + ' directory... ') }
}
currDir += '/' + p[i]
}
if (onlyPath === undefined || onlyPath === false) {
} else {
fs.mkdirSync(path)
}
}
function WriteFile (content, path) {
CreatePath(path)
fs.writeFileSync(path, content)
}
function WriteFileAsync (content, path) {
CreatePath(path)
fs.writeFile(path, content, function (err) {
if (err) {
logger.Log('Error writing file: ' + path + ' (sync)', logger.GetColor('redbg'))
}
})
}
function AppendToFile (data, file) {
CreatePath(file)
try {
fs.appendFileSync(file, '\n' + data)
} catch (e) {
logger.Log('Error appendig to file log file: ' + file + ' (sync)', logger.GetColor('redbg'))
logger.Log(data)
console.log(e)
}
}
function Beep () {
try {
process.stdout.write('\x07')
} catch (e) {
console.log('error beepin')
}
}
function WriteBackup () {
try {
WriteFileAsync(ReadFile(dataFile), 'public/backs/data_' + new Date().toString())
} catch (e) {
logger.Log('Error backing up data json file!', logger.GetColor('redbg'))
console.log(e)
}
}