diff --git a/modules/qmining/qmining.js b/modules/qmining/qmining.js index e9112cb..15a77d6 100644 --- a/modules/qmining/qmining.js +++ b/modules/qmining/qmining.js @@ -37,6 +37,7 @@ const uloadFiles = 'public/f' const dataFile = 'public/data.json' const msgFile = 'stats/msgs' const motdFile = 'public/motd' +const versionFile = 'public/version' let donateURL = '' try { donateURL = utils.ReadFile('./data/donateURL') @@ -64,6 +65,8 @@ app.use(bodyParser.json({ limit: '5mb' })) +var data = actions.LoadJSON(dataFile) + // -------------------------------------------------------------- app.get('/', function (req, res) { @@ -90,14 +93,14 @@ app.get('/manual', function (req, res) { }) app.get('/legacy', function (req, res) { - var f = utils.ReadFile(dataFile) - var d = actions.LoadJSON(f) - let qcount = 0 - for (let i = 0; i < d.length; i++) { qcount += d.Subjects[i].length } - let scount = d.length + let qcount = data.Subjects.reduce((acc, currItem) => { + acc += currItem.length + return acc + }, 0) + let scount = data.length res.render('alldata', { - data: d, + data: data, scount: scount, qcount: qcount, siteurl: url @@ -116,13 +119,6 @@ app.get('/postfeedback', function (req, res) { res.redirect('/') }) -app.post('/isAdding', function (req, res) { - res.end('OK') - logger.LogReq(req) - actions.ProcessIncomingRequest(req.body.datatoadd) - utils.WriteBackup() -}) - app.get('/lred', function (req, res) { res.redirect('/legacy') res.end() @@ -226,6 +222,90 @@ app.route('/badtestsender').post(function (req, res, next) { logger.LogReq(req) }) +// ------------------------------------------------------------------------------------------- +// API + +app.post('/isAdding', function (req, res) { + logger.LogReq(req) + let result = actions.ProcessIncomingRequest(req.body.datatoadd, data) + res.json({ + success: result !== -1, + newQuestions: result + }) + utils.WriteBackup() // TODO: write every n times / after n minutes +}) + +app.get('/q', function (req, res) { + logger.LogReq(req) + if (Object.keys(req.query).length === 0) { + res.send('ask something!') + } else { + if (req.query.q) { + let subj = req.query.subj || '' + let recData = {} + try { + recData = JSON.parse(req.query.data) + } catch (e) { + console.log(e) // TODO: normal error logging + } + let r = data.Search(decodeURIComponent(req.query.q), subj, recData) + + res.json(r) + } else { + res.send('no key "q" in query' + JSON.stringify(req.query)) + } + } +}) + +function getSimplreRes () { + return { + subjects: data.length, + questions: data.Subjects.reduce((acc, subj) => { + return acc + subj.length + }, 0) + } +} +function getDetailedRes () { + return data.Subjects.map((subj) => { + return { + name: subj.Name, + count: subj.length + } + }) +} + +app.get('/datacount', function (req, res) { + logger.LogReq(req) + if (req.query.detailed === 'all') { + res.json({ + detailed: getDetailedRes(), + simple: getSimplreRes() + }) + } else if (req.query.detailed) { + res.json(getDetailedRes()) + } else { + res.json(getSimplreRes()) + } +}) + +app.get('/infos', function (req, res) { + let result = {} + if (req.query.subjinfo) { + result.subjinfo = getSimplreRes() + } + // TODO: watch file, and global variable for motd version + if (req.query.version) { + result.version = utils.ReadFile(versionFile) + } + if (req.query.motd) { + result.motd = utils.ReadFile(motdFile) + } + logger.LogReq(req) + res.json(result) +}) + +// ------------------------------------------------------------------------------------------- + app.get('*', function (req, res) { res.status(404).render('404') }) diff --git a/utils/actions.js b/utils/actions.js index 7250c54..30f8fdf 100755 --- a/utils/actions.js +++ b/utils/actions.js @@ -29,14 +29,17 @@ const versionFile = './public/version' const motdFile = './public/motd' const qaFile = './public/qa' -var logger = require('../utils/logger.js') +const logger = require('../utils/logger.js') const idStats = require('../utils/ids.js') idStats.Load() // TODO: dont always load when actions.js is used -var utils = require('../utils/utils.js') +const utils = require('../utils/utils.js') const classes = require('./question-classes/classes.js') -function ProcessIncomingRequest (data) { - if (data === undefined) { +const writeAfter = 1 // write after # of adds FIXME: set reasonable save rate +var currWrites = 0 + +function ProcessIncomingRequest (recievedData, qdb) { + if (recievedData === undefined) { logger.Log('\tRecieved data is undefined!', logger.GetColor('redbg')) return } @@ -44,7 +47,7 @@ function ProcessIncomingRequest (data) { try { let towrite = logger.GetDateString() + '\n' towrite += '------------------------------------------------------------------------------\n' - towrite += data + towrite += recievedData towrite += '\n------------------------------------------------------------------------------\n' utils.AppendToFile(towrite, recDataFile) } catch (e) { @@ -52,44 +55,49 @@ function ProcessIncomingRequest (data) { } try { - var d = JSON.parse(data) - var allQuestions = [] + // recievedData: { version: "", id: "", subj: "" quiz: {} } + let d = JSON.parse(recievedData) + let allQLength = d.quiz.length + let allQuestions = [] - d = ConvertToNewFormat(d) - - for (let i = 0; i < d.allData.length; i++) { - allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data)) - } + d.quiz.forEach((question) => { + let q = new classes.Question(question.Q, question.A, question.data) + let sames = qdb.Search(q, '') // TODO d.subj + // if it didnt find any question, or every found questions match is lower thatn 80 + let isNew = sames.length === 0 || sames.every(searchResItem => { + return searchResItem.match < classes.minMatchAmmount + }) + if (isNew) { + allQuestions.push(q) + } + }) let color = logger.GetColor('green') - let questions = [] let msg = '' - if (d.data.length > 0) { - msg += `New questions: ${d.data.length} ( All: ${allQuestions.length} )` - } else { - msg += `No new data ( ${allQuestions.length} )` - } - if (d.data.length > 0) { - let qdb = LoadJSON(utils.ReadFile(dataFile)) - d.data.forEach((x) => { - let q = new classes.Question(x.Q, x.A, x.data) - questions.push(q) + if (allQuestions.length > 0) { + color = logger.GetColor('blue') + msg += `New questions: ${allQuestions.length} ( All: ${allQLength} )` + + allQuestions.forEach((q) => { qdb.AddQuestion(d.subj, q) }) - try { - qdb.version = utils.ReadFile(versionFile) - qdb.motd = utils.ReadFile(motdFile) - } catch (e) { - logger.Log('MOTD/Version writing/reading error!') - } - - if (qdb !== undefined && d.data.length > 0) { - utils.WriteBackup() + currWrites++ + console.log(currWrites) + if (currWrites >= writeAfter) { + currWrites = 0 + try { + // TODO: with utils.WatchFile + qdb.version = utils.ReadFile(versionFile) + qdb.motd = utils.ReadFile(motdFile) + } catch (e) { + logger.Log('MOTD/Version writing/reading error!') + } utils.WriteFile(JSON.stringify(qdb), dataFile) - msg += ' - Data file written!' - color = logger.GetColor('blue') + logger.Log('\tData file written', color) } + } else { + msg += `No new data ( ${allQLength} )` } let subjRow = '\t' + d.subj @@ -101,16 +109,18 @@ function ProcessIncomingRequest (data) { if (d.version !== undefined) { msg += '. Version: ' + d.version } logger.Log('\t' + msg, color) + return allQuestions.length } catch (e) { console.log(e) logger.Log('Couldnt parse JSON data', logger.GetColor('redbg')) + return -1 } } // loading stuff -function LoadJSON (resource) { +function LoadJSON (dataFile) { try { - var d = JSON.parse(resource) + var d = JSON.parse(utils.ReadFile(dataFile)) var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) var rt = [] @@ -155,40 +165,3 @@ function ProcessQA () { return r } - -function ConvertToNewFormat (d) { - if (d.allData.length < 0) { - return d - } - - try { - if (!d.allData[0].data) { - logger.Log('\tConverting old data to new...') - d.allData = d.allData.map((x) => { - if (x.I) { - x.data = { - type: 'image', - images: JSON.parse(x.I) - } - delete x.I - } - return x - }) - d.data = d.data.map((x) => { - if (x.I) { - x.data = { - type: 'image', - images: JSON.parse(x.I) - } - delete x.I - } - return x - }) - } - } catch (e) { - console.log(e) - logger.Log('Couldnt convert old data to new!', logger.GetColor('redbg')) - } - - return d -}