From 127575afd5afa7490764a5730c64dfad5c9ce87f Mon Sep 17 00:00:00 2001 From: MrFry Date: Sat, 11 Jan 2020 11:59:55 +0100 Subject: [PATCH 1/3] added data updater, merging with master --- utils/dataUpdater.js | 233 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 utils/dataUpdater.js diff --git a/utils/dataUpdater.js b/utils/dataUpdater.js new file mode 100644 index 0000000..333555e --- /dev/null +++ b/utils/dataUpdater.js @@ -0,0 +1,233 @@ +/* ---------------------------------------------------------------------------- + + Question Server question file merger + GitLab: + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + + ------------------------------------------------------------------------- */ + +const classes = require('./question-classes/classes.js') +const utils = require('./utils.js') + +Main() + +function Main () { + console.clear() + const params = GetParams() + console.log(params) + var db = [] + + PrintLN() + console.log(params[0] + ': ') + try { + db = ParseJSONData(utils.ReadFile(params[0])) + console.log('JSON data added') + } catch (e) { + console.log(e) + console.log('Trying with old format...') + db = ReadData(utils.ReadFile(params[0])).result + } + PrintLN() + + PrintDB(db) + + PrintLN() + + RefactorDb(db) + + utils.WriteFile(JSON.stringify(db), 'newData') +} + +function RefactorDb (db) { + db.Subjects.forEach((subj) => { + subj.Questions.forEach((question) => { + if (question.I) { + question.data = { + type: 'image', + images: question.I + } + delete question.I + } else { + question.data = { + type: 'simple' + } + } + }) + }) +} + +// ----------------------------------------------------------------------------------------- + +function PrintLN () { + console.log('------------------------------------------------------') +} + +function PrintDB (r, olds) { + console.log('Data subject count: ' + r.length) + var maxLength = 0 + for (let i = 0; i < r.length; i++) { + if (maxLength < r.Subjects[i].Name.length) { maxLength = r.Subjects[i].Name.length } + } + + let qcount = 0 + + for (let i = 0; i < r.length; i++) { + let line = i + if (line < 10) { line += ' ' } + + line += ': ' + var currLength = line.length + maxLength + 4 + line += r.Subjects[i].Name + + while (line.length < currLength) { + if (i % 4 === 0) { line += '.' } else { line += ' ' } + } + + if (olds && olds.length > 0) { + // TODO: check if correct row! should be now, but well... + if (olds[i] < 10) { line += ' ' } + if (olds[i] < 100) { line += ' ' } + + line += olds[i] + line += ' -> ' + } + + if (r.Subjects[i].length < 10) { line += ' ' } + if (r.Subjects[i].length < 100) { line += ' ' } + + line += r.Subjects[i].length + qcount += r.Subjects[i].length + + line += ' db' + + console.log(line) + } + + console.log('Total questions: ' + qcount) + + PrintLN() +} + +function GetParams () { + return process.argv.splice(2) +} + +function ParseJSONData (data) { + var d = JSON.parse(data) + var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) + var rt = [] + + for (var i = 0; i < d.Subjects.length; i++) { + let s = new classes.Subject(d.Subjects[i].Name) + var j = 0 + for (j = 0; j < d.Subjects[i].Questions.length; j++) { + var currQ = d.Subjects[i].Questions[j] + s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I)) + } + rt.push({ + name: d.Subjects[i].Name, + count: j + }) + r.AddSubject(s) + } + return r +} + +function ReadData (data) { + const d = data.split('\n') + const r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) + var logs = [] + var currSubj = '' // the current subjects name + var ExpectedIdentifier = ['+', '?'] + let currQuestion = new classes.Question() + + var i = -1 + while (i < d.length) { + let currIdentifier + let skipped = 0 + do { + if (skipped >= 1) { logs.push(i + ': ' + d[i]) } + i++ + if (i >= d.length) { + if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } + return { + result: r, + logs: logs + } + } + currIdentifier = d[i][0] + skipped++ + } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length) + + let currData = d[i].substring(1).trim() + + if (currIdentifier === '+') { + if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } + currQuestion = new classes.Question() + currSubj = currData + ExpectedIdentifier = ['?'] + continue + } + + if (currIdentifier === '?') { + if (currQuestion.IsComplete()) { + r.AddQuestion(currSubj, currQuestion) + currQuestion = new classes.Question() + } + // overwriting is allowed here, bcus: + // ?????!> + currQuestion.Q = currData + ExpectedIdentifier = ['!', '?'] + continue + } + + if (currIdentifier === '!') { + // if dont have question continue + if (!currQuestion.HasQuestion()) { throw new Error('No question! (A)') } + // dont allow overwriting + // ?!!!! + if (!currQuestion.HasAnswer()) { + currData = currData.replace('A helyes válaszok: ', '') + currData = currData.replace('A helyes válasz: ', '') + + currQuestion.A = currData + } + ExpectedIdentifier = ['?', '>', '+'] + continue + } + + if (currIdentifier === '>') { + // if dont have question or answer continue + if (!currQuestion.HasQuestion()) { throw new Error('No question! (I)') } + if (!currQuestion.HasAnswer()) { throw new Error('No asnwer! (I)') } + // dont allow overwriting + // ?!>>> + if (!currQuestion.HasImage()) { + try { + currQuestion.I = JSON.parse(currData) + } catch (e) { + currQuestion.I = currData.split(',') + } + } + ExpectedIdentifier = ['?', '+'] + continue + } + } + + return { + result: r, + logs: logs + } +} From bb23853043f05c3c0f8b976da33e81cb4b8fe5f5 Mon Sep 17 00:00:00 2001 From: MrFry Date: Wed, 15 Jan 2020 08:55:20 +0100 Subject: [PATCH 2/3] data uploader, handling extra data part of questions --- utils/actions.js | 6 +- utils/dataUpdater.js | 422 ++++++++++++++++++++++++++++++++++++++++- utils/question-classes | 2 +- 3 files changed, 417 insertions(+), 13 deletions(-) diff --git a/utils/actions.js b/utils/actions.js index 9bdb2e1..6673476 100644 --- a/utils/actions.js +++ b/utils/actions.js @@ -55,7 +55,7 @@ function ProcessIncomingRequest (data) { var d = JSON.parse(data) var allQuestions = [] for (let i = 0; i < d.allData.length; i++) { - allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].I)) + allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data)) } let color = logger.GetColor('green') @@ -69,7 +69,7 @@ function ProcessIncomingRequest (data) { if (d.data.length > 0) { let qdb = LoadJSON(utils.ReadFile(dataFile)) d.data.forEach((x) => { - let q = new classes.Question(x.Q, x.A, x.I) + let q = new classes.Question(x.Q, x.A, x.data) questions.push(q) qdb.AddQuestion(d.subj, q) }) @@ -116,7 +116,7 @@ function LoadJSON (resource) { var j = 0 for (j = 0; j < d.Subjects[i].Questions.length; j++) { var currQ = d.Subjects[i].Questions[j] - s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I)) + s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.data)) } rt.push({ name: d.Subjects[i].Name, diff --git a/utils/dataUpdater.js b/utils/dataUpdater.js index 333555e..be22220 100644 --- a/utils/dataUpdater.js +++ b/utils/dataUpdater.js @@ -18,7 +18,411 @@ ------------------------------------------------------------------------- */ -const classes = require('./question-classes/classes.js') +const commonUselessAnswerParts = [ + 'A helyes válasz az ', + 'A helyes válasz a ', + 'A helyes válaszok: ', + 'A helyes válaszok:', + 'A helyes válasz: ', + 'A helyes válasz:', + 'The correct answer is:', + '\'' +] +const commonUselessStringParts = [',', '\\.', ':', '!', '\\+'] +const specialChars = [ '&', '\\+' ] +const lengthDiffMultiplier = 10 /* Percent minus for length difference */ +const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */ + +const assert = (val) => { + if (!val) { throw new Error('Assertion failed') } +} + +class StringUtils { + RemoveStuff (value, removableStrings, toReplace) { + removableStrings.forEach((x) => { + var regex = new RegExp(x, 'g') + value = value.replace(regex, toReplace || '') + }) + return value + } + + SimplifyQuery (q) { + assert(q) + + var result = q.replace(/\n/g, ' ').replace(/\s/g, ' ') + return this.RemoveUnnecesarySpaces(result) + } + + ShortenString (toShorten, ammount) { + assert(toShorten) + + var result = '' + var i = 0 + while (i < toShorten.length && i < ammount) { + result += toShorten[i] + i++ + } + return result + } + + ReplaceCharsWithSpace (val, char) { + assert(val) + assert(char) + + var toremove = this.NormalizeSpaces(val) + + var regex = new RegExp(char, 'g') + toremove = toremove.replace(regex, ' ') + + return this.RemoveUnnecesarySpaces(toremove) + } + + // removes whitespace from begining and and, and replaces multiple spaces with one space + RemoveUnnecesarySpaces (toremove) { + assert(toremove) + + toremove = this.NormalizeSpaces(toremove) + while (toremove.includes(' ')) { + toremove = toremove.replace(/ {2}/g, ' ') + } + return toremove.trim() + } + + // simplifies a string for easier comparison + SimplifyStringForComparison (value) { + assert(value) + + value = this.RemoveUnnecesarySpaces(value).toLowerCase() + return this.RemoveStuff(value, commonUselessStringParts) + } + + RemoveSpecialChars (value) { + assert(value) + + return this.RemoveStuff(value, specialChars, ' ') + } + + // if the value is empty, or whitespace + EmptyOrWhiteSpace (value) { + // replaces /n-s with "". then replaces spaces with "". if it equals "", then its empty, or only consists of white space + if (value === undefined) { return true } + return (value.replace(/\n/g, '').replace(/ /g, '').replace(/\s/g, ' ') === '') + } + + // damn nonbreaking space + NormalizeSpaces (input) { + assert(input) + + return input.replace(/\s/g, ' ') + } + + CompareString (s1, s2) { + if (!s1 || !s2) { + return 0 + } + + s1 = this.SimplifyStringForComparison(s1).split(' ') + s2 = this.SimplifyStringForComparison(s2).split(' ') + var match = 0 + for (var i = 0; i < s1.length; i++) { + if (s2.includes(s1[i])) { match++ } + } + var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length) + percent -= lengthDifference * lengthDiffMultiplier + if (percent < 0) { percent = 0 } + return percent + } + + AnswerPreProcessor (value) { + assert(value) + + return this.RemoveStuff( + value, commonUselessAnswerParts) + } + + // 'a. pécsi sör' -> 'pécsi sör' + RemoveAnswerLetters (value) { + assert(value) + + let s = value.split('. ') + if (s[0].length < 2 && s.length > 1) { + s.shift() + return s.join(' ') + } else { + return value + } + } + + SimplifyQA (value, mods) { + if (!value) { return } + + const reducer = (res, fn) => { + return fn(res) + } + + return mods.reduce(reducer, value) + } + + SimplifyAnswer (value) { + return this.SimplifyQA( + value, + [ + this.RemoveSpecialChars.bind(this), + this.RemoveUnnecesarySpaces.bind(this), + this.AnswerPreProcessor.bind(this), + this.RemoveAnswerLetters.bind(this) + ]) + } + + SimplifyQuestion (value) { + return this.SimplifyQA( + value, + [ + this.RemoveSpecialChars.bind(this), + this.RemoveUnnecesarySpaces.bind(this) + ]) + } + + SimplifyStack (stack) { + return this.SimplifyQuery(stack) + } +} + +const SUtils = new StringUtils() + +class Question { + constructor (q, a, i) { + this.Q = SUtils.SimplifyQuestion(q) + this.A = SUtils.SimplifyAnswer(a) + this.I = i + } + + toString () { + var r = '?' + this.Q + '\n!' + this.A + if (this.I) { r += '\n>' + this.I } + return r + } + + HasQuestion () { + return this.Q !== undefined + } + + HasAnswer () { + return this.A !== undefined + } + + HasImage () { + return this.I !== undefined && (typeof this.I === 'string' || Array.isArray(this.I)) + } + + IsComplete () { + return this.HasQuestion() && this.HasAnswer() + } + + Compare (q2, i) { + assert(q2) + + if (typeof q2 === 'string') { + var qmatchpercent = SUtils.CompareString(this.Q, q2) + + if (i === undefined || i.length === 0) { return qmatchpercent } else { + if (this.HasImage()) { + const imatchpercent = this.HasImage() ? SUtils.CompareString(this.I.join(' '), i.join(' ')) + : 0 + return (qmatchpercent + imatchpercent) / 2 + } else { + qmatchpercent -= 30 + if (qmatchpercent < 0) { return 0 } else { return qmatchpercent } + } + } + } else { + const qmatchpercent = SUtils.CompareString(this.Q, q2.Q) + const amatchpercent = SUtils.CompareString(this.A, q2.A) + if (this.I !== undefined) { + const imatchpercent = this.I === undefined ? SUtils.CompareString(this.I.join(' '), q2.I.join( + ' ')) : 0 + return (qmatchpercent + amatchpercent + imatchpercent) / 3 + } else { + return (qmatchpercent + amatchpercent) / 2 + } + } + } +} + +class Subject { + constructor (n) { + assert(n) + + this.Name = n + this.Questions = [] + this.active = false + } + + setIndex (i) { + this.index = i + } + + getIndex () { + return this.index || -1 + } + + get length () { + return this.Questions.length + } + + markActive () { + this.active = true + } + + getIfActive () { + return this.active + } + + AddQuestion (q) { + assert(q) + + this.Questions.push(q) + } + + getSubjNameWithoutYear () { + let t = this.Name.split(' - ') + if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t[1] || '' + } else { + return '' + } + } + + getYear () { + let t = this.Name.split(' - ')[0] + if (t.match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t + } else { + return '' + } + } + + Search (q, img) { + assert(q) + + var r = [] + for (let i = 0; i < this.length; i++) { + let percent = this.Questions[i].Compare(q, img) + if (percent > minMatchAmmount) { + r.push({ + q: this.Questions[i], + match: percent + }) + } + } + + for (let i = 0; i < r.length; i++) { + for (var j = i; j < r.length; j++) { + if (r[i].match < r[j].match) { + var tmp = r[i] + r[i] = r[j] + r[j] = tmp + } + } + } + + return r + } + + toString () { + var r = [] + for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) } + return '+' + this.Name + '\n' + r.join('\n') + } +} + +class QuestionDB { + constructor (getVal, setVal) { + this.Subjects = [] + this.getVal = getVal + this.setVal = setVal + } + + get length () { + return this.Subjects.length + } + + get activeIndexes () { + var r = [] + for (var i = 0; i < this.length; i++) { + if (this.getVal('Is' + i + 'Active')) { + r.push(i) + } + } + return r + } + + GetIfActive (ind) { + return this.getVal('Is' + ind + 'Active') + } + + ChangeActive (i, value) { + this.setVal('Is' + i + 'Active', !!value) + } + + AddQuestion (subj, q) { + assert(subj) + + var i = 0 + while (i < this.Subjects.length && this.Subjects[i].Name !== subj) { i++ } + if (i < this.Subjects.length) { this.Subjects[i].AddQuestion(q) } else { + const n = new Subject(subj) + n.AddQuestion(q) + this.Subjects.push(n) + } + } + + Search (q, img) { + assert(q) + + var r = [] + for (let i = 0; i < this.length; i++) { + if (this.GetIfActive(i)) { r = r.concat(this.Subjects[i].Search(q, img)) } + } + + for (let i = 0; i < r.length; i++) { + for (var j = i; j < r.length; j++) { + if (r[i].match < r[j].match) { + var tmp = r[i] + r[i] = r[j] + r[j] = tmp + } + } + } + + return r + } + + AddSubject (subj) { + assert(subj) + + var i = 0 + while (i < this.length && subj.Name !== this.Subjects[i].Name) { i++ } + + if (i < this.length) { + this.Subjects.concat(subj.Questions) + } else { + this.Subjects.push(subj) + } + } + + toString () { + var r = [] + for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) } + return r.join('\n\n') + } +} + +module.exports.StringUtils = StringUtils +module.exports.Question = Question +module.exports.Subject = Subject +module.exports.QuestionDB = QuestionDB const utils = require('./utils.js') Main() @@ -56,7 +460,7 @@ function RefactorDb (db) { if (question.I) { question.data = { type: 'image', - images: question.I + images: typeof question.I === 'string' ? JSON.parse(question.I) : question.I } delete question.I } else { @@ -126,15 +530,15 @@ function GetParams () { function ParseJSONData (data) { var d = JSON.parse(data) - var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) + var r = new QuestionDB((x) => true, (x, y) => console.log(x, y)) var rt = [] for (var i = 0; i < d.Subjects.length; i++) { - let s = new classes.Subject(d.Subjects[i].Name) + let s = new Subject(d.Subjects[i].Name) var j = 0 for (j = 0; j < d.Subjects[i].Questions.length; j++) { var currQ = d.Subjects[i].Questions[j] - s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I)) + s.AddQuestion(new Question(currQ.Q, currQ.A, currQ.I)) } rt.push({ name: d.Subjects[i].Name, @@ -147,11 +551,11 @@ function ParseJSONData (data) { function ReadData (data) { const d = data.split('\n') - const r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) + const r = new QuestionDB((x) => true, (x, y) => console.log(x, y)) var logs = [] var currSubj = '' // the current subjects name var ExpectedIdentifier = ['+', '?'] - let currQuestion = new classes.Question() + let currQuestion = new Question() var i = -1 while (i < d.length) { @@ -175,7 +579,7 @@ function ReadData (data) { if (currIdentifier === '+') { if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } - currQuestion = new classes.Question() + currQuestion = new Question() currSubj = currData ExpectedIdentifier = ['?'] continue @@ -184,7 +588,7 @@ function ReadData (data) { if (currIdentifier === '?') { if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) - currQuestion = new classes.Question() + currQuestion = new Question() } // overwriting is allowed here, bcus: // ?????!> diff --git a/utils/question-classes b/utils/question-classes index 0cfd9cf..5dac788 160000 --- a/utils/question-classes +++ b/utils/question-classes @@ -1 +1 @@ -Subproject commit 0cfd9cfda377ff8917df50e68f105ec43f5e9311 +Subproject commit 5dac788b18b7240a4194a2a37ecbee857292800d From 2f6f39c294e67735288ecee104ef60ee2824d1c2 Mon Sep 17 00:00:00 2001 From: MrFry Date: Wed, 22 Jan 2020 17:16:11 +0100 Subject: [PATCH 3/3] Handling old data --- .gitmodules | 0 README.md | 0 license | 0 modules/main.js | 0 modules/old.js | 0 modules/qmining.js | 0 modules/sio.js | 0 modules/stuff.js | 0 package.json | 0 public/favicon.ico | Bin server.js | 0 utils/actions.js | 40 ++++++++++++++++ utils/changedataversion.js | 0 utils/dataUpdater.js | 0 utils/ids.js | 0 utils/logger.js | 0 utils/merger.js | 95 +------------------------------------ utils/motd.js | 0 utils/utils.js | 0 views/main/main.ejs | 0 views/qmining/alldata.ejs | 0 views/qmining/allqr.ejs | 0 views/qmining/aludni.ejs | 0 views/qmining/b.ejs | 0 views/qmining/main.ejs | 0 views/qmining/man.ejs | 0 views/qmining/qa.ejs | 0 views/qmining/uploaded.ejs | 0 views/shared/404.ejs | 0 views/sio/uload.ejs | 0 views/stuff/audio.ejs | 0 views/stuff/folders.ejs | 0 views/stuff/nofile.ejs | 0 views/stuff/video.ejs | 0 34 files changed, 41 insertions(+), 94 deletions(-) mode change 100644 => 100755 .gitmodules mode change 100644 => 100755 README.md mode change 100644 => 100755 license mode change 100644 => 100755 modules/main.js mode change 100644 => 100755 modules/old.js mode change 100644 => 100755 modules/qmining.js mode change 100644 => 100755 modules/sio.js mode change 100644 => 100755 modules/stuff.js mode change 100644 => 100755 package.json mode change 100644 => 100755 public/favicon.ico mode change 100644 => 100755 server.js mode change 100644 => 100755 utils/actions.js mode change 100644 => 100755 utils/changedataversion.js mode change 100644 => 100755 utils/dataUpdater.js mode change 100644 => 100755 utils/ids.js mode change 100644 => 100755 utils/logger.js mode change 100644 => 100755 utils/merger.js mode change 100644 => 100755 utils/motd.js mode change 100644 => 100755 utils/utils.js mode change 100644 => 100755 views/main/main.ejs mode change 100644 => 100755 views/qmining/alldata.ejs mode change 100644 => 100755 views/qmining/allqr.ejs mode change 100644 => 100755 views/qmining/aludni.ejs mode change 100644 => 100755 views/qmining/b.ejs mode change 100644 => 100755 views/qmining/main.ejs mode change 100644 => 100755 views/qmining/man.ejs mode change 100644 => 100755 views/qmining/qa.ejs mode change 100644 => 100755 views/qmining/uploaded.ejs mode change 100644 => 100755 views/shared/404.ejs mode change 100644 => 100755 views/sio/uload.ejs mode change 100644 => 100755 views/stuff/audio.ejs mode change 100644 => 100755 views/stuff/folders.ejs mode change 100644 => 100755 views/stuff/nofile.ejs mode change 100644 => 100755 views/stuff/video.ejs diff --git a/.gitmodules b/.gitmodules old mode 100644 new mode 100755 diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/license b/license old mode 100644 new mode 100755 diff --git a/modules/main.js b/modules/main.js old mode 100644 new mode 100755 diff --git a/modules/old.js b/modules/old.js old mode 100644 new mode 100755 diff --git a/modules/qmining.js b/modules/qmining.js old mode 100644 new mode 100755 diff --git a/modules/sio.js b/modules/sio.js old mode 100644 new mode 100755 diff --git a/modules/stuff.js b/modules/stuff.js old mode 100644 new mode 100755 diff --git a/package.json b/package.json old mode 100644 new mode 100755 diff --git a/public/favicon.ico b/public/favicon.ico old mode 100644 new mode 100755 diff --git a/server.js b/server.js old mode 100644 new mode 100755 diff --git a/utils/actions.js b/utils/actions.js old mode 100644 new mode 100755 index 6673476..7250c54 --- a/utils/actions.js +++ b/utils/actions.js @@ -54,6 +54,9 @@ function ProcessIncomingRequest (data) { try { var d = JSON.parse(data) var allQuestions = [] + + d = ConvertToNewFormat(d) + for (let i = 0; i < d.allData.length; i++) { allQuestions.push(new classes.Question(d.allData[i].Q, d.allData[i].A, d.allData[i].data)) } @@ -152,3 +155,40 @@ function ProcessQA () { return r } + +function ConvertToNewFormat (d) { + if (d.allData.length < 0) { + return d + } + + try { + if (!d.allData[0].data) { + logger.Log('\tConverting old data to new...') + d.allData = d.allData.map((x) => { + if (x.I) { + x.data = { + type: 'image', + images: JSON.parse(x.I) + } + delete x.I + } + return x + }) + d.data = d.data.map((x) => { + if (x.I) { + x.data = { + type: 'image', + images: JSON.parse(x.I) + } + delete x.I + } + return x + }) + } + } catch (e) { + console.log(e) + logger.Log('Couldnt convert old data to new!', logger.GetColor('redbg')) + } + + return d +} diff --git a/utils/changedataversion.js b/utils/changedataversion.js old mode 100644 new mode 100755 diff --git a/utils/dataUpdater.js b/utils/dataUpdater.js old mode 100644 new mode 100755 diff --git a/utils/ids.js b/utils/ids.js old mode 100644 new mode 100755 diff --git a/utils/logger.js b/utils/logger.js old mode 100644 new mode 100755 diff --git a/utils/merger.js b/utils/merger.js old mode 100644 new mode 100755 index 45796dc..a306e85 --- a/utils/merger.js +++ b/utils/merger.js @@ -41,8 +41,7 @@ function Main () { console.log('JSON data added') } catch (e) { console.log(e) - console.log('Trying with old format...') - dbs.push(ReadData(utils.ReadFile(params[i])).result) + return } } PrintLN() @@ -155,98 +154,6 @@ function MergeDatabases (dbs) { return db } -/* - * Returns a question database from the given data. - * Parameter should be raw read file in string with "\n"-s - * TODO: ??? -s are not listed as errors, tho works correctly - * */ -function ReadData (data) { - const d = data.split('\n') - const r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) - var logs = [] - var currSubj = '' // the current subjects name - var ExpectedIdentifier = ['+', '?'] - let currQuestion = new classes.Question() - - var i = -1 - while (i < d.length) { - let currIdentifier - let skipped = 0 - do { - if (skipped >= 1) { logs.push(i + ': ' + d[i]) } - i++ - if (i >= d.length) { - if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } - return { - result: r, - logs: logs - } - } - currIdentifier = d[i][0] - skipped++ - } while (!ExpectedIdentifier.includes(currIdentifier) && i < d.length) - - let currData = d[i].substring(1).trim() - - if (currIdentifier === '+') { - if (currQuestion.IsComplete()) { r.AddQuestion(currSubj, currQuestion) } - currQuestion = new classes.Question() - currSubj = currData - ExpectedIdentifier = ['?'] - continue - } - - if (currIdentifier === '?') { - if (currQuestion.IsComplete()) { - r.AddQuestion(currSubj, currQuestion) - currQuestion = new classes.Question() - } - // overwriting is allowed here, bcus: - // ?????!> - currQuestion.Q = currData - ExpectedIdentifier = ['!', '?'] - continue - } - - if (currIdentifier === '!') { - // if dont have question continue - if (!currQuestion.HasQuestion()) { throw new Error('No question! (A)') } - // dont allow overwriting - // ?!!!! - if (!currQuestion.HasAnswer()) { - currData = currData.replace('A helyes válaszok: ', '') - currData = currData.replace('A helyes válasz: ', '') - - currQuestion.A = currData - } - ExpectedIdentifier = ['?', '>', '+'] - continue - } - - if (currIdentifier === '>') { - // if dont have question or answer continue - if (!currQuestion.HasQuestion()) { throw new Error('No question! (I)') } - if (!currQuestion.HasAnswer()) { throw new Error('No asnwer! (I)') } - // dont allow overwriting - // ?!>>> - if (!currQuestion.HasImage()) { - try { - currQuestion.I = JSON.parse(currData) - } catch (e) { - currQuestion.I = currData.split(',') - } - } - ExpectedIdentifier = ['?', '+'] - continue - } - } - - return { - result: r, - logs: logs - } -} - function RemoveDuplicates (dataObj) { for (var i = 0; i < dataObj.length; i++) { RemoveDuplFromSubject(dataObj.Subjects[i]) } return dataObj diff --git a/utils/motd.js b/utils/motd.js old mode 100644 new mode 100755 diff --git a/utils/utils.js b/utils/utils.js old mode 100644 new mode 100755 diff --git a/views/main/main.ejs b/views/main/main.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/alldata.ejs b/views/qmining/alldata.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/allqr.ejs b/views/qmining/allqr.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/aludni.ejs b/views/qmining/aludni.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/b.ejs b/views/qmining/b.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/main.ejs b/views/qmining/main.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/man.ejs b/views/qmining/man.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/qa.ejs b/views/qmining/qa.ejs old mode 100644 new mode 100755 diff --git a/views/qmining/uploaded.ejs b/views/qmining/uploaded.ejs old mode 100644 new mode 100755 diff --git a/views/shared/404.ejs b/views/shared/404.ejs old mode 100644 new mode 100755 diff --git a/views/sio/uload.ejs b/views/sio/uload.ejs old mode 100644 new mode 100755 diff --git a/views/stuff/audio.ejs b/views/stuff/audio.ejs old mode 100644 new mode 100755 diff --git a/views/stuff/folders.ejs b/views/stuff/folders.ejs old mode 100644 new mode 100755 diff --git a/views/stuff/nofile.ejs b/views/stuff/nofile.ejs old mode 100644 new mode 100755 diff --git a/views/stuff/video.ejs b/views/stuff/video.ejs old mode 100644 new mode 100755