diff --git a/README.md b/README.md index 7ed239b..9034e35 100755 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ dataUpdater.js | régifajta adatbázist, amiben még van `.Q` propertyjű kérd changedataversion.js | `data.json`-ban és a ./public/version ban írja át a teszt megoldó kliens aktuális verzióját merger.js | Paraméterként kapott adatbázisból törli az egyező bejegyzéseket, és egyesíti egy fájlba merge.sh | Biztonsági mentést készít, és egyszerűsíti az adatbázist, majd felülírja az újjal -question-classes/classes.js | Összehasonlításhoz és tároláshoz szükséges osztályok +classes.js | Összehasonlításhoz és tároláshoz szükséges osztályok # Egyéb Jelenleg sok optimalizálatlan rész található benne, cél ezek kijavítása, szépítése diff --git a/modules/dataEditor/qmining-data-editor b/modules/dataEditor/qmining-data-editor index 48d531c..0ba12f4 160000 --- a/modules/dataEditor/qmining-data-editor +++ b/modules/dataEditor/qmining-data-editor @@ -1 +1 @@ -Subproject commit 48d531ca1d2d6b48b508879ea4ef2e968feb0eb7 +Subproject commit 0ba12f4d67f2bfb5ba2553f32dec5a2b439960fb diff --git a/modules/qmining/qmining-page b/modules/qmining/qmining-page index a2644bf..f6a6b4e 160000 --- a/modules/qmining/qmining-page +++ b/modules/qmining/qmining-page @@ -1 +1 @@ -Subproject commit a2644bfb91f5d512ef36ddf5d1ce77f7c35c64a4 +Subproject commit f6a6b4e452cddb6150003776113b332026d4354e diff --git a/modules/stuff/stuff.js b/modules/stuff/stuff.js index 6ee5f67..129ae94 100644 --- a/modules/stuff/stuff.js +++ b/modules/stuff/stuff.js @@ -152,7 +152,8 @@ app.get('/*', function (req, res) { logger.LogReq(req) try { - if (fs.lstatSync(curr).isDirectory()) { + const stat = fs.lstatSync(curr) + if (stat.isDirectory() || stat.isSymbolicLink()) { if (curr[curr.length - 1] !== '/') { curr += '/' } let f = [] diff --git a/utils/actions.js b/utils/actions.js index 0fd2818..3d77532 100755 --- a/utils/actions.js +++ b/utils/actions.js @@ -29,7 +29,7 @@ const logger = require('../utils/logger.js') const idStats = require('../utils/ids.js') idStats.Load() // FIXME: dont always load when actions.js is used const utils = require('../utils/utils.js') -const classes = require('./question-classes/classes.js') +const classes = require('./classes.js') classes.initLogger(logger.DebugLog) // if a recievend question doesnt match at least this % to any other question in the db it gets // added to db diff --git a/utils/classes.js b/utils/classes.js new file mode 100755 index 0000000..5006f63 --- /dev/null +++ b/utils/classes.js @@ -0,0 +1,509 @@ +var debugLogger = null + +function initLogger (logger) { + debugLogger = logger +} + +function debugLog (msg, name, lvl) { + if (debugLogger) { + debugLogger(msg, name, lvl) + } +} + +const commonUselessAnswerParts = [ + 'A helyes válasz az ', + 'A helyes válasz a ', + 'A helyes válaszok: ', + 'A helyes válaszok:', + 'A helyes válasz: ', + 'A helyes válasz:', + 'The correct answer is:', + '\'' +] +const commonUselessStringParts = [ + ',', + '\\.', + ':', + '!', + '\\+', + '\\s*\\.' +] +const specialChars = [ '&', '\\+' ] +const lengthDiffMultiplier = 10 /* Percent minus for length difference */ +const minMatchAmmount = 60 /* Minimum ammount to consider that two questions match during answering */ + +const assert = (val) => { + if (!val) { throw new Error('Assertion failed') } +} + +class StringUtils { + GetSubjNameWithoutYear (subjName) { + let t = subjName.split(' - ') + if (t[0].match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t[1] || subjName + } else { + return subjName + } + } + + RemoveStuff (value, removableStrings, toReplace) { + removableStrings.forEach((x) => { + var regex = new RegExp(x, 'g') + value = value.replace(regex, toReplace || '') + }) + return value + } + + SimplifyQuery (q) { + assert(q) + + var result = q.replace(/\n/g, ' ').replace(/\s/g, ' ') + return this.RemoveUnnecesarySpaces(result) + } + + ShortenString (toShorten, ammount) { + assert(toShorten) + + var result = '' + var i = 0 + while (i < toShorten.length && i < ammount) { + result += toShorten[i] + i++ + } + return result + } + + ReplaceCharsWithSpace (val, char) { + assert(val) + assert(char) + + var toremove = this.NormalizeSpaces(val) + + var regex = new RegExp(char, 'g') + toremove = toremove.replace(regex, ' ') + + return this.RemoveUnnecesarySpaces(toremove) + } + + // removes whitespace from begining and and, and replaces multiple spaces with one space + RemoveUnnecesarySpaces (toremove) { + assert(toremove) + + toremove = this.NormalizeSpaces(toremove) + while (toremove.includes(' ')) { + toremove = toremove.replace(/ {2}/g, ' ') + } + return toremove.trim() + } + + // simplifies a string for easier comparison + SimplifyStringForComparison (value) { + assert(value) + + value = this.RemoveUnnecesarySpaces(value).toLowerCase() + return this.RemoveStuff(value, commonUselessStringParts) + } + + RemoveSpecialChars (value) { + assert(value) + + return this.RemoveStuff(value, specialChars, ' ') + } + + // if the value is empty, or whitespace + EmptyOrWhiteSpace (value) { + // replaces /n-s with "". then replaces spaces with "". if it equals "", then its empty, or only consists of white space + if (value === undefined) { return true } + return (value.replace(/\n/g, '').replace(/ /g, '').replace(/\s/g, ' ') === '') + } + + // damn nonbreaking space + NormalizeSpaces (input) { + assert(input) + + return input.replace(/\s/g, ' ') + } + + CompareString (s1, s2) { + if (!s1 || !s2) { + if (!s1 && !s2) { + return 100 + } else { + return 0 + } + } + + s1 = this.SimplifyStringForComparison(s1).split(' ') + s2 = this.SimplifyStringForComparison(s2).split(' ') + var match = 0 + for (var i = 0; i < s1.length; i++) { + if (s2.includes(s1[i])) { match++ } + } + var percent = Math.round(((match / s1.length) * 100).toFixed(2)) // matched words percent + var lengthDifference = Math.abs(s2.length - s1.length) + percent -= lengthDifference * lengthDiffMultiplier + if (percent < 0) { percent = 0 } + return percent + } + + AnswerPreProcessor (value) { + assert(value) + + return this.RemoveStuff( + value, commonUselessAnswerParts) + } + + // 'a. pécsi sör' -> 'pécsi sör' + RemoveAnswerLetters (value) { + assert(value) + + let s = value.split('. ') + if (s[0].length < 2 && s.length > 1) { + s.shift() + return s.join(' ') + } else { + return value + } + } + + SimplifyQA (value, mods) { + if (!value) { return } + + const reducer = (res, fn) => { + return fn(res) + } + + return mods.reduce(reducer, value) + } + + SimplifyAnswer (value) { + return this.SimplifyQA( + value, + [ + this.RemoveSpecialChars.bind(this), + this.RemoveUnnecesarySpaces.bind(this), + this.AnswerPreProcessor.bind(this), + this.RemoveAnswerLetters.bind(this) + ]) + } + + SimplifyQuestion (value) { + return this.SimplifyQA( + value, + [ + this.RemoveSpecialChars.bind(this), + this.RemoveUnnecesarySpaces.bind(this) + ]) + } + + SimplifyStack (stack) { + return this.SimplifyQuery(stack) + } +} + +const SUtils = new StringUtils() + +class Question { + constructor (q, a, data) { + this.Q = SUtils.SimplifyQuestion(q) + this.A = SUtils.SimplifyAnswer(a) + this.data = { ...data } + } + + toString () { + if (this.data.type !== 'simple') { + return '?' + this.Q + '\n!' + this.A + '\n>' + JSON.stringify(this.data) + } else { + return '?' + this.Q + '\n!' + this.A + } + } + + HasQuestion () { + return this.Q !== undefined + } + + HasAnswer () { + return this.A !== undefined + } + + HasImage () { + return this.data.type === 'image' + } + + IsComplete () { + return this.HasQuestion() && this.HasAnswer() + } + + CompareImage (data2) { + return SUtils.CompareString(this.data.images.join(' '), data2.images.join(' ')) + } + + // returns -1 if botth is simple + CompareData (qObj) { + try { + if (qObj.data.type === this.data.type) { + let dataType = qObj.data.type + if (dataType === 'simple') { + return -1 + } else if (dataType === 'image') { + return this.CompareImage(qObj.data) + } else { + debugLog(`Unhandled data type ${dataType}`, 'Compare question data', 1) + debugLog(qObj, 'Compare question data', 2) + } + } else { + return 0 + } + } catch (e) { + debugLog('Error comparing data', 'Compare question data', 1) + debugLog(e.message, 'Compare question data', 1) + debugLog(e, 'Compare question data', 2) + } + return 0 + } + + CompareQuestion (qObj) { + return SUtils.CompareString(this.Q, qObj.Q) + } + + CompareAnswer (qObj) { + return SUtils.CompareString(this.A, qObj.A) + } + + // TODO: return q / a / data match for debuging + Compare (q2, data) { + assert(q2) + let qObj + + if (typeof q2 === 'string') { + qObj = { + Q: q2, + data: data + } + } else { + qObj = q2 + } + + const qMatch = this.CompareQuestion(qObj) + const aMatch = this.CompareAnswer(qObj) + // -1 if botth questions are simple + const dMatch = this.CompareData(qObj) + + let avg = -1 + if (qObj.A) { + if (dMatch === -1) { + avg = (qMatch + aMatch) / 2 + } else { + avg = (qMatch + aMatch + dMatch) / 3 + } + } else { + if (dMatch === -1) { + avg = qMatch + } else { + avg = (qMatch + dMatch) / 2 + } + } + + return { + qMatch: qMatch, + aMatch: aMatch, + dMatch: dMatch, + avg: avg + } + } +} + +class Subject { + constructor (n) { + assert(n) + + this.Name = n + this.Questions = [] + } + + setIndex (i) { + this.index = i + } + + getIndex () { + return this.index + } + + get length () { + return this.Questions.length + } + + AddQuestion (q) { + assert(q) + + this.Questions.push(q) + } + + getSubjNameWithoutYear () { + return SUtils.GetSubjNameWithoutYear(this.Name) + } + + getYear () { + let t = this.Name.split(' - ')[0] + if (t.match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{1}$/i)) { + return t + } else { + return '' + } + } + + Search (q, data) { + assert(q) + + var r = [] + for (let i = 0; i < this.length; i++) { + let percent = this.Questions[i].Compare(q, data) + if (percent.avg > minMatchAmmount) { + r.push({ + q: this.Questions[i], + match: percent.avg, + detailedMatch: percent + }) + } + } + + for (let i = 0; i < r.length; i++) { + for (var j = i; j < r.length; j++) { + if (r[i].match < r[j].match) { + var tmp = r[i] + r[i] = r[j] + r[j] = tmp + } + } + } + + return r + } + + toString () { + var r = [] + for (var i = 0; i < this.Questions.length; i++) { r.push(this.Questions[i].toString()) } + return '+' + this.Name + '\n' + r.join('\n') + } +} + +class QuestionDB { + constructor () { + this.Subjects = [] + } + + get length () { + return this.Subjects.length + } + + AddQuestion (subj, q) { + debugLog('Adding new question with subjName: ' + subj, 'qdb add', 1) + debugLog(q, 'qdb add', 3) + assert(subj) + + var i = 0 + while (i < this.Subjects.length && + !subj.toLowerCase().includes(this.Subjects[i].getSubjNameWithoutYear().toLowerCase())) { + i++ + } + + if (i < this.Subjects.length) { + debugLog('Adding new question to existing subject', 'qdb add', 1) + this.Subjects[i].AddQuestion(q) + } else { + debugLog('Creating new subject for question', 'qdb add', 1) + const n = new Subject(subj) + n.AddQuestion(q) + this.Subjects.push(n) + } + } + + SimplifyQuestion (q) { + if (typeof q === 'string') { + return SUtils.SimplifyQuestion(q) + } else { + q.Q = SUtils.SimplifyQuestion(q.Q) + q.A = SUtils.SimplifyQuestion(q.A) + return q + } + } + + Search (q, subjName, data) { + assert(q) + debugLog('Searching for question', 'qdb search', 1) + debugLog('Question:', 'qdb search', 2) + debugLog(q, 'qdb search', 2) + debugLog(`Subject name: ${subjName}`, 'qdb search', 2) + debugLog('Data:', 'qdb search', 2) + debugLog(data || q.data, 'qdb search', 2) + + if (!data) { + data = q.data || { type: 'simple' } + } + if (!subjName) { + subjName = '' + debugLog('No subject name as param!', 'qdb search', 1) + } + q = this.SimplifyQuestion(q) + + var r = [] + this.Subjects.forEach((subj) => { + if (subjName.toLowerCase().includes(subj.getSubjNameWithoutYear().toLowerCase())) { + debugLog(`Searching in ${subj.Name} `, 2) + r = r.concat(subj.Search(q, data)) + } + }) + + // FIXME: try to remove this? but this is also a good backup plan so idk + if (r.length === 0) { + debugLog('Reqults length is zero when comparing names, trying all subjects', 'qdb search', 1) + this.Subjects.forEach((subj) => { + r = r.concat(subj.Search(q, data)) + }) + if (r.length > 0) { + debugLog(`FIXME: '${subjName}' gave no result but '' did!`, 'qdb search', 1) + console.error(`FIXME: '${subjName}' gave no result but '' did!`) + } + } + + for (let i = 0; i < r.length; i++) { + for (var j = i; j < r.length; j++) { + if (r[i].match < r[j].match) { + var tmp = r[i] + r[i] = r[j] + r[j] = tmp + } + } + } + + debugLog(`QDB search result length: ${r.length}`, 'qdb search', 1) + return r + } + + AddSubject (subj) { + assert(subj) + + var i = 0 + while (i < this.length && subj.Name !== this.Subjects[i].Name) { i++ } + + if (i < this.length) { + this.Subjects.concat(subj.Questions) + } else { + this.Subjects.push(subj) + } + } + + toString () { + var r = [] + for (var i = 0; i < this.Subjects.length; i++) { r.push(this.Subjects[i].toString()) } + return r.join('\n\n') + } +} + +module.exports.StringUtils = StringUtils // TODO: export singleton string utils, remove nea StringUtils from other files +module.exports.SUtils = SUtils +module.exports.Question = Question +module.exports.Subject = Subject +module.exports.QuestionDB = QuestionDB +module.exports.minMatchAmmount = minMatchAmmount +module.exports.initLogger = initLogger diff --git a/utils/rmDuplicates.js b/utils/rmDuplicates.js index ab540e1..e725e3e 100644 --- a/utils/rmDuplicates.js +++ b/utils/rmDuplicates.js @@ -19,7 +19,7 @@ ------------------------------------------------------------------------- */ const utils = require('./utils.js') -const classes = require('./question-classes/classes.js') +const classes = require('./classes.js') const actions = require('./actions.js') const logger = require('./logger.js') @@ -109,12 +109,8 @@ function LogDataCount (data) { function PrintDB (data) { const maxSubjNameLength = MaxLengthOf(data.Subjects, 'Name') - data.Subjects.forEach((subj, i) => { + data.Subjects.forEach((subj) => { let toLog = '' - toLog += C('magenta') - toLog += (i + 1) - toLog += C() - toLog += ': ' toLog += C('green') toLog += GetExactLength(subj.Name, maxSubjNameLength) toLog += C()