From 1280af95a8157dd654240a36ef1e379c0cb331d0 Mon Sep 17 00:00:00 2001 From: MrFry Date: Thu, 26 Mar 2020 15:05:21 +0100 Subject: [PATCH] Removed old merger, updated merger.sh #5 --- utils/merge.sh | 5 +- utils/merger.js | 192 ------------------------------------------------ 2 files changed, 3 insertions(+), 194 deletions(-) delete mode 100755 utils/merger.js diff --git a/utils/merge.sh b/utils/merge.sh index b3289ae..f76e26f 100755 --- a/utils/merge.sh +++ b/utils/merge.sh @@ -1,10 +1,11 @@ #!/bin/bash -p="/home/qminer/nodeServ" +p=$(echo $PWD | rev | cut -d'/' -f2- | rev) cp -v $p/public/data.json /tmp/data.json -node $p/utils/merger.js /tmp/data.json +node $p/utils/rmDuplicates.js /tmp/data.json 2> /dev/null +mkdir -p "$p/public/backs/" mv -v $p/public/data.json "$p/public/backs/data.json $(date)" mv -v $p/utils/newData $p/public/data.json diff --git a/utils/merger.js b/utils/merger.js deleted file mode 100755 index 4a113df..0000000 --- a/utils/merger.js +++ /dev/null @@ -1,192 +0,0 @@ -/* ---------------------------------------------------------------------------- - - Question Server question file merger - GitLab: - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - - ------------------------------------------------------------------------- */ - -// TODO: handle flags -// join json datas, or raw datas -// or something else - -var classes = require('./question-classes/classes.js') -var utils = require('./utils.js') - -Main() - -function Main () { - console.clear() - const params = GetParams() - console.log(params) - var dbs = [] - - for (var i = 0; i < params.length; i++) { - PrintLN() - console.log(params[i] + ': ') - try { - dbs.push(ParseJSONData(utils.ReadFile(params[i]))) - console.log('JSON data added') - } catch (e) { - console.log(e) - return - } - } - PrintLN() - - dbs.forEach((item) => { - PrintDB(item) - }) - - var olds = [] - if (dbs.length === 1) { - for (let i = 0; i < dbs[0].length; i++) { olds.push(dbs[0].Subjects[i].length) } - } - - console.log('Parsed data count: ' + dbs.length) - PrintLN() - - console.log('Merging databases...') - var db = MergeDatabases(dbs) - - console.log('Removing duplicates...') - var r = RemoveDuplicates(db) - - console.log('RESULT:') - PrintDB(r, olds) - - utils.WriteFile(JSON.stringify(r), 'newData') - console.log('File written!') -} - -function PrintLN () { - console.log('------------------------------------------------------') -} - -function PrintDB (r, olds) { - console.log('Data subject count: ' + r.length) - var maxLength = 0 - for (let i = 0; i < r.length; i++) { - if (maxLength < r.Subjects[i].Name.length) { maxLength = r.Subjects[i].Name.length } - } - - let qcount = 0 - - for (let i = 0; i < r.length; i++) { - let line = i - if (line < 10) { line += ' ' } - - line += ': ' - var currLength = line.length + maxLength + 4 - line += r.Subjects[i].Name - - while (line.length < currLength) { - if (i % 4 === 0) { line += '.' } else { line += ' ' } - } - - let tolog = true - if (olds && olds.length > 0) { - if (r.Subjects[i].length !== olds[i]) { - // TODO: check if correct row! should be now, but well... - if (olds[i] < 10) { line += ' ' } - if (olds[i] < 100) { line += ' ' } - - line += olds[i] - line += ' -> ' - } else { - tolog = false - } - } - - if (r.Subjects[i].length < 10) { line += ' ' } - if (r.Subjects[i].length < 100) { line += ' ' } - - line += r.Subjects[i].length - qcount += r.Subjects[i].length - - line += ' db' - - if (tolog) { - console.log(line) - } - } - - console.log('Total questions: ' + qcount) - - PrintLN() -} - -function GetParams () { - return process.argv.splice(2) -} - -function ParseJSONData (data) { - var d = JSON.parse(data) - var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) - var rt = [] - - for (var i = 0; i < d.Subjects.length; i++) { - let s = new classes.Subject(d.Subjects[i].Name) - var j = 0 - for (j = 0; j < d.Subjects[i].Questions.length; j++) { - var currQ = d.Subjects[i].Questions[j] - s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I)) - } - rt.push({ - name: d.Subjects[i].Name, - count: j - }) - r.AddSubject(s) - } - return r -} - -function MergeDatabases (dbs) { - var db = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y)) - for (var i = 0; i < dbs.length; i++) { - for (var j = 0; j < dbs[i].length; j++) { db.AddSubject(dbs[i].Subjects[j]) } - } - return db -} - -function RemoveDuplicates (dataObj) { - for (var i = 0; i < dataObj.length; i++) { RemoveDuplFromSubject(dataObj.Subjects[i]) } - return dataObj -} - -function RemoveDuplFromSubject (subj) { - var cp = subj.Questions - subj.Questions = [] - for (var i = 0; i < cp.length; i++) { - var j = 0 - // Only removes 100% match! - while (j < subj.length && cp[i].Compare(subj.Questions[j]).avg <= 99) { - j++ - } - if (j < subj.length) { - // console.log("----------------------------------------------------------"); - // console.log(cp[i].toString()); - // console.log(" VS "); - // console.log(subj.Questions[j].toString()); - // console.log(cp[i].Compare(subj.Questions[j])); - // console.log(j); - // console.log("removed:"); - // console.log(subj.Questions.splice(j, 1).toString()); - // console.log("----------------------------------------------------------"); - } else { - subj.AddQuestion(cp[i]) - } - } -}