albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Removed old merger, updated merger.sh #5

Browse source
This commit is contained in:
MrFry 2020-03-26 15:05:21 +01:00
parent e6db0f9175
commit 1280af95a8
2 changed files with 3 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

5
utils/merge.sh
View file

@ -1,10 +1,11 @@
#!/bin/bash #!/bin/bash
p="/home/qminer/nodeServ" p=$(echo $PWD | rev | cut -d'/' -f2- | rev)
cp -v $p/public/data.json /tmp/data.json cp -v $p/public/data.json /tmp/data.json
node $p/utils/merger.js /tmp/data.json node $p/utils/rmDuplicates.js /tmp/data.json 2> /dev/null
mkdir -p "$p/public/backs/"
mv -v $p/public/data.json "$p/public/backs/data.json $(date)" mv -v $p/public/data.json "$p/public/backs/data.json $(date)"
mv -v $p/utils/newData $p/public/data.json mv -v $p/utils/newData $p/public/data.json

192
utils/merger.js
View file

@ -1,192 +0,0 @@
/* ----------------------------------------------------------------------------
Question Server question file merger
GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
------------------------------------------------------------------------- */
// TODO: handle flags
// join json datas, or raw datas
// or something else
var classes = require('./question-classes/classes.js')
var utils = require('./utils.js')
Main()
function Main () {
console.clear()
const params = GetParams()
console.log(params)
var dbs = []
for (var i = 0; i < params.length; i++) {
PrintLN()
console.log(params[i] + ': ')
try {
dbs.push(ParseJSONData(utils.ReadFile(params[i])))
console.log('JSON data added')
} catch (e) {
console.log(e)
return
}
}
PrintLN()
dbs.forEach((item) => {
PrintDB(item)
})
var olds = []
if (dbs.length === 1) {
for (let i = 0; i < dbs[0].length; i++) { olds.push(dbs[0].Subjects[i].length) }
}
console.log('Parsed data count: ' + dbs.length)
PrintLN()
console.log('Merging databases...')
var db = MergeDatabases(dbs)
console.log('Removing duplicates...')
var r = RemoveDuplicates(db)
console.log('RESULT:')
PrintDB(r, olds)
utils.WriteFile(JSON.stringify(r), 'newData')
console.log('File written!')
}
function PrintLN () {
console.log('------------------------------------------------------')
}
function PrintDB (r, olds) {
console.log('Data subject count: ' + r.length)
var maxLength = 0
for (let i = 0; i < r.length; i++) {
if (maxLength < r.Subjects[i].Name.length) { maxLength = r.Subjects[i].Name.length }
}
let qcount = 0
for (let i = 0; i < r.length; i++) {
let line = i
if (line < 10) { line += ' ' }
line += ': '
var currLength = line.length + maxLength + 4
line += r.Subjects[i].Name
while (line.length < currLength) {
if (i % 4 === 0) { line += '.' } else { line += ' ' }
}
let tolog = true
if (olds && olds.length > 0) {
if (r.Subjects[i].length !== olds[i]) {
// TODO: check if correct row! should be now, but well...
if (olds[i] < 10) { line += ' ' }
if (olds[i] < 100) { line += ' ' }
line += olds[i]
line += ' -> '
} else {
tolog = false
}
}
if (r.Subjects[i].length < 10) { line += ' ' }
if (r.Subjects[i].length < 100) { line += ' ' }
line += r.Subjects[i].length
qcount += r.Subjects[i].length
line += ' db'
if (tolog) {
console.log(line)
}
}
console.log('Total questions: ' + qcount)
PrintLN()
}
function GetParams () {
return process.argv.splice(2)
}
function ParseJSONData (data) {
var d = JSON.parse(data)
var r = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y))
var rt = []
for (var i = 0; i < d.Subjects.length; i++) {
let s = new classes.Subject(d.Subjects[i].Name)
var j = 0
for (j = 0; j < d.Subjects[i].Questions.length; j++) {
var currQ = d.Subjects[i].Questions[j]
s.AddQuestion(new classes.Question(currQ.Q, currQ.A, currQ.I))
}
rt.push({
name: d.Subjects[i].Name,
count: j
})
r.AddSubject(s)
}
return r
}
function MergeDatabases (dbs) {
var db = new classes.QuestionDB((x) => true, (x, y) => console.log(x, y))
for (var i = 0; i < dbs.length; i++) {
for (var j = 0; j < dbs[i].length; j++) { db.AddSubject(dbs[i].Subjects[j]) }
}
return db
}
function RemoveDuplicates (dataObj) {
for (var i = 0; i < dataObj.length; i++) { RemoveDuplFromSubject(dataObj.Subjects[i]) }
return dataObj
}
function RemoveDuplFromSubject (subj) {
var cp = subj.Questions
subj.Questions = []
for (var i = 0; i < cp.length; i++) {
var j = 0
// Only removes 100% match!
while (j < subj.length && cp[i].Compare(subj.Questions[j]).avg <= 99) {
j++
}
if (j < subj.length) {
// console.log("----------------------------------------------------------");
// console.log(cp[i].toString());
// console.log(" VS ");
// console.log(subj.Questions[j].toString());
// console.log(cp[i].Compare(subj.Questions[j]));
// console.log(j);
// console.log("removed:");
// console.log(subj.Questions.splice(j, 1).toString());
// console.log("----------------------------------------------------------");
} else {
subj.AddQuestion(cp[i])
}
}
}