albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Duplicate remover logging imporvements

Browse source
This commit is contained in:
mrfry 2021-03-17 16:21:28 +01:00
parent 16b2f1a3b9
commit c167dcf5b2
1 changed files with 19 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/standaloneUtils/rmDuplicates.js
View file

@ -4,15 +4,15 @@ const { addQuestion, doSearch } = require('../../dist/utils/classes.js') // esli
const { loadData } = require('../../dist/utils/actions.js') // eslint-disable-line
// Params [ 'publicDirs/qminingPublic/questionDbs/elektro.json' ]
// load: 4.44ms
// load: 2.767ms
// ==============================================================================================
// 1 / 1: Elektronika, 826 questions
// Result length: 0, original length: 826
// [=====================================================================] 826 / 826
// Removed 11 questions
// Removed 19 questions
// ==============================================================================================
// Result length: 815, original length: 826, removed 11 questions
// rmduplicates: 9.349s
// Result length: 807, original length: 826, removed 19 questions
// rmduplicates: 9.527s
// File written
// load: 15.91ms
@ -32,7 +32,9 @@ const resultDbFileName = 'res.json'
const line =
'===================================================================='
const logPath = './duplicateRemovingLog/'
const globalLog = './duplicateRemovingLog/log'
utils.CreatePath(logPath)
utils.WriteFile('', globalLog)
const params = process.argv.splice(2)
@ -46,14 +48,14 @@ console.timeEnd('load')
console.time('rmduplicates')
if (!dbB) {
console.log(`Removing duplicate questions from ${fileA}`)
log(`Removing duplicate questions from ${fileA}`)
rmDuplicates(dbA).then((res) => {
console.timeEnd('rmduplicates')
utils.WriteFile(JSON.stringify(res), resultDbFileName)
console.log('File written')
log('File written')
})
} else {
console.log(
log(
`Removing questions found in ${C('green')}${fileB}${C()} from ${C(
'green'
)}${fileA}${C()}`
@ -61,7 +63,7 @@ if (!dbB) {
difference({ dbA: dbA, dbB: dbB }).then((res) => {
console.timeEnd('rmduplicates')
utils.WriteFile(JSON.stringify(res), resultDbFileName)
console.log('File written')
log('File written')
})
}
@ -95,16 +97,11 @@ async function difference({ dbA, dbB }) {
let removedCount = 0
hr()
console.log(
log(
`${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${subj.Name}, ${C(
'blue'
)}${subj.Questions.length}${C('green')} questions${C()}`
)
console.log(
`${C('green')}Result length: ${C('blue')}${getResultDbLength()}${C(
'green'
)}, original length: ${C('blue')}${dbLength}${C()}`
)
for (let j = 0; j < subj.Questions.length; j++) {
const question = subj.Questions[j]
@ -142,7 +139,7 @@ async function difference({ dbA, dbB }) {
removedTotal++
}
}
console.log(
log(
`${C('yellow')}Removed ${C('red')}${removedCount}${C(
'yellow'
)} questions${C()}`
@ -150,8 +147,7 @@ async function difference({ dbA, dbB }) {
}
hr()
// console.log('RESTUL', JSON.stringify(resultDb, null, 2))
console.log(
log(
`Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions`
)
return resultDb
@ -195,7 +191,12 @@ function hr() {
for (let i = 0; i < process.stdout.columns; i++) {
res += '='
}
console.log(`${C('cyan')}${res}${C()}`)
log(`${C('cyan')}${res}${C()}`)
}
function log(text) {
console.log(text)
utils.AppendToFile(text, globalLog)
}
function writeInSameLine(text, returnToLineStart) {