albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Duplicate remover logging imporvements

Browse source
This commit is contained in:
mrfry 2021-03-17 16:21:28 +01:00
parent 16b2f1a3b9
commit c167dcf5b2
1 changed files with 19 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/standaloneUtils/rmDuplicates.js
View file

@ -4,15 +4,15 @@ const { addQuestion, doSearch } = require('../../dist/utils/classes.js') // esli
const { loadData } = require('../../dist/utils/actions.js') // eslint-disable-line const { loadData } = require('../../dist/utils/actions.js') // eslint-disable-line
// Params [ 'publicDirs/qminingPublic/questionDbs/elektro.json' ] // Params [ 'publicDirs/qminingPublic/questionDbs/elektro.json' ]
// load: 4.44ms // load: 2.767ms
// ============================================================================================== // ==============================================================================================
// 1 / 1: Elektronika, 826 questions // 1 / 1: Elektronika, 826 questions
// Result length: 0, original length: 826 // Result length: 0, original length: 826
// [=====================================================================] 826 / 826 // [=====================================================================] 826 / 826
// Removed 11 questions // Removed 19 questions
// ============================================================================================== // ==============================================================================================
// Result length: 815, original length: 826, removed 11 questions // Result length: 807, original length: 826, removed 19 questions
// rmduplicates: 9.349s // rmduplicates: 9.527s
// File written // File written
// load: 15.91ms // load: 15.91ms
@ -32,7 +32,9 @@ const resultDbFileName = 'res.json'
const line = const line =
'====================================================================' '===================================================================='
const logPath = './duplicateRemovingLog/' const logPath = './duplicateRemovingLog/'
const globalLog = './duplicateRemovingLog/log'
utils.CreatePath(logPath) utils.CreatePath(logPath)
utils.WriteFile('', globalLog)
const params = process.argv.splice(2) const params = process.argv.splice(2)
@ -46,14 +48,14 @@ console.timeEnd('load')
console.time('rmduplicates') console.time('rmduplicates')
if (!dbB) { if (!dbB) {
console.log(`Removing duplicate questions from ${fileA}`) log(`Removing duplicate questions from ${fileA}`)
rmDuplicates(dbA).then((res) => { rmDuplicates(dbA).then((res) => {
console.timeEnd('rmduplicates') console.timeEnd('rmduplicates')
utils.WriteFile(JSON.stringify(res), resultDbFileName) utils.WriteFile(JSON.stringify(res), resultDbFileName)
console.log('File written') log('File written')
}) })
} else { } else {
console.log( log(
`Removing questions found in ${C('green')}${fileB}${C()} from ${C( `Removing questions found in ${C('green')}${fileB}${C()} from ${C(
'green' 'green'
)}${fileA}${C()}` )}${fileA}${C()}`
@ -61,7 +63,7 @@ if (!dbB) {
difference({ dbA: dbA, dbB: dbB }).then((res) => { difference({ dbA: dbA, dbB: dbB }).then((res) => {
console.timeEnd('rmduplicates') console.timeEnd('rmduplicates')
utils.WriteFile(JSON.stringify(res), resultDbFileName) utils.WriteFile(JSON.stringify(res), resultDbFileName)
console.log('File written') log('File written')
}) })
} }
@ -95,16 +97,11 @@ async function difference({ dbA, dbB }) {
let removedCount = 0 let removedCount = 0
hr() hr()
console.log( log(
`${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${subj.Name}, ${C( `${C('blue')}${i + 1} / ${dbA.length}: ${C('green')}${subj.Name}, ${C(
'blue' 'blue'
)}${subj.Questions.length}${C('green')} questions${C()}` )}${subj.Questions.length}${C('green')} questions${C()}`
) )
console.log(
`${C('green')}Result length: ${C('blue')}${getResultDbLength()}${C(
'green'
)}, original length: ${C('blue')}${dbLength}${C()}`
)
for (let j = 0; j < subj.Questions.length; j++) { for (let j = 0; j < subj.Questions.length; j++) {
const question = subj.Questions[j] const question = subj.Questions[j]
@ -142,7 +139,7 @@ async function difference({ dbA, dbB }) {
removedTotal++ removedTotal++
} }
} }
console.log( log(
`${C('yellow')}Removed ${C('red')}${removedCount}${C( `${C('yellow')}Removed ${C('red')}${removedCount}${C(
'yellow' 'yellow'
)} questions${C()}` )} questions${C()}`
@ -150,8 +147,7 @@ async function difference({ dbA, dbB }) {
} }
hr() hr()
// console.log('RESTUL', JSON.stringify(resultDb, null, 2)) log(
console.log(
`Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions` `Result length: ${getResultDbLength()}, original length: ${dbLength}, removed ${removedTotal} questions`
) )
return resultDb return resultDb
@ -195,7 +191,12 @@ function hr() {
for (let i = 0; i < process.stdout.columns; i++) { for (let i = 0; i < process.stdout.columns; i++) {
res += '=' res += '='
} }
console.log(`${C('cyan')}${res}${C()}`) log(`${C('cyan')}${res}${C()}`)
}
function log(text) {
console.log(text)
utils.AppendToFile(text, globalLog)
} }
function writeInSameLine(text, returnToLineStart) { function writeInSameLine(text, returnToLineStart) {