albert/mrfrys-node-server
1
0
Fork 0
mirror of https://gitlab.com/MrFry/mrfrys-node-server synced 2025-04-01 20:24:18 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

more rm duplicates improvements

Browse source
This commit is contained in:
mrfry 2021-04-25 10:30:43 +02:00
parent 9ddff88ebd
commit 0aa9e209c1
1 changed files with 16 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/standaloneUtils/rmDuplicates.js
View file

@ -34,7 +34,6 @@ const fs = require('fs') // eslint-disable-line
// File written // File written
const minpercent = 95 const minpercent = 95
const resultDbFileName = 'res.json'
const line = const line =
'====================================================================' '===================================================================='
const logPath = './duplicateRemovingLog/' const logPath = './duplicateRemovingLog/'
@ -58,6 +57,12 @@ if (stat.isDirectory()) {
const db = pathB ? loadData(pathB) : null const db = pathB ? loadData(pathB) : null
clearPossibleAnswers(pathA, db) clearPossibleAnswers(pathA, db)
log(
`Cleared possible questions from ${C('green')}${pathA}${C()} based on ${C(
'green'
)}${pathB}${C()} db`
)
} else { } else {
removePossibleAnswersDuplicates(pathA) removePossibleAnswersDuplicates(pathA)
} }
@ -71,10 +76,12 @@ if (stat.isDirectory()) {
if (!dbB) { if (!dbB) {
log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`) log(`Removing duplicate questions from ${C('green')}${pathA}${C()}`)
const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
const res = rmDuplicates(dbA) const res = rmDuplicates(dbA)
console.timeEnd('rmduplicates') console.timeEnd('rmduplicates')
writeData(res, resultDbFileName) writeData(res, resultDbFileName + '.res')
log('File written') log('File written')
log(`Removed duplicate questions from ${C('green')}${pathA}${C()}`)
} else { } else {
log( log(
`Removing questions found in ${C('green')}${pathB}${C()} from ${C( `Removing questions found in ${C('green')}${pathB}${C()} from ${C(
@ -83,8 +90,14 @@ if (stat.isDirectory()) {
) )
const res = difference({ dbA: dbA, dbB: dbB }) const res = difference({ dbA: dbA, dbB: dbB })
console.timeEnd('rmduplicates') console.timeEnd('rmduplicates')
writeData(res, resultDbFileName) const resultDbFileName = pathA.split('/')[pathA.split('/').length - 1]
writeData(res, resultDbFileName + '.res')
log('File written') log('File written')
log(
`Removed questions found in ${C('green')}${pathB}${C()} from ${C(
'green'
)}${pathA}${C()}`
)
} }
} }