/* ---------------------------------------------------------------------------- Question Server question file merger GitLab: This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . ------------------------------------------------------------------------- */ const utils = require('./utils.js') const { searchData, addQuestion, getSubjNameWithoutYear, createQuestion, } = require('./classes.js') const actions = require('./actions.js') const logger = require('./logger.js') const resultFileName = 'res.json' const minMatchAmmount = 100 const logPath = './mergeLogs/mergelog_' + GetDateString().replace(/ /g, '_') Main() async function Main() { const params = GetParams() console.log(params) if (params.length === 0) { console.error('No params! Need a path to a question database!') process.exit() } const data = actions.LoadJSON(params[0]) PrintDB(data) console.log(hr('=')) const { res, stats } = await RemoveDuplicates(data) console.log(hr('=')) LogStats(stats, data, res) console.log(hr('=')) console.log('Result database:') PrintDB(res) console.log(hr('=')) utils.WriteFile(JSON.stringify(res), resultFileName) console.log(C('green') + resultFileName + ' written!' + C()) console.log(hr('=')) console.log(C('green') + 'Done' + C()) } function LogStats(stats, oldData, newData) { const maxSubjNameLength = MaxLengthOf(stats, 'name') const maxPrevLength = MaxLengthOf(stats, 'prevQuestions') const maxAddedLength = MaxLengthOf(stats, 'addedQuestions') const maxRemovedLength = MaxLengthOf(stats, 'removedQuestions') stats.forEach((currStat) => { const { name, prevQuestions, addedQuestions, removedQuestions } = currStat let toLog = '' toLog += C('green') toLog += GetExactLength(name, maxSubjNameLength) toLog += C() toLog += ' ' toLog += C('magenta') toLog += GetExactLength(prevQuestions, maxPrevLength) toLog += C() toLog += C('cyan') toLog += ' -> ' toLog += C() toLog += C('green') toLog += GetExactLength(addedQuestions, maxAddedLength) toLog += C() toLog += ' [ ' toLog += C('red') toLog += GetExactLength(removedQuestions, maxRemovedLength) toLog += C() toLog += ' ]' console.log(toLog) }) console.log(hr()) console.log('Old data:') LogDataCount(oldData) console.log('New data:') LogDataCount(newData) } function LogDataCount(data) { const subjLength = data.length const qLength = data.reduce((acc, subj) => { return acc + subj.Questions.length }, 0) console.log( 'Subjects: ' + C('green') + subjLength + C() + ', Questions: ' + C('green') + qLength + C() ) } function PrintDB(data) { const maxSubjNameLength = MaxLengthOf(data, 'Name') data.forEach((subj) => { let toLog = '' toLog += C('green') toLog += GetExactLength(subj.Name, maxSubjNameLength) toLog += C() toLog += ' [ ' toLog += C('cyan') toLog += subj.Questions.length toLog += C() toLog += ' ]' console.log(toLog) }) console.log(hr()) LogDataCount(data) console.log(hr()) } function GetExactLength(string, length) { let toLog = string.toString() const lengthDiff = length - toLog.length for (let i = 0; i < lengthDiff; i++) { toLog += ' ' } return toLog } function MaxLengthOf(prop, key) { return prop.reduce((acc, currStat) => { if (acc < currStat[key].toString().length) { acc = currStat[key].toString().length } return acc }, 0) } async function RemoveDuplicates(data) { console.log(C('yellow') + 'Removing duplicates' + C()) let res = [] const stats = [] for (let i = 0; i < data.length; i++) { const subj = data[i] const logFile = logPath + '/' + subj.Name.replace(/ /g, '_').replace(/\//g, '-') let addedQuestions = 0 let removedQuestions = 0 for (let j = 0; j < subj.Questions.length; j++) { const question = subj.Questions[j] try { // Searching for same question in result database const tempQuestion = createQuestion( question.Q, question.A, question.data ) let result = await searchData(res, tempQuestion) result = result.reduce((acc, res) => { if (res.match >= minMatchAmmount) { acc.push(res) } return acc }, []) // if htere are more that one same questions in the new database if (result.length > 0) { utils.AppendToFile(hr('#'), logFile) utils.AppendToFile('QUESTION', logFile) utils.AppendToFile(JSON.stringify(tempQuestion, null, 2), logFile) utils.AppendToFile(hr(), logFile) utils.AppendToFile('SAMES', logFile) utils.AppendToFile(JSON.stringify(result, null, 2), logFile) removedQuestions++ } else { // if no same questions are fount then adding it to then new db addQuestion(res, getSubjNameWithoutYear(subj.Name), tempQuestion) addedQuestions++ } LogResultProgress( subj, i, j, subj.Questions.length, addedQuestions, removedQuestions, data.length ) } catch (err) { console.log() console.log('ERROR') console.log(err) console.log('QUESTION') console.log(question) console.log() } } stats.push({ name: subj.Name, prevQuestions: subj.Questions.length, addedQuestions: addedQuestions, removedQuestions: removedQuestions, }) } return { res, stats } } function LogResultProgress( subj, i, j, length, addedQuestions, removedQuestions, subjCount ) { process.stdout.write( '[ ' + C('cyan') + (i + 1) + C() + ' / ' + C('green') + subjCount + C() + ' ] ' + C('yellow') + subj.Name + C() + ': ' + C('green') + subj.Questions.length + ' ' + C('cyan') + '-> ' + C('green') + addedQuestions + C() + ', removed: ' + C('red') + removedQuestions + C() ) if (j === length - 1) { process.stdout.write('\n') } else { process.stdout.write('\r') } } function hr(char) { let hr = '' const cols = process.stdout.columns || 20 for (let i = 0; i < cols; i++) { hr += char || '-' } return hr } function C(color) { return logger.C(color) } function GetParams() { return process.argv.splice(2) } function GetDateString() { const date = new Date() const dateString = date.getFullYear() + '-' + ('0' + (date.getMonth() + 1)).slice(-2) + '-' + ('0' + date.getDate()).slice(-2) + ' ' + ('0' + date.getHours()).slice(-2) + ':' + ('0' + date.getMinutes()).slice(-2) + ':' + ('0' + date.getSeconds()).slice(-2) return dateString }