Lotsa typescript bullshit

src/standaloneUtils/rmDuplicates.js

@@ -0,0 +1,309 @@
+/* ----------------------------------------------------------------------------
+
+ Question Server question file merger
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+const utils = require('./utils.js')
+const {
+  searchData,
+  addQuestion,
+  getSubjNameWithoutYear,
+  createQuestion,
+} = require('./classes.js')
+const actions = require('./actions.js')
+const logger = require('./logger.js')
+
+const resultFileName = 'res.json'
+const minMatchAmmount = 100
+
+const logPath = './mergeLogs/mergelog_' + GetDateString().replace(/ /g, '_')
+
+Main()
+
+async function Main() {
+  const params = GetParams()
+  console.log(params)
+  if (params.length === 0) {
+    console.error('No params! Need a path to a question database!')
+    process.exit()
+  }
+  const data = actions.LoadJSON(params[0])
+
+  PrintDB(data)
+  console.log(hr('='))
+
+  const { res, stats } = await RemoveDuplicates(data)
+  console.log(hr('='))
+
+  LogStats(stats, data, res)
+  console.log(hr('='))
+
+  console.log('Result database:')
+  PrintDB(res)
+  console.log(hr('='))
+
+  utils.WriteFile(JSON.stringify(res), resultFileName)
+  console.log(C('green') + resultFileName + ' written!' + C())
+  console.log(hr('='))
+
+  console.log(C('green') + 'Done' + C())
+}
+
+function LogStats(stats, oldData, newData) {
+  const maxSubjNameLength = MaxLengthOf(stats, 'name')
+  const maxPrevLength = MaxLengthOf(stats, 'prevQuestions')
+  const maxAddedLength = MaxLengthOf(stats, 'addedQuestions')
+  const maxRemovedLength = MaxLengthOf(stats, 'removedQuestions')
+
+  stats.forEach((currStat) => {
+    const { name, prevQuestions, addedQuestions, removedQuestions } = currStat
+    let toLog = ''
+
+    toLog += C('green')
+    toLog += GetExactLength(name, maxSubjNameLength)
+    toLog += C()
+    toLog += ' '
+    toLog += C('magenta')
+    toLog += GetExactLength(prevQuestions, maxPrevLength)
+    toLog += C()
+    toLog += C('cyan')
+    toLog += ' -> '
+    toLog += C()
+    toLog += C('green')
+    toLog += GetExactLength(addedQuestions, maxAddedLength)
+    toLog += C()
+    toLog += ' [ '
+    toLog += C('red')
+    toLog += GetExactLength(removedQuestions, maxRemovedLength)
+    toLog += C()
+    toLog += ' ]'
+
+    console.log(toLog)
+  })
+  console.log(hr())
+  console.log('Old data:')
+  LogDataCount(oldData)
+  console.log('New data:')
+  LogDataCount(newData)
+}
+
+function LogDataCount(data) {
+  const subjLength = data.length
+  const qLength = data.reduce((acc, subj) => {
+    return acc + subj.Questions.length
+  }, 0)
+
+  console.log(
+    'Subjects: ' +
+      C('green') +
+      subjLength +
+      C() +
+      ', Questions: ' +
+      C('green') +
+      qLength +
+      C()
+  )
+}
+
+function PrintDB(data) {
+  const maxSubjNameLength = MaxLengthOf(data, 'Name')
+
+  data.forEach((subj) => {
+    let toLog = ''
+    toLog += C('green')
+    toLog += GetExactLength(subj.Name, maxSubjNameLength)
+    toLog += C()
+    toLog += ' [ '
+    toLog += C('cyan')
+    toLog += subj.Questions.length
+    toLog += C()
+    toLog += ' ]'
+
+    console.log(toLog)
+  })
+  console.log(hr())
+  LogDataCount(data)
+  console.log(hr())
+}
+
+function GetExactLength(string, length) {
+  let toLog = string.toString()
+  const lengthDiff = length - toLog.length
+  for (let i = 0; i < lengthDiff; i++) {
+    toLog += ' '
+  }
+
+  return toLog
+}
+
+function MaxLengthOf(prop, key) {
+  return prop.reduce((acc, currStat) => {
+    if (acc < currStat[key].toString().length) {
+      acc = currStat[key].toString().length
+    }
+    return acc
+  }, 0)
+}
+
+async function RemoveDuplicates(data) {
+  console.log(C('yellow') + 'Removing duplicates' + C())
+  let res = []
+  const stats = []
+
+  for (let i = 0; i < data.length; i++) {
+    const subj = data[i]
+    const logFile =
+      logPath + '/' + subj.Name.replace(/ /g, '_').replace(/\//g, '-')
+    let addedQuestions = 0
+    let removedQuestions = 0
+    for (let j = 0; j < subj.Questions.length; j++) {
+      const question = subj.Questions[j]
+      try {
+        // Searching for same question in result database
+        const tempQuestion = createQuestion(
+          question.Q,
+          question.A,
+          question.data
+        )
+        let result = await searchData(res, tempQuestion)
+        result = result.reduce((acc, res) => {
+          if (res.match >= minMatchAmmount) {
+            acc.push(res)
+          }
+          return acc
+        }, [])
+
+        // if htere are more that one same questions in the new database
+        if (result.length > 0) {
+          utils.AppendToFile(hr('#'), logFile)
+          utils.AppendToFile('QUESTION', logFile)
+          utils.AppendToFile(JSON.stringify(tempQuestion, null, 2), logFile)
+          utils.AppendToFile(hr(), logFile)
+          utils.AppendToFile('SAMES', logFile)
+          utils.AppendToFile(JSON.stringify(result, null, 2), logFile)
+          removedQuestions++
+        } else {
+          // if no same questions are fount then adding it to then new db
+          addQuestion(res, getSubjNameWithoutYear(subj.Name), tempQuestion)
+          addedQuestions++
+        }
+        LogResultProgress(
+          subj,
+          i,
+          j,
+          subj.Questions.length,
+          addedQuestions,
+          removedQuestions,
+          data.length
+        )
+      } catch (err) {
+        console.log()
+        console.log('ERROR')
+        console.log(err)
+        console.log('QUESTION')
+        console.log(question)
+        console.log()
+      }
+    }
+    stats.push({
+      name: subj.Name,
+      prevQuestions: subj.Questions.length,
+      addedQuestions: addedQuestions,
+      removedQuestions: removedQuestions,
+    })
+  }
+  return { res, stats }
+}
+
+function LogResultProgress(
+  subj,
+  i,
+  j,
+  length,
+  addedQuestions,
+  removedQuestions,
+  subjCount
+) {
+  process.stdout.write(
+    '[ ' +
+      C('cyan') +
+      (i + 1) +
+      C() +
+      ' / ' +
+      C('green') +
+      subjCount +
+      C() +
+      ' ] ' +
+      C('yellow') +
+      subj.Name +
+      C() +
+      ': ' +
+      C('green') +
+      subj.Questions.length +
+      ' ' +
+      C('cyan') +
+      '->  ' +
+      C('green') +
+      addedQuestions +
+      C() +
+      ', removed: ' +
+      C('red') +
+      removedQuestions +
+      C()
+  )
+
+  if (j === length - 1) {
+    process.stdout.write('\n')
+  } else {
+    process.stdout.write('\r')
+  }
+}
+
+function hr(char) {
+  let hr = ''
+  const cols = process.stdout.columns || 20
+  for (let i = 0; i < cols; i++) {
+    hr += char || '-'
+  }
+  return hr
+}
+
+function C(color) {
+  return logger.C(color)
+}
+
+function GetParams() {
+  return process.argv.splice(2)
+}
+
+function GetDateString() {
+  const date = new Date()
+  const dateString =
+    date.getFullYear() +
+    '-' +
+    ('0' + (date.getMonth() + 1)).slice(-2) +
+    '-' +
+    ('0' + date.getDate()).slice(-2) +
+    ' ' +
+    ('0' + date.getHours()).slice(-2) +
+    ':' +
+    ('0' + date.getMinutes()).slice(-2) +
+    ':' +
+    ('0' + date.getSeconds()).slice(-2)
+  return dateString
+}