Lotsa typescript bullshit

src/standaloneUtils/createUserDataList.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+scp -P 19192 qminer@megacarrot.frylabs.net:/home/qminer/mrfrys-node-server/stats/idstats "./idstats.new"
+scp -P 19192 qminer@megacarrot.frylabs.net:/home/qminer/old.nodeServ/stats/idstats "./idstats.old"
+
+cat idstats.new | jq 'keys' > idstats.new.keys
+cat idstats.old | jq 'keys' > idstats.old.keys
+
+cat idstats.new.keys idstats.old.keys | sort | uniq > keys.tmp
+
+cat keys.tmp | grep -Eo '[0-9]{13}' > ids
+
+rm idstats.new
+rm idstats.new.keys
+rm idstats.old
+rm idstats.old.keys
+rm keys.tmp

src/standaloneUtils/dbSetup.js

@@ -0,0 +1,72 @@
+const utils = require('../utils/utils.js')
+const logger = require('../utils/logger.js')
+const dbtools = require('../utils/dbtools.js')
+const dbStructPath = '../modules/api/apiDBStruct.json'
+const usersDBPath = '../data/dbs/users.db'
+const { v4: uuidv4 } = require('uuid')
+
+let authDB
+
+console.clear()
+CreateDB()
+
+authDB.close()
+
+function CreateDB() {
+  const dbStruct = utils.ReadJSON(dbStructPath)
+  // authDB = dbtools.GetDB(':memory:')
+  authDB = dbtools.GetDB(usersDBPath)
+  authDB.pragma('synchronous = OFF')
+
+  Object.keys(dbStruct).forEach((tableName) => {
+    const tableData = dbStruct[tableName]
+    dbtools.CreateTable(
+      authDB,
+      tableName,
+      tableData.tableStruct,
+      tableData.foreignKey
+    )
+  })
+
+  try {
+    if (utils.FileExists('./ids')) {
+      const uids = utils.ReadFile('./ids').split('\n')
+
+      uids.forEach((cid, i) => {
+        if (!cid) {
+          return
+        }
+        logger.Log(`[ ${i} / ${uids.length} ]`)
+        try {
+          dbtools.Insert(authDB, 'users', {
+            pw: uuidv4(),
+            oldCID: cid,
+            avaiblePWRequests: 4,
+            created: utils.GetDateString(),
+          })
+        } catch (e) {
+          logger.Log('Error during inserting', logger.GetColor('redbg'))
+          console.error(e)
+        }
+      })
+    }
+  } catch (e) {
+    console.error(e)
+  }
+
+  const dir = `./dbSetupResult/${utils.GetDateString().replace(/ /g, '_')}`
+  utils.CreatePath(dir)
+  Object.keys(dbStruct).forEach((key) => {
+    const path = `${dir}/${key}.json`
+    logger.Log(`Writing ${path}...`)
+    utils.WriteFile(
+      JSON.stringify({
+        tableInfo: dbtools.TableInfo(authDB, key),
+        tableRows: dbtools.SelectAll(authDB, key),
+      }),
+      path
+    )
+  })
+
+  logger.Log('Done')
+}

src/standaloneUtils/merge.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+p=$(echo $PWD | rev | cut -d'/' -f2- | rev)
+
+cp -v $p/public/data.json /tmp/data.json
+node $p/utils/rmDuplicates.js /tmp/data.json 2> /dev/null
+
+mkdir -p "$p/public/backs/"
+mv -v $p/public/data.json "$p/public/backs/data.json $(date)"
+mv -v $p/utils/res.json $p/public/data.json
+
+echo Done

src/standaloneUtils/rmDuplicates.js

@@ -0,0 +1,309 @@
+/* ----------------------------------------------------------------------------
+
+ Question Server question file merger
+ GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+ ------------------------------------------------------------------------- */
+
+const utils = require('./utils.js')
+const {
+  searchData,
+  addQuestion,
+  getSubjNameWithoutYear,
+  createQuestion,
+} = require('./classes.js')
+const actions = require('./actions.js')
+const logger = require('./logger.js')
+
+const resultFileName = 'res.json'
+const minMatchAmmount = 100
+
+const logPath = './mergeLogs/mergelog_' + GetDateString().replace(/ /g, '_')
+
+Main()
+
+async function Main() {
+  const params = GetParams()
+  console.log(params)
+  if (params.length === 0) {
+    console.error('No params! Need a path to a question database!')
+    process.exit()
+  }
+  const data = actions.LoadJSON(params[0])
+
+  PrintDB(data)
+  console.log(hr('='))
+
+  const { res, stats } = await RemoveDuplicates(data)
+  console.log(hr('='))
+
+  LogStats(stats, data, res)
+  console.log(hr('='))
+
+  console.log('Result database:')
+  PrintDB(res)
+  console.log(hr('='))
+
+  utils.WriteFile(JSON.stringify(res), resultFileName)
+  console.log(C('green') + resultFileName + ' written!' + C())
+  console.log(hr('='))
+
+  console.log(C('green') + 'Done' + C())
+}
+
+function LogStats(stats, oldData, newData) {
+  const maxSubjNameLength = MaxLengthOf(stats, 'name')
+  const maxPrevLength = MaxLengthOf(stats, 'prevQuestions')
+  const maxAddedLength = MaxLengthOf(stats, 'addedQuestions')
+  const maxRemovedLength = MaxLengthOf(stats, 'removedQuestions')
+
+  stats.forEach((currStat) => {
+    const { name, prevQuestions, addedQuestions, removedQuestions } = currStat
+    let toLog = ''
+
+    toLog += C('green')
+    toLog += GetExactLength(name, maxSubjNameLength)
+    toLog += C()
+    toLog += ' '
+    toLog += C('magenta')
+    toLog += GetExactLength(prevQuestions, maxPrevLength)
+    toLog += C()
+    toLog += C('cyan')
+    toLog += ' -> '
+    toLog += C()
+    toLog += C('green')
+    toLog += GetExactLength(addedQuestions, maxAddedLength)
+    toLog += C()
+    toLog += ' [ '
+    toLog += C('red')
+    toLog += GetExactLength(removedQuestions, maxRemovedLength)
+    toLog += C()
+    toLog += ' ]'
+
+    console.log(toLog)
+  })
+  console.log(hr())
+  console.log('Old data:')
+  LogDataCount(oldData)
+  console.log('New data:')
+  LogDataCount(newData)
+}
+
+function LogDataCount(data) {
+  const subjLength = data.length
+  const qLength = data.reduce((acc, subj) => {
+    return acc + subj.Questions.length
+  }, 0)
+
+  console.log(
+    'Subjects: ' +
+      C('green') +
+      subjLength +
+      C() +
+      ', Questions: ' +
+      C('green') +
+      qLength +
+      C()
+  )
+}
+
+function PrintDB(data) {
+  const maxSubjNameLength = MaxLengthOf(data, 'Name')
+
+  data.forEach((subj) => {
+    let toLog = ''
+    toLog += C('green')
+    toLog += GetExactLength(subj.Name, maxSubjNameLength)
+    toLog += C()
+    toLog += ' [ '
+    toLog += C('cyan')
+    toLog += subj.Questions.length
+    toLog += C()
+    toLog += ' ]'
+
+    console.log(toLog)
+  })
+  console.log(hr())
+  LogDataCount(data)
+  console.log(hr())
+}
+
+function GetExactLength(string, length) {
+  let toLog = string.toString()
+  const lengthDiff = length - toLog.length
+  for (let i = 0; i < lengthDiff; i++) {
+    toLog += ' '
+  }
+
+  return toLog
+}
+
+function MaxLengthOf(prop, key) {
+  return prop.reduce((acc, currStat) => {
+    if (acc < currStat[key].toString().length) {
+      acc = currStat[key].toString().length
+    }
+    return acc
+  }, 0)
+}
+
+async function RemoveDuplicates(data) {
+  console.log(C('yellow') + 'Removing duplicates' + C())
+  let res = []
+  const stats = []
+
+  for (let i = 0; i < data.length; i++) {
+    const subj = data[i]
+    const logFile =
+      logPath + '/' + subj.Name.replace(/ /g, '_').replace(/\//g, '-')
+    let addedQuestions = 0
+    let removedQuestions = 0
+    for (let j = 0; j < subj.Questions.length; j++) {
+      const question = subj.Questions[j]
+      try {
+        // Searching for same question in result database
+        const tempQuestion = createQuestion(
+          question.Q,
+          question.A,
+          question.data
+        )
+        let result = await searchData(res, tempQuestion)
+        result = result.reduce((acc, res) => {
+          if (res.match >= minMatchAmmount) {
+            acc.push(res)
+          }
+          return acc
+        }, [])
+
+        // if htere are more that one same questions in the new database
+        if (result.length > 0) {
+          utils.AppendToFile(hr('#'), logFile)
+          utils.AppendToFile('QUESTION', logFile)
+          utils.AppendToFile(JSON.stringify(tempQuestion, null, 2), logFile)
+          utils.AppendToFile(hr(), logFile)
+          utils.AppendToFile('SAMES', logFile)
+          utils.AppendToFile(JSON.stringify(result, null, 2), logFile)
+          removedQuestions++
+        } else {
+          // if no same questions are fount then adding it to then new db
+          addQuestion(res, getSubjNameWithoutYear(subj.Name), tempQuestion)
+          addedQuestions++
+        }
+        LogResultProgress(
+          subj,
+          i,
+          j,
+          subj.Questions.length,
+          addedQuestions,
+          removedQuestions,
+          data.length
+        )
+      } catch (err) {
+        console.log()
+        console.log('ERROR')
+        console.log(err)
+        console.log('QUESTION')
+        console.log(question)
+        console.log()
+      }
+    }
+    stats.push({
+      name: subj.Name,
+      prevQuestions: subj.Questions.length,
+      addedQuestions: addedQuestions,
+      removedQuestions: removedQuestions,
+    })
+  }
+  return { res, stats }
+}
+
+function LogResultProgress(
+  subj,
+  i,
+  j,
+  length,
+  addedQuestions,
+  removedQuestions,
+  subjCount
+) {
+  process.stdout.write(
+    '[ ' +
+      C('cyan') +
+      (i + 1) +
+      C() +
+      ' / ' +
+      C('green') +
+      subjCount +
+      C() +
+      ' ] ' +
+      C('yellow') +
+      subj.Name +
+      C() +
+      ': ' +
+      C('green') +
+      subj.Questions.length +
+      ' ' +
+      C('cyan') +
+      '->  ' +
+      C('green') +
+      addedQuestions +
+      C() +
+      ', removed: ' +
+      C('red') +
+      removedQuestions +
+      C()
+  )
+
+  if (j === length - 1) {
+    process.stdout.write('\n')
+  } else {
+    process.stdout.write('\r')
+  }
+}
+
+function hr(char) {
+  let hr = ''
+  const cols = process.stdout.columns || 20
+  for (let i = 0; i < cols; i++) {
+    hr += char || '-'
+  }
+  return hr
+}
+
+function C(color) {
+  return logger.C(color)
+}
+
+function GetParams() {
+  return process.argv.splice(2)
+}
+
+function GetDateString() {
+  const date = new Date()
+  const dateString =
+    date.getFullYear() +
+    '-' +
+    ('0' + (date.getMonth() + 1)).slice(-2) +
+    '-' +
+    ('0' + date.getDate()).slice(-2) +
+    ' ' +
+    ('0' + date.getHours()).slice(-2) +
+    ':' +
+    ('0' + date.getMinutes()).slice(-2) +
+    ':' +
+    ('0' + date.getSeconds()).slice(-2)
+  return dateString
+}

src/standaloneUtils/runSqliteCmds.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ "$#" -lt "2" ]; then
+  echo "No params! 2 file required: db, commands file"
+  echo "usage: ./runSqliteCmds db.db commands"
+  exit 1
+fi
+
+echo "Executing:"
+cat $2
+echo
+
+cmd=''
+
+while read p; do
+  cmd="$cmd -cmd \"${p}\" -cmd \".shell echo\""
+done <"$2"
+
+echo "sqlite3 -bail $1 $cmd"
+eval "sqlite3 -bail $1 $cmd" > cmdRes 2> /dev/null
+
+echo "Done, result written to cmdRes file!"

src/standaloneUtils/sqliteBatchCommands/showAll

@@ -0,0 +1,9 @@
+.mode column
+.headers ON
+select * from users 
+select * from sessions
+select * from veteranPWRequests
+select * from accesses
+.tables
+.bail
+select * from EXIT

src/standaloneUtils/sqliteTest.js

@@ -0,0 +1,78 @@
+// https://www.sqlitetutorial.net/sqlite-nodejs/
+// https://github.com/JoshuaWise/better-sqlite3/blob/HEAD/docs/api.md
+
+const dbtools = require('../utils/dbtools.js')
+
+Main()
+
+function Main() {
+  const cols = {
+    uname: {
+      type: 'text',
+    },
+    pw: {
+      type: 'text',
+    },
+    notes: {
+      type: 'text',
+    },
+  }
+  const dbName = 'test'
+
+  const db = dbtools.GetDB('./testdb.db')
+
+  // Creating table
+  dbtools.CreateTable(db, dbName, cols)
+  console.log(dbtools.TableInfo(db, dbName))
+  dbtools.SelectAll(db, dbName)
+  // inserting test val to table
+  dbtools.Insert(db, dbName, {
+    uname: 'mrfry',
+    pw: 'dsads',
+  })
+  // Selecting a record
+  console.log(
+    dbtools.Select(db, dbName, {
+      uname: 'mrfry',
+    })
+  )
+  console.log(dbtools.TableInfo(db, dbName))
+  console.log(dbtools.SelectAll(db, dbName))
+  // Updating record
+  dbtools.Update(
+    db,
+    dbName,
+    {
+      pw: 'sspw',
+    },
+    {
+      uname: 'mrfry',
+    }
+  )
+  console.log(dbtools.SelectAll(db, dbName))
+  // Updating record again
+  dbtools.Update(
+    db,
+    dbName,
+    {
+      notes: 'new note!',
+    },
+    {
+      uname: 'mrfry',
+    }
+  )
+  console.log(dbtools.SelectAll(db, dbName))
+  // Adding new column and
+  dbtools.AddColumn(db, dbName, {
+    test: 'text',
+  })
+  console.log(dbtools.TableInfo(db, dbName))
+  console.log(dbtools.SelectAll(db, dbName))
+  // Deleting stuff
+  dbtools.Delete(db, dbName, {
+    uname: 'mrfry',
+  })
+  console.log(dbtools.SelectAll(db, dbName))
+
+  dbtools.CloseDB(db)
+}