/* ----------------------------------------------------------------------------

 Question Server
 GitLab: <https://gitlab.com/MrFry/mrfrys-node-server>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.

 ------------------------------------------------------------------------- */

import { Response } from 'express'

import logger from '../../../utils/logger'
import {
    Request,
    SubmoduleData,
    Submodule,
    PeerInfo,
    Subject,
    QuestionDb,
    User,
    DataFile,
} from '../../../types/basicTypes'
import utils from '../../../utils/utils'
import { backupData, writeData } from '../../../utils/actions'
import dbtools from '../../../utils/dbtools'
import {
    createKeyPair,
    decrypt,
    encrypt,
    isKeypairValid,
} from '../../../utils/encryption'
import {
    countOfQdb,
    countOfQdbs,
    createQuestion,
    getAvailableQdbIndexes,
    removeCacheFromQuestion,
} from '../../../utils/qdbUtils'
import {
    files,
    paths,
    publicDir,
    readAndValidateFile,
} from '../../../utils/files'
import { GetResult, downloadFile, get } from '../../../utils/networkUtils'
import {
    msgAllWorker,
    queueWork,
    setPendingJobsAlertCount,
} from '../../../worker/workerPool'
import { WorkerResult } from '../../../worker/worker'
import {
    isPeerSameAs,
    loginToPeer,
    peerToString,
    updatePeersFile,
} from '../../../utils/p2putils'
import { Database } from 'better-sqlite3'
import constants from '../../../constants'
import path from 'node:path'
import { UserDirDataFile } from './userFiles'

interface MergeResult {
    newData: Subject[]
    newSubjects: Subject[]
    localQdbIndex: number
    e: Error
}

interface RemotePeerInfo {
    selfInfo: PeerInfo
    myPeers: PeerInfo[]
    serverRevision?: string
    scriptRevision?: string
    qminingPageRevision?: string
    dataEditorRevision?: string
    serverLastCommitDate?: number
    scriptLastCommitDate?: number
    qminingPageLastCommitDate?: number
    dataEditorLastCommitDate?: number
    serverBuildTime?: number
    qminingPageBuildTime?: number
    dataEditorBuildTime?: number
    scriptVersion?: string
    userCount?: number
    qdbInfo?: {
        questionDbCount: number
        subjectCount: number
        questionCount: number
    }
}

interface SyncResult {
    old?: { [key: string]: number }
    added?: { [key: string]: number }
    final?: { [key: string]: number }
    msg?: string
}

interface SyncDataResult {
    remoteInfo?: RemotePeerInfo
    users?: {
        encryptedUsers: string
    }
    questions?: {
        questionDbs: QuestionDb[]
        count: {
            qdbs: number
            subjects: number
            questions: number
        }
    }
    userFiles?: {
        newFiles: {
            [key: string]: {
                [key: string]: UserDirDataFile
            }
        }
    }
}

function updateThirdPartyPeers(
    newVal: Omit<PeerInfo, 'publicKey' | 'name' | 'contact'>[]
) {
    const prevVal = utils.FileExists(paths.thirdPartyPeersFile)
        ? utils.ReadJSON<PeerInfo[]>(paths.thirdPartyPeersFile)
        : []

    const dataToWrite = newVal.reduce((acc, peer) => {
        const isIncluded = acc.find((x) => {
            return peerToString(x) === peerToString(peer)
        })
        if (!isIncluded) {
            return [...acc, peer]
        }
        return acc
    }, prevVal)

    utils.WriteFile(
        JSON.stringify(dataToWrite, null, 2),
        paths.thirdPartyPeersFile
    )
}

export function getNewDataSince(subjects: Subject[], date: number): Subject[] {
    return subjects
        .map((subject) => {
            return {
                ...subject,
                Questions: subject.Questions.filter((question) => {
                    return (question.data.date || 0) >= date
                }).map((question) => removeCacheFromQuestion(question)),
            }
        })
        .filter((subject) => subject.Questions.length !== 0)
}

export function mergeSubjects(
    subjectsToMergeTo: Subject[],
    subjectsToMerge: Subject[],
    newSubjects: Subject[]
): Subject[] {
    return [
        ...subjectsToMergeTo.map((subj) => {
            const newSubjs = subjectsToMerge.filter(
                (subjRes) => subjRes.Name === subj.Name
            )

            if (newSubjs) {
                const newQuestions = newSubjs.flatMap((subj) => {
                    return subj.Questions
                })

                return {
                    ...subj,
                    Questions: [...subj.Questions, ...newQuestions],
                }
            } else {
                return subj
            }
        }),
        ...newSubjects,
    ]
}

export function mergeQdbs(
    qdbToMergeTo: QuestionDb[],
    mergeResults: MergeResult[]
): { mergedQuestionDbs: QuestionDb[]; changedQdbIndexes: number[] } {
    const changedQdbIndexes: number[] = []
    const mergedQuestionDbs = qdbToMergeTo.map((qdb) => {
        const qdbMergeResult = mergeResults.find(
            (mergeRes) => mergeRes.localQdbIndex === qdb.index
        )
        if (
            qdbMergeResult &&
            (qdbMergeResult.newData.length > 0 ||
                qdbMergeResult.newSubjects.length > 0)
        ) {
            const mergedQdb = {
                ...qdb,
                data: mergeSubjects(
                    qdb.data,
                    qdbMergeResult.newData,
                    qdbMergeResult.newSubjects
                ),
            }
            changedQdbIndexes.push(qdb.index)
            return mergedQdb
        } else {
            // unchanged
            return qdb
        }
    })
    return {
        mergedQuestionDbs: mergedQuestionDbs,
        changedQdbIndexes: changedQdbIndexes,
    }
}

async function sendNewDataToWorkers(
    mergeResults: MergeResult[],
    newQuestionDbs: QuestionDb[]
) {
    // FIXME: this might be slow, maybe make a new type of message for workers?
    const updatePromises: Promise<WorkerResult[]>[] = []
    let newQuestionCount = 0
    let newSubjectCount = 0
    let newQuestionDbCount = 0

    mergeResults.forEach((mergeRes) => {
        if (mergeRes.e) {
            logger.Log(`There was an error processing the merge!`, 'redbg')
            console.error(mergeRes.e)
            return
        }

        mergeRes.newData.forEach((subjectWithNewData) => {
            newQuestionCount += subjectWithNewData.Questions.length
            updatePromises.push(
                msgAllWorker({
                    type: 'newQuestions',
                    data: {
                        subjName: subjectWithNewData.Name,
                        qdbIndex: mergeRes.localQdbIndex,
                        newQuestions: subjectWithNewData.Questions,
                    },
                })
            )
        })

        newSubjectCount += mergeRes.newSubjects.length
        mergeRes.newSubjects.forEach((newSubject) => {
            newQuestionCount += newSubject.Questions.length
            updatePromises.push(
                msgAllWorker({
                    type: 'newQuestions',
                    data: {
                        subjName: newSubject.Name,
                        qdbIndex: mergeRes.localQdbIndex,
                        newQuestions: newSubject.Questions,
                    },
                })
            )
        })
    })
    newQuestionDbCount += newQuestionDbs.length
    newQuestionDbs.forEach((newQdb) => {
        const { subjCount: sc, questionCount: qc } = countOfQdb(newQdb)
        newSubjectCount += sc
        newQuestionCount += qc
        msgAllWorker({
            data: newQdb,
            type: 'newdb',
        })
    })

    await Promise.all(updatePromises)

    return {
        newQuestionDbCount: newQuestionDbCount,
        newSubjectCount: newSubjectCount,
        newQuestionCount: newQuestionCount,
    }
}

function writeNewData(
    newQuestionDbs: QuestionDb[],
    changedQuestionDbs: QuestionDb[],
    dbsFilePath: string
) {
    const qdbsToWrite = [...changedQuestionDbs, ...newQuestionDbs]
    const existingQdbs = utils.ReadJSON<DataFile[]>(dbsFilePath)

    const qdbDataToWrite = qdbsToWrite
        .reduce((acc, qdb) => {
            // eslint-disable-next-line @typescript-eslint/no-unused-vars
            const { data, index, ...qdbData } = qdb
            const existingQdbData = acc.find((data) => {
                return data.name === qdbData.name
            })
            if (!existingQdbData) {
                return [...acc, qdbData]
            } else {
                return acc
            }
        }, existingQdbs)
        .map((qdb) => {
            if (qdb.path.includes(publicDir)) {
                return {
                    ...qdb,
                    path: qdb.path.replace(publicDir, ''),
                }
            } else {
                return qdb
            }
        })

    utils.WriteFile(JSON.stringify(qdbDataToWrite, null, 2), dbsFilePath)
    qdbsToWrite.forEach((qdb) => {
        try {
            writeData(qdb.data, qdb.path)
        } catch (e) {
            logger.Log(`Error writing ${qdb.name} qdb to file!`, 'redbg')
            console.error(e)
        }
    })
}

function updateLastSync(selfInfo: PeerInfo, newDate: number) {
    utils.WriteFile(
        JSON.stringify({ ...selfInfo, lastSync: newDate }, null, 2),
        paths.selfInfoFile
    )
}

function setupQuestionsForMerge(qdb: QuestionDb, peer: PeerInfo) {
    return {
        ...qdb,
        data: qdb.data.map((subj) => {
            return {
                ...subj,
                Questions: subj.Questions.map((q) => {
                    const initializedQuestion = q.cache ? q : createQuestion(q)
                    initializedQuestion.data.source = peerToString(peer)
                    return initializedQuestion
                }),
            }
        }),
    }
}

async function authAndGetNewData({
    peer,
    selfInfo,
    allTime,
    shouldSync,
}: {
    peer: PeerInfo
    selfInfo: PeerInfo
    allTime?: boolean
    shouldSync: {
        questions: boolean
        users: boolean
        userFiles: boolean
    }
}): Promise<GetResult<SyncDataResult> & { peer: PeerInfo }> {
    try {
        const syncAll =
            !shouldSync ||
            Object.values(shouldSync).filter((x) => x).length === 0
        let sessionCookie = peer.sessionCookie

        const login = async () => {
            const loginResult = await loginToPeer(peer)
            if (typeof loginResult === 'string') {
                sessionCookie = loginResult
                updatePeersFile(peer, { sessionCookie: loginResult })
            } else {
                throw {
                    error: loginResult,
                    data: {
                        peer: peer,
                    },
                }
            }
        }

        if (!sessionCookie) {
            await login()
        }

        const getData = async <T>(path: string) => {
            return get<T>(
                {
                    headers: {
                        cookie: `sessionID=${sessionCookie}`,
                    },
                    host: peer.host,
                    port: peer.port,
                    path: path,
                },
                peer.http
            )
        }

        let result: GetResult<SyncDataResult>

        const setResult = async () => {
            let url = `/api/getnewdata?host=${encodeURIComponent(
                peerToString(selfInfo)
            )}`

            if (!allTime) {
                url += `&questionsSince=${peer.lastQuestionsSync}`
                url += `&usersSince=${peer.lastUsersSync}`
                url += `&userFilesSince=${peer.lastUserFilesSync}`
            }

            if (!syncAll) {
                if (shouldSync.questions) {
                    url += '&questions=true'
                }
                if (shouldSync.users) {
                    url += '&users=true'
                }
                if (shouldSync.userFiles) {
                    url += '&userFiles=true'
                }
            }

            result = await getData<SyncDataResult>(url)
        }

        await setResult()

        const hasNoUser = Object.values(result).find((res) => {
            return res.data?.result === 'nouser'
        })
        if (hasNoUser) {
            await login()
            result = {}
            await setResult()
        }

        return { ...result, peer: peer }
    } catch (e) {
        console.error(e)
        return { error: e, peer: peer }
    }
}

function addUsersToDb(
    users: User[],
    userDB: Database,
    extraProps: Partial<User>
) {
    let addedUserCount = 0
    users.forEach((remoteUser) => {
        // eslint-disable-next-line @typescript-eslint/no-unused-vars
        const { id, ...remoteUserWithoutId } = remoteUser
        const localUser = dbtools.Select(userDB, 'users', {
            pw: remoteUser.pw,
        })
        if (localUser.length === 0) {
            addedUserCount += 1
            // FIXME: users will not have consistend id across servers. This may be
            // harmless, will see
            dbtools.Insert(userDB, 'users', {
                ...(remoteUserWithoutId as Omit<User, 'id'>),
                ...extraProps,
            })
        }
    })
    return addedUserCount
}

function getNewUserFilesSince(since: number) {
    const newData: SyncDataResult['userFiles']['newFiles'] = {}

    const dirs = utils.ReadDir(paths.userFilesDir)
    dirs.forEach((dir) => {
        const userDirPath = path.join(paths.userFilesDir, dir)
        const dataFilePath = path.join(
            userDirPath,
            constants.userFilesDataFileName
        )

        if (!utils.FileExists(dataFilePath)) {
            return
        }
        const dataFile =
            utils.ReadJSON<Map<string, UserDirDataFile>>(dataFilePath)
        Object.entries(dataFile).forEach(([fileName, data]) => {
            const mtime = utils.statFile(path.join(userDirPath, fileName)).mtime
            if (mtime.getTime() >= since) {
                if (!newData[dir]) {
                    newData[dir] = {}
                }
                newData[dir][fileName] = data
            }
        })
    })

    return newData
}

function setup(data: SubmoduleData): Submodule {
    const {
        app,
        userDB,
        moduleSpecificData: { setQuestionDbs, getQuestionDbs, dbsFile },
    } = data

    let syncInProgress = false

    // ---------------------------------------------------------------------------------------
    // SETUP
    // ---------------------------------------------------------------------------------------

    let publicKey: string
    let privateKey: string

    if (
        !utils.FileExists(paths.keyFile + '.priv') ||
        !utils.FileExists(paths.keyFile + '.pub')
    ) {
        createKeyPair().then(({ publicKey: pubk, privateKey: privk }) => {
            // at first start there won't be a keypair available until this finishes
            utils.WriteFile(pubk, paths.keyFile + '.pub')
            utils.WriteFile(privk, paths.keyFile + '.priv')

            publicKey = pubk
            privateKey = privk
        })
        logger.Log(
            'There were no public / private keys for p2p functionality, created new ones',
            'yellowbg'
        )
    } else {
        publicKey = utils.ReadFile(paths.keyFile + '.pub')
        privateKey = utils.ReadFile(paths.keyFile + '.priv')
        // checking only here, because if it got generated in the other branch then it must be good
        if (!isKeypairValid(publicKey, privateKey)) {
            logger.Log('Loaded keypair is not valid!', 'redbg')
        }
    }

    let peers: PeerInfo[] = utils.ReadJSON(paths.peersFile)
    let selfInfo: PeerInfo = utils.ReadJSON(paths.selfInfoFile)
    selfInfo.publicKey = publicKey

    const filesToWatch = [
        {
            fname: paths.peersFile,
            logMsg: 'Peers file updated',
            action: () => {
                const newVal = readAndValidateFile<PeerInfo[]>(files.peersFile)
                if (newVal) {
                    peers = newVal
                }
            },
        },
        {
            fname: paths.selfInfoFile,
            logMsg: 'P2P self info file changed',
            action: () => {
                const newVal = readAndValidateFile<PeerInfo>(files.selfInfoFile)
                if (newVal) {
                    selfInfo = newVal
                }
            },
        },
    ]

    filesToWatch.forEach((ftw) => {
        if (utils.FileExists(ftw.fname)) {
            utils.WatchFile(ftw.fname, () => {
                logger.Log(ftw.logMsg)
                ftw.action()
            })
            ftw.action()
        } else {
            logger.Log(`File ${ftw.fname} does not exists to watch!`, 'redbg')
        }
    })

    if (peers.length === 0) {
        logger.Log(
            `Warning: peers file is empty. You probably want to fill it`,
            'yellowbg'
        )
    }

    // ---------------------------------------------------------------------------------------
    // FUNCTIONS
    // ---------------------------------------------------------------------------------------

    function getSelfInfo(includeVerboseInfo?: boolean) {
        const result: RemotePeerInfo = {
            selfInfo: { ...selfInfo, publicKey: publicKey },
            myPeers: peers.map((peer) => {
                // eslint-disable-next-line @typescript-eslint/no-unused-vars
                const { pw, sessionCookie, ...restOfPeer } = peer
                return restOfPeer
            }),
        }

        if (includeVerboseInfo) {
            const serverRevision = utils.getGitInfo(__dirname)
            result.serverRevision = serverRevision.revision
            result.serverLastCommitDate = serverRevision.lastCommitDate

            const scriptRevision = utils.getGitInfo(
                paths.moodleTestUserscriptDir
            )
            result.scriptRevision = scriptRevision.revision
            result.scriptLastCommitDate = scriptRevision.lastCommitDate

            const qminingPageRevision = utils.getGitInfo(paths.qminingPageDir)
            result.qminingPageRevision = qminingPageRevision.revision
            result.qminingPageLastCommitDate =
                qminingPageRevision.lastCommitDate

            const dataEditorRevision = utils.getGitInfo(paths.dataEditorPageDir)
            result.dataEditorRevision = dataEditorRevision.revision
            result.dataEditorLastCommitDate = dataEditorRevision.lastCommitDate

            result.qminingPageBuildTime = utils
                .statFile(paths.qminingIndexPath)
                ?.mtime.getTime()
            result.serverBuildTime = utils
                .statFile(paths.serverPath)
                ?.mtime.getTime()
            result.dataEditorBuildTime = utils
                .statFile(paths.dataEditorIndexPath)
                ?.mtime.getTime()
            result.scriptVersion = utils.getScriptVersion()
            result.userCount = dbtools.TableInfo(userDB, 'users').dataCount

            const questionDbCount = getQuestionDbs().length
            const { subjCount, questionCount } = countOfQdbs(getQuestionDbs())
            result.qdbInfo = {
                questionDbCount: questionDbCount,
                subjectCount: subjCount,
                questionCount: questionCount,
            }
        }

        return result
    }

    function getNewUsersSince(since: number) {
        const users: User[] = dbtools.runStatement(
            userDB,
            `SELECT *
            FROM users
            WHERE created >= ${since}
            AND id != 1
            AND isAdmin is null
            ;`
        )
        return users
    }

    function updateQdbForLocalUse(qdb: QuestionDb[]) {
        const availableIndexes = getAvailableQdbIndexes(
            getQuestionDbs(),
            qdb.length
        )
        return qdb.map((qdb, i) => {
            return {
                ...qdb,
                index: availableIndexes[i],
                path: `${publicDir}questionDbs/${qdb.name}.json`,
            }
        })
    }

    async function getMergeResults(remoteQuestionDbs: QuestionDb[]) {
        const mergeJobs: Promise<any>[] = []
        const rawNewQuestionDbs: QuestionDb[] = []
        remoteQuestionDbs.forEach((remoteQdb) => {
            const localQdb = getQuestionDbs().find(
                (lqdb) => lqdb.name === remoteQdb.name
            )

            if (!localQdb) {
                rawNewQuestionDbs.push(remoteQdb)
            } else {
                mergeJobs.push(
                    queueWork({
                        type: 'merge',
                        data: {
                            localQdbIndex: localQdb.index,
                            remoteQdb: remoteQdb,
                        },
                    })
                )
            }
        })

        const mergeResults: MergeResult[] = await Promise.all(mergeJobs)

        return {
            mergeResults: mergeResults,
            rawNewQuestionDbs: rawNewQuestionDbs,
        }
    }

    async function syncData({
        shouldSync,
        allTime,
    }: {
        shouldSync: {
            questions: boolean
            users: boolean
            userFiles: boolean
        }
        allTime: boolean
    }) {
        if (peers.length === 0) {
            logger.Log(
                `There are no peers specified in ${paths.peersFile}, aborting sync`,
                'yellowbg'
            )
            return {
                msg: 'No peers specified, aborting',
            }
        }
        // FIXME: this might be blocking the main thread, but not sure how much
        logger.Log(
            `\tStarting data sync, getting new data from ${logger.C('green')}${
                peers.length
            }${logger.C()} peers`
        )

        const syncAll =
            !shouldSync ||
            Object.values(shouldSync).filter((x) => x).length === 0

        logger.Log(
            `\tSyncing: ${
                syncAll
                    ? 'everything'
                    : Object.entries(shouldSync)
                          // eslint-disable-next-line @typescript-eslint/no-unused-vars
                          .filter(([_key, value]) => value)
                          .map(([key]) => key)
                          .join(', ')
            }`,
            'green'
        )

        if (allTime) {
            logger.Log(`\tSyncing since all time!`, 'yellowbg')
        }

        const lastSync = selfInfo.lastSync
        logger.Log(
            `\tLast sync date: ${logger.C('blue')}${
                lastSync ? new Date(lastSync).toLocaleString() : 'never'
            }${logger.C()}`
        )
        const syncStart = new Date().getTime()
        const lastSyncInfos = peers.map((peer) => {
            return [
                peerToString(peer),
                peer.lastSync
                    ? new Date(peer.lastSync).toLocaleString()
                    : 'never',
            ]
        })
        logger.Log(`\tLast sync with peers:`)
        logger.logTable([['', 'Date'], ...lastSyncInfos], {
            colWidth: [20],
            rowPrefix: '\t',
        })

        const requests = peers.map((peer) => {
            return authAndGetNewData({
                peer: peer,
                selfInfo: selfInfo,
                allTime: allTime,
                shouldSync: shouldSync,
            })
        })

        const allResults = await Promise.all(requests)

        // -------------------------------------------------------------------------------------------------------
        // filtering, transforming, and counting responses
        // -------------------------------------------------------------------------------------------------------
        allResults.forEach((res) => {
            if (res?.error) {
                logger.Log(
                    `\tError syncing with ${peerToString(res.peer)}: ${
                        res.error.message
                    }`,
                    'red'
                )
            }
        })

        const resultDataWithoutErrors = allResults.filter(
            (resData) => !resData.error && resData.data
        )

        if (resultDataWithoutErrors.length === 0) {
            logger.Log(
                `No peers returned data without error, aborting sync`,
                'redbg'
            )
            return {
                msg: 'No peers returned data without error, aborting sync',
            }
        }

        // -------------------------------------------------------------------------------------------------------
        // third party peers handling
        // -------------------------------------------------------------------------------------------------------
        const peersHosts = [...peers, selfInfo]
        const thirdPartyPeers = resultDataWithoutErrors
            .map((res) => res.data.remoteInfo)
            .flatMap((res) => res.myPeers)
            .filter((res) => {
                return !peersHosts.some((localPeer) => {
                    return isPeerSameAs(localPeer, res)
                })
            })

        if (thirdPartyPeers.length > 0) {
            updateThirdPartyPeers(thirdPartyPeers)
            logger.Log(
                `\tPeers reported ${logger.C('green')}${
                    thirdPartyPeers.length
                }${logger.C()} third party peer(s) not connected to this server. See ${logger.C(
                    'blue'
                )}${paths.thirdPartyPeersFile}${logger.C()} for details`
            )
        }

        // -------------------------------------------------------------------------------------------------------
        // data syncing
        // -------------------------------------------------------------------------------------------------------

        const getData = <T extends keyof SyncDataResult>(key: T) => {
            return resultDataWithoutErrors
                .map((x) => ({ ...x.data[key], peer: x.peer }))
                .filter((x) => Object.keys(x).length > 1)
        }

        const syncResults: SyncResult[] = []

        const questionData = getData('questions')
        if (questionData && questionData.length > 0) {
            const res = await syncQuestions(questionData, syncStart)
            syncResults.push(res)
        }

        const userData = getData('users')
        if (userData && userData.length > 0) {
            const res = await syncUsers(userData, syncStart)
            syncResults.push(res)
        }

        const userFilesData = getData('userFiles')
        if (userFilesData && userFilesData.length > 0) {
            const res = await syncUserFiles(userFilesData, syncStart)
            syncResults.push(res)
        }

        return syncResults.reduce(
            (acc, x) => {
                return {
                    old: { ...acc.old, ...x.old },
                    added: { ...acc.added, ...x.added },
                    final: { ...acc.final, ...x.final },
                }
            },
            { old: {}, added: {}, final: {} }
        )
    }

    async function syncUsers(
        userData: (SyncDataResult['users'] & { peer: PeerInfo })[],
        syncStart: number
    ): Promise<SyncResult> {
        logger.Log('Syncing users...')
        let totalRecievedUsers = 0
        const resultsCount: {
            [key: string]: {
                newUsers?: number
            }
        } = {}
        const oldUserCount = dbtools.SelectAll(userDB, 'users').length

        try {
            userData.forEach((res) => {
                if (res.encryptedUsers) {
                    const decryptedUsers: User[] = JSON.parse(
                        decrypt(privateKey, res.encryptedUsers)
                    )
                    const addedUserCount = addUsersToDb(
                        decryptedUsers,
                        userDB,
                        {
                            sourceHost: peerToString(res.peer),
                        }
                    )
                    resultsCount[peerToString(res.peer)] = {
                        newUsers: addedUserCount,
                    }
                    totalRecievedUsers += decryptedUsers.length
                    updatePeersFile(res.peer, {
                        lastUsersSync: syncStart,
                    })
                }
            })
        } catch (e) {
            logger.Log(
                '\tError while trying to sync users: ' + e.message,
                'redbg'
            )
            console.error(e)
        }
        const newUserCount = dbtools.SelectAll(userDB, 'users').length

        if (totalRecievedUsers === 0) {
            logger.Log(
                `No peers returned any new users. User sync successfully finished!`,
                'green'
            )
        } else {
            logger.logTable(
                [
                    ['', 'Users'],
                    ['Old', oldUserCount],
                    ...Object.entries(resultsCount).map(([key, result]) => {
                        return [key, result.newUsers]
                    }),
                    ['Added total', newUserCount - oldUserCount],
                    ['Final', newUserCount],
                ],
                { colWidth: [20], rowPrefix: '\t' }
            )
            logger.Log(`Successfully synced users!`, 'green')
        }

        return {
            old: {
                users: oldUserCount,
            },
            added: {
                users: newUserCount - oldUserCount,
            },
            final: {
                users: newUserCount,
            },
        }
    }

    async function syncQuestions(
        questionData: (SyncDataResult['questions'] & { peer: PeerInfo })[],
        syncStart: number
    ): Promise<SyncResult> {
        logger.Log('Syncing questions...')
        const recievedDataCounts: (number | string)[][] = []
        // all results statistics
        const resultsCount: {
            [key: string]: {
                newQuestionDbs?: number
                newSubjects?: number
                newQuestions?: number
            }
        } = {}

        const resultDataWithoutEmptyDbs: (SyncDataResult['questions'] & {
            peer: PeerInfo
        })[] = []
        questionData.forEach((res) => {
            const qdbCount = res.questionDbs.length
            const { subjCount, questionCount } = countOfQdbs(res.questionDbs)

            recievedDataCounts.push([
                peerToString(res.peer),
                qdbCount,
                subjCount,
                questionCount,
            ])

            if (questionCount > 0) {
                resultDataWithoutEmptyDbs.push(res)
            } else {
                updatePeersFile(res.peer, {
                    lastQuestionsSync: syncStart,
                })
            }
        })

        const resultData = resultDataWithoutEmptyDbs.map((res) => {
            return {
                ...res,
                questionDbs: res.questionDbs.map((qdb) => {
                    return setupQuestionsForMerge(qdb, res.peer)
                }),
            }
        })

        const hasNewData = resultData.length > 0
        if (!hasNewData) {
            logger.Log(
                `No peers returned any new questions. Question sync successfully finished!`,
                'green'
            )
            updateLastSync(selfInfo, syncStart)
            return {
                msg: 'No peers returned any new questions',
            }
        }

        logger.Log(`\tRecieved data from peers:`)
        logger.logTable(
            [['', 'QDBs', 'Subjs', 'Questions'], ...recievedDataCounts],
            {
                colWidth: [20],
                rowPrefix: '\t',
            }
        )

        // -------------------------------------------------------------------------------------------------------
        // backup
        // -------------------------------------------------------------------------------------------------------
        const { subjCount: oldSubjCount, questionCount: oldQuestionCount } =
            countOfQdbs(getQuestionDbs())
        const oldQuestionDbCount = getQuestionDbs().length
        backupData(getQuestionDbs())
        logger.Log('\tOld data backed up!')

        // -------------------------------------------------------------------------------------------------------
        // adding questions to db
        // -------------------------------------------------------------------------------------------------------
        for (let i = 0; i < resultData.length; i++) {
            const { questionDbs: remoteQuestionDbs, peer } = resultData[i]
            logger.Log(
                `\tProcessing result from "${logger.C('blue')}${peerToString(
                    peer
                )}${logger.C()}" (${logger.C('green')}${
                    resultData.length
                }${logger.C()}/${logger.C('green')}${i + 1}${logger.C()})`
            )
            // FIXME: if remoteQuestionDbs contain multiple dbs with the same name, then the merging
            // process could get wonky. Ideally it should not contain, but we will see

            const { rawNewQuestionDbs, mergeResults } = await getMergeResults(
                remoteQuestionDbs
            )

            const newQuestionDbs = updateQdbForLocalUse(rawNewQuestionDbs)

            const { mergedQuestionDbs, changedQdbIndexes } = mergeQdbs(
                getQuestionDbs(),
                mergeResults
            )
            // setting new index & path
            writeNewData(
                newQuestionDbs,
                getQuestionDbs().filter((qdb) => {
                    return changedQdbIndexes.includes(qdb.index)
                }),
                dbsFile
            )

            setQuestionDbs([...mergedQuestionDbs, ...newQuestionDbs])

            const { newQuestionDbCount, newSubjectCount, newQuestionCount } =
                await sendNewDataToWorkers(mergeResults, newQuestionDbs)

            resultsCount[peerToString(peer)] = {
                ...(resultsCount[peerToString(peer)] || { newUsers: 0 }),
                newQuestionDbs: newQuestionDbCount,
                newSubjects: newSubjectCount,
                newQuestions: newQuestionCount,
            }
            // Processing result data is successfull
            updatePeersFile(peer, {
                lastQuestionsSync: syncStart,
            })
        }

        // -------------------------------------------------------------------------------------------------------
        updateLastSync(selfInfo, syncStart)

        const newQdb = getQuestionDbs()
        const { subjCount: newSubjCount, questionCount: newQuestionCount } =
            countOfQdbs(newQdb)
        const newQuestionDbCount = newQdb.length

        const resultsTable = Object.entries(resultsCount).map(
            ([key, value]) => {
                return [
                    key.length > 14 ? key.substring(0, 14) + '...' : key,
                    value.newQuestionDbs,
                    value.newSubjects,
                    value.newQuestions,
                ]
            }
        )

        const sumNewCount = (key: string) => {
            return Object.values(resultsCount).reduce(
                (acc, val) => acc + val[key],
                0
            )
        }

        const totalNewQuestions = sumNewCount('newQuestions')
        const totalNewSubjects = sumNewCount('newSubjects')
        const totalNewQdbs = sumNewCount('newQuestionDbs')

        logger.logTable(
            [
                ['', 'QDBs', 'Subjs', 'Questions'],
                ['Old', oldQuestionDbCount, oldSubjCount, oldQuestionCount],
                ...resultsTable,
                [
                    'Added total',
                    totalNewQdbs,
                    totalNewSubjects,
                    totalNewQuestions,
                ],
                ['Final', newQuestionDbCount, newSubjCount, newQuestionCount],
            ],
            { colWidth: [20], rowPrefix: '\t' }
        )

        logger.Log(`Successfully synced questions!`, 'green')

        return {
            old: {
                questionDbs: oldQuestionDbCount,
                subjects: oldSubjCount,
                questions: oldQuestionCount,
            },
            added: {
                questionDbs: totalNewQdbs,
                subjects: totalNewSubjects,
                questions: totalNewQuestions,
            },
            final: {
                questionDbs: newQuestionDbCount,
                subjects: newSubjCount,
                questions: newQuestionCount,
            },
        }
    }

    async function syncUserFiles(
        newData: (SyncDataResult['userFiles'] & { peer: PeerInfo })[],
        syncStart: number
    ): Promise<SyncResult> {
        logger.Log('Syncing user files...')

        const recievedUserFilesCount: (string | number)[][] = []
        let totalRecievedd = 0
        newData.forEach((res) => {
            const count = Object.values(res.newFiles).reduce((acc, data) => {
                totalRecievedd += Object.keys(data).length
                return acc + Object.keys(data).length
            }, 0)
            recievedUserFilesCount.push([peerToString(res.peer), count])
        })

        if (totalRecievedd === 0) {
            logger.Log(
                `No peers returned any new files. User file sync successfully finished!`,
                'green'
            )
            return {
                old: {
                    userFiles: 0,
                },
                added: {
                    userFiles: 0,
                },
                final: {
                    userFiles: 0,
                },
            }
        }

        logger.logTable([['', 'Files'], ...recievedUserFilesCount], {
            colWidth: [20],
            rowPrefix: '\t',
        })

        const filesToGet: {
            fileName: string
            dir: string
            filePath: string
            data: UserDirDataFile
            peer: PeerInfo
        }[] = []
        newData.forEach((res) => {
            Object.entries(res.newFiles).forEach(([dirName, userFilesDir]) => {
                Object.entries(userFilesDir).forEach(([fileName, data]) => {
                    filesToGet.push({
                        fileName: fileName,
                        dir: dirName,
                        filePath: path.join(
                            paths.userFilesDir,
                            dirName,
                            fileName
                        ),
                        data: data,
                        peer: res.peer,
                    })
                })
            })
        })

        let addedFiles = 0
        for (const fileToGet of filesToGet) {
            const { peer, dir, fileName, filePath, data } = fileToGet

            try {
                await downloadFile(
                    {
                        host: peer.host,
                        port: peer.port,
                        path: `/api/userFiles/${dir}/${fileName}`,
                    },
                    filePath,
                    peer.http
                )

                const dataFilePath = path.join(
                    paths.userFilesDir,
                    dir,
                    constants.userFilesDataFileName
                )
                if (!utils.FileExists(dataFilePath)) {
                    utils.WriteFile(JSON.stringify({}), dataFilePath)
                }
                const dataFile = utils.ReadJSON<{
                    [key: string]: UserDirDataFile
                }>(dataFilePath)

                if (dataFile[fileName]) {
                    // dataFile[fileName].views += data.views // views are not unique
                    dataFile[fileName].upvotes = dataFile[fileName].upvotes
                        .concat(data.upvotes)
                        .reduce((acc, x) => {
                            if (acc.includes(x)) return acc
                            return [...acc, x]
                        }, [])
                    dataFile[fileName].downvotes = dataFile[fileName].downvotes
                        .concat(data.downvotes)
                        .reduce((acc, x) => {
                            if (acc.includes(x)) return acc
                            return [...acc, x]
                        }, [])
                } else {
                    dataFile[fileName] = data
                }

                utils.WriteFile(JSON.stringify(dataFile), dataFilePath)
                addedFiles += 1
            } catch (e) {
                logger.Log(`Unable to download "${fileName}": ${e.message}`)
            }
        }

        newData.forEach((res) => {
            updatePeersFile(res.peer, {
                lastUserFilesSync: syncStart,
            })
        })

        logger.Log(
            `Successfully synced user files! Added ${addedFiles} files`,
            'green'
        )
        return {
            old: {
                userFiles: 0,
            },
            added: {
                userFiles: addedFiles,
            },
            final: {
                userFiles: 0,
            },
        }
    }

    function handleNewThirdPartyPeer(remoteHost: string) {
        logger.Log(
            'Couldn\'t find remote peer info based on remoteHost: "' +
                remoteHost +
                '". This could mean that the host uses this server as peer, but this server does not ' +
                'use it as a peer.',
            'yellowbg'
        )
        if (remoteHost.includes(':')) {
            const [host, port] = remoteHost.split(':')
            updateThirdPartyPeers([
                {
                    host: host,
                    port: +port,
                },
            ])
            logger.Log('Host info written to host info file')
        }
    }

    // ---------------------------------------------------------------------------------------
    // APP SETUP
    // ---------------------------------------------------------------------------------------
    app.get('/selfInfo', (_req: Request, res: Response<PeerInfo>) => {
        res.json({ ...selfInfo, publicKey: publicKey })
    })

    app.get('/p2pinfo', (_req: Request, res: Response<RemotePeerInfo>) => {
        res.json(getSelfInfo(true))
    })

    app.get('/getnewdata', (req: Request, res: Response<any>) => {
        logger.LogReq(req)
        const questionsSince = Number.isNaN(+req.query.questionsSince)
            ? 0
            : +req.query.questionsSince
        const usersSince = Number.isNaN(+req.query.usersSince)
            ? 0
            : +req.query.usersSince
        const userFilesSince = Number.isNaN(+req.query.userFilesSince)
            ? 0
            : +req.query.userFilesSince

        const questions = !!req.query.questions
        const users = !!req.query.users
        const userFiles = !!req.query.userFiles
        const remoteHost = req.query.host

        const sendAll = !questions && !users && !userFiles

        let hostToLog = remoteHost || 'Unknown host'
        let remotePeerInfo: PeerInfo = null

        if (remoteHost) {
            remotePeerInfo = peers.find((peer) => {
                return peerToString(peer) === remoteHost
            })
            if (!remotePeerInfo) {
                handleNewThirdPartyPeer(remoteHost)
            } else {
                hostToLog = peerToString(remotePeerInfo)
            }
        }

        const result: SyncDataResult = {
            remoteInfo: getSelfInfo(),
        }

        if (questions || sendAll) {
            const questionDbsWithNewQuestions = Number.isNaN(questionsSince)
                ? getQuestionDbs()
                : getQuestionDbs()
                      .map((qdb) => {
                          return {
                              ...qdb,
                              data: getNewDataSince(qdb.data, questionsSince),
                          }
                      })
                      .filter((qdb) => {
                          const { questionCount: questionCount } =
                              countOfQdb(qdb)
                          return questionCount > 0
                      })

            const { subjCount: subjects, questionCount: questions } =
                countOfQdbs(questionDbsWithNewQuestions)

            result.questions = {
                questionDbs: questionDbsWithNewQuestions,
                count: {
                    qdbs: questionDbsWithNewQuestions.length,
                    subjects: subjects,
                    questions: questions,
                },
            }

            const questionsSinceDate = questionsSince
                ? new Date(questionsSince).toLocaleString()
                : 'all time'

            logger.Log(
                `\tSending new data to ${logger.C(
                    'blue'
                )}${hostToLog}${logger.C()} since ${logger.C(
                    'blue'
                )}${questionsSinceDate}${logger.C()}`
            )
            logger.logTable(
                [
                    ['', 'QDBs', 'Subjs', 'Questions'],
                    [
                        'Count',
                        result.questions.questionDbs.length,
                        result.questions.count.subjects,
                        result.questions.count.questions,
                    ],
                ],
                { rowPrefix: '\t' }
            )
        }

        if (users || sendAll) {
            let sentUsers = 0
            if (!remoteHost) {
                res.json({
                    ...result,
                    success: false,
                    message: 'remoteHost key is missing from body',
                })
                return
            }
            if (!remotePeerInfo) {
                res.json({
                    success: false,
                    message:
                        "couldn't find remote peer info based on remoteHost",
                })
                return
            }
            const remotePublicKey = remotePeerInfo.publicKey
            if (remotePublicKey) {
                // FIXME: sign data?
                const newUsers = getNewUsersSince(usersSince)
                sentUsers = newUsers.length
                result.users = {
                    encryptedUsers: encrypt(
                        remotePublicKey,
                        JSON.stringify(newUsers)
                    ),
                }

                const usersSinceDate = usersSince
                    ? new Date(usersSince).toLocaleString()
                    : 'all time'

                logger.Log(
                    `\tSending new users to ${logger.C(
                        'blue'
                    )}${hostToLog}${logger.C()} since ${logger.C(
                        'blue'
                    )}${usersSinceDate}${logger.C()}. Sent users: ${logger.C(
                        'blue'
                    )}${sentUsers}${logger.C()}`
                )
            } else if (remotePeerInfo) {
                logger.Log(
                    `Warning: "${hostToLog}" has no public key saved!`,
                    'yellowbg'
                )
            }
        }

        if (userFiles || sendAll) {
            const newFiles = getNewUserFilesSince(userFilesSince)
            const sentFilesCount = Object.values(newFiles).reduce(
                (acc, data) => {
                    return acc + Object.keys(data).length
                },
                0
            )
            result.userFiles = { newFiles: newFiles }

            const userFilesSinceDate = questionsSince
                ? new Date(questionsSince).toLocaleString()
                : 'all time'

            logger.Log(
                `\tSending new user files to ${logger.C(
                    'blue'
                )}${hostToLog}${logger.C()} since ${logger.C(
                    'blue'
                )}${userFilesSinceDate}${logger.C()} Sent files: ${logger.C(
                    'blue'
                )}${sentFilesCount}${logger.C()}`
            )
        }

        res.json(result)
    })

    app.get('/syncp2pdata', (req: Request, res: Response) => {
        logger.LogReq(req)
        const questions = !!req.query.questions
        const users = !!req.query.users
        const userFiles = !!req.query.userFiles

        const allTime = !!req.query.allTime
        const user = req.session.user

        if (!user || user.id !== 1) {
            res.json({
                status: 'error',
                message: 'only user 1 can call this EP',
            })
            return
        }

        // FIXME: /syncResult EP if this EP times out, but we still need the result
        if (syncInProgress) {
            res.json({
                error: 'A sync is already in progress!',
            })
            return
        }

        syncInProgress = true
        setPendingJobsAlertCount(5000)
        syncData({
            shouldSync: {
                questions: questions,
                users: users,
                userFiles: userFiles,
            },
            allTime: allTime,
        })
            .then((syncResult) => {
                res.json({
                    msg: 'sync successfull',
                    ...syncResult,
                })
                setPendingJobsAlertCount()
                syncInProgress = false
            })
            .catch((e) => {
                console.error(e)
                res.json({
                    error: e,
                    msg: e.message,
                })
                setPendingJobsAlertCount()
                syncInProgress = false
            })
    })

    app.post(
        '/newusercreated',
        (req: Request<{ host: string; newUsers: string }>, res: Response) => {
            logger.LogReq(req)

            const encryptedNewUsers = req.body.newUsers
            const remoteHost = req.body.host

            if (!encryptedNewUsers || !remoteHost) {
                res.json({
                    success: false,
                    message:
                        'encryptedNewUsers or remoteHost key are missing from body',
                })
                return
            }

            const remotePeerInfo = peers.find((peer) => {
                return peerToString(peer) === remoteHost
            })

            if (!remotePeerInfo) {
                res.json({
                    success: false,
                    message:
                        "couldn't find remote peer info based on remoteHost",
                })
                return
            }

            const decryptedUsers: User[] = JSON.parse(
                decrypt(privateKey, encryptedNewUsers)
            )

            const addedUserCount = addUsersToDb(decryptedUsers, userDB, {
                sourceHost: peerToString(remotePeerInfo),
            })

            if (addedUserCount > 0) {
                logger.Log(
                    `\tAdded ${addedUserCount} new users from "${peerToString(
                        remotePeerInfo
                    )}"`,
                    'cyan'
                )
            }

            res.json({ success: true, addedUserCount: addedUserCount })
        }
    )

    logger.Log(
        'P2P functionality set up. Peers (' +
            peers.length +
            '): ' +
            peers.map((peer) => peerToString(peer)).join(', '),
        'blue'
    )

    return {}
}

export default {
    setup: setup,
}