sending new users to peers on user creation

src/modules/api/submodules/p2p.ts

@@ -60,6 +60,7 @@ import {
     peerToString,
     updatePeersFile,
 } from '../../../utils/p2putils'
+import { Database } from 'better-sqlite3'
 
 interface MergeResult {
     newData: Subject[]
@@ -456,6 +457,31 @@ async function authAndGetNewData({
     }
 }
 
+function addUsersToDb(
+    users: User[],
+    userDB: Database,
+    extraProps: Partial<User>
+) {
+    let addedUserCount = 0
+    users.forEach((remoteUser) => {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { id, ...remoteUserWithoutId } = remoteUser
+        const localUser = dbtools.Select(userDB, 'users', {
+            pw: remoteUser.pw,
+        })
+        if (localUser.length === 0) {
+            addedUserCount += 1
+            // FIXME: users will not have consistend id across servers. This may be
+            // harmless, will see
+            dbtools.Insert(userDB, 'users', {
+                ...(remoteUserWithoutId as Omit<User, 'id'>),
+                ...extraProps,
+            })
+        }
+    })
+    return addedUserCount
+}
+
 function setup(data: SubmoduleData): Submodule {
     const {
         app,
@@ -696,6 +722,7 @@ function setup(data: SubmoduleData): Submodule {
                 syncAll
                     ? 'everything'
                     : Object.entries(shouldSync)
+                          // eslint-disable-next-line @typescript-eslint/no-unused-vars
                           .filter(([_key, value]) => value)
                           .map(([key]) => key)
                           .join(', ')
@@ -870,26 +897,16 @@ function setup(data: SubmoduleData): Submodule {
         try {
             userData.forEach((res) => {
                 if (res.encryptedUsers) {
-                    let addedUserCount = 0
                     const decryptedUsers: User[] = JSON.parse(
                         decrypt(privateKey, res.encryptedUsers)
                     )
-                    decryptedUsers.forEach((remoteUser) => {
+                    const addedUserCount = addUsersToDb(
-                        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+                        decryptedUsers,
-                        const { id, ...remoteUserWithoutId } = remoteUser
+                        userDB,
-                        const localUser = dbtools.Select(userDB, 'users', {
+                        {
-                            pw: remoteUser.pw,
+                            sourceHost: peerToString(res.peer),
-                        })
-                        if (localUser.length === 0) {
-                            addedUserCount += 1
-                            // FIXME: users will not have consistend id across servers. This may be
-                            // harmless, will see
-                            dbtools.Insert(userDB, 'users', {
-                                ...(remoteUserWithoutId as Omit<User, 'id'>),
-                                sourceHost: peerToString(res.peer),
-                            })
                         }
-                    })
+                    )
                     resultsCount[peerToString(res.peer)] = {
                         newUsers: addedUserCount,
                     }
@@ -1169,39 +1186,17 @@ function setup(data: SubmoduleData): Submodule {
     app.get('/getnewfilessince', (req: Request, res: Response<any>) => {
         const since = Number.isNaN(+req.query.since) ? 0 : +req.query.since
 
-        const remoteHost = req.query.host
+        res.json({ since: since, message: 'unimplemented' })
-        const hostToLog = remoteHost || 'Unknown host'
-
-        const result: any = {
-            remoteInfo: getSelfInfo(),
-        }
-
-        if (remoteHost) {
-            const remotePeerInfo = peers.find((peer) => {
-                return peerToString(peer) === remoteHost
-            })
-            if (!remotePeerInfo) {
-                handleNewThirdPartyPeer(remoteHost)
-            }
-        }
-        const usersSinceDate = since
-            ? new Date(since).toLocaleString()
-            : 'all time'
-
-        logger.Log(
-            `\tSending new files to ${logger.C(
-                'blue'
-            )}${hostToLog}${logger.C()} since ${logger.C(
-                'blue'
-            )}${usersSinceDate}${logger.C()}`
-        )
-
-        res.json(result)
     })
 
     app.get(
         '/getnewuserssince',
-        (req: Request, res: Response<UserSyncDataRes>) => {
+        (
+            req: Request,
+            res: Response<
+                UserSyncDataRes & { message?: string; success?: boolean }
+            >
+        ) => {
             logger.LogReq(req)
             const since = Number.isNaN(+req.query.since) ? 0 : +req.query.since
 
@@ -1213,47 +1208,61 @@ function setup(data: SubmoduleData): Submodule {
                 remoteInfo: getSelfInfo(),
             }
 
-            if (remoteHost) {
+            if (!remoteHost) {
-                const remotePeerInfo = peers.find((peer) => {
+                res.json({
-                    return peerToString(peer) === remoteHost
+                    ...result,
+                    success: false,
+                    message: 'remoteHost key is missing from body',
                 })
-                if (!remotePeerInfo) {
+                return
-                    handleNewThirdPartyPeer(remoteHost)
+            }
-                } else {
-                    hostToLog = peerToString(remotePeerInfo)
-                }
 
-                if (remotePeerInfo) {
+            const remotePeerInfo = peers.find((peer) => {
-                    const remotePublicKey = remotePeerInfo?.publicKey
+                return peerToString(peer) === remoteHost
-                    if (remotePublicKey) {
+            })
-                        // FIXME: sign data?
+            if (!remotePeerInfo) {
-                        const newUsers = getNewUsersSince(since)
+                handleNewThirdPartyPeer(remoteHost)
-                        sentUsers = newUsers.length
+            } else {
-                        result.encryptedUsers = encrypt(
+                hostToLog = peerToString(remotePeerInfo)
-                            remotePublicKey,
+            }
-                            JSON.stringify(newUsers)
-                        )
 
-                        const usersSinceDate = since
+            if (!remotePeerInfo) {
-                            ? new Date(since).toLocaleString()
+                res.json({
-                            : 'all time'
+                    success: false,
+                    message:
+                        "couldn't find remote peer info based on remoteHost",
+                })
+                return
+            }
 
-                        logger.Log(
+            const remotePublicKey = remotePeerInfo.publicKey
-                            `\tSending new users to ${logger.C(
+            if (remotePublicKey) {
-                                'blue'
+                // FIXME: sign data?
-                            )}${hostToLog}${logger.C()} since ${logger.C(
+                const newUsers = getNewUsersSince(since)
-                                'blue'
+                sentUsers = newUsers.length
-                            )}${usersSinceDate}${logger.C()}. Sent users: ${logger.C(
+                result.encryptedUsers = encrypt(
-                                'blue'
+                    remotePublicKey,
-                            )}${sentUsers}${logger.C()}`
+                    JSON.stringify(newUsers)
-                        )
+                )
-                    } else if (remotePeerInfo) {
+
-                        logger.Log(
+                const usersSinceDate = since
-                            `Warning: "${hostToLog}" has no public key saved!`,
+                    ? new Date(since).toLocaleString()
-                            'yellowbg'
+                    : 'all time'
-                        )
+
-                    }
+                logger.Log(
-                }
+                    `\tSending new users to ${logger.C(
+                        'blue'
+                    )}${hostToLog}${logger.C()} since ${logger.C(
+                        'blue'
+                    )}${usersSinceDate}${logger.C()}. Sent users: ${logger.C(
+                        'blue'
+                    )}${sentUsers}${logger.C()}`
+                )
+            } else if (remotePeerInfo) {
+                logger.Log(
+                    `Warning: "${hostToLog}" has no public key saved!`,
+                    'yellowbg'
+                )
             }
 
             res.json(result)
@@ -1389,6 +1398,57 @@ function setup(data: SubmoduleData): Submodule {
             })
     })
 
+    app.post(
+        '/newusercreated',
+        (req: Request<{ host: string; newUsers: string }>, res: Response) => {
+            logger.LogReq(req)
+
+            const encryptedNewUsers = req.body.newUsers
+            const remoteHost = req.body.host
+
+            if (!encryptedNewUsers || !remoteHost) {
+                res.json({
+                    success: false,
+                    message:
+                        'encryptedNewUsers or remoteHost key are missing from body',
+                })
+                return
+            }
+
+            const remotePeerInfo = peers.find((peer) => {
+                return peerToString(peer) === remoteHost
+            })
+
+            if (!remotePeerInfo) {
+                res.json({
+                    success: false,
+                    message:
+                        "couldn't find remote peer info based on remoteHost",
+                })
+                return
+            }
+
+            const decryptedUsers: User[] = JSON.parse(
+                decrypt(privateKey, encryptedNewUsers)
+            )
+
+            const addedUserCount = addUsersToDb(decryptedUsers, userDB, {
+                sourceHost: peerToString(remotePeerInfo),
+            })
+
+            if (addedUserCount > 0) {
+                logger.Log(
+                    `\tAdded ${addedUserCount} new users from "${peerToString(
+                        remotePeerInfo
+                    )}"`,
+                    'cyan'
+                )
+            }
+
+            res.json({ success: true, addedUserCount: addedUserCount })
+        }
+    )
+
     logger.Log(
         'P2P functionality set up. Peers (' +
             peers.length +

src/modules/api/submodules/qminingapi.ts

@@ -73,9 +73,8 @@ interface SavedQuestionData {
 
 export interface QuestionAddResponse {
     success: boolean
-    newQuestions: number
     totalNewQuestions: number
-    result?: string
+    result: string
 }
 
 const line = '====================================================' // lol

src/modules/api/submodules/userManagement.ts

@@ -30,6 +30,7 @@ import {
     Submodule,
 } from '../../../types/basicTypes'
 import dbtools from '../../../utils/dbtools'
+import { queueWork } from '../../../worker/workerPool'
 
 const minimumAlowwedSessions = 2 // how many sessions are allowed for a user
 const usersDbBackupPath = 'data/dbs/backup'
@@ -150,12 +151,14 @@ function setup(data: SubmoduleData): Submodule {
         )
 
         const pw = uuidv4()
-        const insertRes = dbtools.Insert(userDB, 'users', {
+        const newUser: Omit<User, 'id'> = {
             pw: pw,
             avaiblePWRequests: 0,
             created: new Date().getTime(),
             createdBy: requestingUser.id,
-        })
+        }
+
+        const insertRes = dbtools.Insert(userDB, 'users', newUser)
 
         logger.Log(
             `User #${requestingUser.id} created new user #${insertRes.lastInsertRowid}`,
@@ -173,6 +176,13 @@ function setup(data: SubmoduleData): Submodule {
             dayDiff: getDayDiff(requestingUser.created),
             userCount: dbtools.TableInfo(userDB, 'users').dataCount,
         })
+
+        queueWork({
+            type: 'sendUsersToPeers',
+            data: {
+                newUsers: [newUser],
+            },
+        })
     })
 
     app.post('/login', (req: Request, res: any) => {

src/types/basicTypes.ts

@@ -89,14 +89,14 @@ export interface User {
     id: number
     pw: string
     notes?: string
-    loginCount: number
+    loginCount?: number
     avaiblePWRequests: number
-    pwRequestCount: number
+    pwRequestCount?: number
     createdBy: number
     created: number
-    lastLogin: number
+    lastLogin?: number
-    lastAccess: number
+    lastAccess?: number
-    sourceHost?: number
+    sourceHost?: string
     //     isAdmin: boolean // TODO
 }
 

src/utils/p2putils.ts

@@ -1,6 +1,7 @@
 import { PeerInfo } from '../types/basicTypes'
 import { files, paths, readAndValidateFile } from './files'
-import { parseCookie, post } from './networkUtils'
+import logger from './logger'
+import { PostResult, parseCookie, post } from './networkUtils'
 import utils from './utils'
 
 export function peerToString(peer: {
@@ -62,3 +63,86 @@ export async function loginToPeer(peer: PeerInfo): Promise<string | Error> {
     const parsedCookies = parseCookie(cookie)
     return parsedCookies.sessionID
 }
+
+export async function loginAndPostDataToAllPeers<
+    T extends { result?: string; success?: boolean }
+>(
+    peers: PeerInfo[],
+    postDataFn: (
+        peer: PeerInfo,
+        sessionCookie: string
+    ) => Promise<PostResult<T>>,
+    resultCallback?: (peer: PeerInfo, res: PostResult<T>) => void
+): Promise<void> {
+    const results: {
+        errors: PeerInfo[]
+        sent: PeerInfo[]
+        loginErrors: PeerInfo[]
+    } = {
+        errors: [],
+        sent: [],
+        loginErrors: [],
+    }
+
+    for (const peer of peers) {
+        try {
+            let sessionCookie = peer.sessionCookie
+
+            const login = async (peer: PeerInfo) => {
+                const loginResult = await loginToPeer(peer)
+                if (typeof loginResult === 'string') {
+                    sessionCookie = loginResult
+                    updatePeersFile(peer, { sessionCookie: loginResult })
+                } else {
+                    throw new Error('Error logging in to' + peerToString(peer))
+                }
+            }
+
+            if (!sessionCookie) {
+                await login(peer)
+            }
+
+            let res = await postDataFn(peer, sessionCookie)
+
+            if (res.data?.result === 'nouser' && sessionCookie) {
+                await login(peer)
+
+                res = await postDataFn(peer, sessionCookie)
+            }
+
+            if (res.error || !res.data?.success) {
+                results.errors.push(peer)
+                console.error(res.error || JSON.stringify(res.data))
+            } else {
+                results.sent.push(peer)
+            }
+
+            if (resultCallback) resultCallback(peer, res)
+        } catch (e) {
+            results.loginErrors.push(peer)
+        }
+    }
+
+    const logMsg: string[] = []
+    const addToLogMsg = (
+        peerResult: PeerInfo[],
+        prefix: string,
+        color: string
+    ) => {
+        if (peerResult.length > 0) {
+            logMsg.push(
+                `${logger.C(color)}${prefix}:${logger.C()} ` +
+                    peerResult.map((x) => peerToString(x)).join(', ')
+            )
+        }
+    }
+    addToLogMsg(results.loginErrors, 'Login error', 'red')
+    addToLogMsg(results.errors, 'Error', 'red')
+    addToLogMsg(results.sent, 'Sent', 'green')
+
+    logger.Log(
+        `\t${logger.C('green')}Sent data to peers${logger.C()}; ${logMsg.join(
+            ', '
+        )}`
+    )
+}

src/worker/handlers/handleQuestionsToPeers.ts

@@ -6,22 +6,9 @@ import { RecievedData } from '../../utils/actions'
 import { removeCacheFromQuestion } from '../../utils/qdbUtils'
 import { QuestionAddResponse } from '../../modules/api/submodules/qminingapi'
 import logger from '../../utils/logger'
-import {
+import { peerToString, loginAndPostDataToAllPeers } from '../../utils/p2putils'
-    loginToPeer,
-    peerToString,
-    updatePeersFile,
-} from '../../utils/p2putils'
 import { post } from '../../utils/networkUtils'
 
-const login = async (peer: PeerInfo): Promise<string> => {
-    const loginResult = await loginToPeer(peer)
-    if (typeof loginResult === 'string') {
-        return loginResult
-    } else {
-        return null
-    }
-}
-
 export type QuestionsToPeersTaskObject = {
     type: 'sendQuestionsToPeers'
     data: {
@@ -66,18 +53,6 @@ export const handleQuestionsToPeers = async (
         }),
     }
 
-    const results: {
-        errors: PeerInfo[]
-        hasNew: PeerInfo[]
-        sent: PeerInfo[]
-        loginErrors: PeerInfo[]
-    } = {
-        errors: [],
-        hasNew: [],
-        sent: [],
-        loginErrors: [],
-    }
-
     const postData = (peer: PeerInfo, sessionCookie: string) => {
         return post<QuestionAddResponse>({
             hostname: peer.host,
@@ -89,63 +64,17 @@ export const handleQuestionsToPeers = async (
         })
     }
 
-    for (const peer of peers) {
+    const hadNewQuestions: string[] = []
-        let sessionCookie = peer.sessionCookie
+    loginAndPostDataToAllPeers<QuestionAddResponse & { success: boolean }>(
-
+        peers,
-        if (!sessionCookie) {
+        postData,
-            sessionCookie = await login(peer)
+        (peer, res) => {
-            if (!sessionCookie) {
+            if (res.data?.totalNewQuestions > 0) {
-                results.loginErrors.push(peer)
+                hadNewQuestions.push(peerToString(peer))
-                continue
             }
-            updatePeersFile(peer, { sessionCookie: sessionCookie })
         }
-
-        let res = await postData(peer, sessionCookie)
-
-        if (res.data?.result === 'nouser' && sessionCookie) {
-            sessionCookie = await login(peer)
-            if (!sessionCookie) {
-                results.loginErrors.push(peer)
-                continue
-            }
-            updatePeersFile(peer, { sessionCookie: sessionCookie })
-            res = await postData(peer, sessionCookie)
-        }
-
-        if (res.error || !res.data?.success) {
-            results.errors.push(peer)
-            console.error(res.error || JSON.stringify(res.data))
-        } else if (res.data?.totalNewQuestions > 0) {
-            results.hasNew.push(peer)
-        } else {
-            results.sent.push(peer)
-        }
-    }
-
-    const logMsg: string[] = []
-    const addToLogMsg = (
-        peerResult: PeerInfo[],
-        prefix: string,
-        color: string
-    ) => {
-        if (peerResult.length > 0) {
-            logMsg.push(
-                `${logger.C(color)}${prefix}:${logger.C()} ` +
-                    peerResult.map((x) => peerToString(x)).join(', ')
-            )
-        }
-    }
-    addToLogMsg(results.loginErrors, 'Login error', 'red')
-    addToLogMsg(results.errors, 'Error', 'red')
-    addToLogMsg(results.hasNew, 'Had new questions', 'blue')
-    addToLogMsg(results.sent, 'Sent', 'green')
-
-    logger.Log(
-        `\t${logger.C(
-            'green'
-        )}Sent new questions to peers${logger.C()}; ${logMsg.join(', ')}`
     )
+    logger.Log(`Peers that added new questions: ${hadNewQuestions.join(', ')}`)
 
     parentPort.postMessage({
         msg: `From thread #${workerIndex}: sendQuestionsToPeers done`,

src/worker/handlers/handleUsersToPeers.ts

@@ -0,0 +1,92 @@
+import { parentPort } from 'node:worker_threads'
+import { PeerInfo, QuestionDb, User } from '../../types/basicTypes'
+import { files, readAndValidateFile } from '../../utils/files'
+import logger from '../../utils/logger'
+import { peerToString, loginAndPostDataToAllPeers } from '../../utils/p2putils'
+import { post } from '../../utils/networkUtils'
+import { encrypt } from '../../utils/encryption'
+
+export type UsersToPeersTaskObject = {
+    type: 'sendUsersToPeers'
+    data: {
+        newUsers: (Omit<User, 'id'> & { id?: number })[]
+    }
+}
+
+export const handleUsersToPeers = async (
+    _qdbs: QuestionDb[],
+    msg: UsersToPeersTaskObject,
+    workerIndex: number
+): Promise<void> => {
+    const { newUsers } = msg.data
+
+    const selfInfo = readAndValidateFile<PeerInfo>(files.selfInfoFile)
+    const host = peerToString(selfInfo)
+    const peers = readAndValidateFile<PeerInfo[]>(files.peersFile)
+
+    if (!peers || peers.length === 0 || newUsers.length === 0) {
+        parentPort.postMessage({
+            msg: `From thread #${workerIndex}: sendUsersToPeers done`,
+            workerIndex: workerIndex,
+        })
+        return
+    }
+
+    const postData = (peer: PeerInfo, sessionCookie: string) => {
+        if (!peer.publicKey) {
+            logger.Log(
+                `"${peerToString(peer)}" has no public key saved!`,
+                'yellowbg'
+            )
+
+            return Promise.resolve({
+                error: new Error(
+                    `"${peerToString(peer)}" has no public key saved!`
+                ),
+            })
+        }
+
+        const encryptedUsers = encrypt(peer.publicKey, JSON.stringify(newUsers))
+
+        const dataToSend: { host: string; newUsers: string } = {
+            host: host,
+            newUsers: encryptedUsers,
+        }
+
+        return post<{
+            addedUserCount?: number
+            result?: string
+            success?: boolean
+        }>({
+            hostname: peer.host,
+            port: peer.port,
+            http: peer.http,
+            path: '/api/newusercreated',
+            bodyObject: dataToSend,
+            cookie: `sessionID=${sessionCookie}`,
+        })
+    }
+
+    const newUserAdded: string[] = []
+    loginAndPostDataToAllPeers<{
+        addedUserCount?: number
+        result?: string
+        success?: boolean
+    }>(peers, postData, (peer, res) => {
+        if (res.data?.addedUserCount > 0) {
+            newUserAdded.push(peerToString(peer))
+        }
+    })
+
+    if (newUserAdded.length > 0) {
+        logger.Log(
+            `Peers that saved new users: ${newUserAdded.join(', ')}`,
+            'cyan'
+        )
+    }
+
+    parentPort.postMessage({
+        msg: `From thread #${workerIndex}: sendUsersToPeers done`,
+        workerIndex: workerIndex,
+    })
+}

src/worker/worker.ts

@@ -12,6 +12,7 @@ import { handleNewDb } from './handlers/handleNewDb'
 import { handleDbClean } from './handlers/handleDbClean'
 import { handleQuestionsToPeers } from './handlers/handleQuestionsToPeers'
 import { handleRmQuestions } from './handlers/handleRmQuestions'
+import { handleUsersToPeers } from './handlers/handleUsersToPeers'
 
 export interface WorkerResult {
     msg: string
@@ -81,6 +82,8 @@ async function handleMessage(
         await handleRmQuestions(qdbs, msg, workerIndex, setQdbs)
     } else if (msg.type === 'sendQuestionsToPeers') {
         await handleQuestionsToPeers(qdbs, msg, workerIndex)
+    } else if (msg.type === 'sendUsersToPeers') {
+        await handleUsersToPeers(qdbs, msg, workerIndex)
     } else {
         logger.Log(`Invalid msg type!`, logger.GetColor('redbg'))
         console.error(msg)

src/worker/workerPool.ts

@@ -34,6 +34,7 @@ import { MergeTaskObject } from './handlers/handleMerge'
 import { QuestionsToPeersTaskObject } from './handlers/handleQuestionsToPeers'
 import { WorkerResult } from './worker'
 import logger from '../utils/logger'
+import { UsersToPeersTaskObject } from './handlers/handleUsersToPeers'
 
 const threadCount = +process.env.NS_THREAD_COUNT || os.cpus().length
 
@@ -52,6 +53,7 @@ export type TaskObject =
     | RmQuestionsTaskObject
     | MergeTaskObject
     | QuestionsToPeersTaskObject
+    | UsersToPeersTaskObject
 
 interface PendingJob {
     workData: TaskObject