p2p user files sync login fix

README.md

@@ -77,9 +77,9 @@ files `./src/modules.json`.
 
 This server implements P2P functionality. It can fetch question databases and users from other
 server instances, and merge the response data to its own databases. The server also instantly sends
-new questions received from users and new users to all registered peers. The sync feature should be
+new questions received from users, new users and uploaded user files to all registered peers. The
-used for initialization, new user getting, and rarely for catching up, since all new questions
+sync feature should be used for initialization and rarely for catching up, since important data is
-should be received instantly.
+received instantly.
 
 To setup P2P functionality you have to create a few files in `./data/p2p`:
 
@@ -104,7 +104,7 @@ To setup P2P functionality you have to create a few files in `./data/p2p`:
    Public key is optional, but needed to encrypt and add the users database in the response, so they
    can be synced too.
 
-   New keys will be added during certain actions, such as: `sessionCookie` and `lastSync`
+   New keys will be added during certain actions, such as: `sessionCookie` and `last${key}Sync`
 
 ### Using `/syncp2pdata`
 

src/modules/api/p2p/p2putils.ts

@@ -1,7 +1,12 @@
 import { PeerInfo, QuestionDb } from '../../../types/basicTypes'
 import { files, paths, readAndValidateFile } from '../../../utils/files'
 import logger from '../../../utils/logger'
-import { PostResult, parseCookie, post } from '../../../utils/networkUtils'
+import {
+    PostResult,
+    downloadFile,
+    parseCookie,
+    post,
+} from '../../../utils/networkUtils'
 import utils from '../../../utils/utils'
 import { UserDirDataFile } from '../submodules/userFiles'
 
@@ -199,7 +204,11 @@ export async function loginAndPostDataToAllPeers<
                 res = await postDataFn(peer, sessionCookie)
             }
 
-            if (res.error || !res.data?.success) {
+            if (
+                res.error ||
+                !res.data?.success ||
+                res.data?.result === 'nouser'
+            ) {
                 results.errors.push(peer)
                 console.error(
                     `Error: posting data to ${peerToString(peer)}`,
@@ -238,3 +247,60 @@ export async function loginAndPostDataToAllPeers<
         )}`
     )
 }
+
+export async function loginAndDownloadFile(
+    peer: PeerInfo,
+    destination: string,
+    fileName: string,
+    dir: string
+): Promise<{ success: boolean; message?: string }> {
+    const download = (sessionCookie: string) => {
+        return downloadFile(
+            {
+                host: peer.host,
+                port: peer.port,
+                path: `/api/userFiles/${encodeURIComponent(
+                    dir
+                )}/${encodeURIComponent(fileName)}`,
+            },
+            destination,
+            `sessionID=${sessionCookie}`,
+            peer.http
+        )
+    }
+
+    try {
+        let sessionCookie = peer.sessionCookie
+
+        const login = async (peer: PeerInfo) => {
+            const loginResult = await loginToPeer(peer)
+            if (typeof loginResult === 'string') {
+                sessionCookie = loginResult
+                updatePeersFile(peer, { sessionCookie: loginResult })
+            } else {
+                throw new Error('Error logging in to' + peerToString(peer))
+            }
+        }
+
+        if (!sessionCookie) {
+            await login(peer)
+        }
+
+        let res = await download(sessionCookie)
+
+        if (res.result === 'nouser' && sessionCookie) {
+            await login(peer)
+
+            res = await download(sessionCookie)
+        } else if (!res.success) {
+            throw new Error(res.message)
+        }
+
+        if (res.result === 'nouser') {
+            throw new Error(`Unable to login to peer: ${peerToString(peer)}`)
+        }
+        return { success: true }
+    } catch (e) {
+        return { success: false, message: e.message }
+    }
+}

src/modules/api/p2p/userFiles.ts

@@ -6,12 +6,12 @@ import {
     SyncDataResult,
     SyncResponseBase,
     SyncResult,
+    loginAndDownloadFile,
     peerToString,
     updatePeersFile,
 } from './p2putils'
 import constants from '../../../constants'
 import { PeerInfo } from '../../../types/basicTypes'
-import { downloadFile } from '../../../utils/networkUtils'
 import logger from '../../../utils/logger'
 
 interface UserFileToGet {
@@ -101,18 +101,17 @@ async function downloadUserFiles(filesToGet: UserFileToGet[]) {
         const { peer, dir, fileName, filePath, data } = fileToGet
 
         try {
-            await downloadFile(
+            const { success, message } = await loginAndDownloadFile(
-                {
+                peer,
-                    host: peer.host,
-                    port: peer.port,
-                    path: `/api/userFiles/${encodeURIComponent(
-                        dir
-                    )}/${encodeURIComponent(fileName)}`,
-                },
                 filePath,
-                peer.http
+                fileName,
+                dir
             )
 
+            if (!success) {
+                throw new Error(message)
+            }
+
             const dataFilePath = path.join(
                 paths.userFilesDir,
                 dir,
@@ -150,7 +149,11 @@ async function downloadUserFiles(filesToGet: UserFileToGet[]) {
             utils.WriteFile(JSON.stringify(dataFile), dataFilePath)
             addedFiles += 1
         } catch (e) {
-            logger.Log(`Unable to download "${fileName}": ${e.message}`)
+            logger.Log(
+                `Unable to download "${fileName}" from "${peerToString(
+                    peer
+                )}": "${e.message}"`
+            )
             console.error(e)
         }
     }
@@ -264,6 +267,7 @@ export async function syncUserFiles(
         }
     }
 
+    logger.Log('\tRecieved user files:', 'green')
     logger.logTable([['', 'Files'], ...recievedUserFilesCount], {
         colWidth: [20],
         rowPrefix: '\t',
@@ -274,13 +278,18 @@ export async function syncUserFiles(
         filesToGet.push(...setupFilesToGet(res.newFiles, res.peer))
     })
 
-    const addedFiles = await downloadUserFiles(filesToGet)
+    let addedFiles = 0
+    if (filesToGet.length > 0) {
+        logger.Log(`\tDownloading new files ...`)
+
+        addedFiles = await downloadUserFiles(filesToGet)
 
         newData.forEach((res) => {
             updatePeersFile(res.peer, {
                 lastUserFilesSync: syncStart,
             })
         })
+    }
 
     logger.Log(
         `Successfully synced user files! Added ${addedFiles} files`,

src/modules/api/submodules/p2p.ts

@@ -457,13 +457,15 @@ function setup(data: SubmoduleData): Submodule {
         const getData = <T extends keyof Omit<SyncDataResult, 'remoteInfo'>>(
             key: T
         ) => {
+            const shouldHaveSynced = shouldSync[key] || syncAll
+
             let data = resultDataWithoutErrors.map((x) => ({
                 ...x.data[key],
                 peer: x.peer,
             }))
 
             data.forEach((x) => {
-                if (!x.success) {
+                if (!x.success && shouldHaveSynced) {
                     logger.Log(
                         `Error syncing "${key}" with ${peerToString(
                             x.peer
@@ -473,7 +475,7 @@ function setup(data: SubmoduleData): Submodule {
                 }
             })
 
-            if ((!data || data.length === 0) && (shouldSync[key] || syncAll)) {
+            if ((!data || data.length === 0) && shouldHaveSynced) {
                 logger.Log(
                     `"${key}" data was requested, but not received!`,
                     'yellowbg'
@@ -668,15 +670,15 @@ function setup(data: SubmoduleData): Submodule {
         const userFiles = !!req.query.userFiles
 
         const allTime = !!req.query.allTime
-        const user = req.session.user
+        // const user = req.session.user
 
-        if (!user || user.id !== 1) {
+        // if (!user || user.id !== 1) {
-            res.json({
+        //     res.json({
-                status: 'error',
+        //         status: 'error',
-                message: 'only user 1 can call this EP',
+        //         message: 'only user 1 can call this EP',
-            })
+        //     })
-            return
+        //     return
-        }
+        // }
 
         // FIXME: /syncResult EP if this EP times out, but we still need the result
         if (syncInProgress) {

src/utils/networkUtils.ts

@@ -1,4 +1,4 @@
-import http, { request as httpRequest } from 'http'
+import http, { IncomingMessage, request as httpRequest } from 'http'
 import https, { request as httpsRequest } from 'https'
 import fs from 'node:fs'
 import utils from './utils'
@@ -9,50 +9,73 @@ export interface GetResult<T> {
     options?: http.RequestOptions
 }
 
-export function get<T = any>(
+export function getRaw(
     options: http.RequestOptions,
     useHttp?: boolean
-): Promise<GetResult<T>> {
+): Promise<GetResult<Buffer> & { response?: IncomingMessage }> {
     const provider = useHttp ? http : https
 
     return new Promise((resolve) => {
-        const req = provider.get(
+        const req = provider.get(options, function (res) {
-            {
-                ...options,
-                headers: {
-                    ...options?.headers,
-                    'Content-Type': 'application/json',
-                },
-            },
-            function (res) {
             const bodyChunks: Uint8Array[] = []
             res.on('data', (chunk) => {
                 bodyChunks.push(chunk)
             }).on('end', () => {
-                    const body = Buffer.concat(bodyChunks).toString()
+                const body = Buffer.concat(bodyChunks)
                 try {
                     if (res.statusCode === 200) {
-                            resolve({ data: JSON.parse(body) })
+                        resolve({ data: body, response: res })
                     } else {
                         resolve({
-                                data: JSON.parse(body),
+                            data: body,
                             error: new Error(
                                 `HTTP response code: ${res.statusCode}`
                             ),
+                            response: res,
                         })
                     }
                 } catch (e) {
-                        resolve({ error: e, options: options })
+                    resolve({ error: e, options: options, response: res })
                 }
             })
-            }
+        })
-        )
         req.on('error', function (e) {
             resolve({ error: e, options: options })
         })
     })
 }
 
+export async function get<T = any>(
+    options: http.RequestOptions,
+    useHttp?: boolean
+): Promise<GetResult<T>> {
+    const { data, response } = await getRaw(
+        {
+            ...options,
+            headers: {
+                'Content-Type': 'application/json',
+                ...options?.headers,
+            },
+        },
+        useHttp
+    )
+
+    const body = data.toString()
+
+    try {
+        if (response.statusCode === 200) {
+            return { data: JSON.parse(body) }
+        } else {
+            return {
+                data: JSON.parse(body),
+                error: new Error(`HTTP response code: ${response.statusCode}`),
+            }
+        }
+    } catch (e) {
+        return { error: e, options: options }
+    }
+}
+
 export interface PostResult<T> {
     data?: T
     error?: Error
@@ -128,27 +151,59 @@ export function post<T = any>({
 export function downloadFile(
     options: http.RequestOptions,
     destination: string,
+    cookie: string,
     useHttp?: boolean
-): Promise<void> {
+): Promise<{ message?: string; result?: string; success: boolean }> {
     const provider = useHttp ? http : https
 
+    if (utils.FileExists(destination)) {
+        return Promise.resolve({
+            success: true,
+            message: `\tDownload file: "${destination}" already esxists, skipping download`,
+        })
+    }
+
+    return new Promise((resolve, reject) => {
+        provider.get(
+            {
+                ...options,
+                headers: {
+                    'Content-Type': 'application/json',
+                    ...options?.headers,
+                    ...(cookie
+                        ? {
+                              cookie: cookie,
+                          }
+                        : {}),
+                },
+            },
+            function (res) {
+                if (res.statusCode === 200) {
                     utils.createDirsForFile(destination)
                     const file = fs.createWriteStream(destination)
-    return new Promise((resolve, reject) => {
+
-        provider.get(options, function (response) {
+                    res.pipe(file)
-            response.pipe(file)
 
                     file.on('finish', () => {
                         file.close()
-                resolve()
+                        resolve({ success: true })
                     })
 
-            response.on('error', (e) => {
+                    res.on('error', (e) => {
                         file.close()
-                fs.unlinkSync(destination)
+                        utils.deleteFile(destination)
                         reject(e)
                     })
+                } else if (res.statusCode === 401) {
+                    resolve({ success: false, result: 'nouser' })
+                } else {
+                    resolve({
+                        success: false,
+                        message: `Unhandled status code: ${res.statusCode}`,
                     })
+                }
+            }
+        )
     })
 }
 

submodules/qmining-page

@@ -1 +1 @@
-Subproject commit 055a732733b05d4579fa8e9a85da6b97c29957de
+Subproject commit 96d1dafe90a55a476876958b384958b3d394f963