import path from 'node:path' import { paths } from '../../../utils/files' import utils from '../../../utils/utils' import { UserDirDataFile } from '../submodules/userFiles' import { SyncDataResult, SyncResponseBase, SyncResult, peerToString, updatePeersFile, } from './p2putils' import constants from '../../../constants' import { PeerInfo } from '../../../types/basicTypes' import { downloadFile } from '../../../utils/networkUtils' import logger from '../../../utils/logger' interface UserFileToGet { fileName: string dir: string filePath: string data: UserDirDataFile peer: PeerInfo } export interface NewUserFilesRequestBody { host: string newFiles: { [key: string]: { // key: dir [key: string]: UserDirDataFile // key: file name } } } // --------------------------------------------------------------------------------------------- // Getting // --------------------------------------------------------------------------------------------- export function getUserFiles(since: number): SyncResponseBase & { newFiles: { [key: string]: { [key: string]: UserDirDataFile } } } { const newFiles: SyncDataResult['userFiles']['newFiles'] = {} const dirs = utils.ReadDir(paths.userFilesDir) dirs.forEach((dir) => { const userDirPath = path.join(paths.userFilesDir, dir) const dataFilePath = path.join( userDirPath, constants.userFilesDataFileName ) if (!utils.FileExists(dataFilePath)) { return } const dataFile = utils.ReadJSON>(dataFilePath) Object.entries(dataFile).forEach(([fileName, data]) => { const mtime = utils.statFile(path.join(userDirPath, fileName)).mtime if (mtime.getTime() >= since) { if (!newFiles[dir]) { newFiles[dir] = {} } newFiles[dir][fileName] = data } }) }) return { success: true, newFiles: newFiles } } function setupFilesToGet( newFiles: SyncDataResult['userFiles']['newFiles'], peer: PeerInfo ): UserFileToGet[] { const filesToGet: UserFileToGet[] = [] Object.entries(newFiles).forEach(([dirName, userFilesDir]) => { Object.entries(userFilesDir).forEach(([fileName, data]) => { filesToGet.push({ fileName: fileName, dir: dirName, filePath: path.join(paths.userFilesDir, dirName, fileName), data: data, peer: peer, }) }) }) return filesToGet } async function downloadUserFiles(filesToGet: UserFileToGet[]) { let addedFiles = 0 for (const fileToGet of filesToGet) { const { peer, dir, fileName, filePath, data } = fileToGet try { await downloadFile( { host: peer.host, port: peer.port, path: `/api/userFiles/${dir}/${fileName}`, }, filePath, peer.http ) const dataFilePath = path.join( paths.userFilesDir, dir, constants.userFilesDataFileName ) if (!utils.FileExists(dataFilePath)) { utils.WriteFile(JSON.stringify({}), dataFilePath) } const dataFile = utils.ReadJSON<{ [key: string]: UserDirDataFile }>(dataFilePath) if (dataFile[fileName]) { // dataFile[fileName].views += data.views // views are not unique dataFile[fileName].upvotes = dataFile[fileName].upvotes ? dataFile[fileName].upvotes .concat(data.upvotes) .reduce((acc, x) => { if (acc.includes(x)) return acc return [...acc, x] }, []) : [] dataFile[fileName].downvotes = dataFile[fileName].downvotes ? dataFile[fileName].downvotes .concat(data.downvotes) .reduce((acc, x) => { if (acc.includes(x)) return acc return [...acc, x] }, []) : [] } else { dataFile[fileName] = data } utils.WriteFile(JSON.stringify(dataFile), dataFilePath) addedFiles += 1 } catch (e) { logger.Log(`Unable to download "${fileName}": ${e.message}`) console.error(e) } } return addedFiles } // --------------------------------------------------------------------------------------------- // Adding new // --------------------------------------------------------------------------------------------- export async function handleNewUserFiles( props: NewUserFilesRequestBody & { peers: PeerInfo[] } ): Promise<{ success: boolean addedFileCount?: number message?: string }> { const result = await addNewUserFiles(props) if (!result.success) { logger.Log( `Error while adding new user files: "${result.message}", from host: "${props.host}"`, 'yellowbg' ) } return result } async function addNewUserFiles({ newFiles, host, peers, }: NewUserFilesRequestBody & { peers: PeerInfo[] }): Promise<{ success: boolean addedFileCount?: number message?: string }> { if (!newFiles || !host) { return { success: false, message: 'newFiles or host key are missing from body', } } const remotePeerInfo = peers.find((peer) => { return peerToString(peer) === host }) if (!remotePeerInfo) { return { success: false, message: "couldn't find remote peer info based on host", } } try { const filesToGet = setupFilesToGet(newFiles, remotePeerInfo) const addedFileCount = await downloadUserFiles(filesToGet) logger.Log( `\tAdded ${logger.C( 'blue' )}${addedFileCount}${logger.C()} new files from ${logger.C( 'blue' )}${peerToString(remotePeerInfo)}${logger.C()}` ) return { success: true, addedFileCount: addedFileCount } } catch (e) { return { success: false, message: e.message } } } // --------------------------------------------------------------------------------------------- // Syncing // --------------------------------------------------------------------------------------------- export async function syncUserFiles( newData: (SyncDataResult['userFiles'] & { peer: PeerInfo })[], syncStart: number ): Promise { logger.Log('Syncing user files...') const recievedUserFilesCount: (string | number)[][] = [] let totalRecievedd = 0 newData.forEach((res) => { const count = Object.values(res.newFiles).reduce((acc, data) => { totalRecievedd += Object.keys(data).length return acc + Object.keys(data).length }, 0) recievedUserFilesCount.push([peerToString(res.peer), count]) }) if (totalRecievedd === 0) { logger.Log( `No peers returned any new files. User file sync successfully finished!`, 'green' ) return { old: { userFiles: 0, }, added: { userFiles: 0, }, final: { userFiles: 0, }, } } logger.logTable([['', 'Files'], ...recievedUserFilesCount], { colWidth: [20], rowPrefix: '\t', }) const filesToGet: UserFileToGet[] = [] newData.forEach((res) => { filesToGet.push(...setupFilesToGet(res.newFiles, res.peer)) }) const addedFiles = await downloadUserFiles(filesToGet) newData.forEach((res) => { updatePeersFile(res.peer, { lastUserFilesSync: syncStart, }) }) logger.Log( `Successfully synced user files! Added ${addedFiles} files`, 'green' ) return { old: { userFiles: 0, }, added: { userFiles: addedFiles, }, final: { userFiles: 0, }, } }