init claude-code

2026-04-01 17:32:37 +02:00
commit 73b208c009
1902 changed files with 513237 additions and 0 deletions
@@ -0,0 +1,287 @@
+/**
+ * File persistence orchestrator
+ *
+ * This module provides the main orchestration logic for persisting files
+ * at the end of each turn:
+ * - BYOC mode: Upload files to Files API and collect file IDs
+ * - 1P/Cloud mode: Query Files API listDirectory for file IDs (rclone handles sync)
+ */
+
+import { feature } from 'bun:bundle'
+import { join, relative } from 'path'
+import {
+  type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  logEvent,
+} from '../../services/analytics/index.js'
+import {
+  type FilesApiConfig,
+  uploadSessionFiles,
+} from '../../services/api/filesApi.js'
+import { getCwd } from '../cwd.js'
+import { errorMessage } from '../errors.js'
+import { logError } from '../log.js'
+import { getSessionIngressAuthToken } from '../sessionIngressAuth.js'
+import {
+  findModifiedFiles,
+  getEnvironmentKind,
+  logDebug,
+} from './outputsScanner.js'
+import {
+  DEFAULT_UPLOAD_CONCURRENCY,
+  type FailedPersistence,
+  FILE_COUNT_LIMIT,
+  type FilesPersistedEventData,
+  OUTPUTS_SUBDIR,
+  type PersistedFile,
+  type TurnStartTime,
+} from './types.js'
+
+/**
+ * Execute file persistence for modified files in the outputs directory.
+ *
+ * Assembles all config internally:
+ * - Checks environment kind (CLAUDE_CODE_ENVIRONMENT_KIND)
+ * - Retrieves session access token
+ * - Requires CLAUDE_CODE_REMOTE_SESSION_ID for session ID
+ *
+ * @param turnStartTime - The timestamp when the turn started
+ * @param signal - Optional abort signal for cancellation
+ * @returns Event data, or null if not enabled or no files to persist
+ */
+export async function runFilePersistence(
+  turnStartTime: TurnStartTime,
+  signal?: AbortSignal,
+): Promise<FilesPersistedEventData | null> {
+  const environmentKind = getEnvironmentKind()
+  if (environmentKind !== 'byoc') {
+    return null
+  }
+
+  const sessionAccessToken = getSessionIngressAuthToken()
+  if (!sessionAccessToken) {
+    return null
+  }
+
+  const sessionId = process.env.CLAUDE_CODE_REMOTE_SESSION_ID
+  if (!sessionId) {
+    logError(
+      new Error(
+        'File persistence enabled but CLAUDE_CODE_REMOTE_SESSION_ID is not set',
+      ),
+    )
+    return null
+  }
+
+  const config: FilesApiConfig = {
+    oauthToken: sessionAccessToken,
+    sessionId,
+  }
+
+  const outputsDir = join(getCwd(), sessionId, OUTPUTS_SUBDIR)
+
+  // Check if aborted
+  if (signal?.aborted) {
+    logDebug('Persistence aborted before processing')
+    return null
+  }
+
+  const startTime = Date.now()
+  logEvent('tengu_file_persistence_started', {
+    mode: environmentKind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+  })
+
+  try {
+    let result: FilesPersistedEventData
+    if (environmentKind === 'byoc') {
+      result = await executeBYOCPersistence(
+        turnStartTime,
+        config,
+        outputsDir,
+        signal,
+      )
+    } else {
+      result = await executeCloudPersistence()
+    }
+
+    // Nothing to report
+    if (result.files.length === 0 && result.failed.length === 0) {
+      return null
+    }
+
+    const durationMs = Date.now() - startTime
+    logEvent('tengu_file_persistence_completed', {
+      success_count: result.files.length,
+      failure_count: result.failed.length,
+      duration_ms: durationMs,
+      mode: environmentKind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    })
+
+    return result
+  } catch (error) {
+    logError(error)
+    logDebug(`File persistence failed: ${error}`)
+
+    const durationMs = Date.now() - startTime
+    logEvent('tengu_file_persistence_completed', {
+      success_count: 0,
+      failure_count: 0,
+      duration_ms: durationMs,
+      mode: environmentKind as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+      error:
+        'exception' as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS,
+    })
+
+    return {
+      files: [],
+      failed: [
+        {
+          filename: outputsDir,
+          error: errorMessage(error),
+        },
+      ],
+    }
+  }
+}
+
+/**
+ * Execute BYOC mode persistence: scan local filesystem for modified files,
+ * then upload to Files API.
+ */
+async function executeBYOCPersistence(
+  turnStartTime: TurnStartTime,
+  config: FilesApiConfig,
+  outputsDir: string,
+  signal?: AbortSignal,
+): Promise<FilesPersistedEventData> {
+  // Find modified files via local filesystem scan
+  // Uses same directory structure as downloads: {cwd}/{sessionId}/outputs
+  const modifiedFiles = await findModifiedFiles(turnStartTime, outputsDir)
+
+  if (modifiedFiles.length === 0) {
+    logDebug('No modified files to persist')
+    return { files: [], failed: [] }
+  }
+
+  logDebug(`Found ${modifiedFiles.length} modified files`)
+
+  if (signal?.aborted) {
+    return { files: [], failed: [] }
+  }
+
+  // Enforce file count limit
+  if (modifiedFiles.length > FILE_COUNT_LIMIT) {
+    logDebug(
+      `File count limit exceeded: ${modifiedFiles.length} > ${FILE_COUNT_LIMIT}`,
+    )
+    logEvent('tengu_file_persistence_limit_exceeded', {
+      file_count: modifiedFiles.length,
+      limit: FILE_COUNT_LIMIT,
+    })
+    return {
+      files: [],
+      failed: [
+        {
+          filename: outputsDir,
+          error: `Too many files modified (${modifiedFiles.length}). Maximum: ${FILE_COUNT_LIMIT}.`,
+        },
+      ],
+    }
+  }
+
+  const filesToProcess = modifiedFiles
+    .map(filePath => ({
+      path: filePath,
+      relativePath: relative(outputsDir, filePath),
+    }))
+    .filter(({ relativePath }) => {
+      // Security: skip files that resolve outside the outputs directory
+      if (relativePath.startsWith('..')) {
+        logDebug(`Skipping file outside outputs directory: ${relativePath}`)
+        return false
+      }
+      return true
+    })
+
+  logDebug(`BYOC mode: uploading ${filesToProcess.length} files`)
+
+  // Upload files in parallel
+  const results = await uploadSessionFiles(
+    filesToProcess,
+    config,
+    DEFAULT_UPLOAD_CONCURRENCY,
+  )
+
+  // Separate successful and failed uploads
+  const persistedFiles: PersistedFile[] = []
+  const failedFiles: FailedPersistence[] = []
+
+  for (const result of results) {
+    if (result.success) {
+      persistedFiles.push({
+        filename: result.path,
+        file_id: result.fileId,
+      })
+    } else {
+      failedFiles.push({
+        filename: result.path,
+        error: result.error,
+      })
+    }
+  }
+
+  logDebug(
+    `BYOC persistence complete: ${persistedFiles.length} uploaded, ${failedFiles.length} failed`,
+  )
+
+  return {
+    files: persistedFiles,
+    failed: failedFiles,
+  }
+}
+
+/**
+ * Execute Cloud (1P) mode persistence.
+ * TODO: Read file_id from xattr on output files. xattr-based file IDs are
+ * currently being added for 1P environments.
+ */
+function executeCloudPersistence(): FilesPersistedEventData {
+  logDebug('Cloud mode: xattr-based file ID reading not yet implemented')
+  return { files: [], failed: [] }
+}
+
+/**
+ * Execute file persistence and emit result via callback.
+ * Handles errors internally.
+ */
+export async function executeFilePersistence(
+  turnStartTime: TurnStartTime,
+  signal: AbortSignal,
+  onResult: (result: FilesPersistedEventData) => void,
+): Promise<void> {
+  try {
+    const result = await runFilePersistence(turnStartTime, signal)
+    if (result) {
+      onResult(result)
+    }
+  } catch (error) {
+    logError(error)
+  }
+}
+
+/**
+ * Check if file persistence is enabled.
+ * Requires: feature flag ON, valid environment kind, session access token,
+ * and CLAUDE_CODE_REMOTE_SESSION_ID.
+ * This ensures only public-api/sessions users trigger file persistence,
+ * not normal Claude Code CLI users.
+ */
+export function isFilePersistenceEnabled(): boolean {
+  if (feature('FILE_PERSISTENCE')) {
+    return (
+      getEnvironmentKind() === 'byoc' &&
+      !!getSessionIngressAuthToken() &&
+      !!process.env.CLAUDE_CODE_REMOTE_SESSION_ID
+    )
+  }
+  return false
+}
@@ -0,0 +1,126 @@
+/**
+ * Outputs directory scanner for file persistence
+ *
+ * This module provides utilities to:
+ * - Detect the session type from environment variables
+ * - Capture turn start timestamp
+ * - Find modified files by comparing file mtimes against turn start time
+ */
+
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import { logForDebugging } from '../debug.js'
+import type { EnvironmentKind } from '../teleport/environments.js'
+import type { TurnStartTime } from './types.js'
+
+/** Shared debug logger for file persistence modules */
+export function logDebug(message: string): void {
+  logForDebugging(`[file-persistence] ${message}`)
+}
+
+/**
+ * Get the environment kind from CLAUDE_CODE_ENVIRONMENT_KIND.
+ * Returns null if not set or not a recognized value.
+ */
+export function getEnvironmentKind(): EnvironmentKind | null {
+  const kind = process.env.CLAUDE_CODE_ENVIRONMENT_KIND
+  if (kind === 'byoc' || kind === 'anthropic_cloud') {
+    return kind
+  }
+  return null
+}
+
+function hasParentPath(
+  entry: object,
+): entry is { parentPath: string; name: string } {
+  return 'parentPath' in entry && typeof entry.parentPath === 'string'
+}
+
+function hasPath(entry: object): entry is { path: string; name: string } {
+  return 'path' in entry && typeof entry.path === 'string'
+}
+
+function getEntryParentPath(entry: object, fallback: string): string {
+  if (hasParentPath(entry)) {
+    return entry.parentPath
+  }
+  if (hasPath(entry)) {
+    return entry.path
+  }
+  return fallback
+}
+
+/**
+ * Find files that have been modified since the turn started.
+ * Returns paths of files with mtime >= turnStartTime.
+ *
+ * Uses recursive directory listing and parallelized stat calls for efficiency.
+ *
+ * @param turnStartTime - The timestamp when the turn started
+ * @param outputsDir - The directory to scan for modified files
+ */
+export async function findModifiedFiles(
+  turnStartTime: TurnStartTime,
+  outputsDir: string,
+): Promise<string[]> {
+  // Use recursive flag to get all entries in one call
+  let entries: Awaited<ReturnType<typeof fs.readdir>>
+  try {
+    entries = await fs.readdir(outputsDir, {
+      withFileTypes: true,
+      recursive: true,
+    })
+  } catch {
+    // Directory doesn't exist or is not accessible
+    return []
+  }
+
+  // Filter to regular files only (skip symlinks for security) and build full paths
+  const filePaths: string[] = []
+  for (const entry of entries) {
+    if (entry.isSymbolicLink()) {
+      continue
+    }
+    if (entry.isFile()) {
+      // entry.parentPath is available in Node 20+, fallback to entry.path for older versions
+      const parentPath = getEntryParentPath(entry, outputsDir)
+      filePaths.push(path.join(parentPath, entry.name))
+    }
+  }
+
+  if (filePaths.length === 0) {
+    logDebug('No files found in outputs directory')
+    return []
+  }
+
+  // Parallelize stat calls for all files
+  const statResults = await Promise.all(
+    filePaths.map(async filePath => {
+      try {
+        const stat = await fs.lstat(filePath)
+        // Skip if it became a symlink between readdir and stat (race condition)
+        if (stat.isSymbolicLink()) {
+          return null
+        }
+        return { filePath, mtimeMs: stat.mtimeMs }
+      } catch {
+        // File may have been deleted between readdir and stat
+        return null
+      }
+    }),
+  )
+
+  // Filter to files modified since turn start
+  const modifiedFiles: string[] = []
+  for (const result of statResults) {
+    if (result && result.mtimeMs >= turnStartTime) {
+      modifiedFiles.push(result.filePath)
+    }
+  }
+
+  logDebug(
+    `Found ${modifiedFiles.length} modified files since turn start (scanned ${filePaths.length} total)`,
+  )
+
+  return modifiedFiles
+}