diff --git a/Dockerfile b/Dockerfile index bdd4ef2..5e2546f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM --platform=$BUILDPLATFORM golang:1.25.6 AS builder WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download + COPY . . ARG TARGETOS diff --git a/go.mod b/go.mod index 9580abc..71cefde 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,7 @@ module github.com/skidoodle/safebin go 1.25.6 + +require go.etcd.io/bbolt v1.4.3 + +require golang.org/x/sys v0.29.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..56ca229 --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= +go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/config.go b/internal/app/config.go index b22e05b..b0a5be3 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -8,6 +8,8 @@ import ( "os" "strconv" "time" + + "go.etcd.io/bbolt" ) const ( @@ -31,6 +33,10 @@ const ( TempExpiry = 4 * time.Hour MinRetention = 24 * time.Hour MaxRetention = 365 * 24 * time.Hour + + DBFileName = "safebin.db" + DBBucketName = "files" + TempDirName = "tmp" ) type Config struct { @@ -43,6 +49,7 @@ type App struct { Conf Config Tmpl *template.Template Logger *slog.Logger + DB *bbolt.DB } func LoadConfig() Config { diff --git a/internal/app/db.go b/internal/app/db.go new file mode 100644 index 0000000..6e974ed --- /dev/null +++ b/internal/app/db.go @@ -0,0 +1,35 @@ +package app + +import ( + "path/filepath" + "time" + + "go.etcd.io/bbolt" +) + +type FileMeta struct { + ID string `json:"id"` + Size int64 `json:"size"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` +} + +func InitDB(storageDir string) (*bbolt.DB, error) { + path := filepath.Join(storageDir, DBFileName) + db, err := bbolt.Open(path, 0600, &bbolt.Options{Timeout: 1 * time.Second}) + if err != nil { + return nil, err + } + + err = db.Update(func(tx *bbolt.Tx) error { + _, err := tx.CreateBucketIfNotExists([]byte(DBBucketName)) + return err + }) + + if err != nil { + db.Close() + return nil, err + } + + return db, nil +} diff --git a/internal/app/db_test.go b/internal/app/db_test.go new file mode 100644 index 0000000..2dd0278 --- /dev/null +++ b/internal/app/db_test.go @@ -0,0 +1,85 @@ +package app + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "go.etcd.io/bbolt" +) + +func TestInitDB(t *testing.T) { + tmpDir := t.TempDir() + + db, err := InitDB(tmpDir) + if err != nil { + t.Fatalf("InitDB failed: %v", err) + } + defer db.Close() + + dbPath := filepath.Join(tmpDir, DBFileName) + if _, err := os.Stat(dbPath); os.IsNotExist(err) { + t.Error("Database file was not created") + } + + err = db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + if b == nil { + t.Errorf("Bucket '%s' was not created", DBBucketName) + } + return nil + }) + if err != nil { + t.Errorf("View failed: %v", err) + } +} + +func TestDB_MetadataLifecycle(t *testing.T) { + tmpDir := t.TempDir() + db, err := InitDB(tmpDir) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + app := &App{ + Conf: Config{StorageDir: tmpDir, MaxMB: 100}, + DB: db, + } + + fileID := "test-file-id" + fileSize := int64(1024) + + if err := app.RegisterFile(fileID, fileSize); err != nil { + t.Fatalf("RegisterFile failed: %v", err) + } + + err = db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + data := b.Get([]byte(fileID)) + if data == nil { + t.Fatal("Metadata not found in DB") + } + + var meta FileMeta + if err := json.Unmarshal(data, &meta); err != nil { + t.Fatalf("Failed to unmarshal meta: %v", err) + } + + if meta.ID != fileID { + t.Errorf("Want ID %s, got %s", fileID, meta.ID) + } + if meta.Size != fileSize { + t.Errorf("Want Size %d, got %d", fileSize, meta.Size) + } + if meta.ExpiresAt.Before(time.Now()) { + t.Error("Expiration time is in the past") + } + return nil + }) + if err != nil { + t.Error(err) + } +} diff --git a/internal/app/server_test.go b/internal/app/server_test.go index 0d7d19f..4a34343 100644 --- a/internal/app/server_test.go +++ b/internal/app/server_test.go @@ -17,7 +17,7 @@ import ( func setupTestApp(t *testing.T) (*App, string) { storageDir := t.TempDir() - os.MkdirAll(filepath.Join(storageDir, "tmp"), 0700) + os.MkdirAll(filepath.Join(storageDir, TempDirName), 0700) tmplDir := filepath.Join(storageDir, "templates") os.MkdirAll(tmplDir, 0700) @@ -26,6 +26,12 @@ func setupTestApp(t *testing.T) (*App, string) { tmpl := template.Must(template.New("base").Parse(`{{define "base"}}OK{{end}}`)) + db, err := InitDB(storageDir) + if err != nil { + t.Fatalf("Failed to init db: %v", err) + } + t.Cleanup(func() { db.Close() }) + app := &App{ Conf: Config{ StorageDir: storageDir, @@ -33,6 +39,7 @@ func setupTestApp(t *testing.T) (*App, string) { }, Logger: discardLogger(), Tmpl: tmpl, + DB: db, } return app, storageDir diff --git a/internal/app/storage.go b/internal/app/storage.go index d37e1ae..f707787 100644 --- a/internal/app/storage.go +++ b/internal/app/storage.go @@ -2,6 +2,7 @@ package app import ( "context" + "encoding/json" "fmt" "io" "math" @@ -11,6 +12,7 @@ import ( "time" "github.com/skidoodle/safebin/internal/crypto" + "go.etcd.io/bbolt" ) func (app *App) StartCleanupTask(ctx context.Context) { @@ -22,14 +24,14 @@ func (app *App) StartCleanupTask(ctx context.Context) { ticker.Stop() return case <-ticker.C: - app.CleanStorage(app.Conf.StorageDir) - app.CleanTemp(filepath.Join(app.Conf.StorageDir, "tmp")) + app.CleanStorage() + app.CleanTemp(filepath.Join(app.Conf.StorageDir, TempDirName)) } } } func (app *App) saveChunk(uid string, idx int, src io.Reader) error { - dir := filepath.Join(app.Conf.StorageDir, "tmp", uid) + dir := filepath.Join(app.Conf.StorageDir, TempDirName, uid) if err := os.MkdirAll(dir, PermUserRWX); err != nil { return fmt.Errorf("create chunk dir: %w", err) @@ -54,7 +56,7 @@ func (app *App) saveChunk(uid string, idx int, src io.Reader) error { } func (app *App) mergeChunks(uid string, total int) (string, error) { - tmpPath := filepath.Join(app.Conf.StorageDir, "tmp", "m_"+uid) + tmpPath := filepath.Join(app.Conf.StorageDir, TempDirName, "m_"+uid) merged, err := os.Create(tmpPath) if err != nil { @@ -71,7 +73,7 @@ func (app *App) mergeChunks(uid string, total int) (string, error) { var written int64 for i := range total { - partPath := filepath.Join(app.Conf.StorageDir, "tmp", uid, strconv.Itoa(i)) + partPath := filepath.Join(app.Conf.StorageDir, TempDirName, uid, strconv.Itoa(i)) part, err := os.Open(partPath) if err != nil { @@ -139,26 +141,72 @@ func (app *App) encryptAndSave(src io.Reader, key []byte, finalPath string) erro return nil } -func (app *App) CleanStorage(path string) { - entries, err := os.ReadDir(path) +func (app *App) RegisterFile(id string, size int64) error { + retention := CalculateRetention(size, app.Conf.MaxMB) + meta := FileMeta{ + ID: id, + Size: size, + CreatedAt: time.Now(), + ExpiresAt: time.Now().Add(retention), + } + + return app.DB.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + data, err := json.Marshal(meta) + if err != nil { + return err + } + return b.Put([]byte(id), data) + }) +} + +func (app *App) CleanStorage() { + now := time.Now() + var toDelete []string + + err := app.DB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + c := b.Cursor() + + for k, v := c.First(); k != nil; k, v = c.Next() { + var meta FileMeta + if err := json.Unmarshal(v, &meta); err != nil { + continue + } + + if now.After(meta.ExpiresAt) { + toDelete = append(toDelete, string(k)) + } + } + return nil + }) + if err != nil { - app.Logger.Error("Failed to read storage dir", "err", err) + app.Logger.Error("Failed to view DB for cleanup", "err", err) return } - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - continue - } + if len(toDelete) == 0 { + return + } - expiry := CalculateRetention(info.Size(), app.Conf.MaxMB) + err = app.DB.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + for _, id := range toDelete { + path := filepath.Join(app.Conf.StorageDir, id) + if err := os.RemoveAll(path); err != nil { + app.Logger.Error("Failed to remove expired file", "path", id, "err", err) + } - if time.Since(info.ModTime()) > expiry { - if err := os.RemoveAll(filepath.Join(path, entry.Name())); err != nil { - app.Logger.Error("Failed to remove expired file", "path", entry.Name(), "err", err) + if err := b.Delete([]byte(id)); err != nil { + app.Logger.Error("Failed to delete metadata", "id", id, "err", err) } } + return nil + }) + + if err != nil { + app.Logger.Error("Failed to update DB during cleanup", "err", err) } } diff --git a/internal/app/storage_test.go b/internal/app/storage_test.go index ae7d286..5e72409 100644 --- a/internal/app/storage_test.go +++ b/internal/app/storage_test.go @@ -1,20 +1,27 @@ package app import ( + "encoding/json" "os" "path/filepath" "testing" "time" + + "go.etcd.io/bbolt" ) func TestCleanup_AbandonedMerge(t *testing.T) { tmpDir := t.TempDir() - tmpStorage := filepath.Join(tmpDir, "tmp") + tmpStorage := filepath.Join(tmpDir, TempDirName) os.MkdirAll(tmpStorage, 0700) + db, _ := InitDB(tmpDir) + defer db.Close() + app := &App{ - Conf: Config{StorageDir: tmpDir}, + Conf: Config{StorageDir: tmpDir}, Logger: discardLogger(), + DB: db, } abandonedFile := filepath.Join(tmpStorage, "m_abandoned_upload_id") @@ -36,12 +43,16 @@ func TestCleanup_AbandonedMerge(t *testing.T) { func TestCleanup_AbandonedChunks(t *testing.T) { tmpDir := t.TempDir() - tmpStorage := filepath.Join(tmpDir, "tmp") + tmpStorage := filepath.Join(tmpDir, TempDirName) os.MkdirAll(tmpStorage, 0700) + db, _ := InitDB(tmpDir) + defer db.Close() + app := &App{ - Conf: Config{StorageDir: tmpDir}, + Conf: Config{StorageDir: tmpDir}, Logger: discardLogger(), + DB: db, } chunkDir := filepath.Join(tmpStorage, "some_upload_id") @@ -60,26 +71,48 @@ func TestCleanup_AbandonedChunks(t *testing.T) { func TestCleanup_ExpiredStorage(t *testing.T) { storageDir := t.TempDir() + db, _ := InitDB(storageDir) + defer db.Close() + app := &App{ Conf: Config{ StorageDir: storageDir, MaxMB: 100, }, Logger: discardLogger(), + DB: db, } filename := "large_file_id" path := filepath.Join(storageDir, filename) f, _ := os.Create(path) - f.Truncate(100 * MegaByte) // Max size + f.Truncate(100 * MegaByte) f.Close() - oldTime := time.Now().Add(-MinRetention - time.Hour) - os.Chtimes(path, oldTime, oldTime) + expiredMeta := FileMeta{ + ID: filename, + Size: 100 * MegaByte, + CreatedAt: time.Now().Add(-MinRetention - 2*time.Hour), + ExpiresAt: time.Now().Add(-time.Hour), + } - app.CleanStorage(storageDir) + app.DB.Update(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + data, _ := json.Marshal(expiredMeta) + return b.Put([]byte(filename), data) + }) + + app.CleanStorage() if _, err := os.Stat(path); !os.IsNotExist(err) { t.Error("Cleanup failed to remove expired large file") } + + app.DB.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte(DBBucketName)) + if v := b.Get([]byte(filename)); v != nil { + t.Error("Cleanup failed to remove metadata") + } + return nil + }) } diff --git a/internal/app/upload.go b/internal/app/upload.go index 91be669..bc710a7 100644 --- a/internal/app/upload.go +++ b/internal/app/upload.go @@ -36,7 +36,7 @@ func (app *App) HandleUpload(writer http.ResponseWriter, request *http.Request) } }() - tmp, err := os.CreateTemp(filepath.Join(app.Conf.StorageDir, "tmp"), "up_*") + tmp, err := os.CreateTemp(filepath.Join(app.Conf.StorageDir, TempDirName), "up_*") if err != nil { app.Logger.Error("Failed to create temp file", "err", err) @@ -160,7 +160,7 @@ func (app *App) HandleFinish(writer http.ResponseWriter, request *http.Request) app.FinalizeFile(writer, request, mergedRead, request.FormValue("filename")) - if err := os.RemoveAll(filepath.Join(app.Conf.StorageDir, "tmp", uid)); err != nil { + if err := os.RemoveAll(filepath.Join(app.Conf.StorageDir, TempDirName, uid)); err != nil { app.Logger.Error("Failed to remove chunk dir", "err", err) } } @@ -184,7 +184,10 @@ func (app *App) FinalizeFile(writer http.ResponseWriter, request *http.Request, id := crypto.GetID(key, ext) finalPath := filepath.Join(app.Conf.StorageDir, id) - if _, err := os.Stat(finalPath); err == nil { + if info, err := os.Stat(finalPath); err == nil { + if err := app.RegisterFile(id, info.Size()); err != nil { + app.Logger.Error("Failed to update metadata for existing file", "err", err) + } app.RespondWithLink(writer, request, key, filename) return } @@ -201,5 +204,13 @@ func (app *App) FinalizeFile(writer http.ResponseWriter, request *http.Request, return } + if info, err := os.Stat(finalPath); err == nil { + if err := app.RegisterFile(id, info.Size()); err != nil { + app.Logger.Error("Failed to save metadata", "err", err) + } + } else { + app.Logger.Error("Failed to stat new file", "err", err) + } + app.RespondWithLink(writer, request, key, filename) } diff --git a/main.go b/main.go index ed1fdb3..50965ed 100644 --- a/main.go +++ b/main.go @@ -26,16 +26,28 @@ func main() { "max_file_size", fmt.Sprintf("%dMB", cfg.MaxMB), ) - tmpDir := filepath.Join(cfg.StorageDir, "tmp") + tmpDir := filepath.Join(cfg.StorageDir, app.TempDirName) if err := os.MkdirAll(tmpDir, app.PermUserRWX); err != nil { logger.Error("Failed to initialize storage directory", "err", err) os.Exit(1) } + db, err := app.InitDB(cfg.StorageDir) + if err != nil { + logger.Error("Failed to initialize database", "err", err) + os.Exit(1) + } + defer func() { + if err := db.Close(); err != nil { + logger.Error("Failed to close database", "err", err) + } + }() + application := &app.App{ Conf: cfg, Logger: logger, Tmpl: app.ParseTemplates(), + DB: db, } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)