From d18ef48bd409dfcfde5bcb6e2dd6bc52f53d08b9 Mon Sep 17 00:00:00 2001 From: skidoodle Date: Sun, 18 Jan 2026 22:10:07 +0100 Subject: [PATCH] perf(storage)!: optimize cleanup with secondary index BREAKING CHANGE: This change requires a fresh database. Existing databases will lack the index, and the cleanup routine will not function correctly for pre-existing files. Signed-off-by: skidoodle --- internal/app/config.go | 7 +++--- internal/app/db.go | 9 +++++-- internal/app/db_test.go | 15 +++++++++-- internal/app/storage.go | 48 ++++++++++++++++++++++++------------ internal/app/storage_test.go | 21 +++++++++++++--- 5 files changed, 73 insertions(+), 27 deletions(-) diff --git a/internal/app/config.go b/internal/app/config.go index 203bab8..7f87cf2 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -35,9 +35,10 @@ const ( MinRetention = 24 * time.Hour MaxRetention = 365 * 24 * time.Hour - DBFileName = "safebin.db" - DBBucketName = "files" - TempDirName = "tmp" + DBFileName = "safebin.db" + DBBucketName = "files" + DBBucketIndexName = "expiry_index" + TempDirName = "tmp" ) type Config struct { diff --git a/internal/app/db.go b/internal/app/db.go index 317e040..d191354 100644 --- a/internal/app/db.go +++ b/internal/app/db.go @@ -22,8 +22,13 @@ func InitDB(storageDir string) (*bbolt.DB, error) { } err = db.Update(func(tx *bbolt.Tx) error { - _, err := tx.CreateBucketIfNotExists([]byte(DBBucketName)) - return err + if _, err := tx.CreateBucketIfNotExists([]byte(DBBucketName)); err != nil { + return err + } + if _, err := tx.CreateBucketIfNotExists([]byte(DBBucketIndexName)); err != nil { + return err + } + return nil }) if err != nil { diff --git a/internal/app/db_test.go b/internal/app/db_test.go index b57acd3..495ab7c 100644 --- a/internal/app/db_test.go +++ b/internal/app/db_test.go @@ -29,10 +29,12 @@ func TestInitDB(t *testing.T) { } err = db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(DBBucketName)) - if b == nil { + if b := tx.Bucket([]byte(DBBucketName)); b == nil { t.Errorf("Bucket '%s' was not created", DBBucketName) } + if b := tx.Bucket([]byte(DBBucketIndexName)); b == nil { + t.Errorf("Bucket '%s' was not created", DBBucketIndexName) + } return nil }) if err != nil { @@ -85,6 +87,15 @@ func TestDB_MetadataLifecycle(t *testing.T) { if meta.ExpiresAt.Before(time.Now()) { t.Error("Expiration time is in the past") } + + bIndex := tx.Bucket([]byte(DBBucketIndexName)) + indexKey := []byte(meta.ExpiresAt.Format(time.RFC3339) + "_" + fileID) + if val := bIndex.Get(indexKey); val == nil { + t.Error("Index entry not found") + } else if string(val) != fileID { + t.Errorf("Index value mismatch: want %s, got %s", fileID, string(val)) + } + return nil }) if err != nil { diff --git a/internal/app/storage.go b/internal/app/storage.go index f1be775..d4ae71a 100644 --- a/internal/app/storage.go +++ b/internal/app/storage.go @@ -129,32 +129,42 @@ func (app *App) RegisterFile(id string, size int64) error { } return app.DB.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(DBBucketName)) + bFiles := tx.Bucket([]byte(DBBucketName)) + bIndex := tx.Bucket([]byte(DBBucketIndexName)) + data, err := json.Marshal(meta) if err != nil { return err } - return b.Put([]byte(id), data) + + if err := bFiles.Put([]byte(id), data); err != nil { + return err + } + + indexKey := []byte(meta.ExpiresAt.Format(time.RFC3339) + "_" + id) + return bIndex.Put(indexKey, []byte(id)) }) } func (app *App) CleanStorage() { - now := time.Now() - var toDelete []string + now := time.Now().Format(time.RFC3339) + var toDeleteIDs []string + var toDeleteKeys []string err := app.DB.View(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(DBBucketName)) - c := b.Cursor() + bIndex := tx.Bucket([]byte(DBBucketIndexName)) + if bIndex == nil { + return nil + } + c := bIndex.Cursor() for k, v := c.First(); k != nil; k, v = c.Next() { - var meta FileMeta - if err := json.Unmarshal(v, &meta); err != nil { - continue + if string(k) > now { + break } - if now.After(meta.ExpiresAt) { - toDelete = append(toDelete, string(k)) - } + toDeleteKeys = append(toDeleteKeys, string(k)) + toDeleteIDs = append(toDeleteIDs, string(v)) } return nil }) @@ -164,21 +174,27 @@ func (app *App) CleanStorage() { return } - if len(toDelete) == 0 { + if len(toDeleteIDs) == 0 { return } err = app.DB.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(DBBucketName)) - for _, id := range toDelete { + bFiles := tx.Bucket([]byte(DBBucketName)) + bIndex := tx.Bucket([]byte(DBBucketIndexName)) + + for i, id := range toDeleteIDs { path := filepath.Join(app.Conf.StorageDir, id) if err := os.RemoveAll(path); err != nil { app.Logger.Error("Failed to remove expired file", "path", id, "err", err) } - if err := b.Delete([]byte(id)); err != nil { + if err := bFiles.Delete([]byte(id)); err != nil { app.Logger.Error("Failed to delete metadata", "id", id, "err", err) } + + if err := bIndex.Delete([]byte(toDeleteKeys[i])); err != nil { + app.Logger.Error("Failed to delete index", "key", toDeleteKeys[i], "err", err) + } } return nil }) diff --git a/internal/app/storage_test.go b/internal/app/storage_test.go index 3600501..80c5c9e 100644 --- a/internal/app/storage_test.go +++ b/internal/app/storage_test.go @@ -95,9 +95,16 @@ func TestCleanup_ExpiredStorage(t *testing.T) { } if err := app.DB.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(DBBucketName)) + bFiles := tx.Bucket([]byte(DBBucketName)) + bIndex := tx.Bucket([]byte(DBBucketIndexName)) + data, _ := json.Marshal(expiredMeta) - return b.Put([]byte(filename), data) + if err := bFiles.Put([]byte(filename), data); err != nil { + return err + } + + indexKey := []byte(expiredMeta.ExpiresAt.Format(time.RFC3339) + "_" + filename) + return bIndex.Put(indexKey, []byte(filename)) }); err != nil { t.Fatalf("DB Update failed: %v", err) } @@ -109,10 +116,16 @@ func TestCleanup_ExpiredStorage(t *testing.T) { } if err := app.DB.View(func(tx *bbolt.Tx) error { - b := tx.Bucket([]byte(DBBucketName)) - if v := b.Get([]byte(filename)); v != nil { + bFiles := tx.Bucket([]byte(DBBucketName)) + if v := bFiles.Get([]byte(filename)); v != nil { t.Error("Cleanup failed to remove metadata") } + + bIndex := tx.Bucket([]byte(DBBucketIndexName)) + indexKey := []byte(expiredMeta.ExpiresAt.Format(time.RFC3339) + "_" + filename) + if v := bIndex.Get(indexKey); v != nil { + t.Error("Cleanup failed to remove index entry") + } return nil }); err != nil { t.Fatalf("DB View failed: %v", err)