perf(storage)!: optimize cleanup with secondary index

BREAKING CHANGE: This change requires a fresh database. Existing
databases will lack the index, and the cleanup routine will not function
correctly for pre-existing files.

Signed-off-by: skidoodle <contact@albert.lol>

internal/app/config.go

@@ -35,9 +35,10 @@ const (
 	MinRetention    = 24 * time.Hour
 	MaxRetention    = 365 * 24 * time.Hour
 
-	DBFileName   = "safebin.db"
-	DBBucketName = "files"
-	TempDirName  = "tmp"
+	DBFileName        = "safebin.db"
+	DBBucketName      = "files"
+	DBBucketIndexName = "expiry_index"
+	TempDirName       = "tmp"
 )
 
 type Config struct {

internal/app/db.go

@@ -22,8 +22,13 @@ func InitDB(storageDir string) (*bbolt.DB, error) {
 	}
 
 	err = db.Update(func(tx *bbolt.Tx) error {
-		_, err := tx.CreateBucketIfNotExists([]byte(DBBucketName))
-		return err
+		if _, err := tx.CreateBucketIfNotExists([]byte(DBBucketName)); err != nil {
+			return err
+		}
+		if _, err := tx.CreateBucketIfNotExists([]byte(DBBucketIndexName)); err != nil {
+			return err
+		}
+		return nil
 	})
 
 	if err != nil {

internal/app/db_test.go

@@ -29,10 +29,12 @@ func TestInitDB(t *testing.T) {
 	}
 
 	err = db.View(func(tx *bbolt.Tx) error {
-		b := tx.Bucket([]byte(DBBucketName))
-		if b == nil {
+		if b := tx.Bucket([]byte(DBBucketName)); b == nil {
 			t.Errorf("Bucket '%s' was not created", DBBucketName)
 		}
+		if b := tx.Bucket([]byte(DBBucketIndexName)); b == nil {
+			t.Errorf("Bucket '%s' was not created", DBBucketIndexName)
+		}
 		return nil
 	})
 	if err != nil {
@@ -85,6 +87,15 @@ func TestDB_MetadataLifecycle(t *testing.T) {
 		if meta.ExpiresAt.Before(time.Now()) {
 			t.Error("Expiration time is in the past")
 		}
+
+		bIndex := tx.Bucket([]byte(DBBucketIndexName))
+		indexKey := []byte(meta.ExpiresAt.Format(time.RFC3339) + "_" + fileID)
+		if val := bIndex.Get(indexKey); val == nil {
+			t.Error("Index entry not found")
+		} else if string(val) != fileID {
+			t.Errorf("Index value mismatch: want %s, got %s", fileID, string(val))
+		}
+
 		return nil
 	})
 	if err != nil {

internal/app/storage.go

@@ -129,32 +129,42 @@ func (app *App) RegisterFile(id string, size int64) error {
 	}
 
 	return app.DB.Update(func(tx *bbolt.Tx) error {
-		b := tx.Bucket([]byte(DBBucketName))
+		bFiles := tx.Bucket([]byte(DBBucketName))
+		bIndex := tx.Bucket([]byte(DBBucketIndexName))
+
 		data, err := json.Marshal(meta)
 		if err != nil {
 			return err
 		}
-		return b.Put([]byte(id), data)
+
+		if err := bFiles.Put([]byte(id), data); err != nil {
+			return err
+		}
+
+		indexKey := []byte(meta.ExpiresAt.Format(time.RFC3339) + "_" + id)
+		return bIndex.Put(indexKey, []byte(id))
 	})
 }
 
 func (app *App) CleanStorage() {
-	now := time.Now()
-	var toDelete []string
+	now := time.Now().Format(time.RFC3339)
+	var toDeleteIDs []string
+	var toDeleteKeys []string
 
 	err := app.DB.View(func(tx *bbolt.Tx) error {
-		b := tx.Bucket([]byte(DBBucketName))
-		c := b.Cursor()
+		bIndex := tx.Bucket([]byte(DBBucketIndexName))
+		if bIndex == nil {
+			return nil
+		}
+		c := bIndex.Cursor()
 
 		for k, v := c.First(); k != nil; k, v = c.Next() {
-			var meta FileMeta
-			if err := json.Unmarshal(v, &meta); err != nil {
-				continue
+			if string(k) > now {
+				break
 			}
 
-			if now.After(meta.ExpiresAt) {
-				toDelete = append(toDelete, string(k))
-			}
+			toDeleteKeys = append(toDeleteKeys, string(k))
+			toDeleteIDs = append(toDeleteIDs, string(v))
 		}
 		return nil
 	})
@@ -164,21 +174,27 @@ func (app *App) CleanStorage() {
 		return
 	}
 
-	if len(toDelete) == 0 {
+	if len(toDeleteIDs) == 0 {
 		return
 	}
 
 	err = app.DB.Update(func(tx *bbolt.Tx) error {
-		b := tx.Bucket([]byte(DBBucketName))
-		for _, id := range toDelete {
+		bFiles := tx.Bucket([]byte(DBBucketName))
+		bIndex := tx.Bucket([]byte(DBBucketIndexName))
+
+		for i, id := range toDeleteIDs {
 			path := filepath.Join(app.Conf.StorageDir, id)
 			if err := os.RemoveAll(path); err != nil {
 				app.Logger.Error("Failed to remove expired file", "path", id, "err", err)
 			}
 
-			if err := b.Delete([]byte(id)); err != nil {
+			if err := bFiles.Delete([]byte(id)); err != nil {
 				app.Logger.Error("Failed to delete metadata", "id", id, "err", err)
 			}
+
+			if err := bIndex.Delete([]byte(toDeleteKeys[i])); err != nil {
+				app.Logger.Error("Failed to delete index", "key", toDeleteKeys[i], "err", err)
+			}
 		}
 		return nil
 	})

internal/app/storage_test.go

@@ -95,9 +95,16 @@ func TestCleanup_ExpiredStorage(t *testing.T) {
 	}
 
 	if err := app.DB.Update(func(tx *bbolt.Tx) error {
-		b := tx.Bucket([]byte(DBBucketName))
+		bFiles := tx.Bucket([]byte(DBBucketName))
+		bIndex := tx.Bucket([]byte(DBBucketIndexName))
+
 		data, _ := json.Marshal(expiredMeta)
-		return b.Put([]byte(filename), data)
+		if err := bFiles.Put([]byte(filename), data); err != nil {
+			return err
+		}
+
+		indexKey := []byte(expiredMeta.ExpiresAt.Format(time.RFC3339) + "_" + filename)
+		return bIndex.Put(indexKey, []byte(filename))
 	}); err != nil {
 		t.Fatalf("DB Update failed: %v", err)
 	}
@@ -109,10 +116,16 @@ func TestCleanup_ExpiredStorage(t *testing.T) {
 	}
 
 	if err := app.DB.View(func(tx *bbolt.Tx) error {
-		b := tx.Bucket([]byte(DBBucketName))
-		if v := b.Get([]byte(filename)); v != nil {
+		bFiles := tx.Bucket([]byte(DBBucketName))
+		if v := bFiles.Get([]byte(filename)); v != nil {
 			t.Error("Cleanup failed to remove metadata")
 		}
+
+		bIndex := tx.Bucket([]byte(DBBucketIndexName))
+		indexKey := []byte(expiredMeta.ExpiresAt.Format(time.RFC3339) + "_" + filename)
+		if v := bIndex.Get(indexKey); v != nil {
+			t.Error("Cleanup failed to remove index entry")
+		}
 		return nil
 	}); err != nil {
 		t.Fatalf("DB View failed: %v", err)