feat: replace fs scans with bbolt for fast, persistent metadata management

Signed-off-by: skidoodle <contact@albert.lol>

Dockerfile

@@ -2,6 +2,9 @@ FROM --platform=$BUILDPLATFORM golang:1.25.6 AS builder
 
 WORKDIR /app
 
+COPY go.mod go.sum ./
+RUN go mod download
+
 COPY . .
 
 ARG TARGETOS

go.mod

@@ -1,3 +1,7 @@
 module github.com/skidoodle/safebin
 
 go 1.25.6
+
+require go.etcd.io/bbolt v1.4.3
+
+require golang.org/x/sys v0.29.0 // indirect

go.sum

@@ -0,0 +1,14 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
+go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app/config.go

@@ -8,6 +8,8 @@ import (
 	"os"
 	"strconv"
 	"time"
+
+	"go.etcd.io/bbolt"
 )
 
 const (
@@ -31,6 +33,10 @@ const (
 	TempExpiry      = 4 * time.Hour
 	MinRetention    = 24 * time.Hour
 	MaxRetention    = 365 * 24 * time.Hour
+
+	DBFileName   = "safebin.db"
+	DBBucketName = "files"
+	TempDirName = "tmp"
 )
 
 type Config struct {
@@ -43,6 +49,7 @@ type App struct {
 	Conf   Config
 	Tmpl   *template.Template
 	Logger *slog.Logger
+	DB     *bbolt.DB
 }
 
 func LoadConfig() Config {

internal/app/db.go

@@ -0,0 +1,35 @@
+package app
+
+import (
+	"path/filepath"
+	"time"
+
+	"go.etcd.io/bbolt"
+)
+
+type FileMeta struct {
+	ID        string    `json:"id"`
+	Size      int64     `json:"size"`
+	CreatedAt time.Time `json:"created_at"`
+	ExpiresAt time.Time `json:"expires_at"`
+}
+
+func InitDB(storageDir string) (*bbolt.DB, error) {
+	path := filepath.Join(storageDir, DBFileName)
+	db, err := bbolt.Open(path, 0600, &bbolt.Options{Timeout: 1 * time.Second})
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.Update(func(tx *bbolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(DBBucketName))
+		return err
+	})
+
+	if err != nil {
+		db.Close()
+		return nil, err
+	}
+
+	return db, nil
+}

internal/app/db_test.go

@@ -0,0 +1,85 @@
+package app
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go.etcd.io/bbolt"
+)
+
+func TestInitDB(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	db, err := InitDB(tmpDir)
+	if err != nil {
+		t.Fatalf("InitDB failed: %v", err)
+	}
+	defer db.Close()
+
+	dbPath := filepath.Join(tmpDir, DBFileName)
+	if _, err := os.Stat(dbPath); os.IsNotExist(err) {
+		t.Error("Database file was not created")
+	}
+
+	err = db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		if b == nil {
+			t.Errorf("Bucket '%s' was not created", DBBucketName)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Errorf("View failed: %v", err)
+	}
+}
+
+func TestDB_MetadataLifecycle(t *testing.T) {
+	tmpDir := t.TempDir()
+	db, err := InitDB(tmpDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	app := &App{
+		Conf: Config{StorageDir: tmpDir, MaxMB: 100},
+		DB:   db,
+	}
+
+	fileID := "test-file-id"
+	fileSize := int64(1024)
+
+	if err := app.RegisterFile(fileID, fileSize); err != nil {
+		t.Fatalf("RegisterFile failed: %v", err)
+	}
+
+	err = db.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		data := b.Get([]byte(fileID))
+		if data == nil {
+			t.Fatal("Metadata not found in DB")
+		}
+
+		var meta FileMeta
+		if err := json.Unmarshal(data, &meta); err != nil {
+			t.Fatalf("Failed to unmarshal meta: %v", err)
+		}
+
+		if meta.ID != fileID {
+			t.Errorf("Want ID %s, got %s", fileID, meta.ID)
+		}
+		if meta.Size != fileSize {
+			t.Errorf("Want Size %d, got %d", fileSize, meta.Size)
+		}
+		if meta.ExpiresAt.Before(time.Now()) {
+			t.Error("Expiration time is in the past")
+		}
+		return nil
+	})
+	if err != nil {
+		t.Error(err)
+	}
+}

internal/app/server_test.go

@@ -17,7 +17,7 @@ import (
 
 func setupTestApp(t *testing.T) (*App, string) {
 	storageDir := t.TempDir()
-	os.MkdirAll(filepath.Join(storageDir, "tmp"), 0700)
+	os.MkdirAll(filepath.Join(storageDir, TempDirName), 0700)
 
 	tmplDir := filepath.Join(storageDir, "templates")
 	os.MkdirAll(tmplDir, 0700)
@@ -26,6 +26,12 @@ func setupTestApp(t *testing.T) (*App, string) {
 
 	tmpl := template.Must(template.New("base").Parse(`{{define "base"}}OK{{end}}`))
 
+	db, err := InitDB(storageDir)
+	if err != nil {
+		t.Fatalf("Failed to init db: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+
 	app := &App{
 		Conf: Config{
 			StorageDir: storageDir,
@@ -33,6 +39,7 @@ func setupTestApp(t *testing.T) (*App, string) {
 		},
 		Logger: discardLogger(),
 		Tmpl:   tmpl,
+		DB:     db,
 	}
 
 	return app, storageDir

internal/app/storage.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math"
@@ -11,6 +12,7 @@ import (
 	"time"
 
 	"github.com/skidoodle/safebin/internal/crypto"
+	"go.etcd.io/bbolt"
 )
 
 func (app *App) StartCleanupTask(ctx context.Context) {
@@ -22,14 +24,14 @@ func (app *App) StartCleanupTask(ctx context.Context) {
 			ticker.Stop()
 			return
 		case <-ticker.C:
-			app.CleanStorage(app.Conf.StorageDir)
+			app.CleanStorage()
-			app.CleanTemp(filepath.Join(app.Conf.StorageDir, "tmp"))
+			app.CleanTemp(filepath.Join(app.Conf.StorageDir, TempDirName))
 		}
 	}
 }
 
 func (app *App) saveChunk(uid string, idx int, src io.Reader) error {
-	dir := filepath.Join(app.Conf.StorageDir, "tmp", uid)
+	dir := filepath.Join(app.Conf.StorageDir, TempDirName, uid)
 
 	if err := os.MkdirAll(dir, PermUserRWX); err != nil {
 		return fmt.Errorf("create chunk dir: %w", err)
@@ -54,7 +56,7 @@ func (app *App) saveChunk(uid string, idx int, src io.Reader) error {
 }
 
 func (app *App) mergeChunks(uid string, total int) (string, error) {
-	tmpPath := filepath.Join(app.Conf.StorageDir, "tmp", "m_"+uid)
+	tmpPath := filepath.Join(app.Conf.StorageDir, TempDirName, "m_"+uid)
 
 	merged, err := os.Create(tmpPath)
 	if err != nil {
@@ -71,7 +73,7 @@ func (app *App) mergeChunks(uid string, total int) (string, error) {
 	var written int64
 
 	for i := range total {
-		partPath := filepath.Join(app.Conf.StorageDir, "tmp", uid, strconv.Itoa(i))
+		partPath := filepath.Join(app.Conf.StorageDir, TempDirName, uid, strconv.Itoa(i))
 
 		part, err := os.Open(partPath)
 		if err != nil {
@@ -139,26 +141,72 @@ func (app *App) encryptAndSave(src io.Reader, key []byte, finalPath string) erro
 	return nil
 }
 
-func (app *App) CleanStorage(path string) {
+func (app *App) RegisterFile(id string, size int64) error {
-	entries, err := os.ReadDir(path)
+	retention := CalculateRetention(size, app.Conf.MaxMB)
+	meta := FileMeta{
+		ID:        id,
+		Size:      size,
+		CreatedAt: time.Now(),
+		ExpiresAt: time.Now().Add(retention),
+	}
+
+	return app.DB.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		data, err := json.Marshal(meta)
+		if err != nil {
+			return err
+		}
+		return b.Put([]byte(id), data)
+	})
+}
+
+func (app *App) CleanStorage() {
+	now := time.Now()
+	var toDelete []string
+
+	err := app.DB.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		c := b.Cursor()
+
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			var meta FileMeta
+			if err := json.Unmarshal(v, &meta); err != nil {
+				continue
+			}
+
+			if now.After(meta.ExpiresAt) {
+				toDelete = append(toDelete, string(k))
+			}
+		}
+		return nil
+	})
+
 	if err != nil {
-		app.Logger.Error("Failed to read storage dir", "err", err)
+		app.Logger.Error("Failed to view DB for cleanup", "err", err)
 		return
 	}
 
-	for _, entry := range entries {
+	if len(toDelete) == 0 {
-		info, err := entry.Info()
+		return
-		if err != nil {
+	}
-			continue
-		}
 
-		expiry := CalculateRetention(info.Size(), app.Conf.MaxMB)
+	err = app.DB.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		for _, id := range toDelete {
+			path := filepath.Join(app.Conf.StorageDir, id)
+			if err := os.RemoveAll(path); err != nil {
+				app.Logger.Error("Failed to remove expired file", "path", id, "err", err)
+			}
 
-		if time.Since(info.ModTime()) > expiry {
+			if err := b.Delete([]byte(id)); err != nil {
-			if err := os.RemoveAll(filepath.Join(path, entry.Name())); err != nil {
+				app.Logger.Error("Failed to delete metadata", "id", id, "err", err)
-				app.Logger.Error("Failed to remove expired file", "path", entry.Name(), "err", err)
 			}
 		}
+		return nil
+	})
+
+	if err != nil {
+		app.Logger.Error("Failed to update DB during cleanup", "err", err)
 	}
 }
 

internal/app/storage_test.go

@@ -1,20 +1,27 @@
 package app
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
+
+	"go.etcd.io/bbolt"
 )
 
 func TestCleanup_AbandonedMerge(t *testing.T) {
 	tmpDir := t.TempDir()
-	tmpStorage := filepath.Join(tmpDir, "tmp")
+	tmpStorage := filepath.Join(tmpDir, TempDirName)
 	os.MkdirAll(tmpStorage, 0700)
 
+	db, _ := InitDB(tmpDir)
+	defer db.Close()
+
 	app := &App{
-		Conf: Config{StorageDir: tmpDir},
+		Conf:   Config{StorageDir: tmpDir},
 		Logger: discardLogger(),
+		DB:     db,
 	}
 
 	abandonedFile := filepath.Join(tmpStorage, "m_abandoned_upload_id")
@@ -36,12 +43,16 @@ func TestCleanup_AbandonedMerge(t *testing.T) {
 
 func TestCleanup_AbandonedChunks(t *testing.T) {
 	tmpDir := t.TempDir()
-	tmpStorage := filepath.Join(tmpDir, "tmp")
+	tmpStorage := filepath.Join(tmpDir, TempDirName)
 	os.MkdirAll(tmpStorage, 0700)
 
+	db, _ := InitDB(tmpDir)
+	defer db.Close()
+
 	app := &App{
-		Conf: Config{StorageDir: tmpDir},
+		Conf:   Config{StorageDir: tmpDir},
 		Logger: discardLogger(),
+		DB:     db,
 	}
 
 	chunkDir := filepath.Join(tmpStorage, "some_upload_id")
@@ -60,26 +71,48 @@ func TestCleanup_AbandonedChunks(t *testing.T) {
 
 func TestCleanup_ExpiredStorage(t *testing.T) {
 	storageDir := t.TempDir()
+	db, _ := InitDB(storageDir)
+	defer db.Close()
+
 	app := &App{
 		Conf: Config{
 			StorageDir: storageDir,
 			MaxMB:      100,
 		},
 		Logger: discardLogger(),
+		DB:     db,
 	}
 
 	filename := "large_file_id"
 	path := filepath.Join(storageDir, filename)
 	f, _ := os.Create(path)
-	f.Truncate(100 * MegaByte) // Max size
+	f.Truncate(100 * MegaByte)
 	f.Close()
 
-	oldTime := time.Now().Add(-MinRetention - time.Hour)
+	expiredMeta := FileMeta{
-	os.Chtimes(path, oldTime, oldTime)
+		ID:        filename,
+		Size:      100 * MegaByte,
+		CreatedAt: time.Now().Add(-MinRetention - 2*time.Hour),
+		ExpiresAt: time.Now().Add(-time.Hour),
+	}
 
-	app.CleanStorage(storageDir)
+	app.DB.Update(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		data, _ := json.Marshal(expiredMeta)
+		return b.Put([]byte(filename), data)
+	})
+
+	app.CleanStorage()
 
 	if _, err := os.Stat(path); !os.IsNotExist(err) {
 		t.Error("Cleanup failed to remove expired large file")
 	}
+
+	app.DB.View(func(tx *bbolt.Tx) error {
+		b := tx.Bucket([]byte(DBBucketName))
+		if v := b.Get([]byte(filename)); v != nil {
+			t.Error("Cleanup failed to remove metadata")
+		}
+		return nil
+	})
 }

internal/app/upload.go

@@ -36,7 +36,7 @@ func (app *App) HandleUpload(writer http.ResponseWriter, request *http.Request)
 		}
 	}()
 
-	tmp, err := os.CreateTemp(filepath.Join(app.Conf.StorageDir, "tmp"), "up_*")
+	tmp, err := os.CreateTemp(filepath.Join(app.Conf.StorageDir, TempDirName), "up_*")
 
 	if err != nil {
 		app.Logger.Error("Failed to create temp file", "err", err)
@@ -160,7 +160,7 @@ func (app *App) HandleFinish(writer http.ResponseWriter, request *http.Request)
 
 	app.FinalizeFile(writer, request, mergedRead, request.FormValue("filename"))
 
-	if err := os.RemoveAll(filepath.Join(app.Conf.StorageDir, "tmp", uid)); err != nil {
+	if err := os.RemoveAll(filepath.Join(app.Conf.StorageDir, TempDirName, uid)); err != nil {
 		app.Logger.Error("Failed to remove chunk dir", "err", err)
 	}
 }
@@ -184,7 +184,10 @@ func (app *App) FinalizeFile(writer http.ResponseWriter, request *http.Request,
 	id := crypto.GetID(key, ext)
 	finalPath := filepath.Join(app.Conf.StorageDir, id)
 
-	if _, err := os.Stat(finalPath); err == nil {
+	if info, err := os.Stat(finalPath); err == nil {
+		if err := app.RegisterFile(id, info.Size()); err != nil {
+			app.Logger.Error("Failed to update metadata for existing file", "err", err)
+		}
 		app.RespondWithLink(writer, request, key, filename)
 		return
 	}
@@ -201,5 +204,13 @@ func (app *App) FinalizeFile(writer http.ResponseWriter, request *http.Request,
 		return
 	}
 
+	if info, err := os.Stat(finalPath); err == nil {
+		if err := app.RegisterFile(id, info.Size()); err != nil {
+			app.Logger.Error("Failed to save metadata", "err", err)
+		}
+	} else {
+		app.Logger.Error("Failed to stat new file", "err", err)
+	}
+
 	app.RespondWithLink(writer, request, key, filename)
 }

main.go

@@ -26,16 +26,28 @@ func main() {
 		"max_file_size", fmt.Sprintf("%dMB", cfg.MaxMB),
 	)
 
-	tmpDir := filepath.Join(cfg.StorageDir, "tmp")
+	tmpDir := filepath.Join(cfg.StorageDir, app.TempDirName)
 	if err := os.MkdirAll(tmpDir, app.PermUserRWX); err != nil {
 		logger.Error("Failed to initialize storage directory", "err", err)
 		os.Exit(1)
 	}
 
+	db, err := app.InitDB(cfg.StorageDir)
+	if err != nil {
+		logger.Error("Failed to initialize database", "err", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := db.Close(); err != nil {
+			logger.Error("Failed to close database", "err", err)
+		}
+	}()
+
 	application := &app.App{
 		Conf:   cfg,
 		Logger: logger,
 		Tmpl:   app.ParseTemplates(),
+		DB:     db,
 	}
 
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)