x/safebin
1
0
Fork 0
mirror of https://github.com/skidoodle/safebin.git synced 2026-04-28 03:07:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: replace fs scans with bbolt for fast, persistent metadata management

Browse Source 
Signed-off-by: skidoodle <contact@albert.lol>
This commit is contained in:
albert
2026-01-18 20:27:33 +01:00
parent 5a3846266e
commit 954aec6d8e
11 changed files with 289 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+3
View File
@@ -2,6 +2,9 @@ FROM --platform=$BUILDPLATFORM golang:1.25.6 AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
ARG TARGETOS
go.mod
+4
View File
@@ -1,3 +1,7 @@
module github.com/skidoodle/safebin
go 1.25.6
require go.etcd.io/bbolt v1.4.3
require golang.org/x/sys v0.29.0 // indirect
go.sum
+14
View File
@@ -0,0 +1,14 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
internal/app/config.go
+7
View File
@@ -8,6 +8,8 @@ import (
"os"
"strconv"
"time"
"go.etcd.io/bbolt"
)
const (
@@ -31,6 +33,10 @@ const (
TempExpiry = 4 * time.Hour
MinRetention = 24 * time.Hour
MaxRetention = 365 * 24 * time.Hour
DBFileName = "safebin.db"
DBBucketName = "files"
TempDirName = "tmp"
)
type Config struct {
@@ -43,6 +49,7 @@ type App struct {
Conf Config
Tmpl *template.Template
Logger *slog.Logger
DB *bbolt.DB
}
func LoadConfig() Config {
internal/app/db.go
+35
View File
@@ -0,0 +1,35 @@
package app
import (
"path/filepath"
"time"
"go.etcd.io/bbolt"
)
type FileMeta struct {
ID string `json:"id"`
Size int64 `json:"size"`
CreatedAt time.Time `json:"created_at"`
ExpiresAt time.Time `json:"expires_at"`
}
func InitDB(storageDir string) (*bbolt.DB, error) {
path := filepath.Join(storageDir, DBFileName)
db, err := bbolt.Open(path, 0600, &bbolt.Options{Timeout: 1 * time.Second})
if err != nil {
return nil, err
}
err = db.Update(func(tx *bbolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(DBBucketName))
return err
})
if err != nil {
db.Close()
return nil, err
}
return db, nil
}
internal/app/db_test.go
+85
View File
@@ -0,0 +1,85 @@
package app
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"go.etcd.io/bbolt"
)
func TestInitDB(t *testing.T) {
tmpDir := t.TempDir()
db, err := InitDB(tmpDir)
if err != nil {
t.Fatalf("InitDB failed: %v", err)
}
defer db.Close()
dbPath := filepath.Join(tmpDir, DBFileName)
if _, err := os.Stat(dbPath); os.IsNotExist(err) {
t.Error("Database file was not created")
}
err = db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
if b == nil {
t.Errorf("Bucket '%s' was not created", DBBucketName)
}
return nil
})
if err != nil {
t.Errorf("View failed: %v", err)
}
}
func TestDB_MetadataLifecycle(t *testing.T) {
tmpDir := t.TempDir()
db, err := InitDB(tmpDir)
if err != nil {
t.Fatal(err)
}
defer db.Close()
app := &App{
Conf: Config{StorageDir: tmpDir, MaxMB: 100},
DB: db,
}
fileID := "test-file-id"
fileSize := int64(1024)
if err := app.RegisterFile(fileID, fileSize); err != nil {
t.Fatalf("RegisterFile failed: %v", err)
}
err = db.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
data := b.Get([]byte(fileID))
if data == nil {
t.Fatal("Metadata not found in DB")
}
var meta FileMeta
if err := json.Unmarshal(data, &meta); err != nil {
t.Fatalf("Failed to unmarshal meta: %v", err)
}
if meta.ID != fileID {
t.Errorf("Want ID %s, got %s", fileID, meta.ID)
}
if meta.Size != fileSize {
t.Errorf("Want Size %d, got %d", fileSize, meta.Size)
}
if meta.ExpiresAt.Before(time.Now()) {
t.Error("Expiration time is in the past")
}
return nil
})
if err != nil {
t.Error(err)
}
}
internal/app/server_test.go
+8 -1
View File
@@ -17,7 +17,7 @@ import (
func setupTestApp(t *testing.T) (*App, string) {
storageDir := t.TempDir()
os.MkdirAll(filepath.Join(storageDir, "tmp"), 0700)
os.MkdirAll(filepath.Join(storageDir, TempDirName), 0700)
tmplDir := filepath.Join(storageDir, "templates")
os.MkdirAll(tmplDir, 0700)
@@ -26,6 +26,12 @@ func setupTestApp(t *testing.T) (*App, string) {
tmpl := template.Must(template.New("base").Parse(`{{define "base"}}OK{{end}}`))
db, err := InitDB(storageDir)
if err != nil {
t.Fatalf("Failed to init db: %v", err)
}
t.Cleanup(func() { db.Close() })
app := &App{
Conf: Config{
StorageDir: storageDir,
@@ -33,6 +39,7 @@ func setupTestApp(t *testing.T) (*App, string) {
},
Logger: discardLogger(),
Tmpl: tmpl,
DB: db,
}
return app, storageDir
internal/app/storage.go
+64 -16
View File
@@ -2,6 +2,7 @@ package app
import (
"context"
"encoding/json"
"fmt"
"io"
"math"
@@ -11,6 +12,7 @@ import (
"time"
"github.com/skidoodle/safebin/internal/crypto"
"go.etcd.io/bbolt"
)
func (app *App) StartCleanupTask(ctx context.Context) {
@@ -22,14 +24,14 @@ func (app *App) StartCleanupTask(ctx context.Context) {
ticker.Stop()
return
case <-ticker.C:
app.CleanStorage(app.Conf.StorageDir)
app.CleanTemp(filepath.Join(app.Conf.StorageDir, "tmp"))
app.CleanStorage()
app.CleanTemp(filepath.Join(app.Conf.StorageDir, TempDirName))
}
}
}
func (app *App) saveChunk(uid string, idx int, src io.Reader) error {
dir := filepath.Join(app.Conf.StorageDir, "tmp", uid)
dir := filepath.Join(app.Conf.StorageDir, TempDirName, uid)
if err := os.MkdirAll(dir, PermUserRWX); err != nil {
return fmt.Errorf("create chunk dir: %w", err)
@@ -54,7 +56,7 @@ func (app *App) saveChunk(uid string, idx int, src io.Reader) error {
}
func (app *App) mergeChunks(uid string, total int) (string, error) {
tmpPath := filepath.Join(app.Conf.StorageDir, "tmp", "m_"+uid)
tmpPath := filepath.Join(app.Conf.StorageDir, TempDirName, "m_"+uid)
merged, err := os.Create(tmpPath)
if err != nil {
@@ -71,7 +73,7 @@ func (app *App) mergeChunks(uid string, total int) (string, error) {
var written int64
for i := range total {
partPath := filepath.Join(app.Conf.StorageDir, "tmp", uid, strconv.Itoa(i))
partPath := filepath.Join(app.Conf.StorageDir, TempDirName, uid, strconv.Itoa(i))
part, err := os.Open(partPath)
if err != nil {
@@ -139,26 +141,72 @@ func (app *App) encryptAndSave(src io.Reader, key []byte, finalPath string) erro
return nil
}
func (app *App) CleanStorage(path string) {
entries, err := os.ReadDir(path)
if err != nil {
app.Logger.Error("Failed to read storage dir", "err", err)
return
func (app *App) RegisterFile(id string, size int64) error {
retention := CalculateRetention(size, app.Conf.MaxMB)
meta := FileMeta{
ID: id,
Size: size,
CreatedAt: time.Now(),
ExpiresAt: time.Now().Add(retention),
}
for _, entry := range entries {
info, err := entry.Info()
return app.DB.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
data, err := json.Marshal(meta)
if err != nil {
return err
}
return b.Put([]byte(id), data)
})
}
func (app *App) CleanStorage() {
now := time.Now()
var toDelete []string
err := app.DB.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
c := b.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
var meta FileMeta
if err := json.Unmarshal(v, &meta); err != nil {
continue
}
expiry := CalculateRetention(info.Size(), app.Conf.MaxMB)
if now.After(meta.ExpiresAt) {
toDelete = append(toDelete, string(k))
}
}
return nil
})
if time.Since(info.ModTime()) > expiry {
if err := os.RemoveAll(filepath.Join(path, entry.Name())); err != nil {
app.Logger.Error("Failed to remove expired file", "path", entry.Name(), "err", err)
if err != nil {
app.Logger.Error("Failed to view DB for cleanup", "err", err)
return
}
if len(toDelete) == 0 {
return
}
err = app.DB.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
for _, id := range toDelete {
path := filepath.Join(app.Conf.StorageDir, id)
if err := os.RemoveAll(path); err != nil {
app.Logger.Error("Failed to remove expired file", "path", id, "err", err)
}
if err := b.Delete([]byte(id)); err != nil {
app.Logger.Error("Failed to delete metadata", "id", id, "err", err)
}
}
return nil
})
if err != nil {
app.Logger.Error("Failed to update DB during cleanup", "err", err)
}
}
internal/app/storage_test.go
+39 -6
View File
@@ -1,20 +1,27 @@
package app
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"go.etcd.io/bbolt"
)
func TestCleanup_AbandonedMerge(t *testing.T) {
tmpDir := t.TempDir()
tmpStorage := filepath.Join(tmpDir, "tmp")
tmpStorage := filepath.Join(tmpDir, TempDirName)
os.MkdirAll(tmpStorage, 0700)
db, _ := InitDB(tmpDir)
defer db.Close()
app := &App{
Conf: Config{StorageDir: tmpDir},
Logger: discardLogger(),
DB: db,
}
abandonedFile := filepath.Join(tmpStorage, "m_abandoned_upload_id")
@@ -36,12 +43,16 @@ func TestCleanup_AbandonedMerge(t *testing.T) {
func TestCleanup_AbandonedChunks(t *testing.T) {
tmpDir := t.TempDir()
tmpStorage := filepath.Join(tmpDir, "tmp")
tmpStorage := filepath.Join(tmpDir, TempDirName)
os.MkdirAll(tmpStorage, 0700)
db, _ := InitDB(tmpDir)
defer db.Close()
app := &App{
Conf: Config{StorageDir: tmpDir},
Logger: discardLogger(),
DB: db,
}
chunkDir := filepath.Join(tmpStorage, "some_upload_id")
@@ -60,26 +71,48 @@ func TestCleanup_AbandonedChunks(t *testing.T) {
func TestCleanup_ExpiredStorage(t *testing.T) {
storageDir := t.TempDir()
db, _ := InitDB(storageDir)
defer db.Close()
app := &App{
Conf: Config{
StorageDir: storageDir,
MaxMB: 100,
},
Logger: discardLogger(),
DB: db,
}
filename := "large_file_id"
path := filepath.Join(storageDir, filename)
f, _ := os.Create(path)
f.Truncate(100 * MegaByte) // Max size
f.Truncate(100 * MegaByte)
f.Close()
oldTime := time.Now().Add(-MinRetention - time.Hour)
os.Chtimes(path, oldTime, oldTime)
expiredMeta := FileMeta{
ID: filename,
Size: 100 * MegaByte,
CreatedAt: time.Now().Add(-MinRetention - 2*time.Hour),
ExpiresAt: time.Now().Add(-time.Hour),
}
app.CleanStorage(storageDir)
app.DB.Update(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
data, _ := json.Marshal(expiredMeta)
return b.Put([]byte(filename), data)
})
app.CleanStorage()
if _, err := os.Stat(path); !os.IsNotExist(err) {
t.Error("Cleanup failed to remove expired large file")
}
app.DB.View(func(tx *bbolt.Tx) error {
b := tx.Bucket([]byte(DBBucketName))
if v := b.Get([]byte(filename)); v != nil {
t.Error("Cleanup failed to remove metadata")
}
return nil
})
}
internal/app/upload.go
+14 -3
View File
@@ -36,7 +36,7 @@ func (app *App) HandleUpload(writer http.ResponseWriter, request *http.Request)
}
}()
tmp, err := os.CreateTemp(filepath.Join(app.Conf.StorageDir, "tmp"), "up_*")
tmp, err := os.CreateTemp(filepath.Join(app.Conf.StorageDir, TempDirName), "up_*")
if err != nil {
app.Logger.Error("Failed to create temp file", "err", err)
@@ -160,7 +160,7 @@ func (app *App) HandleFinish(writer http.ResponseWriter, request *http.Request)
app.FinalizeFile(writer, request, mergedRead, request.FormValue("filename"))
if err := os.RemoveAll(filepath.Join(app.Conf.StorageDir, "tmp", uid)); err != nil {
if err := os.RemoveAll(filepath.Join(app.Conf.StorageDir, TempDirName, uid)); err != nil {
app.Logger.Error("Failed to remove chunk dir", "err", err)
}
}
@@ -184,7 +184,10 @@ func (app *App) FinalizeFile(writer http.ResponseWriter, request *http.Request,
id := crypto.GetID(key, ext)
finalPath := filepath.Join(app.Conf.StorageDir, id)
if _, err := os.Stat(finalPath); err == nil {
if info, err := os.Stat(finalPath); err == nil {
if err := app.RegisterFile(id, info.Size()); err != nil {
app.Logger.Error("Failed to update metadata for existing file", "err", err)
}
app.RespondWithLink(writer, request, key, filename)
return
}
@@ -201,5 +204,13 @@ func (app *App) FinalizeFile(writer http.ResponseWriter, request *http.Request,
return
}
if info, err := os.Stat(finalPath); err == nil {
if err := app.RegisterFile(id, info.Size()); err != nil {
app.Logger.Error("Failed to save metadata", "err", err)
}
} else {
app.Logger.Error("Failed to stat new file", "err", err)
}
app.RespondWithLink(writer, request, key, filename)
}
main.go
+13 -1
View File
@@ -26,16 +26,28 @@ func main() {
"max_file_size", fmt.Sprintf("%dMB", cfg.MaxMB),
)
tmpDir := filepath.Join(cfg.StorageDir, "tmp")
tmpDir := filepath.Join(cfg.StorageDir, app.TempDirName)
if err := os.MkdirAll(tmpDir, app.PermUserRWX); err != nil {
logger.Error("Failed to initialize storage directory", "err", err)
os.Exit(1)
}
db, err := app.InitDB(cfg.StorageDir)
if err != nil {
logger.Error("Failed to initialize database", "err", err)
os.Exit(1)
}
defer func() {
if err := db.Close(); err != nil {
logger.Error("Failed to close database", "err", err)
}
}()
application := &app.App{
Conf: cfg,
Logger: logger,
Tmpl: app.ParseTemplates(),
DB: db,
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)