feat(metadata): SQLite backend, storage reconciliation and local defaults
- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls - SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx) - separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite - cmd/reconcile compares object storage with the journal (report, then --delete) - storage.Backend.List and photos.Repository.AllStorageKeys - DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults (sqlite ./galo.db, fs ./data) - Makefile reconcile/reconcile-delete; docs and .env.example updated
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
// Command reconcile compares the object storage with the metadata journal and
|
||||
// reports orphaned blobs (present in storage, absent from the journal). With
|
||||
// --delete it removes orphans older than --min-age. It never touches missing
|
||||
// keys (in the journal but not in storage) and never runs automatically.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"flag"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
assets "shared-album"
|
||||
"shared-album/internal/config"
|
||||
"shared-album/internal/db"
|
||||
"shared-album/internal/reconcile"
|
||||
"shared-album/internal/storage"
|
||||
"shared-album/internal/store"
|
||||
)
|
||||
|
||||
const objectPrefix = "objects/"
|
||||
|
||||
func main() {
|
||||
loadDotEnv(".env")
|
||||
loadDotEnv("../.env")
|
||||
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})))
|
||||
|
||||
deleteFlag := flag.Bool("delete", false, "delete orphaned blobs older than --min-age")
|
||||
minAge := flag.Duration("min-age", time.Hour, "minimum blob age before deletion is allowed")
|
||||
sample := flag.Int("sample", 20, "number of example keys to print")
|
||||
flag.Parse()
|
||||
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
slog.Error("config error", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
repos, closer, err := openRepos(ctx, cfg)
|
||||
if err != nil {
|
||||
slog.Error("metadata open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer closer()
|
||||
|
||||
blob, err := storage.Open(ctx, cfg)
|
||||
if err != nil {
|
||||
slog.Error("storage init failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
objects, err := blob.List(ctx, objectPrefix)
|
||||
if err != nil {
|
||||
slog.Error("list storage", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
journaled, err := repos.Photos.AllStorageKeys(ctx)
|
||||
if err != nil {
|
||||
slog.Error("list journal", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
res := reconcile.Classify(objects, journaled)
|
||||
slog.Info("reconciliation report",
|
||||
"storage", cfg.Storage.Backend,
|
||||
"storage_objects", len(objects),
|
||||
"journal_keys", len(journaled),
|
||||
"orphans", len(res.Orphans),
|
||||
"missing", len(res.Missing))
|
||||
printSamples("orphan", res.Orphans, *sample)
|
||||
printSamples("missing", res.Missing, *sample)
|
||||
|
||||
if !*deleteFlag {
|
||||
slog.Info("dry run: nothing deleted (pass --delete to remove orphans)")
|
||||
return
|
||||
}
|
||||
|
||||
targets := reconcile.Deletable(objects, res, objectPrefix, *minAge, time.Now())
|
||||
if len(targets) == 0 {
|
||||
slog.Info("no orphaned blobs old enough to delete", "min_age", minAge.String())
|
||||
return
|
||||
}
|
||||
if err := blob.Delete(ctx, targets...); err != nil {
|
||||
slog.Error("delete orphans", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("deleted orphaned blobs", "count", len(targets), "min_age", minAge.String())
|
||||
}
|
||||
|
||||
// openRepos builds the metadata repositories for the configured backend.
|
||||
func openRepos(ctx context.Context, cfg *config.Config) (store.Repos, func(), error) {
|
||||
if cfg.Database.Backend == "sqlite" {
|
||||
migFS, err := fs.Sub(assets.Migrations, "migrations/sqlite")
|
||||
if err != nil {
|
||||
return store.Repos{}, nil, err
|
||||
}
|
||||
if err := db.MigrateSQLite(cfg.Database.SQLitePath, migFS); err != nil {
|
||||
return store.Repos{}, nil, err
|
||||
}
|
||||
database, err := db.OpenSQLite(cfg.Database.SQLitePath)
|
||||
if err != nil {
|
||||
return store.Repos{}, nil, err
|
||||
}
|
||||
return store.NewSQLite(database), func() { _ = database.Close() }, nil
|
||||
}
|
||||
|
||||
migFS, err := fs.Sub(assets.Migrations, "migrations")
|
||||
if err != nil {
|
||||
return store.Repos{}, nil, err
|
||||
}
|
||||
if err := db.Migrate(cfg.Postgres.DSN(), migFS); err != nil {
|
||||
return store.Repos{}, nil, err
|
||||
}
|
||||
pool, err := db.Connect(ctx, cfg.Postgres.DSN())
|
||||
if err != nil {
|
||||
return store.Repos{}, nil, err
|
||||
}
|
||||
return store.NewPostgres(pool), pool.Close, nil
|
||||
}
|
||||
|
||||
func printSamples(label string, keys []string, n int) {
|
||||
if len(keys) == 0 {
|
||||
return
|
||||
}
|
||||
if n > len(keys) {
|
||||
n = len(keys)
|
||||
}
|
||||
for _, k := range keys[:n] {
|
||||
slog.Info("sample", "kind", label, "key", k)
|
||||
}
|
||||
}
|
||||
|
||||
func loadDotEnv(path string) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
scanner := bufio.NewScanner(f)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key, value, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
value = strings.TrimSpace(value)
|
||||
if _, exists := os.LookupEnv(key); !exists {
|
||||
_ = os.Setenv(key, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
+49
-20
@@ -19,6 +19,7 @@ import (
|
||||
"shared-album/internal/db"
|
||||
"shared-album/internal/sse"
|
||||
"shared-album/internal/storage"
|
||||
"shared-album/internal/store"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -37,37 +38,65 @@ func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
migFS, err := fs.Sub(assets.Migrations, "migrations")
|
||||
if err != nil {
|
||||
slog.Error("migrations fs", "error", err)
|
||||
os.Exit(1)
|
||||
var (
|
||||
repos store.Repos
|
||||
ping func(ctx context.Context) error
|
||||
closer func()
|
||||
)
|
||||
switch cfg.Database.Backend {
|
||||
case "sqlite":
|
||||
migFS, err := fs.Sub(assets.Migrations, "migrations/sqlite")
|
||||
if err != nil {
|
||||
slog.Error("migrations fs", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.MigrateSQLite(cfg.Database.SQLitePath, migFS); err != nil {
|
||||
slog.Error("migration failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
database, err := db.OpenSQLite(cfg.Database.SQLitePath)
|
||||
if err != nil {
|
||||
slog.Error("database open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
repos = store.NewSQLite(database)
|
||||
ping = database.PingContext
|
||||
closer = func() { _ = database.Close() }
|
||||
default:
|
||||
migFS, err := fs.Sub(assets.Migrations, "migrations")
|
||||
if err != nil {
|
||||
slog.Error("migrations fs", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := db.Migrate(cfg.Postgres.DSN(), migFS); err != nil {
|
||||
slog.Error("migration failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
pool, err := db.Connect(ctx, cfg.Postgres.DSN())
|
||||
if err != nil {
|
||||
slog.Error("database connect failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
repos = store.NewPostgres(pool)
|
||||
ping = pool.Ping
|
||||
closer = pool.Close
|
||||
}
|
||||
if err := db.Migrate(cfg.Postgres.DSN(), migFS); err != nil {
|
||||
slog.Error("migration failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
slog.Info("migrations applied")
|
||||
defer closer()
|
||||
slog.Info("migrations applied", "db_backend", cfg.Database.Backend)
|
||||
|
||||
pool, err := db.Connect(ctx, cfg.Postgres.DSN())
|
||||
if err != nil {
|
||||
slog.Error("database connect failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
store, err := storage.Open(ctx, cfg)
|
||||
blob, err := storage.Open(ctx, cfg)
|
||||
if err != nil {
|
||||
slog.Error("storage init failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := store.EnsureBucket(ctx); err != nil {
|
||||
if err := blob.EnsureBucket(ctx); err != nil {
|
||||
slog.Warn("could not ensure bucket", "error", err)
|
||||
} else {
|
||||
slog.Info("storage bucket ready", "bucket", store.Bucket())
|
||||
slog.Info("storage ready", "location", blob.Bucket())
|
||||
}
|
||||
|
||||
hub := sse.NewHub(cfg.SSE.ClientBuffer)
|
||||
server := app.New(cfg, pool, store, hub)
|
||||
server := app.New(cfg, repos, ping, blob, hub)
|
||||
|
||||
go server.StartWorker(ctx)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user