ds-galo/backend/cmd/reconcile/main.go

// Command reconcile compares the object storage with the metadata journal and
// reports orphaned blobs (present in storage, absent from the journal). With
// --delete it removes orphans older than --min-age. It never touches missing
// keys (in the journal but not in storage) and never runs automatically.
package main

import (
	"bufio"
	"context"
	"flag"
	"io/fs"
	"log/slog"
	"os"
	"strings"
	"time"

	assets "galo"
	"galo/internal/config"
	"galo/internal/db"
	"galo/internal/reconcile"
	"galo/internal/settings"
	"galo/internal/storage"
	"galo/internal/store"
)

const objectPrefix = "objects/"

func main() {
	loadDotEnv(".env")
	loadDotEnv("../.env")

	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})))

	deleteFlag := flag.Bool("delete", false, "delete orphaned blobs older than --min-age")
	minAge := flag.Duration("min-age", time.Hour, "minimum blob age before deletion is allowed")
	sample := flag.Int("sample", 20, "number of example keys to print")
	flag.Parse()

	cfg, err := config.Load()
	if err != nil {
		slog.Error("config error", "error", err)
		os.Exit(1)
	}

	ctx := context.Background()

	repos, closer, err := openRepos(ctx, cfg)
	if err != nil {
		slog.Error("metadata open failed", "error", err)
		os.Exit(1)
	}
	defer closer()

	secret, err := repos.Settings.Ensure(ctx, settings.JWTSecretKey, settings.GenerateSecret)
	if err != nil {
		slog.Error("secret init failed", "error", err)
		os.Exit(1)
	}

	blob, err := storage.Open(ctx, cfg, secret)
	if err != nil {
		slog.Error("storage init failed", "error", err)
		os.Exit(1)
	}

	objects, err := blob.List(ctx, objectPrefix)
	if err != nil {
		slog.Error("list storage", "error", err)
		os.Exit(1)
	}
	journaled, err := repos.Photos.AllStorageKeys(ctx)
	if err != nil {
		slog.Error("list journal", "error", err)
		os.Exit(1)
	}

	res := reconcile.Classify(objects, journaled)
	slog.Info("reconciliation report",
		"storage", cfg.Storage.Backend,
		"storage_objects", len(objects),
		"journal_keys", len(journaled),
		"orphans", len(res.Orphans),
		"missing", len(res.Missing))
	printSamples("orphan", res.Orphans, *sample)
	printSamples("missing", res.Missing, *sample)

	if !*deleteFlag {
		slog.Info("dry run: nothing deleted (pass --delete to remove orphans)")
		return
	}

	targets := reconcile.Deletable(objects, res, objectPrefix, *minAge, time.Now())
	if len(targets) == 0 {
		slog.Info("no orphaned blobs old enough to delete", "min_age", minAge.String())
		return
	}
	if err := blob.Delete(ctx, targets...); err != nil {
		slog.Error("delete orphans", "error", err)
		os.Exit(1)
	}
	slog.Info("deleted orphaned blobs", "count", len(targets), "min_age", minAge.String())
}

// openRepos builds the metadata repositories for the configured backend.
func openRepos(ctx context.Context, cfg *config.Config) (store.Repos, func(), error) {
	if cfg.Database.Backend == "sqlite" {
		migFS, err := fs.Sub(assets.Migrations, "migrations/sqlite")
		if err != nil {
			return store.Repos{}, nil, err
		}
		if err := db.MigrateSQLite(cfg.Database.SQLitePath, migFS); err != nil {
			return store.Repos{}, nil, err
		}
		database, err := db.OpenSQLite(cfg.Database.SQLitePath)
		if err != nil {
			return store.Repos{}, nil, err
		}
		return store.NewSQLite(database), func() { _ = database.Close() }, nil
	}

	migFS, err := fs.Sub(assets.Migrations, "migrations")
	if err != nil {
		return store.Repos{}, nil, err
	}
	if err := db.Migrate(cfg.Postgres.DSN(), migFS); err != nil {
		return store.Repos{}, nil, err
	}
	pool, err := db.Connect(ctx, cfg.Postgres.DSN())
	if err != nil {
		return store.Repos{}, nil, err
	}
	return store.NewPostgres(pool), pool.Close, nil
}

func printSamples(label string, keys []string, n int) {
	if len(keys) == 0 {
		return
	}
	if n > len(keys) {
		n = len(keys)
	}
	for _, k := range keys[:n] {
		slog.Info("sample", "kind", label, "key", k)
	}
}

func loadDotEnv(path string) {
	f, err := os.Open(path)
	if err != nil {
		return
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		key, value, ok := strings.Cut(line, "=")
		if !ok {
			continue
		}
		key = strings.TrimSpace(key)
		value = strings.TrimSpace(value)
		if _, exists := os.LookupEnv(key); !exists {
			_ = os.Setenv(key, value)
		}
	}
}