feat(metadata): SQLite backend, storage reconciliation and local defaults
- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls - SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx) - separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite - cmd/reconcile compares object storage with the journal (report, then --delete) - storage.Backend.List and photos.Repository.AllStorageKeys - DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults (sqlite ./galo.db, fs ./data) - Makefile reconcile/reconcile-delete; docs and .env.example updated
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
// Package reconcile computes the difference between the object storage and the
|
||||
// metadata journal so orphaned blobs can be reported and, on request, removed.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"shared-album/internal/storage"
|
||||
)
|
||||
|
||||
// Result holds the two kinds of divergence between storage and the journal.
|
||||
type Result struct {
|
||||
// Orphans are keys present in storage but absent from the journal; they are
|
||||
// candidates for deletion.
|
||||
Orphans []string
|
||||
// Missing are keys present in the journal but absent from storage; they are
|
||||
// reported only and never modified.
|
||||
Missing []string
|
||||
}
|
||||
|
||||
// Classify compares the listed storage objects with the journaled keys.
|
||||
func Classify(objects []storage.ObjectInfo, journaled []string) Result {
|
||||
have := make(map[string]struct{}, len(journaled))
|
||||
for _, k := range journaled {
|
||||
have[k] = struct{}{}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(objects))
|
||||
var orphans []string
|
||||
for _, o := range objects {
|
||||
if _, dup := seen[o.Key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[o.Key] = struct{}{}
|
||||
if _, ok := have[o.Key]; !ok {
|
||||
orphans = append(orphans, o.Key)
|
||||
}
|
||||
}
|
||||
var missing []string
|
||||
for _, k := range journaled {
|
||||
if _, ok := seen[k]; !ok {
|
||||
missing = append(missing, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(orphans)
|
||||
sort.Strings(missing)
|
||||
return Result{Orphans: orphans, Missing: missing}
|
||||
}
|
||||
|
||||
// Deletable selects orphaned keys that are safe to remove: within prefix and
|
||||
// older than minAge (a guard against racing an in-flight upload whose row is
|
||||
// not written yet).
|
||||
func Deletable(objects []storage.ObjectInfo, res Result, prefix string, minAge time.Duration, now time.Time) []string {
|
||||
orphans := make(map[string]struct{}, len(res.Orphans))
|
||||
for _, k := range res.Orphans {
|
||||
orphans[k] = struct{}{}
|
||||
}
|
||||
var out []string
|
||||
for _, o := range objects {
|
||||
if _, ok := orphans[o.Key]; !ok {
|
||||
continue
|
||||
}
|
||||
if prefix != "" && !strings.HasPrefix(o.Key, prefix) {
|
||||
continue
|
||||
}
|
||||
if now.Sub(o.ModTime) < minAge {
|
||||
continue
|
||||
}
|
||||
out = append(out, o.Key)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"shared-album/internal/storage"
|
||||
)
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
now := time.Now()
|
||||
objects := []storage.ObjectInfo{
|
||||
{Key: "objects/a/original.jpg", ModTime: now},
|
||||
{Key: "objects/b/original.jpg", ModTime: now},
|
||||
}
|
||||
journaled := []string{"objects/b/original.jpg", "objects/c/original.jpg"}
|
||||
|
||||
got := Classify(objects, journaled)
|
||||
if !reflect.DeepEqual(got.Orphans, []string{"objects/a/original.jpg"}) {
|
||||
t.Fatalf("orphans = %v", got.Orphans)
|
||||
}
|
||||
if !reflect.DeepEqual(got.Missing, []string{"objects/c/original.jpg"}) {
|
||||
t.Fatalf("missing = %v", got.Missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeletableRespectsAgeAndPrefix(t *testing.T) {
|
||||
now := time.Now()
|
||||
objects := []storage.ObjectInfo{
|
||||
{Key: "objects/old/original.jpg", ModTime: now.Add(-2 * time.Hour)},
|
||||
{Key: "objects/new/original.jpg", ModTime: now.Add(-1 * time.Minute)},
|
||||
{Key: "misc/other.bin", ModTime: now.Add(-2 * time.Hour)},
|
||||
}
|
||||
res := Result{Orphans: []string{"objects/old/original.jpg", "objects/new/original.jpg", "misc/other.bin"}}
|
||||
|
||||
got := Deletable(objects, res, "objects/", time.Hour, now)
|
||||
if !reflect.DeepEqual(got, []string{"objects/old/original.jpg"}) {
|
||||
t.Fatalf("deletable = %v, want only the old in-prefix orphan", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user