feat(metadata): SQLite backend, storage reconciliation and local defaults
- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls - SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx) - separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite - cmd/reconcile compares object storage with the journal (report, then --delete) - storage.Backend.List and photos.Repository.AllStorageKeys - DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults (sqlite ./galo.db, fs ./data) - Makefile reconcile/reconcile-delete; docs and .env.example updated
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
// Package reconcile computes the difference between the object storage and the
|
||||
// metadata journal so orphaned blobs can be reported and, on request, removed.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"shared-album/internal/storage"
|
||||
)
|
||||
|
||||
// Result holds the two kinds of divergence between storage and the journal.
|
||||
type Result struct {
|
||||
// Orphans are keys present in storage but absent from the journal; they are
|
||||
// candidates for deletion.
|
||||
Orphans []string
|
||||
// Missing are keys present in the journal but absent from storage; they are
|
||||
// reported only and never modified.
|
||||
Missing []string
|
||||
}
|
||||
|
||||
// Classify compares the listed storage objects with the journaled keys.
|
||||
func Classify(objects []storage.ObjectInfo, journaled []string) Result {
|
||||
have := make(map[string]struct{}, len(journaled))
|
||||
for _, k := range journaled {
|
||||
have[k] = struct{}{}
|
||||
}
|
||||
seen := make(map[string]struct{}, len(objects))
|
||||
var orphans []string
|
||||
for _, o := range objects {
|
||||
if _, dup := seen[o.Key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[o.Key] = struct{}{}
|
||||
if _, ok := have[o.Key]; !ok {
|
||||
orphans = append(orphans, o.Key)
|
||||
}
|
||||
}
|
||||
var missing []string
|
||||
for _, k := range journaled {
|
||||
if _, ok := seen[k]; !ok {
|
||||
missing = append(missing, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(orphans)
|
||||
sort.Strings(missing)
|
||||
return Result{Orphans: orphans, Missing: missing}
|
||||
}
|
||||
|
||||
// Deletable selects orphaned keys that are safe to remove: within prefix and
|
||||
// older than minAge (a guard against racing an in-flight upload whose row is
|
||||
// not written yet).
|
||||
func Deletable(objects []storage.ObjectInfo, res Result, prefix string, minAge time.Duration, now time.Time) []string {
|
||||
orphans := make(map[string]struct{}, len(res.Orphans))
|
||||
for _, k := range res.Orphans {
|
||||
orphans[k] = struct{}{}
|
||||
}
|
||||
var out []string
|
||||
for _, o := range objects {
|
||||
if _, ok := orphans[o.Key]; !ok {
|
||||
continue
|
||||
}
|
||||
if prefix != "" && !strings.HasPrefix(o.Key, prefix) {
|
||||
continue
|
||||
}
|
||||
if now.Sub(o.ModTime) < minAge {
|
||||
continue
|
||||
}
|
||||
out = append(out, o.Key)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user