iu
176a8d8ced
- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls - SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx) - separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite - cmd/reconcile compares object storage with the journal (report, then --delete) - storage.Backend.List and photos.Repository.AllStorageKeys - DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults (sqlite ./galo.db, fs ./data) - Makefile reconcile/reconcile-delete; docs and .env.example updated
75 lines
2.0 KiB
Go
75 lines
2.0 KiB
Go
// Package reconcile computes the difference between the object storage and the
|
|
// metadata journal so orphaned blobs can be reported and, on request, removed.
|
|
package reconcile
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"shared-album/internal/storage"
|
|
)
|
|
|
|
// Result holds the two kinds of divergence between storage and the journal.
|
|
type Result struct {
|
|
// Orphans are keys present in storage but absent from the journal; they are
|
|
// candidates for deletion.
|
|
Orphans []string
|
|
// Missing are keys present in the journal but absent from storage; they are
|
|
// reported only and never modified.
|
|
Missing []string
|
|
}
|
|
|
|
// Classify compares the listed storage objects with the journaled keys.
|
|
func Classify(objects []storage.ObjectInfo, journaled []string) Result {
|
|
have := make(map[string]struct{}, len(journaled))
|
|
for _, k := range journaled {
|
|
have[k] = struct{}{}
|
|
}
|
|
seen := make(map[string]struct{}, len(objects))
|
|
var orphans []string
|
|
for _, o := range objects {
|
|
if _, dup := seen[o.Key]; dup {
|
|
continue
|
|
}
|
|
seen[o.Key] = struct{}{}
|
|
if _, ok := have[o.Key]; !ok {
|
|
orphans = append(orphans, o.Key)
|
|
}
|
|
}
|
|
var missing []string
|
|
for _, k := range journaled {
|
|
if _, ok := seen[k]; !ok {
|
|
missing = append(missing, k)
|
|
}
|
|
}
|
|
sort.Strings(orphans)
|
|
sort.Strings(missing)
|
|
return Result{Orphans: orphans, Missing: missing}
|
|
}
|
|
|
|
// Deletable selects orphaned keys that are safe to remove: within prefix and
|
|
// older than minAge (a guard against racing an in-flight upload whose row is
|
|
// not written yet).
|
|
func Deletable(objects []storage.ObjectInfo, res Result, prefix string, minAge time.Duration, now time.Time) []string {
|
|
orphans := make(map[string]struct{}, len(res.Orphans))
|
|
for _, k := range res.Orphans {
|
|
orphans[k] = struct{}{}
|
|
}
|
|
var out []string
|
|
for _, o := range objects {
|
|
if _, ok := orphans[o.Key]; !ok {
|
|
continue
|
|
}
|
|
if prefix != "" && !strings.HasPrefix(o.Key, prefix) {
|
|
continue
|
|
}
|
|
if now.Sub(o.ModTime) < minAge {
|
|
continue
|
|
}
|
|
out = append(out, o.Key)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|