feat(metadata): SQLite backend, storage reconciliation and local defaults

- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls
- SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx)
- separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite
- cmd/reconcile compares object storage with the journal (report, then --delete)
- storage.Backend.List and photos.Repository.AllStorageKeys
- DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults
  (sqlite ./galo.db, fs ./data)
- Makefile reconcile/reconcile-delete; docs and .env.example updated
This commit is contained in:
iu
2026-10-11 10:08:31 +03:00
parent ec8d907565
commit 176a8d8ced
69 changed files with 4174 additions and 95 deletions
+32
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"mime"
"net/http"
"net/url"
@@ -144,6 +145,37 @@ func (f *FS) URL(_ context.Context, key string) (string, error) {
return f.signer.URL(key), nil
}
// List returns every blob under prefix as a key relative to the storage root.
func (f *FS) List(_ context.Context, prefix string) ([]ObjectInfo, error) {
var out []ObjectInfo
err := filepath.WalkDir(f.root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(f.root, path)
if err != nil {
return err
}
rel = filepath.ToSlash(rel)
if prefix != "" && !strings.HasPrefix(rel, prefix) {
return nil
}
info, err := d.Info()
if err != nil {
return err
}
out = append(out, ObjectInfo{Key: rel, Size: info.Size(), ModTime: info.ModTime()})
return nil
})
if err != nil {
return nil, err
}
return out, nil
}
// ServeObject validates the signed URL and serves the blob. It supports range
// requests and sets the content type from the key extension. It writes the
// response itself and returns nil on success.
+30
View File
@@ -172,3 +172,33 @@ func TestContentTypeForKey(t *testing.T) {
}
}
}
func TestFSList(t *testing.T) {
ctx := context.Background()
fs := newTestFS(t)
for _, key := range []string{"objects/a/original.jpg", "objects/b/original.jpg", "misc/readme.txt"} {
if err := fs.Put(ctx, key, strings.NewReader("x"), 1, "text/plain"); err != nil {
t.Fatalf("put %s: %v", key, err)
}
}
all, err := fs.List(ctx, "")
if err != nil || len(all) != 3 {
t.Fatalf("list all: %v len=%d", err, len(all))
}
objs, err := fs.List(ctx, "objects/")
if err != nil {
t.Fatalf("list objects: %v", err)
}
if len(objs) != 2 {
t.Fatalf("list objects len=%d, want 2", len(objs))
}
for _, o := range objs {
if !strings.HasPrefix(o.Key, "objects/") {
t.Fatalf("unexpected key %q", o.Key)
}
if o.Size != 1 || o.ModTime.IsZero() {
t.Fatalf("unexpected info %+v", o)
}
}
}
+27
View File
@@ -117,6 +117,33 @@ func TestS3ServeObject(t *testing.T) {
})
}
func TestS3List(t *testing.T) {
ctx := context.Background()
cl, _ := newTestS3(t)
prefix := "objects/" + strconv.FormatInt(time.Now().UnixNano(), 10) + "/"
keys := []string{prefix + "original.jpg", prefix + "thumb.webp"}
for _, k := range keys {
if err := cl.Put(ctx, k, strings.NewReader("x"), 1, "image/jpeg"); err != nil {
t.Fatalf("put %s: %v", k, err)
}
}
t.Cleanup(func() { _ = cl.Delete(context.Background(), keys...) })
got, err := cl.List(ctx, prefix)
if err != nil {
t.Fatalf("list: %v", err)
}
found := map[string]bool{}
for _, o := range got {
found[o.Key] = true
}
for _, k := range keys {
if !found[k] {
t.Fatalf("key %s not listed: %+v", k, got)
}
}
}
func TestS3ServeObjectHead(t *testing.T) {
ctx := context.Background()
cl, signer := newTestS3(t)
+40
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
@@ -29,6 +30,13 @@ var ErrObjectNotFound = errors.New("object not found")
// ErrRangeNotSatisfiable is returned when a requested byte range is invalid.
var ErrRangeNotSatisfiable = errors.New("requested range not satisfiable")
// ObjectInfo describes one stored blob for inventory and reconciliation.
type ObjectInfo struct {
Key string
Size int64
ModTime time.Time
}
// Backend is the object storage abstraction. Keys are opaque and independent of
// the backend: every implementation stores and serves the same logical blob.
// Blobs are always delivered through the app.
@@ -41,6 +49,8 @@ type Backend interface {
// ServeObject validates the signed URL and writes the blob, including
// Content-Type and range handling, to the response.
ServeObject(w http.ResponseWriter, r *http.Request, key string) error
// List returns the blobs under prefix for inventory and reconciliation.
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
Ping(ctx context.Context) error
EnsureBucket(ctx context.Context) error
// Bucket returns a human-readable identifier for the storage location.
@@ -183,6 +193,36 @@ func (c *S3) URL(_ context.Context, key string) (string, error) {
return c.signer.URL(key), nil
}
// List returns every object under prefix using paginated ListObjectsV2.
func (c *S3) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
var out []ObjectInfo
var token *string
for {
in := &s3.ListObjectsV2Input{Bucket: aws.String(c.bucket)}
if prefix != "" {
in.Prefix = aws.String(prefix)
}
if token != nil {
in.ContinuationToken = token
}
res, err := c.s3.ListObjectsV2(ctx, in)
if err != nil {
return nil, fmt.Errorf("list objects: %w", err)
}
for _, o := range res.Contents {
out = append(out, ObjectInfo{
Key: aws.ToString(o.Key),
Size: aws.ToInt64(o.Size),
ModTime: aws.ToTime(o.LastModified),
})
}
if res.IsTruncated == nil || !*res.IsTruncated {
return out, nil
}
token = res.NextContinuationToken
}
}
// ServeObject validates the signed URL and streams the blob from S3. It passes
// the client's Range header through to S3, so byte ranges are handled natively,
// and answers HEAD via HeadObject. It writes the response itself.