feat(metadata): SQLite backend, storage reconciliation and local defaults
- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls - SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx) - separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite - cmd/reconcile compares object storage with the journal (report, then --delete) - storage.Backend.List and photos.Repository.AllStorageKeys - DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults (sqlite ./galo.db, fs ./data) - Makefile reconcile/reconcile-delete; docs and .env.example updated
This commit is contained in:
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -144,6 +145,37 @@ func (f *FS) URL(_ context.Context, key string) (string, error) {
|
||||
return f.signer.URL(key), nil
|
||||
}
|
||||
|
||||
// List returns every blob under prefix as a key relative to the storage root.
|
||||
func (f *FS) List(_ context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
var out []ObjectInfo
|
||||
err := filepath.WalkDir(f.root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(f.root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rel = filepath.ToSlash(rel)
|
||||
if prefix != "" && !strings.HasPrefix(rel, prefix) {
|
||||
return nil
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out = append(out, ObjectInfo{Key: rel, Size: info.Size(), ModTime: info.ModTime()})
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ServeObject validates the signed URL and serves the blob. It supports range
|
||||
// requests and sets the content type from the key extension. It writes the
|
||||
// response itself and returns nil on success.
|
||||
|
||||
@@ -172,3 +172,33 @@ func TestContentTypeForKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSList(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := newTestFS(t)
|
||||
for _, key := range []string{"objects/a/original.jpg", "objects/b/original.jpg", "misc/readme.txt"} {
|
||||
if err := fs.Put(ctx, key, strings.NewReader("x"), 1, "text/plain"); err != nil {
|
||||
t.Fatalf("put %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
all, err := fs.List(ctx, "")
|
||||
if err != nil || len(all) != 3 {
|
||||
t.Fatalf("list all: %v len=%d", err, len(all))
|
||||
}
|
||||
objs, err := fs.List(ctx, "objects/")
|
||||
if err != nil {
|
||||
t.Fatalf("list objects: %v", err)
|
||||
}
|
||||
if len(objs) != 2 {
|
||||
t.Fatalf("list objects len=%d, want 2", len(objs))
|
||||
}
|
||||
for _, o := range objs {
|
||||
if !strings.HasPrefix(o.Key, "objects/") {
|
||||
t.Fatalf("unexpected key %q", o.Key)
|
||||
}
|
||||
if o.Size != 1 || o.ModTime.IsZero() {
|
||||
t.Fatalf("unexpected info %+v", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,6 +117,33 @@ func TestS3ServeObject(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestS3List(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cl, _ := newTestS3(t)
|
||||
prefix := "objects/" + strconv.FormatInt(time.Now().UnixNano(), 10) + "/"
|
||||
keys := []string{prefix + "original.jpg", prefix + "thumb.webp"}
|
||||
for _, k := range keys {
|
||||
if err := cl.Put(ctx, k, strings.NewReader("x"), 1, "image/jpeg"); err != nil {
|
||||
t.Fatalf("put %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
t.Cleanup(func() { _ = cl.Delete(context.Background(), keys...) })
|
||||
|
||||
got, err := cl.List(ctx, prefix)
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
found := map[string]bool{}
|
||||
for _, o := range got {
|
||||
found[o.Key] = true
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !found[k] {
|
||||
t.Fatalf("key %s not listed: %+v", k, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3ServeObjectHead(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cl, signer := newTestS3(t)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
awsconfig "github.com/aws/aws-sdk-go-v2/config"
|
||||
@@ -29,6 +30,13 @@ var ErrObjectNotFound = errors.New("object not found")
|
||||
// ErrRangeNotSatisfiable is returned when a requested byte range is invalid.
|
||||
var ErrRangeNotSatisfiable = errors.New("requested range not satisfiable")
|
||||
|
||||
// ObjectInfo describes one stored blob for inventory and reconciliation.
|
||||
type ObjectInfo struct {
|
||||
Key string
|
||||
Size int64
|
||||
ModTime time.Time
|
||||
}
|
||||
|
||||
// Backend is the object storage abstraction. Keys are opaque and independent of
|
||||
// the backend: every implementation stores and serves the same logical blob.
|
||||
// Blobs are always delivered through the app.
|
||||
@@ -41,6 +49,8 @@ type Backend interface {
|
||||
// ServeObject validates the signed URL and writes the blob, including
|
||||
// Content-Type and range handling, to the response.
|
||||
ServeObject(w http.ResponseWriter, r *http.Request, key string) error
|
||||
// List returns the blobs under prefix for inventory and reconciliation.
|
||||
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
|
||||
Ping(ctx context.Context) error
|
||||
EnsureBucket(ctx context.Context) error
|
||||
// Bucket returns a human-readable identifier for the storage location.
|
||||
@@ -183,6 +193,36 @@ func (c *S3) URL(_ context.Context, key string) (string, error) {
|
||||
return c.signer.URL(key), nil
|
||||
}
|
||||
|
||||
// List returns every object under prefix using paginated ListObjectsV2.
|
||||
func (c *S3) List(ctx context.Context, prefix string) ([]ObjectInfo, error) {
|
||||
var out []ObjectInfo
|
||||
var token *string
|
||||
for {
|
||||
in := &s3.ListObjectsV2Input{Bucket: aws.String(c.bucket)}
|
||||
if prefix != "" {
|
||||
in.Prefix = aws.String(prefix)
|
||||
}
|
||||
if token != nil {
|
||||
in.ContinuationToken = token
|
||||
}
|
||||
res, err := c.s3.ListObjectsV2(ctx, in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list objects: %w", err)
|
||||
}
|
||||
for _, o := range res.Contents {
|
||||
out = append(out, ObjectInfo{
|
||||
Key: aws.ToString(o.Key),
|
||||
Size: aws.ToInt64(o.Size),
|
||||
ModTime: aws.ToTime(o.LastModified),
|
||||
})
|
||||
}
|
||||
if res.IsTruncated == nil || !*res.IsTruncated {
|
||||
return out, nil
|
||||
}
|
||||
token = res.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
// ServeObject validates the signed URL and streams the blob from S3. It passes
|
||||
// the client's Range header through to S3, so byte ranges are handled natively,
|
||||
// and answers HEAD via HeadObject. It writes the response itself.
|
||||
|
||||
Reference in New Issue
Block a user