feat(photos): separate object storage from metadata layer

Move physical file facts out of the photos table into an objects ledger:

- objects / photo_objects / variant_jobs tables (migration 000008)
- drop storage_key/thumb_key/preview_key/size/width/height/mime_type from
  photos (migration 000009); they are derived via joins to the original object
- album-free storage keys: objects/{id}/{role}.{ext}
- explicit GC via pending/ready/deleting states; objects deleted only when no
  photo references them
- background worker drives variant generation (durable variant_jobs) and GC
- copy/move are metadata-only; storage objects are shared between photos
- remove unused storage.DeletePrefix

Non-destructive to the external JSON API: size/dimensions/mime and URLs are
still returned, just sourced from the object layer.

OpenSpec change: media-object-layer
This commit is contained in:
iu
2026-10-10 23:31:41 +03:00
parent 9effc0fc82
commit 96268ee16c
30 changed files with 1835 additions and 136 deletions
+6 -4
View File
@@ -44,11 +44,12 @@ func (s *Store) GetByID(ctx context.Context, id string) (*models.Album, error) {
var a models.Album
err := s.pool.QueryRow(ctx,
`SELECT a.id, a.owner_id, a.title, a.description, a.cover_photo_id, a.created_at, a.updated_at,
COALESCE(p.thumb_key, ''),
COALESCE(co.storage_key, ''),
(SELECT count(*) FROM photos WHERE album_id = a.id),
(SELECT count(*) FROM album_members WHERE album_id = a.id)
FROM albums a
LEFT JOIN photos p ON p.id = a.cover_photo_id
LEFT JOIN photo_objects cpo ON cpo.photo_id = a.cover_photo_id AND cpo.role = 'thumb'
LEFT JOIN objects co ON co.id = cpo.object_id
WHERE a.id = $1`, id).
Scan(&a.ID, &a.OwnerID, &a.Title, &a.Description, &a.CoverPhotoID, &a.CreatedAt, &a.UpdatedAt, &a.CoverThumbKey, &a.PhotosCount, &a.MembersCount)
if err != nil {
@@ -63,12 +64,13 @@ func (s *Store) GetByID(ctx context.Context, id string) (*models.Album, error) {
func (s *Store) ListForUser(ctx context.Context, userID string) ([]models.Album, error) {
rows, err := s.pool.Query(ctx,
`SELECT a.id, a.owner_id, a.title, a.description, a.cover_photo_id, a.created_at, a.updated_at,
COALESCE(p.thumb_key, ''),
COALESCE(co.storage_key, ''),
(SELECT count(*) FROM photos WHERE album_id = a.id),
(SELECT count(*) FROM album_members WHERE album_id = a.id)
FROM albums a
LEFT JOIN album_members m ON m.album_id = a.id AND m.user_id = $1
LEFT JOIN photos p ON p.id = a.cover_photo_id
LEFT JOIN photo_objects cpo ON cpo.photo_id = a.cover_photo_id AND cpo.role = 'thumb'
LEFT JOIN objects co ON co.id = cpo.object_id
WHERE a.owner_id = $1 OR m.user_id IS NOT NULL
ORDER BY a.created_at DESC`, userID)
if err != nil {
+11
View File
@@ -1,6 +1,7 @@
package app
import (
"context"
"io/fs"
"net/http"
"strings"
@@ -37,6 +38,7 @@ type Server struct {
membersSvc *members.Service
photos *photos.Service
worker *photos.Worker
social *social.Service
@@ -61,6 +63,7 @@ func New(cfg *config.Config, pool *pgxpool.Pool, st *storage.Client, hub *sse.Hu
photosStore := photos.NewStore(pool)
photosSvc := photos.NewService(photosStore, albumsSvc, st, hub)
photosWorker := photos.NewWorker(photosSvc, cfg.Worker.Interval, cfg.Worker.Batch, cfg.Worker.PendingTTL)
socialStore := social.NewStore(pool)
socialSvc := social.NewService(socialStore, albumsSvc, photosStore, hub, notifSvc)
@@ -78,12 +81,20 @@ func New(cfg *config.Config, pool *pgxpool.Pool, st *storage.Client, hub *sse.Hu
auth: authSvc,
membersSvc: membersSvc,
photos: photosSvc,
worker: photosWorker,
social: socialSvc,
notifications: notifSvc,
share: shareSvc,
}
}
// StartWorker runs the background reconciler until ctx is cancelled.
func (s *Server) StartWorker(ctx context.Context) {
if s.worker != nil {
s.worker.Run(ctx)
}
}
func (s *Server) Router() *gin.Engine {
r := gin.New()
r.Use(gin.Recovery())
+27
View File
@@ -48,12 +48,19 @@ type SSE struct {
ClientBuffer int
}
type Worker struct {
Interval time.Duration
Batch int
PendingTTL time.Duration
}
type Config struct {
Postgres Postgres
S3 S3
Auth Auth
Server Server
SSE SSE
Worker Worker
}
func Load() (*Config, error) {
@@ -92,6 +99,21 @@ func Load() (*Config, error) {
return nil, fmt.Errorf("invalid S3_USE_PATH_STYLE: %w", err)
}
workerInterval, err := time.ParseDuration(getEnv("WORKER_INTERVAL", "5s"))
if err != nil {
return nil, fmt.Errorf("invalid WORKER_INTERVAL: %w", err)
}
workerBatch, err := strconv.Atoi(getEnv("WORKER_BATCH", "10"))
if err != nil {
return nil, fmt.Errorf("invalid WORKER_BATCH: %w", err)
}
workerPendingTTL, err := time.ParseDuration(getEnv("WORKER_PENDING_TTL", "10m"))
if err != nil {
return nil, fmt.Errorf("invalid WORKER_PENDING_TTL: %w", err)
}
cfg := &Config{
Postgres: Postgres{
Host: getEnv("PG_HOST", "pg.localnet"),
@@ -124,6 +146,11 @@ func Load() (*Config, error) {
HeartbeatInterval: heartbeat,
ClientBuffer: bufSize,
},
Worker: Worker{
Interval: workerInterval,
Batch: workerBatch,
PendingTTL: workerPendingTTL,
},
}
if cfg.Auth.JWTSecret == "" {
+92
View File
@@ -0,0 +1,92 @@
package photos
import (
"context"
"testing"
"time"
)
func TestCopySharesStorageObjects(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
svc, cl := newTestService(t, pool)
user := createUser(t, ctx, pool)
albumA := createAlbum(t, ctx, pool, user)
albumB := createAlbum(t, ctx, pool, user)
p, err := svc.Upload(ctx, UploadInput{
AlbumID: albumA, UserID: user, Filename: "p.png", MimeType: "image/png", Data: makePNG(t),
})
if err != nil {
t.Fatalf("upload: %v", err)
}
t.Cleanup(func() { cleanupPhoto(t, ctx, pool, cl, p.ID) })
w := NewWorker(svc, time.Second, 10, time.Minute)
w.processVariantJobs(ctx)
srcIDs, err := svc.store.PhotoObjectIDs(ctx, p.ID)
if err != nil || len(srcIDs) != 3 {
t.Fatalf("source objects = %v err=%v", srcIDs, err)
}
clones, err := svc.CopyBatch(ctx, []string{p.ID}, user, albumB)
if err != nil || len(clones) != 1 {
t.Fatalf("copy: %v len=%d", err, len(clones))
}
clone := clones[0]
t.Cleanup(func() { cleanupPhoto(t, ctx, pool, cl, clone.ID) })
cloneIDs, err := svc.store.PhotoObjectIDs(ctx, clone.ID)
if err != nil || len(cloneIDs) != 3 {
t.Fatalf("clone objects = %v err=%v", cloneIDs, err)
}
if !sameSet(srcIDs, cloneIDs) {
t.Fatalf("clone does not share objects: src=%v clone=%v", srcIDs, cloneIDs)
}
for _, id := range cloneIDs {
if n, err := svc.store.ObjectRefCount(ctx, id); err != nil || n != 2 {
t.Fatalf("object %s ref count = %d err=%v, want 2", id, n, err)
}
}
// Move the source into the same album; the storage key and shared objects
// must be untouched.
beforeKey := p.StorageKey
moved, err := svc.MoveBatch(ctx, []string{p.ID}, user, albumB)
if err != nil || moved != 1 {
t.Fatalf("move: %v moved=%d", err, moved)
}
after, err := svc.store.GetByID(ctx, p.ID, "")
if err != nil {
t.Fatalf("get after move: %v", err)
}
if after.AlbumID != albumB {
t.Fatalf("photo not moved: album=%q", after.AlbumID)
}
if after.StorageKey != beforeKey {
t.Fatalf("move changed storage key: %q -> %q", beforeKey, after.StorageKey)
}
for _, id := range cloneIDs {
if n, err := svc.store.ObjectRefCount(ctx, id); err != nil || n != 2 {
t.Fatalf("object %s ref count after move = %d err=%v, want 2", id, n, err)
}
}
}
func sameSet(a, b []string) bool {
if len(a) != len(b) {
return false
}
m := make(map[string]int, len(a))
for _, v := range a {
m[v]++
}
for _, v := range b {
m[v]--
if m[v] < 0 {
return false
}
}
return true
}
+149
View File
@@ -0,0 +1,149 @@
package photos
import (
"bytes"
"context"
"image"
"image/color"
"image/png"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"shared-album/internal/auth"
"shared-album/internal/config"
"shared-album/internal/models"
"shared-album/internal/roles"
"shared-album/internal/sse"
"shared-album/internal/storage"
)
func testPool(t *testing.T) *pgxpool.Pool {
t.Helper()
dsn := os.Getenv("TEST_DATABASE_URL")
if dsn == "" {
t.Skip("TEST_DATABASE_URL not set")
}
pool, err := pgxpool.New(context.Background(), dsn)
if err != nil {
t.Skipf("cannot connect: %v", err)
}
if err := pool.Ping(context.Background()); err != nil {
t.Skipf("cannot ping: %v", err)
}
t.Cleanup(pool.Close)
return pool
}
func testEnvOr(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
type stubAlbums struct{}
func (stubAlbums) Access(ctx context.Context, albumID, userID string) (*models.Album, string, error) {
return &models.Album{ID: albumID}, roles.Owner, nil
}
func createUser(t *testing.T, ctx context.Context, pool *pgxpool.Pool) string {
t.Helper()
id := auth.NewID()
if _, err := pool.Exec(ctx,
`INSERT INTO users (id, email, password_hash, name) VALUES ($1, $2, 'x', 't')`,
id, id+"@test.local"); err != nil {
t.Fatalf("create user: %v", err)
}
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM users WHERE id = $1`, id) })
return id
}
func createAlbum(t *testing.T, ctx context.Context, pool *pgxpool.Pool, ownerID string) string {
t.Helper()
id := auth.NewID()
if _, err := pool.Exec(ctx,
`INSERT INTO albums (id, owner_id, title) VALUES ($1, $2, 't')`, id, ownerID); err != nil {
t.Fatalf("create album: %v", err)
}
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM albums WHERE id = $1`, id) })
return id
}
func createPhoto(t *testing.T, ctx context.Context, pool *pgxpool.Pool, uploaderID string) string {
t.Helper()
id := auth.NewID()
if _, err := pool.Exec(ctx,
`INSERT INTO photos (id, album_id, uploader_id) VALUES ($1, NULL, $2)`, id, uploaderID); err != nil {
t.Fatalf("create photo: %v", err)
}
return id
}
func makePNG(t *testing.T) []byte {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, 4, 4))
for x := 0; x < 4; x++ {
for y := 0; y < 4; y++ {
img.Set(x, y, color.RGBA{R: uint8(x * 40), G: uint8(y * 40), B: 128, A: 255})
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("encode png: %v", err)
}
return buf.Bytes()
}
// newTestService builds a service backed by the real dev storage. It skips when
// S3 env is absent so unit runs stay hermetic.
func newTestService(t *testing.T, pool *pgxpool.Pool) (*Service, *storage.Client) {
t.Helper()
endpoint := os.Getenv("S3_ENDPOINT")
if endpoint == "" {
t.Skip("S3_ENDPOINT not set")
}
cl, err := storage.New(context.Background(), config.S3{
Endpoint: endpoint,
Region: testEnvOr("S3_REGION", "garage"),
AccessKey: os.Getenv("S3_ACCESS_KEY"),
SecretKey: os.Getenv("S3_SECRET_KEY"),
Bucket: testEnvOr("S3_BUCKET", "galo-data"),
UsePathStyle: testEnvOr("S3_USE_PATH_STYLE", "true") != "false",
PresignTTL: time.Hour,
})
if err != nil {
t.Skipf("storage init: %v", err)
}
return NewService(NewStore(pool), stubAlbums{}, cl, sse.NewHub(1)), cl
}
// cleanupPhoto removes a photo, its objects (rows and blobs) and bindings.
func cleanupPhoto(t *testing.T, ctx context.Context, pool *pgxpool.Pool, cl *storage.Client, photoID string) {
t.Helper()
rows, err := pool.Query(ctx,
`SELECT o.id, o.storage_key FROM photo_objects po JOIN objects o ON o.id = po.object_id
WHERE po.photo_id = $1`, photoID)
if err != nil {
return
}
var ids, keys []string
for rows.Next() {
var id, key string
if rows.Scan(&id, &key) == nil {
ids = append(ids, id)
keys = append(keys, key)
}
}
rows.Close()
_, _ = pool.Exec(ctx, `DELETE FROM photos WHERE id = $1`, photoID)
for _, id := range ids {
_, _ = pool.Exec(ctx, `DELETE FROM objects WHERE id = $1`, id)
}
if cl != nil && len(keys) > 0 {
_ = cl.Delete(ctx, keys...)
}
}
+17
View File
@@ -0,0 +1,17 @@
package photos
import (
"fmt"
"strings"
)
// objectKey builds an opaque, album-free storage key for a physical object.
// The key never encodes album, photo or any other metadata entity; the mapping
// between a key and its photo binding lives only in the metadata layer. The
// role segment is cosmetic (for debugging) and is not authoritative.
func objectKey(objectID, role, ext string) string {
if ext != "" && !strings.HasPrefix(ext, ".") {
ext = "." + ext
}
return fmt.Sprintf("objects/%s/%s%s", objectID, role, ext)
}
+32
View File
@@ -0,0 +1,32 @@
package photos
import (
"strings"
"testing"
)
func TestObjectKeyIsAlbumFree(t *testing.T) {
cases := []struct {
name string
id string
role string
ext string
want string
}{
{"original with dot", "obj1", RoleOriginal, ".jpg", "objects/obj1/original.jpg"},
{"thumb without dot", "obj1", RoleThumb, "webp", "objects/obj1/thumb.webp"},
{"preview", "obj2", RolePreview, ".webp", "objects/obj2/preview.webp"},
{"empty ext", "obj3", RoleOriginal, "", "objects/obj3/original"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := objectKey(tc.id, tc.role, tc.ext)
if got != tc.want {
t.Fatalf("objectKey = %q, want %q", got, tc.want)
}
if strings.Contains(got, "album") {
t.Fatalf("key must not encode an album entity: %q", got)
}
})
}
}
+370
View File
@@ -0,0 +1,370 @@
package photos
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
)
// Object lifecycle states. The DB row is the ledger entry that governs the
// lifecycle of the corresponding blob in the storage layer.
const (
ObjectPending = "pending"
ObjectReady = "ready"
ObjectDeleting = "deleting"
)
// Object roles as bound to a photo. The role is a property of the binding.
const (
RoleOriginal = "original"
RoleThumb = "thumb"
RolePreview = "preview"
)
var (
// ErrObjectNotFound is returned when an object row does not exist.
ErrObjectNotFound = errors.New("object not found")
// ErrObjectNotReady is returned when binding an object that is not ready.
ErrObjectNotReady = errors.New("object is not ready")
)
// Object is a physical file in the storage layer, described by metadata.
type Object struct {
ID string
StorageKey string
MimeType string
Size int64
Width int
Height int
Checksum string
State string
CreatedAt time.Time
}
// CreateObject inserts a new object row in the pending state.
func (s *Store) CreateObject(ctx context.Context, o *Object) error {
if o.State == "" {
o.State = ObjectPending
}
return s.pool.QueryRow(ctx,
`INSERT INTO objects (id, storage_key, mime_type, size, width, height, checksum, state)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING created_at`,
o.ID, o.StorageKey, o.MimeType, o.Size, o.Width, o.Height, o.Checksum, o.State).Scan(&o.CreatedAt)
}
// SetObjectState updates an object's lifecycle state.
func (s *Store) SetObjectState(ctx context.Context, id, state string) error {
tag, err := s.pool.Exec(ctx, `UPDATE objects SET state = $2 WHERE id = $1`, id, state)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return ErrObjectNotFound
}
return nil
}
// ActivateObject marks an object ready after its bytes have been stored.
func (s *Store) ActivateObject(ctx context.Context, id string) error {
return s.SetObjectState(ctx, id, ObjectReady)
}
// MarkObjectDeleting marks an object as pending removal from storage.
func (s *Store) MarkObjectDeleting(ctx context.Context, id string) error {
return s.SetObjectState(ctx, id, ObjectDeleting)
}
// DeleteObjectRow permanently removes an object row (after its blob is gone).
func (s *Store) DeleteObjectRow(ctx context.Context, id string) error {
tag, err := s.pool.Exec(ctx, `DELETE FROM objects WHERE id = $1`, id)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return ErrObjectNotFound
}
return nil
}
// BindObject attaches a ready object to a photo under the given role. It locks
// the object row and refuses objects that are not ready, so an object being
// garbage-collected can never be bound.
func (s *Store) BindObject(ctx context.Context, photoID, objectID, role string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
var state string
if err := tx.QueryRow(ctx, `SELECT state FROM objects WHERE id = $1 FOR UPDATE`, objectID).Scan(&state); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return ErrObjectNotFound
}
return err
}
if state != ObjectReady {
return ErrObjectNotReady
}
if _, err := tx.Exec(ctx,
`INSERT INTO photo_objects (photo_id, object_id, role) VALUES ($1, $2, $3)
ON CONFLICT (photo_id, role) DO UPDATE SET object_id = EXCLUDED.object_id`,
photoID, objectID, role); err != nil {
return err
}
return tx.Commit(ctx)
}
// PhotoObjectIDs returns the object ids currently bound to a photo.
func (s *Store) PhotoObjectIDs(ctx context.Context, photoID string) ([]string, error) {
rows, err := s.pool.Query(ctx, `SELECT object_id FROM photo_objects WHERE photo_id = $1`, photoID)
if err != nil {
return nil, err
}
defer rows.Close()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, rows.Err()
}
// ObjectRefCount counts how many photo bindings reference an object.
func (s *Store) ObjectRefCount(ctx context.Context, objectID string) (int, error) {
var n int
err := s.pool.QueryRow(ctx, `SELECT count(*) FROM photo_objects WHERE object_id = $1`, objectID).Scan(&n)
return n, err
}
// OriginalObject returns the original object bound to a photo.
func (s *Store) OriginalObject(ctx context.Context, photoID string) (*Object, error) {
var o Object
err := s.pool.QueryRow(ctx,
`SELECT o.id, o.storage_key, o.mime_type, o.size, o.width, o.height, o.checksum, o.state, o.created_at
FROM photo_objects po JOIN objects o ON o.id = po.object_id
WHERE po.photo_id = $1 AND po.role = $2`, photoID, RoleOriginal).
Scan(&o.ID, &o.StorageKey, &o.MimeType, &o.Size, &o.Width, &o.Height, &o.Checksum, &o.State, &o.CreatedAt)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrObjectNotFound
}
return nil, err
}
return &o, nil
}
// PhotosByOriginalObject returns the ids of all photos bound to the given
// original object, so generated variants can be shared between copies.
func (s *Store) PhotosByOriginalObject(ctx context.Context, originalObjectID string) ([]string, error) {
rows, err := s.pool.Query(ctx,
`SELECT photo_id FROM photo_objects WHERE object_id = $1 AND role = $2`, originalObjectID, RoleOriginal)
if err != nil {
return nil, err
}
defer rows.Close()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, rows.Err()
}
// ClaimGarbage claims collectible objects: ready objects with no bindings and
// objects already marked deleting. Rows are locked with SKIP LOCKED so
// concurrent workers do not contend and a concurrent bind cannot slip in.
func (s *Store) ClaimGarbage(ctx context.Context, limit int) ([]Object, error) {
if limit <= 0 {
limit = 10
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
rows, err := tx.Query(ctx,
`SELECT o.id, o.storage_key FROM objects o
WHERE o.state = ANY($1)
AND NOT EXISTS (SELECT 1 FROM photo_objects po WHERE po.object_id = o.id)
ORDER BY o.created_at
LIMIT $2
FOR UPDATE SKIP LOCKED`, []string{ObjectReady, ObjectDeleting}, limit)
if err != nil {
return nil, err
}
var objs []Object
for rows.Next() {
var o Object
if err := rows.Scan(&o.ID, &o.StorageKey); err != nil {
rows.Close()
return nil, err
}
objs = append(objs, o)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
if len(objs) == 0 {
return nil, tx.Commit(ctx)
}
ids := make([]string, len(objs))
for i, o := range objs {
ids[i] = o.ID
}
if _, err := tx.Exec(ctx, `UPDATE objects SET state = $2 WHERE id = ANY($1)`, ids, ObjectDeleting); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
return objs, nil
}
// ClaimStalePending marks abandoned pending objects (whose upload never
// completed) as deleting and returns them for cleanup.
func (s *Store) ClaimStalePending(ctx context.Context, olderThan time.Duration, limit int) ([]Object, error) {
if limit <= 0 {
limit = 10
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
rows, err := tx.Query(ctx,
`SELECT id, storage_key FROM objects
WHERE state = $1 AND created_at < $2
ORDER BY created_at
LIMIT $3
FOR UPDATE SKIP LOCKED`, ObjectPending, time.Now().Add(-olderThan), limit)
if err != nil {
return nil, err
}
var objs []Object
for rows.Next() {
var o Object
if err := rows.Scan(&o.ID, &o.StorageKey); err != nil {
rows.Close()
return nil, err
}
objs = append(objs, o)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
if len(objs) == 0 {
return nil, tx.Commit(ctx)
}
ids := make([]string, len(objs))
for i, o := range objs {
ids[i] = o.ID
}
if _, err := tx.Exec(ctx, `UPDATE objects SET state = $2 WHERE id = ANY($1)`, ids, ObjectDeleting); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
return objs, nil
}
// ClaimVariantJobs leases due variant jobs and returns their photo ids. The
// lease pushes next_attempt_at forward so the same job is not picked up twice
// while it is being processed.
func (s *Store) ClaimVariantJobs(ctx context.Context, limit int, lease time.Duration) ([]string, error) {
if limit <= 0 {
limit = 10
}
tx, err := s.pool.Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(ctx) }()
rows, err := tx.Query(ctx,
`SELECT photo_id FROM variant_jobs
WHERE next_attempt_at <= now()
ORDER BY next_attempt_at
LIMIT $1
FOR UPDATE SKIP LOCKED`, limit)
if err != nil {
return nil, err
}
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
rows.Close()
return nil, err
}
ids = append(ids, id)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
if len(ids) == 0 {
return nil, tx.Commit(ctx)
}
if _, err := tx.Exec(ctx,
`UPDATE variant_jobs SET next_attempt_at = $2 WHERE photo_id = ANY($1)`,
ids, time.Now().Add(lease)); err != nil {
return nil, err
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
return ids, nil
}
// CompleteVariantJob removes a finished job.
func (s *Store) CompleteVariantJob(ctx context.Context, photoID string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM variant_jobs WHERE photo_id = $1`, photoID)
return err
}
// FailVariantJob records a failure and schedules a retry with exponential
// backoff. After maxAttempts the job is abandoned so a permanently broken photo
// cannot loop forever.
func (s *Store) FailVariantJob(ctx context.Context, photoID, errMsg string, maxAttempts int) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
var attempts int
err = tx.QueryRow(ctx,
`UPDATE variant_jobs
SET attempts = attempts + 1,
next_attempt_at = now() + make_interval(secs => least(300, 10 * power(2, attempts))),
last_error = $2
WHERE photo_id = $1
RETURNING attempts`, photoID, errMsg).Scan(&attempts)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return tx.Commit(ctx)
}
return err
}
if maxAttempts > 0 && attempts >= maxAttempts {
if _, err := tx.Exec(ctx, `DELETE FROM variant_jobs WHERE photo_id = $1`, photoID); err != nil {
return err
}
}
return tx.Commit(ctx)
}
+187
View File
@@ -0,0 +1,187 @@
package photos
import (
"context"
"errors"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"shared-album/internal/auth"
)
func TestObjectStateTransitions(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
st := NewStore(pool)
objID := auth.NewID()
obj := &Object{ID: objID, StorageKey: "objects/" + objID + "/original.jpg", MimeType: "image/jpeg", Size: 42}
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM objects WHERE id = $1`, objID) })
if err := st.CreateObject(ctx, obj); err != nil {
t.Fatalf("create object: %v", err)
}
if obj.State != ObjectPending {
t.Fatalf("new object state = %q, want %q", obj.State, ObjectPending)
}
if obj.CreatedAt.IsZero() {
t.Fatalf("created_at not populated")
}
if err := st.ActivateObject(ctx, objID); err != nil {
t.Fatalf("activate: %v", err)
}
if got := objectState(t, ctx, pool, objID); got != ObjectReady {
t.Fatalf("state after activate = %q", got)
}
if err := st.MarkObjectDeleting(ctx, objID); err != nil {
t.Fatalf("mark deleting: %v", err)
}
if got := objectState(t, ctx, pool, objID); got != ObjectDeleting {
t.Fatalf("state after mark = %q", got)
}
if err := st.DeleteObjectRow(ctx, objID); err != nil {
t.Fatalf("delete row: %v", err)
}
if err := st.DeleteObjectRow(ctx, objID); !errors.Is(err, ErrObjectNotFound) {
t.Fatalf("second delete = %v, want ErrObjectNotFound", err)
}
}
func TestClaimStalePending(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
st := NewStore(pool)
objID := auth.NewID()
obj := &Object{ID: objID, StorageKey: "objects/" + objID + "/original.jpg"}
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM objects WHERE id = $1`, objID) })
if err := st.CreateObject(ctx, obj); err != nil {
t.Fatalf("create object: %v", err)
}
if _, err := pool.Exec(ctx, `UPDATE objects SET created_at = now() - interval '1 hour' WHERE id = $1`, objID); err != nil {
t.Fatalf("age object: %v", err)
}
objs, err := st.ClaimStalePending(ctx, 10*time.Minute, 100)
if err != nil {
t.Fatalf("claim stale: %v", err)
}
if !containsObject(objs, objID) {
t.Fatalf("stale pending object not claimed: %+v", objs)
}
if got := objectState(t, ctx, pool, objID); got != ObjectDeleting {
t.Fatalf("claimed stale object state = %q, want deleting", got)
}
}
func TestBindObjectRequiresReadyAndShares(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
st := NewStore(pool)
user := createUser(t, ctx, pool)
p1 := createPhoto(t, ctx, pool, user)
p2 := createPhoto(t, ctx, pool, user)
objID := auth.NewID()
obj := &Object{ID: objID, StorageKey: "objects/" + objID + "/original.jpg"}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM photo_objects WHERE object_id = $1`, objID)
_, _ = pool.Exec(ctx, `DELETE FROM objects WHERE id = $1`, objID)
})
if err := st.CreateObject(ctx, obj); err != nil {
t.Fatalf("create object: %v", err)
}
if err := st.BindObject(ctx, p1, objID, RoleOriginal); !errors.Is(err, ErrObjectNotReady) {
t.Fatalf("bind pending object = %v, want ErrObjectNotReady", err)
}
if err := st.ActivateObject(ctx, objID); err != nil {
t.Fatalf("activate: %v", err)
}
if err := st.BindObject(ctx, p1, objID, RoleOriginal); err != nil {
t.Fatalf("bind p1: %v", err)
}
if err := st.BindObject(ctx, p2, objID, RoleOriginal); err != nil {
t.Fatalf("bind p2: %v", err)
}
if n, err := st.ObjectRefCount(ctx, objID); err != nil || n != 2 {
t.Fatalf("ref count = %d err=%v, want 2", n, err)
}
siblings, err := st.PhotosByOriginalObject(ctx, objID)
if err != nil || len(siblings) != 2 {
t.Fatalf("siblings = %v err=%v, want 2", siblings, err)
}
}
func TestClaimGarbageRespectsReferences(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
st := NewStore(pool)
user := createUser(t, ctx, pool)
photo := createPhoto(t, ctx, pool, user)
boundID := auth.NewID()
freeID := auth.NewID()
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM photo_objects WHERE object_id IN ($1, $2)`, boundID, freeID)
_, _ = pool.Exec(ctx, `DELETE FROM objects WHERE id IN ($1, $2)`, boundID, freeID)
})
for _, id := range []string{boundID, freeID} {
if err := st.CreateObject(ctx, &Object{ID: id, StorageKey: "objects/" + id + "/original.jpg"}); err != nil {
t.Fatalf("create object: %v", err)
}
if err := st.ActivateObject(ctx, id); err != nil {
t.Fatalf("activate: %v", err)
}
}
if err := st.BindObject(ctx, photo, boundID, RoleOriginal); err != nil {
t.Fatalf("bind: %v", err)
}
claimed, err := st.ClaimGarbage(ctx, 100)
if err != nil {
t.Fatalf("claim garbage: %v", err)
}
if !containsObject(claimed, freeID) {
t.Fatalf("unreferenced object not claimed: %+v", claimed)
}
if containsObject(claimed, boundID) {
t.Fatalf("referenced object was claimed for deletion")
}
// Removing the last reference makes both collectible.
if _, err := pool.Exec(ctx, `DELETE FROM photos WHERE id = $1`, photo); err != nil {
t.Fatalf("delete photo: %v", err)
}
claimed, err = st.ClaimGarbage(ctx, 100)
if err != nil {
t.Fatalf("claim garbage 2: %v", err)
}
if !containsObject(claimed, boundID) {
t.Fatalf("object with no remaining references not claimed: %+v", claimed)
}
}
func objectState(t *testing.T, ctx context.Context, pool *pgxpool.Pool, id string) string {
t.Helper()
var state string
if err := pool.QueryRow(ctx, `SELECT state FROM objects WHERE id = $1`, id).Scan(&state); err != nil {
t.Fatalf("read state: %v", err)
}
return state
}
func containsObject(objs []Object, id string) bool {
for _, o := range objs {
if o.ID == id {
return true
}
}
return false
}
+8
View File
@@ -2,6 +2,8 @@ package photos
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"image"
"strconv"
@@ -138,3 +140,9 @@ func buildVariants(data []byte) (thumb, preview []byte, err error) {
}
return thumbBuf.Bytes(), previewBuf.Bytes(), nil
}
// checksum returns the hex-encoded SHA-256 of the given bytes.
func checksum(data []byte) string {
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:])
}
+34 -47
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"path/filepath"
"strings"
@@ -64,22 +63,20 @@ func (s *Service) Upload(ctx context.Context, in UploadInput) (*models.Photo, er
}
meta := extractEXIF(in.Data)
id := auth.NewID()
ext := strings.ToLower(filepath.Ext(in.Filename))
if ext == "" {
ext = ".bin"
}
storageKey := fmt.Sprintf("albums/%s/original/%s%s", in.AlbumID, id, ext)
if err := s.storage.Put(ctx, storageKey, bytes.NewReader(in.Data), int64(len(in.Data)), mime); err != nil {
return nil, err
}
photoID := auth.NewID()
objectID := auth.NewID()
key := objectKey(objectID, RoleOriginal, ext)
p := &models.Photo{
ID: id,
ID: photoID,
AlbumID: in.AlbumID,
UploaderID: in.UserID,
StorageKey: storageKey,
StorageKey: key,
OriginalFilename: in.Filename,
Size: int64(len(in.Data)),
Width: width,
@@ -90,12 +87,28 @@ func (s *Service) Upload(ctx context.Context, in UploadInput) (*models.Photo, er
Longitude: meta.Longitude,
EXIF: meta.Attributes,
}
if err := s.store.Create(ctx, p); err != nil {
return nil, err
obj := &Object{
ID: objectID,
StorageKey: key,
MimeType: mime,
Size: int64(len(in.Data)),
Width: width,
Height: height,
Checksum: checksum(in.Data),
}
data := in.Data
go s.processVariants(p.ID, in.AlbumID, data)
// The metadata layer is the ledger of record: the row is written first,
// then the bytes are stored, then the object is activated.
if err := s.store.CreateWithOriginal(ctx, p, obj); err != nil {
return nil, err
}
if err := s.storage.Put(ctx, key, bytes.NewReader(in.Data), int64(len(in.Data)), mime); err != nil {
_ = s.store.DeleteUpload(ctx, photoID, objectID)
return nil, err
}
if err := s.store.ActivateObject(ctx, objectID); err != nil {
return nil, err
}
if err := s.attachURLs(ctx, p); err != nil {
return nil, err
@@ -103,33 +116,6 @@ func (s *Service) Upload(ctx context.Context, in UploadInput) (*models.Photo, er
return p, nil
}
func (s *Service) processVariants(photoID, albumID string, data []byte) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
thumb, preview, err := buildVariants(data)
if err != nil {
return
}
thumbKey := fmt.Sprintf("albums/%s/thumb/%s.webp", albumID, photoID)
previewKey := fmt.Sprintf("albums/%s/preview/%s.webp", albumID, photoID)
if err := s.storage.Put(ctx, thumbKey, bytes.NewReader(thumb), int64(len(thumb)), "image/webp"); err != nil {
return
}
if err := s.storage.Put(ctx, previewKey, bytes.NewReader(preview), int64(len(preview)), "image/webp"); err != nil {
return
}
if err := s.store.SetVariants(ctx, photoID, thumbKey, previewKey); err != nil {
return
}
s.store.SetCoverIfEmpty(ctx, albumID, photoID)
if p, err := s.store.GetByID(ctx, photoID, ""); err == nil {
if err := s.attachURLs(ctx, p); err == nil {
s.hub.Publish(albumID, "photo.created", p)
}
}
}
func (s *Service) attachURLs(ctx context.Context, p *models.Photo) error {
if p.StorageKey != "" {
url, err := s.storage.PresignGet(ctx, p.StorageKey)
@@ -416,21 +402,22 @@ func (s *Service) PermanentDelete(ctx context.Context, photoID, userID string) e
if !canDeletePhoto(role, p.UploaderID == userID) {
return httpx.Forbidden("forbidden", "you cannot delete this photo")
}
objectIDs, err := s.store.PhotoObjectIDs(ctx, photoID)
if err != nil {
return err
}
if err := s.store.HardDelete(ctx, photoID); err != nil {
return err
}
if p.AlbumID != "" {
s.publishDeleted(p)
}
if n, err := s.store.StorageKeyRefCount(ctx, p.StorageKey); err == nil && n == 0 {
keys := []string{p.StorageKey}
if p.ThumbKey != "" {
keys = append(keys, p.ThumbKey)
// Mark objects that no longer have any binding for garbage collection.
// The background worker removes the blobs from storage.
for _, oid := range objectIDs {
if n, err := s.store.ObjectRefCount(ctx, oid); err == nil && n == 0 {
_ = s.store.MarkObjectDeleting(ctx, oid)
}
if p.PreviewKey != "" {
keys = append(keys, p.PreviewKey)
}
_ = s.storage.Delete(ctx, keys...)
}
return nil
}
+99 -45
View File
@@ -23,9 +23,14 @@ type Store struct {
func NewStore(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
const photoColumns = `p.id, COALESCE(p.album_id, ''), p.uploader_id, p.storage_key,
COALESCE(p.thumb_key, ''), COALESCE(p.preview_key, ''),
p.original_filename, p.size, p.width, p.height, p.mime_type,
// photoColumns selects photo rows together with the keys and physical facts of
// their bound objects (original, thumb, preview) and the original's size,
// dimensions and mime type.
const photoColumns = `p.id, COALESCE(p.album_id, ''), p.uploader_id,
COALESCE(o_o.storage_key, ''),
COALESCE(o_t.storage_key, ''),
COALESCE(o_p.storage_key, ''),
p.original_filename, COALESCE(o_o.size, 0), COALESCE(o_o.width, 0), COALESCE(o_o.height, 0), COALESCE(o_o.mime_type, ''),
p.taken_at, p.latitude, p.longitude, COALESCE(p.caption, ''), p.created_at,
u.id, u.name, COALESCE(u.avatar_url, ''),
(SELECT count(*) FROM likes l WHERE l.photo_id = p.id),
@@ -36,6 +41,14 @@ const photoColumns = `p.id, COALESCE(p.album_id, ''), p.uploader_id, p.storage_k
FROM photo_tags pt JOIN tags t ON t.id = pt.tag_id
WHERE pt.photo_id = p.id AND t.user_id = $2), '[]')::text`
const photoJoins = `
LEFT JOIN photo_objects po_o ON po_o.photo_id = p.id AND po_o.role = 'original'
LEFT JOIN objects o_o ON o_o.id = po_o.object_id
LEFT JOIN photo_objects po_t ON po_t.photo_id = p.id AND po_t.role = 'thumb'
LEFT JOIN objects o_t ON o_t.id = po_t.object_id
LEFT JOIN photo_objects po_p ON po_p.photo_id = p.id AND po_p.role = 'preview'
LEFT JOIN objects o_p ON o_p.id = po_p.object_id`
func scanPhoto(row pgx.Row) (*models.Photo, error) {
var p models.Photo
var exifText, tagsText string
@@ -76,31 +89,68 @@ func setExif(p *models.Photo, raw string) {
}
}
func (s *Store) Create(ctx context.Context, p *models.Photo) error {
// CreateWithOriginal records a new photo together with its pending original
// object, the original binding and a variant job — all in one transaction. The
// caller stores the bytes and then activates the object. The metadata layer is
// the ledger of record; the storage layer follows.
func (s *Store) CreateWithOriginal(ctx context.Context, p *models.Photo, obj *Object) error {
exifJSON, err := json.Marshal(p.EXIF)
if err != nil || len(exifJSON) == 0 {
exifJSON = []byte("{}")
}
return s.pool.QueryRow(ctx,
`INSERT INTO photos (id, album_id, uploader_id, storage_key, original_filename,
size, width, height, mime_type, taken_at, latitude, longitude, caption, exif)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14::jsonb)
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx,
`INSERT INTO objects (id, storage_key, mime_type, size, width, height, checksum, state)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
obj.ID, obj.StorageKey, obj.MimeType, obj.Size, obj.Width, obj.Height, obj.Checksum, ObjectPending); err != nil {
return err
}
if err := tx.QueryRow(ctx,
`INSERT INTO photos (id, album_id, uploader_id, original_filename, taken_at, latitude, longitude, caption, exif)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)
RETURNING created_at`,
p.ID, p.AlbumID, p.UploaderID, p.StorageKey, p.OriginalFilename,
p.Size, p.Width, p.Height, p.MimeType, p.TakenAt, p.Latitude, p.Longitude, p.Caption, string(exifJSON),
).Scan(&p.CreatedAt)
p.ID, p.AlbumID, p.UploaderID, p.OriginalFilename, p.TakenAt, p.Latitude, p.Longitude, p.Caption, string(exifJSON),
).Scan(&p.CreatedAt); err != nil {
return err
}
if _, err := tx.Exec(ctx,
`INSERT INTO photo_objects (photo_id, object_id, role) VALUES ($1, $2, $3)`,
p.ID, obj.ID, RoleOriginal); err != nil {
return err
}
if _, err := tx.Exec(ctx,
`INSERT INTO variant_jobs (photo_id) VALUES ($1) ON CONFLICT (photo_id) DO NOTHING`, p.ID); err != nil {
return err
}
return tx.Commit(ctx)
}
func (s *Store) SetVariants(ctx context.Context, id, thumbKey, previewKey string) error {
_, err := s.pool.Exec(ctx,
`UPDATE photos SET thumb_key = $2, preview_key = $3 WHERE id = $1`, id, thumbKey, previewKey)
return err
// DeleteUpload rolls back a failed upload: the photo (and its bindings and job)
// plus the pending object row are removed.
func (s *Store) DeleteUpload(ctx context.Context, photoID, objectID string) error {
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx, `DELETE FROM photos WHERE id = $1`, photoID); err != nil {
return err
}
if _, err := tx.Exec(ctx, `DELETE FROM objects WHERE id = $1`, objectID); err != nil {
return err
}
return tx.Commit(ctx)
}
func (s *Store) GetByID(ctx context.Context, id, viewerID string) (*models.Photo, error) {
return scanPhoto(s.pool.QueryRow(ctx,
`SELECT `+photoColumns+`
FROM photos p JOIN users u ON u.id = p.uploader_id
FROM photos p JOIN users u ON u.id = p.uploader_id`+photoJoins+`
WHERE p.id = $1`, id, viewerID))
}
@@ -118,7 +168,7 @@ func (s *Store) List(ctx context.Context, in ListParams) ([]models.Photo, error)
}
rows, err := s.pool.Query(ctx,
`SELECT `+photoColumns+`
FROM photos p JOIN users u ON u.id = p.uploader_id
FROM photos p JOIN users u ON u.id = p.uploader_id`+photoJoins+`
WHERE p.album_id = $1
AND p.deleted_at IS NULL
AND ($3 = '' OR (p.created_at, p.id) < (SELECT created_at, id FROM photos WHERE id = $3))
@@ -181,30 +231,36 @@ func (s *Store) UpdateAlbum(ctx context.Context, id, albumID string) error {
return nil
}
// Clone creates a new photo row that references the same stored objects as src.
// Clone creates a new photo row in albumID that references the same objects as
// src. No storage operation is performed: the clone shares the physical files.
func (s *Store) Clone(ctx context.Context, src *models.Photo, albumID, uploaderID, id string) error {
exifJSON, err := json.Marshal(src.EXIF)
if err != nil || len(exifJSON) == 0 {
exifJSON = []byte("{}")
}
_, err = s.pool.Exec(ctx,
`INSERT INTO photos (id, album_id, uploader_id, storage_key, thumb_key, preview_key,
original_filename, size, width, height, mime_type, taken_at, latitude, longitude, caption, exif)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16::jsonb)`,
id, albumID, uploaderID, src.StorageKey, nullIfEmpty(src.ThumbKey), nullIfEmpty(src.PreviewKey),
src.OriginalFilename, src.Size, src.Width, src.Height, src.MimeType,
src.TakenAt, src.Latitude, src.Longitude, src.Caption, string(exifJSON))
return err
tx, err := s.pool.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(ctx) }()
if _, err := tx.Exec(ctx,
`INSERT INTO photos (id, album_id, uploader_id, original_filename, taken_at, latitude, longitude, caption, exif)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb)`,
id, albumID, uploaderID, src.OriginalFilename, src.TakenAt, src.Latitude, src.Longitude, src.Caption, string(exifJSON)); err != nil {
return err
}
if _, err := tx.Exec(ctx,
`INSERT INTO photo_objects (photo_id, object_id, role)
SELECT $1, object_id, role FROM photo_objects WHERE photo_id = $2`,
id, src.ID); err != nil {
return err
}
return tx.Commit(ctx)
}
// StorageKeyRefCount counts photo rows (any state) referencing the stored object.
func (s *Store) StorageKeyRefCount(ctx context.Context, storageKey string) (int, error) {
var n int
err := s.pool.QueryRow(ctx, `SELECT count(*) FROM photos WHERE storage_key = $1`, storageKey).Scan(&n)
return n, err
}
// HardDelete permanently removes the photo row.
// HardDelete permanently removes the photo row. Bound objects are not touched
// here; the caller marks orphaned objects for collection.
func (s *Store) HardDelete(ctx context.Context, id string) error {
tag, err := s.pool.Exec(ctx, `DELETE FROM photos WHERE id = $1`, id)
if err != nil {
@@ -223,7 +279,7 @@ func (s *Store) ListTrash(ctx context.Context, userID string) ([]models.Photo, e
`SELECT `+photoColumns+`, COALESCE(a.title, '')
FROM photos p
JOIN users u ON u.id = p.uploader_id
LEFT JOIN albums a ON a.id = p.album_id
LEFT JOIN albums a ON a.id = p.album_id`+photoJoins+`
WHERE p.deleted_at IS NOT NULL
AND (
(a.id IS NOT NULL AND (a.owner_id = $1 OR EXISTS(
@@ -362,13 +418,6 @@ func (s *Store) TagOwnedBy(ctx context.Context, tagID, userID string) (bool, err
return ok, err
}
func nullIfEmpty(s string) *string {
if s == "" {
return nil
}
return &s
}
func (s *Store) SetCoverIfEmpty(ctx context.Context, albumID, photoID string) {
_, _ = s.pool.Exec(ctx,
`UPDATE albums SET cover_photo_id = $2 WHERE id = $1 AND cover_photo_id IS NULL`,
@@ -391,11 +440,16 @@ type backfillRow struct {
StorageKey string
}
// BackfillEXIF re-reads the original object of every photo with empty exif and
// fills the exif/geo fields on the photo.
func (s *Store) BackfillEXIF(ctx context.Context, fetch func(context.Context, string) ([]byte, error)) (int, error) {
rows, err := s.pool.Query(ctx,
`SELECT id, storage_key FROM photos
WHERE exif IS NULL OR exif = '{}'::jsonb
ORDER BY created_at`)
`SELECT p.id, o.storage_key
FROM photos p
JOIN photo_objects po ON po.photo_id = p.id AND po.role = 'original'
JOIN objects o ON o.id = po.object_id
WHERE p.exif IS NULL OR p.exif = '{}'::jsonb
ORDER BY p.created_at`)
if err != nil {
return 0, err
}
+70
View File
@@ -0,0 +1,70 @@
package photos
import (
"context"
"strings"
"testing"
)
func TestUploadCreatesLedgerAndAlbumFreeKey(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
svc, cl := newTestService(t, pool)
user := createUser(t, ctx, pool)
album := createAlbum(t, ctx, pool, user)
p, err := svc.Upload(ctx, UploadInput{
AlbumID: album,
UserID: user,
Filename: "photo.png",
MimeType: "image/png",
Data: makePNG(t),
})
if err != nil {
t.Fatalf("upload: %v", err)
}
t.Cleanup(func() { cleanupPhoto(t, ctx, pool, cl, p.ID) })
if !strings.HasPrefix(p.StorageKey, "objects/") {
t.Fatalf("storage key %q is not an object key", p.StorageKey)
}
if strings.Contains(p.StorageKey, album) {
t.Fatalf("storage key leaks album id: %q", p.StorageKey)
}
if p.OriginalURL == "" {
t.Fatalf("original url not set")
}
if p.ThumbURL != "" || p.PreviewURL != "" {
t.Fatalf("variants should be pending, got thumb=%q preview=%q", p.ThumbURL, p.PreviewURL)
}
orig, err := svc.store.OriginalObject(ctx, p.ID)
if err != nil {
t.Fatalf("original object: %v", err)
}
if orig.State != ObjectReady {
t.Fatalf("original object state = %q, want ready", orig.State)
}
if n, err := svc.store.ObjectRefCount(ctx, orig.ID); err != nil || n != 1 {
t.Fatalf("original ref count = %d err=%v, want 1", n, err)
}
var jobs int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM variant_jobs WHERE photo_id = $1`, p.ID).Scan(&jobs); err != nil {
t.Fatalf("count jobs: %v", err)
}
if jobs != 1 {
t.Fatalf("variant jobs = %d, want 1", jobs)
}
list, _, err := svc.List(ctx, album, user, "", nil, 50)
if err != nil || len(list) != 1 {
t.Fatalf("list: %v len=%d", err, len(list))
}
if list[0].MimeType != "image/png" || list[0].Width != 4 || list[0].Height != 4 {
t.Fatalf("derived fields wrong: mime=%q %dx%d", list[0].MimeType, list[0].Width, list[0].Height)
}
if list[0].OriginalURL == "" {
t.Fatalf("list original url not set")
}
}
+210
View File
@@ -0,0 +1,210 @@
package photos
import (
"bytes"
"context"
"errors"
"log/slog"
"time"
"shared-album/internal/auth"
)
const maxVariantAttempts = 5
// Worker is the background reconciler. It drives the variant generation queue
// and the object garbage collector entirely from the state recorded in the
// metadata layer, so it resumes correctly after a restart.
type Worker struct {
svc *Service
interval time.Duration
batch int
pendingTTL time.Duration
lease time.Duration
}
// NewWorker builds a worker. Parameters are clamped to sane defaults.
func NewWorker(svc *Service, interval time.Duration, batch int, pendingTTL time.Duration) *Worker {
if interval <= 0 {
interval = 5 * time.Second
}
if batch <= 0 {
batch = 10
}
if pendingTTL <= 0 {
pendingTTL = 10 * time.Minute
}
return &Worker{
svc: svc,
interval: interval,
batch: batch,
pendingTTL: pendingTTL,
lease: interval * 4,
}
}
// Run loops until the context is cancelled.
func (w *Worker) Run(ctx context.Context) {
slog.Info("background worker started", "interval", w.interval)
defer slog.Info("background worker stopped")
ticker := time.NewTicker(w.interval)
defer ticker.Stop()
for {
w.tick(ctx)
select {
case <-ctx.Done():
return
case <-ticker.C:
}
}
}
func (w *Worker) tick(ctx context.Context) {
w.processVariantJobs(ctx)
w.collectGarbage(ctx)
w.reapStalePending(ctx)
}
func (w *Worker) processVariantJobs(ctx context.Context) {
ids, err := w.svc.store.ClaimVariantJobs(ctx, w.batch, w.lease)
if err != nil {
slog.Error("claim variant jobs", "error", err)
return
}
for _, id := range ids {
w.generateVariants(ctx, id)
}
}
func (w *Worker) generateVariants(ctx context.Context, photoID string) {
orig, err := w.svc.store.OriginalObject(ctx, photoID)
if err != nil {
// The photo or its original binding is gone; nothing to do.
_ = w.svc.store.CompleteVariantJob(ctx, photoID)
return
}
if orig.State != ObjectReady {
w.fail(ctx, photoID, "original object is not ready")
return
}
data, err := w.svc.storage.Get(ctx, orig.StorageKey)
if err != nil {
w.fail(ctx, photoID, "fetch original: "+err.Error())
return
}
thumb, preview, err := buildVariants(data)
if err != nil {
w.fail(ctx, photoID, "build variants: "+err.Error())
return
}
siblings, err := w.svc.store.PhotosByOriginalObject(ctx, orig.ID)
if err != nil || len(siblings) == 0 {
siblings = []string{photoID}
}
if err := w.createVariant(ctx, siblings, RoleThumb, thumb); err != nil {
w.fail(ctx, photoID, "store thumb: "+err.Error())
return
}
if err := w.createVariant(ctx, siblings, RolePreview, preview); err != nil {
w.fail(ctx, photoID, "store preview: "+err.Error())
return
}
_ = w.svc.store.CompleteVariantJob(ctx, photoID)
for _, pid := range siblings {
w.notify(ctx, pid)
}
}
// createVariant stores one derived object and binds it to every photo sharing
// the original, so copies reuse the same physical variant.
func (w *Worker) createVariant(ctx context.Context, photoIDs []string, role string, data []byte) error {
objID := auth.NewID()
key := objectKey(objID, role, ".webp")
obj := &Object{
ID: objID,
StorageKey: key,
MimeType: "image/webp",
Size: int64(len(data)),
Checksum: checksum(data),
}
if err := w.svc.store.CreateObject(ctx, obj); err != nil {
return err
}
if err := w.svc.storage.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "image/webp"); err != nil {
_ = w.svc.store.DeleteObjectRow(ctx, objID)
return err
}
if err := w.svc.store.ActivateObject(ctx, objID); err != nil {
return err
}
bound := false
for _, pid := range photoIDs {
if err := w.svc.store.BindObject(ctx, pid, objID, role); err != nil {
if errors.Is(err, ErrObjectNotReady) || errors.Is(err, ErrObjectNotFound) {
continue
}
return err
}
bound = true
}
if !bound {
// No photo could take the variant; let the collector reclaim it.
_ = w.svc.store.MarkObjectDeleting(ctx, objID)
}
return nil
}
func (w *Worker) notify(ctx context.Context, photoID string) {
p, err := w.svc.store.GetByID(ctx, photoID, "")
if err != nil {
return
}
if err := w.svc.attachURLs(ctx, p); err != nil {
return
}
w.svc.hub.Publish(p.AlbumID, "photo.created", p)
}
func (w *Worker) fail(ctx context.Context, photoID, msg string) {
slog.Warn("variant job failed", "photo", photoID, "error", msg)
_ = w.svc.store.FailVariantJob(ctx, photoID, msg, maxVariantAttempts)
}
func (w *Worker) collectGarbage(ctx context.Context) {
objs, err := w.svc.store.ClaimGarbage(ctx, w.batch)
if err != nil {
slog.Error("claim garbage", "error", err)
return
}
for _, o := range objs {
w.deleteObject(ctx, o)
}
}
func (w *Worker) reapStalePending(ctx context.Context) {
objs, err := w.svc.store.ClaimStalePending(ctx, w.pendingTTL, w.batch)
if err != nil {
slog.Error("claim stale pending", "error", err)
return
}
for _, o := range objs {
w.deleteObject(ctx, o)
}
}
// deleteObject removes an object's blob then its row. If a binding appeared
// meanwhile the object is returned to ready; if the storage delete fails the
// row stays deleting and is retried on the next pass.
func (w *Worker) deleteObject(ctx context.Context, o Object) {
if n, err := w.svc.store.ObjectRefCount(ctx, o.ID); err == nil && n > 0 {
_ = w.svc.store.SetObjectState(ctx, o.ID, ObjectReady)
return
}
if err := w.svc.storage.Delete(ctx, o.StorageKey); err != nil {
slog.Warn("delete object blob", "object", o.ID, "error", err)
return
}
if err := w.svc.store.DeleteObjectRow(ctx, o.ID); err != nil && !errors.Is(err, ErrObjectNotFound) {
slog.Warn("delete object row", "object", o.ID, "error", err)
}
}
+79
View File
@@ -0,0 +1,79 @@
package photos
import (
"context"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestWorkerGeneratesVariantsAndCollects(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
svc, cl := newTestService(t, pool)
user := createUser(t, ctx, pool)
album := createAlbum(t, ctx, pool, user)
p, err := svc.Upload(ctx, UploadInput{
AlbumID: album,
UserID: user,
Filename: "photo.png",
MimeType: "image/png",
Data: makePNG(t),
})
if err != nil {
t.Fatalf("upload: %v", err)
}
t.Cleanup(func() { cleanupPhoto(t, ctx, pool, cl, p.ID) })
w := NewWorker(svc, time.Second, 10, time.Minute)
w.processVariantJobs(ctx)
refreshed, err := svc.store.GetByID(ctx, p.ID, "")
if err != nil {
t.Fatalf("get refreshed: %v", err)
}
if refreshed.ThumbKey == "" || refreshed.PreviewKey == "" {
t.Fatalf("variants not bound: thumb=%q preview=%q", refreshed.ThumbKey, refreshed.PreviewKey)
}
if err := svc.attachURLs(ctx, refreshed); err != nil {
t.Fatalf("attach urls: %v", err)
}
if refreshed.ThumbURL == "" || refreshed.PreviewURL == "" {
t.Fatalf("variant urls not set")
}
var jobs int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM variant_jobs WHERE photo_id = $1`, p.ID).Scan(&jobs); err != nil {
t.Fatalf("count jobs: %v", err)
}
if jobs != 0 {
t.Fatalf("variant job not completed: %d left", jobs)
}
objectIDs, err := svc.store.PhotoObjectIDs(ctx, p.ID)
if err != nil || len(objectIDs) != 3 {
t.Fatalf("expected 3 bound objects, got %v err=%v", objectIDs, err)
}
if err := svc.PermanentDelete(ctx, p.ID, user); err != nil {
t.Fatalf("permanent delete: %v", err)
}
// Run the collector until the objects are gone (bounded to avoid hangs).
for attempt := 0; attempt < 100 && countObjects(t, ctx, pool, objectIDs) > 0; attempt++ {
w.collectGarbage(ctx)
}
if n := countObjects(t, ctx, pool, objectIDs); n != 0 {
t.Fatalf("%d objects were not collected", n)
}
}
func countObjects(t *testing.T, ctx context.Context, pool *pgxpool.Pool, ids []string) int {
t.Helper()
var n int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM objects WHERE id = ANY($1)`, ids).Scan(&n); err != nil {
t.Fatalf("count objects: %v", err)
}
return n
}
+25 -14
View File
@@ -29,6 +29,7 @@ func testPool(t *testing.T) *pgxpool.Pool {
if err := pool.Ping(context.Background()); err != nil {
t.Skipf("cannot ping: %v", err)
}
t.Cleanup(pool.Close)
return pool
}
@@ -48,7 +49,6 @@ func newUser(t *testing.T, ctx context.Context, pool *pgxpool.Pool, name string)
func TestUserRepository(t *testing.T) {
pool := testPool(t)
defer pool.Close()
ctx := context.Background()
store := auth.NewStore(pool)
@@ -82,7 +82,6 @@ func TestUserRepository(t *testing.T) {
func TestAlbumRepository(t *testing.T) {
pool := testPool(t)
defer pool.Close()
ctx := context.Background()
store := albums.NewStore(pool)
@@ -126,7 +125,6 @@ func TestAlbumRepository(t *testing.T) {
func TestAlbumCoverRepository(t *testing.T) {
pool := testPool(t)
defer pool.Close()
ctx := context.Background()
store := albums.NewStore(pool)
@@ -138,18 +136,33 @@ func TestAlbumCoverRepository(t *testing.T) {
t.Fatalf("create album: %v", err)
}
insertPhoto := func(id, thumbKey string) {
insertPhoto := func(id string) string {
t.Helper()
origObj := auth.NewID()
thumbObj := auth.NewID()
thumbKey := "objects/" + thumbObj + "/thumb.webp"
if _, err := pool.Exec(ctx,
`INSERT INTO photos (id, album_id, uploader_id, storage_key, thumb_key)
VALUES ($1, $2, $3, $4, $5)`,
id, a.ID, owner.ID, "albums/"+a.ID+"/original/"+id+".jpg", thumbKey); err != nil {
`INSERT INTO objects (id, storage_key, mime_type, state) VALUES
($1, $2, 'image/jpeg', 'ready'),
($3, $4, 'image/webp', 'ready')`,
origObj, "objects/"+origObj+"/original.jpg", thumbObj, thumbKey); err != nil {
t.Fatalf("insert objects: %v", err)
}
if _, err := pool.Exec(ctx,
`INSERT INTO photos (id, album_id, uploader_id) VALUES ($1, $2, $3)`,
id, a.ID, owner.ID); err != nil {
t.Fatalf("insert photo: %v", err)
}
if _, err := pool.Exec(ctx,
`INSERT INTO photo_objects (photo_id, object_id, role) VALUES ($1, $2, 'original'), ($1, $3, 'thumb')`,
id, origObj, thumbObj); err != nil {
t.Fatalf("insert bindings: %v", err)
}
return thumbKey
}
photo1 := auth.NewID()
insertPhoto(photo1, "albums/"+a.ID+"/thumb/"+photo1+".webp")
thumb1 := insertPhoto(photo1)
got, err := store.GetByID(ctx, a.ID)
if err != nil || got.CoverThumbKey != "" {
@@ -162,12 +175,12 @@ func TestAlbumCoverRepository(t *testing.T) {
if got.CoverPhotoID == nil || *got.CoverPhotoID != photo1 {
t.Fatalf("cover not set: %+v", got.CoverPhotoID)
}
if want := "albums/" + a.ID + "/thumb/" + photo1 + ".webp"; got.CoverThumbKey != want {
t.Fatalf("cover thumb key = %q want %q", got.CoverThumbKey, want)
if got.CoverThumbKey != thumb1 {
t.Fatalf("cover thumb key = %q want %q", got.CoverThumbKey, thumb1)
}
photo2 := auth.NewID()
insertPhoto(photo2, "thumb-2")
thumb2 := insertPhoto(photo2)
ps.SetCoverIfEmpty(ctx, a.ID, photo2)
got, _ = store.GetByID(ctx, a.ID)
if got.CoverPhotoID == nil || *got.CoverPhotoID != photo1 {
@@ -184,14 +197,13 @@ func TestAlbumCoverRepository(t *testing.T) {
ps.SetCoverIfEmpty(ctx, a.ID, photo2)
list, err := store.ListForUser(ctx, owner.ID)
if err != nil || len(list) != 1 || list[0].CoverThumbKey != "thumb-2" {
if err != nil || len(list) != 1 || list[0].CoverThumbKey != thumb2 {
t.Fatalf("list cover key: %v %+v", err, list)
}
}
func TestMemberRepository(t *testing.T) {
pool := testPool(t)
defer pool.Close()
ctx := context.Background()
owner := newUser(t, ctx, pool, "Owner")
@@ -255,7 +267,6 @@ func TestMemberRepository(t *testing.T) {
func TestShareLinkRepository(t *testing.T) {
pool := testPool(t)
defer pool.Close()
ctx := context.Background()
owner := newUser(t, ctx, pool, "Owner")
-18
View File
@@ -104,24 +104,6 @@ func (c *Client) Delete(ctx context.Context, keys ...string) error {
return nil
}
func (c *Client) DeletePrefix(ctx context.Context, prefix string) error {
var keys []string
paginator := s3.NewListObjectsV2Paginator(c.s3, &s3.ListObjectsV2Input{
Bucket: aws.String(c.bucket),
Prefix: aws.String(prefix),
})
for paginator.HasMorePages() {
page, err := paginator.NextPage(ctx)
if err != nil {
return fmt.Errorf("list objects: %w", err)
}
for _, o := range page.Contents {
keys = append(keys, aws.ToString(o.Key))
}
}
return c.Delete(ctx, keys...)
}
func (c *Client) Get(ctx context.Context, key string) ([]byte, error) {
out, err := c.s3.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(c.bucket),