feat(photos): separate object storage from metadata layer

Move physical file facts out of the photos table into an objects ledger:

- objects / photo_objects / variant_jobs tables (migration 000008)
- drop storage_key/thumb_key/preview_key/size/width/height/mime_type from
  photos (migration 000009); they are derived via joins to the original object
- album-free storage keys: objects/{id}/{role}.{ext}
- explicit GC via pending/ready/deleting states; objects deleted only when no
  photo references them
- background worker drives variant generation (durable variant_jobs) and GC
- copy/move are metadata-only; storage objects are shared between photos
- remove unused storage.DeletePrefix

Non-destructive to the external JSON API: size/dimensions/mime and URLs are
still returned, just sourced from the object layer.

OpenSpec change: media-object-layer
This commit is contained in:
iu
2026-10-10 23:31:41 +03:00
parent 9effc0fc82
commit 96268ee16c
30 changed files with 1835 additions and 136 deletions
+34 -47
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"path/filepath"
"strings"
@@ -64,22 +63,20 @@ func (s *Service) Upload(ctx context.Context, in UploadInput) (*models.Photo, er
}
meta := extractEXIF(in.Data)
id := auth.NewID()
ext := strings.ToLower(filepath.Ext(in.Filename))
if ext == "" {
ext = ".bin"
}
storageKey := fmt.Sprintf("albums/%s/original/%s%s", in.AlbumID, id, ext)
if err := s.storage.Put(ctx, storageKey, bytes.NewReader(in.Data), int64(len(in.Data)), mime); err != nil {
return nil, err
}
photoID := auth.NewID()
objectID := auth.NewID()
key := objectKey(objectID, RoleOriginal, ext)
p := &models.Photo{
ID: id,
ID: photoID,
AlbumID: in.AlbumID,
UploaderID: in.UserID,
StorageKey: storageKey,
StorageKey: key,
OriginalFilename: in.Filename,
Size: int64(len(in.Data)),
Width: width,
@@ -90,12 +87,28 @@ func (s *Service) Upload(ctx context.Context, in UploadInput) (*models.Photo, er
Longitude: meta.Longitude,
EXIF: meta.Attributes,
}
if err := s.store.Create(ctx, p); err != nil {
return nil, err
obj := &Object{
ID: objectID,
StorageKey: key,
MimeType: mime,
Size: int64(len(in.Data)),
Width: width,
Height: height,
Checksum: checksum(in.Data),
}
data := in.Data
go s.processVariants(p.ID, in.AlbumID, data)
// The metadata layer is the ledger of record: the row is written first,
// then the bytes are stored, then the object is activated.
if err := s.store.CreateWithOriginal(ctx, p, obj); err != nil {
return nil, err
}
if err := s.storage.Put(ctx, key, bytes.NewReader(in.Data), int64(len(in.Data)), mime); err != nil {
_ = s.store.DeleteUpload(ctx, photoID, objectID)
return nil, err
}
if err := s.store.ActivateObject(ctx, objectID); err != nil {
return nil, err
}
if err := s.attachURLs(ctx, p); err != nil {
return nil, err
@@ -103,33 +116,6 @@ func (s *Service) Upload(ctx context.Context, in UploadInput) (*models.Photo, er
return p, nil
}
func (s *Service) processVariants(photoID, albumID string, data []byte) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
thumb, preview, err := buildVariants(data)
if err != nil {
return
}
thumbKey := fmt.Sprintf("albums/%s/thumb/%s.webp", albumID, photoID)
previewKey := fmt.Sprintf("albums/%s/preview/%s.webp", albumID, photoID)
if err := s.storage.Put(ctx, thumbKey, bytes.NewReader(thumb), int64(len(thumb)), "image/webp"); err != nil {
return
}
if err := s.storage.Put(ctx, previewKey, bytes.NewReader(preview), int64(len(preview)), "image/webp"); err != nil {
return
}
if err := s.store.SetVariants(ctx, photoID, thumbKey, previewKey); err != nil {
return
}
s.store.SetCoverIfEmpty(ctx, albumID, photoID)
if p, err := s.store.GetByID(ctx, photoID, ""); err == nil {
if err := s.attachURLs(ctx, p); err == nil {
s.hub.Publish(albumID, "photo.created", p)
}
}
}
func (s *Service) attachURLs(ctx context.Context, p *models.Photo) error {
if p.StorageKey != "" {
url, err := s.storage.PresignGet(ctx, p.StorageKey)
@@ -416,21 +402,22 @@ func (s *Service) PermanentDelete(ctx context.Context, photoID, userID string) e
if !canDeletePhoto(role, p.UploaderID == userID) {
return httpx.Forbidden("forbidden", "you cannot delete this photo")
}
objectIDs, err := s.store.PhotoObjectIDs(ctx, photoID)
if err != nil {
return err
}
if err := s.store.HardDelete(ctx, photoID); err != nil {
return err
}
if p.AlbumID != "" {
s.publishDeleted(p)
}
if n, err := s.store.StorageKeyRefCount(ctx, p.StorageKey); err == nil && n == 0 {
keys := []string{p.StorageKey}
if p.ThumbKey != "" {
keys = append(keys, p.ThumbKey)
// Mark objects that no longer have any binding for garbage collection.
// The background worker removes the blobs from storage.
for _, oid := range objectIDs {
if n, err := s.store.ObjectRefCount(ctx, oid); err == nil && n == 0 {
_ = s.store.MarkObjectDeleting(ctx, oid)
}
if p.PreviewKey != "" {
keys = append(keys, p.PreviewKey)
}
_ = s.storage.Delete(ctx, keys...)
}
return nil
}