feat(storage): serve media through backend and add local FS backend

- introduce storage.Backend with S3 and local file-system implementations
- deliver all blobs via signed GET/HEAD /api/objects/<key> (Range, Content-Type)
- stream S3 objects through the backend; remove presigned URL path
- share shared HMAC Signer between backends
- remove S3_PUBLIC_URL; rename S3_PRESIGN_TTL to OBJECT_URL_TTL
- make storage label in health dynamic; nginx proxy_buffering off for objects
This commit is contained in:
iu
2026-10-11 09:15:03 +03:00
parent 7ae9bcf305
commit ec8d907565
28 changed files with 1714 additions and 65 deletions
+29 -6
View File
@@ -2,6 +2,7 @@ package app
import (
"context"
"errors"
"io/fs"
"net/http"
"strings"
@@ -27,7 +28,7 @@ import (
type Server struct {
cfg *config.Config
pool *pgxpool.Pool
storage *storage.Client
storage storage.Backend
hub *sse.Hub
albumsStore *albums.Store
@@ -47,7 +48,7 @@ type Server struct {
share *sharelinks.Service
}
func New(cfg *config.Config, pool *pgxpool.Pool, st *storage.Client, hub *sse.Hub) *Server {
func New(cfg *config.Config, pool *pgxpool.Pool, st storage.Backend, hub *sse.Hub) *Server {
authStore := auth.NewStore(pool)
tm := auth.NewTokenManager(cfg.Auth.JWTSecret, cfg.Auth.AccessTTL, cfg.Auth.RefreshTTL)
authSvc := auth.NewService(authStore, tm)
@@ -109,6 +110,8 @@ func (s *Server) Router() *gin.Engine {
api := r.Group("/api")
api.GET("/health", s.health)
api.GET("/objects/*key", s.serveObject)
api.HEAD("/objects/*key", s.serveObject)
api.POST("/auth/register", authHandler.Register)
api.POST("/auth/login", authHandler.Login)
@@ -183,17 +186,37 @@ func (s *Server) health(c *gin.Context) {
if err := s.pool.Ping(ctx); err != nil {
dbStatus = "error"
}
s3Status := "ok"
storageLabel := s.cfg.Storage.Backend
if storageLabel == "" {
storageLabel = "s3"
}
storeStatus := "ok"
if err := s.storage.Ping(ctx); err != nil {
s3Status = "error"
storeStatus = "error"
}
status := http.StatusOK
overall := "ok"
if dbStatus != "ok" || s3Status != "ok" {
if dbStatus != "ok" || storeStatus != "ok" {
status = http.StatusServiceUnavailable
overall = "degraded"
}
c.JSON(status, gin.H{"status": overall, "db": dbStatus, "s3": s3Status})
c.JSON(status, gin.H{"status": overall, "db": dbStatus, storageLabel: storeStatus})
}
func (s *Server) serveObject(c *gin.Context) {
key := strings.TrimPrefix(c.Param("key"), "/")
switch err := s.storage.ServeObject(c.Writer, c.Request, key); {
case err == nil:
return
case errors.Is(err, storage.ErrSignedURLInvalid):
c.Status(http.StatusForbidden)
case errors.Is(err, storage.ErrObjectNotFound):
httpx.RespondError(c, httpx.NotFound("object_not_found", "object not found"))
case errors.Is(err, storage.ErrRangeNotSatisfiable):
c.Status(http.StatusRequestedRangeNotSatisfiable)
default:
c.Status(http.StatusInternalServerError)
}
}
func (s *Server) albumSSE(c *gin.Context) {
+71
View File
@@ -0,0 +1,71 @@
package app
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/jackc/pgx/v5/pgxpool"
"shared-album/internal/config"
"shared-album/internal/storage"
)
func mustSigner(t *testing.T) *storage.Signer {
t.Helper()
signer, err := storage.NewSigner("http://app.test", "secret", time.Hour)
if err != nil {
t.Fatalf("signer: %v", err)
}
return signer
}
func lazyPool(t *testing.T) *pgxpool.Pool {
t.Helper()
pool, err := pgxpool.New(context.Background(), "postgres://u:p@127.0.0.1:1/x?sslmode=disable")
if err != nil {
t.Fatalf("pool: %v", err)
}
t.Cleanup(pool.Close)
return pool
}
func TestHealthStorageLabel(t *testing.T) {
gin.SetMode(gin.TestMode)
fs, err := storage.NewFS(t.TempDir(), mustSigner(t))
if err != nil {
t.Fatalf("fs: %v", err)
}
srv := &Server{cfg: &config.Config{}, pool: lazyPool(t), storage: fs}
call := func() map[string]any {
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
c.Request = httptest.NewRequest(http.MethodGet, "/api/health", nil)
srv.health(c)
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("decode: %v", err)
}
return body
}
srv.cfg.Storage.Backend = "fs"
if body := call(); body["fs"] != "ok" {
t.Fatalf("expected fs label ok, got %v", body)
}
srv.cfg.Storage.Backend = "s3"
if body := call(); body["s3"] != "ok" {
t.Fatalf("expected s3 label ok, got %v", body)
}
srv.cfg.Storage.Backend = ""
if body := call(); body["s3"] != "ok" {
t.Fatalf("expected default s3 label, got %v", body)
}
}
+110
View File
@@ -0,0 +1,110 @@
package app
import (
"context"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"shared-album/internal/config"
"shared-album/internal/sse"
"shared-album/internal/storage"
)
func newTestServerWithFS(t *testing.T) (*storage.FS, *gin.Engine) {
t.Helper()
gin.SetMode(gin.TestMode)
signer, err := storage.NewSigner("http://app.test", "secret", time.Hour)
if err != nil {
t.Fatalf("signer: %v", err)
}
fs, err := storage.NewFS(t.TempDir(), signer)
if err != nil {
t.Fatalf("new fs: %v", err)
}
srv := New(&config.Config{}, nil, fs, sse.NewHub(1))
return fs, srv.Router()
}
func TestServeObjectRoute(t *testing.T) {
ctx := context.Background()
fs, r := newTestServerWithFS(t)
key := "objects/abc/original.jpg"
if err := fs.Put(ctx, key, strings.NewReader("0123456789"), 10, "image/jpeg"); err != nil {
t.Fatalf("put: %v", err)
}
raw, err := fs.URL(ctx, key)
if err != nil {
t.Fatalf("url: %v", err)
}
u, err := url.Parse(raw)
if err != nil {
t.Fatalf("parse: %v", err)
}
do := func(target string, rangeHeader string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodGet, target, nil)
if rangeHeader != "" {
req.Header.Set("Range", rangeHeader)
}
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
return rec
}
t.Run("valid", func(t *testing.T) {
rec := do(u.Path+"?"+u.RawQuery, "")
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", rec.Code)
}
if rec.Body.String() != "0123456789" {
t.Fatalf("body = %q", rec.Body.String())
}
})
t.Run("range", func(t *testing.T) {
rec := do(u.Path+"?"+u.RawQuery, "bytes=2-5")
if rec.Code != http.StatusPartialContent {
t.Fatalf("status = %d, want 206", rec.Code)
}
if rec.Body.String() != "2345" {
t.Fatalf("body = %q, want 2345", rec.Body.String())
}
})
t.Run("expired", func(t *testing.T) {
q := u.Query()
q.Set("e", strconv.FormatInt(time.Now().Add(-time.Minute).Unix(), 10))
rec := do(u.Path+"?"+q.Encode(), "")
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403", rec.Code)
}
})
t.Run("forged", func(t *testing.T) {
q := u.Query()
q.Set("s", "deadbeef")
rec := do(u.Path+"?"+q.Encode(), "")
if rec.Code != http.StatusForbidden {
t.Fatalf("status = %d, want 403", rec.Code)
}
})
t.Run("missing", func(t *testing.T) {
raw, err := fs.URL(ctx, "objects/def/original.jpg")
if err != nil {
t.Fatalf("url: %v", err)
}
mu, _ := url.Parse(raw)
rec := do(mu.Path+"?"+mu.RawQuery, "")
if rec.Code != http.StatusNotFound {
t.Fatalf("status = %d, want 404", rec.Code)
}
})
}
+211
View File
@@ -0,0 +1,211 @@
package app
import (
"bytes"
"context"
"encoding/json"
"image"
"image/color"
"image/png"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/jackc/pgx/v5/pgxpool"
"shared-album/internal/albums"
"shared-album/internal/auth"
"shared-album/internal/config"
"shared-album/internal/members"
"shared-album/internal/models"
"shared-album/internal/photos"
"shared-album/internal/sharelinks"
"shared-album/internal/sse"
"shared-album/internal/storage"
)
func integrationPool(t *testing.T) *pgxpool.Pool {
t.Helper()
dsn := os.Getenv("TEST_DATABASE_URL")
if dsn == "" {
t.Skip("TEST_DATABASE_URL not set")
}
pool, err := pgxpool.New(context.Background(), dsn)
if err != nil {
t.Skipf("cannot connect: %v", err)
}
if err := pool.Ping(context.Background()); err != nil {
t.Skipf("cannot ping: %v", err)
}
t.Cleanup(pool.Close)
return pool
}
func makeTestPNG(t *testing.T) []byte {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, 4, 4))
for x := 0; x < 4; x++ {
for y := 0; y < 4; y++ {
img.Set(x, y, color.RGBA{R: uint8(x * 40), G: uint8(y * 40), B: 128, A: 255})
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("encode png: %v", err)
}
return buf.Bytes()
}
// runShareFlow uploads a photo and validates the read-only share flow returns a
// usable photo URL. verify inspects/serves the URL according to the backend.
func runShareFlow(t *testing.T, backend storage.Backend, verify func(t *testing.T, router *gin.Engine, raw string)) {
t.Helper()
ctx := context.Background()
pool := integrationPool(t)
gin.SetMode(gin.TestMode)
hub := sse.NewHub(1)
srv := New(&config.Config{}, pool, backend, hub)
router := srv.Router()
authStore := auth.NewStore(pool)
owner := &models.User{ID: auth.NewID(), Email: auth.NewID()[:8] + "@test.local", PasswordHash: "x", Name: "Owner"}
if err := authStore.CreateUser(ctx, owner); err != nil {
t.Fatalf("create user: %v", err)
}
albumsStore := albums.NewStore(pool)
albumsSvc := albums.NewService(albumsStore, members.NewStore(pool), hub, backend)
album, err := albumsSvc.Create(ctx, owner.ID, albums.CreateInput{Title: "Share Album"})
if err != nil {
t.Fatalf("create album: %v", err)
}
photosSvc := photos.NewService(photos.NewStore(pool), albumsSvc, backend, hub)
pngData := makeTestPNG(t)
if _, err := photosSvc.Upload(ctx, photos.UploadInput{
AlbumID: album.ID, UserID: owner.ID, Filename: "pic.png", Data: pngData,
}); err != nil {
t.Fatalf("upload: %v", err)
}
var keys []string
rows, err := pool.Query(ctx,
`SELECT o.storage_key FROM photo_objects po
JOIN objects o ON o.id = po.object_id
JOIN photos p ON p.id = po.photo_id
WHERE p.album_id = $1`, album.ID)
if err == nil {
for rows.Next() {
var k string
if rows.Scan(&k) == nil {
keys = append(keys, k)
}
}
rows.Close()
}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM photos WHERE album_id = $1`, album.ID)
for _, k := range keys {
_, _ = pool.Exec(ctx, `DELETE FROM objects WHERE storage_key = $1`, k)
}
_ = backend.Delete(context.Background(), keys...)
_, _ = pool.Exec(ctx, `DELETE FROM albums WHERE id = $1`, album.ID)
_, _ = pool.Exec(ctx, `DELETE FROM users WHERE id = $1`, owner.ID)
})
shareSvc := sharelinks.NewService(sharelinks.NewStore(pool), albumsSvc)
link, err := shareSvc.Create(ctx, album.ID, owner.ID, time.Hour)
if err != nil {
t.Fatalf("share create: %v", err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/share/"+link.Token+"/photos", nil))
if rec.Code != http.StatusOK {
t.Fatalf("share photos status = %d body=%s", rec.Code, rec.Body.String())
}
var body struct {
Photos []models.Photo `json:"photos"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
t.Fatalf("decode: %v", err)
}
if len(body.Photos) != 1 {
t.Fatalf("photos len = %d, want 1", len(body.Photos))
}
if body.Photos[0].OriginalURL == "" {
t.Fatalf("empty original url")
}
verify(t, router, body.Photos[0].OriginalURL)
}
func TestShareFlowWithFS(t *testing.T) {
signer, err := storage.NewSigner("http://app.test", "secret", time.Hour)
if err != nil {
t.Fatalf("signer: %v", err)
}
fs, err := storage.NewFS(t.TempDir(), signer)
if err != nil {
t.Fatalf("new fs: %v", err)
}
runShareFlow(t, fs, func(t *testing.T, router *gin.Engine, raw string) {
if !strings.Contains(raw, "/api/objects/") {
t.Fatalf("expected fs signed url, got %q", raw)
}
u, err := url.Parse(raw)
if err != nil {
t.Fatalf("parse url: %v", err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, u.Path+"?"+u.RawQuery, nil))
if rec.Code != http.StatusOK {
t.Fatalf("fs object fetch status = %d", rec.Code)
}
})
}
func TestShareFlowWithS3(t *testing.T) {
endpoint := os.Getenv("S3_ENDPOINT")
if endpoint == "" {
t.Skip("S3_ENDPOINT not set")
}
signer, err := storage.NewSigner("http://app.test", "secret", time.Hour)
if err != nil {
t.Skipf("signer init: %v", err)
}
cl, err := storage.NewS3(context.Background(), config.S3{
Endpoint: endpoint,
Region: os.Getenv("S3_REGION"),
AccessKey: os.Getenv("S3_ACCESS_KEY"),
SecretKey: os.Getenv("S3_SECRET_KEY"),
Bucket: os.Getenv("S3_BUCKET"),
UsePathStyle: true,
}, signer)
if err != nil {
t.Skipf("storage init: %v", err)
}
runShareFlow(t, cl, func(t *testing.T, router *gin.Engine, raw string) {
if !strings.Contains(raw, "/api/objects/") {
t.Fatalf("expected backend url, got %q", raw)
}
if strings.Contains(raw, "X-Amz-Signature") {
t.Fatalf("presigned url leaked to client: %q", raw)
}
u, err := url.Parse(raw)
if err != nil {
t.Fatalf("parse url: %v", err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, u.Path+"?"+u.RawQuery, nil))
if rec.Code != http.StatusOK {
t.Fatalf("s3 object fetch status = %d", rec.Code)
}
})
}