feat(storage): serve media through backend and add local FS backend
- introduce storage.Backend with S3 and local file-system implementations - deliver all blobs via signed GET/HEAD /api/objects/<key> (Range, Content-Type) - stream S3 objects through the backend; remove presigned URL path - share shared HMAC Signer between backends - remove S3_PUBLIC_URL; rename S3_PRESIGN_TTL to OBJECT_URL_TTL - make storage label in health dynamic; nginx proxy_buffering off for objects
This commit is contained in:
@@ -0,0 +1,174 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newTestFS(t *testing.T) *FS {
|
||||
t.Helper()
|
||||
signer, err := NewSigner("http://app.test", "test-secret", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("signer: %v", err)
|
||||
}
|
||||
fs, err := NewFS(t.TempDir(), signer)
|
||||
if err != nil {
|
||||
t.Fatalf("new fs: %v", err)
|
||||
}
|
||||
return fs
|
||||
}
|
||||
|
||||
func TestFSRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := newTestFS(t)
|
||||
key := "objects/abc/original.jpg"
|
||||
data := []byte("hello blob")
|
||||
|
||||
if err := fs.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "image/jpeg"); err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
got, err := fs.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatalf("get = %q, want %q", got, data)
|
||||
}
|
||||
if err := fs.Delete(ctx, key); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
if _, err := fs.Get(ctx, key); !errors.Is(err, ErrObjectNotFound) {
|
||||
t.Fatalf("get after delete = %v, want ErrObjectNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSPathTraversalRejected(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := newTestFS(t)
|
||||
for _, key := range []string{"../escape", "../../etc/passwd", "/etc/passwd", ""} {
|
||||
if err := fs.Put(ctx, key, strings.NewReader("x"), 1, "text/plain"); err == nil {
|
||||
t.Errorf("put %q: expected rejection", key)
|
||||
}
|
||||
if _, err := fs.Get(ctx, key); err == nil {
|
||||
t.Errorf("get %q: expected rejection", key)
|
||||
}
|
||||
if _, err := fs.URL(ctx, key); err == nil {
|
||||
t.Errorf("url %q: expected rejection", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSURLAndServe(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := newTestFS(t)
|
||||
key := "objects/abc/thumb.webp"
|
||||
if err := fs.Put(ctx, key, strings.NewReader("webp-bytes"), 10, "image/webp"); err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
|
||||
raw, err := fs.URL(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("url: %v", err)
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse url: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(u.Path, "/api/objects/objects/abc/thumb.webp") {
|
||||
t.Fatalf("unexpected path: %s", u.Path)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, raw, nil)
|
||||
if err := fs.ServeObject(rec, req, strings.TrimPrefix(u.Path, "/api/objects/")); err != nil {
|
||||
t.Fatalf("serve: %v", err)
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if ct := rec.Header().Get("Content-Type"); ct != "image/webp" {
|
||||
t.Fatalf("content-type = %q, want image/webp", ct)
|
||||
}
|
||||
if rec.Body.String() != "webp-bytes" {
|
||||
t.Fatalf("body = %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSServeExpiredAndForged(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := newTestFS(t)
|
||||
key := "objects/abc/original.jpg"
|
||||
if err := fs.Put(ctx, key, strings.NewReader("x"), 1, "image/jpeg"); err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
|
||||
past := time.Now().Add(-time.Minute).Unix()
|
||||
expired := "/api/objects/" + key + "?e=" + strconv.FormatInt(past, 10) + "&s=" + fs.signer.sign(key, past)
|
||||
rec := httptest.NewRecorder()
|
||||
err := fs.ServeObject(rec, httptest.NewRequest(http.MethodGet, expired, nil), key)
|
||||
if !errors.Is(err, ErrSignedURLInvalid) {
|
||||
t.Fatalf("expired = %v, want ErrSignedURLInvalid", err)
|
||||
}
|
||||
|
||||
future := time.Now().Add(time.Hour).Unix()
|
||||
forged := "/api/objects/" + key + "?e=" + strconv.FormatInt(future, 10) + "&s=deadbeef"
|
||||
rec = httptest.NewRecorder()
|
||||
err = fs.ServeObject(rec, httptest.NewRequest(http.MethodGet, forged, nil), key)
|
||||
if !errors.Is(err, ErrSignedURLInvalid) {
|
||||
t.Fatalf("forged = %v, want ErrSignedURLInvalid", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSServeRangeAndMissing(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := newTestFS(t)
|
||||
key := "objects/abc/original.jpg"
|
||||
if err := fs.Put(ctx, key, strings.NewReader("0123456789"), 10, "image/jpeg"); err != nil {
|
||||
t.Fatalf("put: %v", err)
|
||||
}
|
||||
exp := time.Now().Add(time.Hour).Unix()
|
||||
base := "/api/objects/" + key + "?e=" + strconv.FormatInt(exp, 10) + "&s=" + fs.signer.sign(key, exp)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, base, nil)
|
||||
req.Header.Set("Range", "bytes=2-5")
|
||||
rec := httptest.NewRecorder()
|
||||
if err := fs.ServeObject(rec, req, key); err != nil {
|
||||
t.Fatalf("serve range: %v", err)
|
||||
}
|
||||
if rec.Code != http.StatusPartialContent {
|
||||
t.Fatalf("status = %d, want 206", rec.Code)
|
||||
}
|
||||
if body, _ := io.ReadAll(rec.Body); string(body) != "2345" {
|
||||
t.Fatalf("range body = %q, want 2345", body)
|
||||
}
|
||||
|
||||
missingKey := "objects/def/original.jpg"
|
||||
missing := "/api/objects/" + missingKey + "?e=" + strconv.FormatInt(exp, 10) + "&s=" + fs.signer.sign(missingKey, exp)
|
||||
rec = httptest.NewRecorder()
|
||||
err := fs.ServeObject(rec, httptest.NewRequest(http.MethodGet, missing, nil), missingKey)
|
||||
if !errors.Is(err, ErrObjectNotFound) {
|
||||
t.Fatalf("missing = %v, want ErrObjectNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentTypeForKey(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
"objects/a/original.jpg": "image/jpeg",
|
||||
"objects/a/thumb.webp": "image/webp",
|
||||
"objects/a/odd.unknownext": "application/octet-stream",
|
||||
}
|
||||
for key, want := range cases {
|
||||
if got := contentTypeForKey(key); got != want {
|
||||
t.Errorf("contentTypeForKey(%q) = %q, want %q", key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user