feat(storage): serve media through backend and add local FS backend

- introduce storage.Backend with S3 and local file-system implementations
- deliver all blobs via signed GET/HEAD /api/objects/<key> (Range, Content-Type)
- stream S3 objects through the backend; remove presigned URL path
- share shared HMAC Signer between backends
- remove S3_PUBLIC_URL; rename S3_PRESIGN_TTL to OBJECT_URL_TTL
- make storage label in health dynamic; nginx proxy_buffering off for objects
This commit is contained in:
iu
2026-10-11 09:15:03 +03:00
parent 7ae9bcf305
commit ec8d907565
28 changed files with 1714 additions and 65 deletions
+26 -6
View File
@@ -28,8 +28,6 @@ type S3 struct {
SecretKey string
Bucket string
UsePathStyle bool
PublicURL string
PresignTTL time.Duration
}
type Auth struct {
@@ -54,9 +52,17 @@ type Worker struct {
PendingTTL time.Duration
}
type Storage struct {
Backend string
FSRoot string
URLSecret string
URLTTL time.Duration
}
type Config struct {
Postgres Postgres
S3 S3
Storage Storage
Auth Auth
Server Server
SSE SSE
@@ -79,9 +85,9 @@ func Load() (*Config, error) {
return nil, fmt.Errorf("invalid JWT_REFRESH_TTL: %w", err)
}
presignTTL, err := time.ParseDuration(getEnv("S3_PRESIGN_TTL", "1h"))
urlTTL, err := time.ParseDuration(getEnv("OBJECT_URL_TTL", "1h"))
if err != nil {
return nil, fmt.Errorf("invalid S3_PRESIGN_TTL: %w", err)
return nil, fmt.Errorf("invalid OBJECT_URL_TTL: %w", err)
}
heartbeat, err := time.ParseDuration(getEnv("SSE_HEARTBEAT_INTERVAL", "25s"))
@@ -130,8 +136,12 @@ func Load() (*Config, error) {
SecretKey: getEnv("S3_SECRET_KEY", ""),
Bucket: getEnv("S3_BUCKET", "albums"),
UsePathStyle: pathStyle,
PublicURL: getEnv("S3_PUBLIC_URL", ""),
PresignTTL: presignTTL,
},
Storage: Storage{
Backend: getEnv("STORAGE_BACKEND", "s3"),
FSRoot: getEnv("FS_ROOT", ""),
URLSecret: getEnv("URL_SECRET", ""),
URLTTL: urlTTL,
},
Auth: Auth{
JWTSecret: getEnv("JWT_SECRET", ""),
@@ -160,6 +170,16 @@ func Load() (*Config, error) {
return nil, fmt.Errorf("PG_PASSWORD is required")
}
switch cfg.Storage.Backend {
case "s3":
case "fs":
if cfg.Storage.FSRoot == "" {
return nil, fmt.Errorf("FS_ROOT is required when STORAGE_BACKEND=fs")
}
default:
return nil, fmt.Errorf("invalid STORAGE_BACKEND %q (want s3 or fs)", cfg.Storage.Backend)
}
return cfg, nil
}
+70
View File
@@ -0,0 +1,70 @@
package config
import (
"testing"
"time"
)
func setRequiredEnv(t *testing.T) {
t.Helper()
t.Setenv("JWT_SECRET", "test-secret")
t.Setenv("PG_PASSWORD", "test-pass")
}
func TestLoadStorageBackend(t *testing.T) {
t.Run("defaults to s3", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("STORAGE_BACKEND", "")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.Storage.Backend != "s3" {
t.Fatalf("backend = %q, want s3", cfg.Storage.Backend)
}
})
t.Run("fs with root", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("STORAGE_BACKEND", "fs")
t.Setenv("FS_ROOT", "/var/lib/galo/objects")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.Storage.Backend != "fs" || cfg.Storage.FSRoot != "/var/lib/galo/objects" {
t.Fatalf("storage = %+v", cfg.Storage)
}
})
t.Run("fs without root", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("STORAGE_BACKEND", "fs")
t.Setenv("FS_ROOT", "")
t.Setenv("URL_SECRET", "")
if _, err := Load(); err == nil {
t.Fatal("expected error for fs without FS_ROOT")
}
})
t.Run("invalid backend", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("STORAGE_BACKEND", "gcs")
if _, err := Load(); err == nil {
t.Fatal("expected error for invalid backend")
}
})
t.Run("object url ttl", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("STORAGE_BACKEND", "")
t.Setenv("OBJECT_URL_TTL", "30m")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.Storage.URLTTL != 30*time.Minute {
t.Fatalf("URLTTL = %v, want 30m", cfg.Storage.URLTTL)
}
})
}