feat(metadata): SQLite backend, storage reconciliation and local defaults
- pluggable per-domain Repository interfaces; PostgreSQL and SQLite impls - SQLite on modernc.org/sqlite (WAL, busy_timeout, foreign_keys, immediate tx) - separate migrations/sqlite and db.MigrateSQLite; store.NewPostgres/NewSQLite - cmd/reconcile compares object storage with the journal (report, then --delete) - storage.Backend.List and photos.Repository.AllStorageKeys - DB_BACKEND/SQLITE_PATH and STORAGE_BACKEND/FS_ROOT selectable; local defaults (sqlite ./galo.db, fs ./data) - Makefile reconcile/reconcile-delete; docs and .env.example updated
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
assets "shared-album"
|
||||
"shared-album/internal/albums"
|
||||
"shared-album/internal/auth"
|
||||
"shared-album/internal/db"
|
||||
"shared-album/internal/photos"
|
||||
"shared-album/internal/sse"
|
||||
"shared-album/internal/storage"
|
||||
"shared-album/internal/store"
|
||||
)
|
||||
|
||||
func makePNG(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 4, 4))
|
||||
for x := 0; x < 4; x++ {
|
||||
for y := 0; y < 4; y++ {
|
||||
img.Set(x, y, color.RGBA{R: uint8(x * 40), G: uint8(y * 40), B: 128, A: 255})
|
||||
}
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := png.Encode(&buf, img); err != nil {
|
||||
t.Fatalf("encode png: %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type stack struct {
|
||||
db *sql.DB
|
||||
photos *photos.Service
|
||||
albums *albums.Service
|
||||
auth *auth.Service
|
||||
}
|
||||
|
||||
// openStack opens a migrated SQLite database at path and builds the services.
|
||||
func openStack(t *testing.T, path, blobRoot string) (*stack, func()) {
|
||||
t.Helper()
|
||||
migFS, err := fs.Sub(assets.Migrations, "migrations/sqlite")
|
||||
if err != nil {
|
||||
t.Fatalf("migrations fs: %v", err)
|
||||
}
|
||||
if err := db.MigrateSQLite(path, migFS); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
database, err := db.OpenSQLite(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
repos := store.NewSQLite(database)
|
||||
hub := sse.NewHub(1)
|
||||
signer, err := storage.NewSigner("http://app.test", "secret", time.Hour)
|
||||
if err != nil {
|
||||
t.Fatalf("signer: %v", err)
|
||||
}
|
||||
blob, err := storage.NewFS(blobRoot, signer)
|
||||
if err != nil {
|
||||
t.Fatalf("fs: %v", err)
|
||||
}
|
||||
authSvc := auth.NewService(repos.Auth, auth.NewTokenManager("secret", 15*time.Minute, time.Hour))
|
||||
albumsSvc := albums.NewService(repos.Albums, repos.Members, hub, blob)
|
||||
photosSvc := photos.NewService(repos.Photos, albumsSvc, blob, hub)
|
||||
return &stack{db: database, photos: photosSvc, albums: albumsSvc, auth: authSvc}, func() { database.Close() }
|
||||
}
|
||||
|
||||
func TestSQLiteEndToEndWithRestart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "galo.db")
|
||||
blobs := t.TempDir()
|
||||
|
||||
st, closeDB := openStack(t, path, blobs)
|
||||
worker := photos.NewWorker(st.photos, 20*time.Millisecond, 10, time.Minute)
|
||||
wctx, cancel := context.WithCancel(ctx)
|
||||
go worker.Run(wctx)
|
||||
|
||||
user, _, err := st.auth.Register(ctx, "e2e@test.local", "password123", "E2E")
|
||||
if err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
album, err := st.albums.Create(ctx, user.ID, albums.CreateInput{Title: "Trip"})
|
||||
if err != nil {
|
||||
t.Fatalf("create album: %v", err)
|
||||
}
|
||||
p, err := st.photos.Upload(ctx, photos.UploadInput{AlbumID: album.ID, UserID: user.ID, Filename: "a.png", Data: makePNG(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("upload: %v", err)
|
||||
}
|
||||
waitFor(t, "variants of first photo", func() bool {
|
||||
got, err := st.photos.Get(ctx, p.ID, user.ID)
|
||||
return err == nil && got.ThumbURL != "" && got.PreviewURL != ""
|
||||
})
|
||||
|
||||
if err := st.photos.PermanentDelete(ctx, p.ID, user.ID); err != nil {
|
||||
t.Fatalf("permanent delete: %v", err)
|
||||
}
|
||||
waitFor(t, "garbage collection", func() bool {
|
||||
var n int
|
||||
return st.db.QueryRowContext(ctx, `SELECT count(*) FROM objects`).Scan(&n) == nil && n == 0
|
||||
})
|
||||
|
||||
// Stop the worker, then queue another upload whose variant job must survive
|
||||
// the restart and be processed by a fresh worker over the same file.
|
||||
cancel()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
p2, err := st.photos.Upload(ctx, photos.UploadInput{AlbumID: album.ID, UserID: user.ID, Filename: "b.png", Data: makePNG(t)})
|
||||
if err != nil {
|
||||
t.Fatalf("upload p2: %v", err)
|
||||
}
|
||||
got2, _ := st.photos.Get(ctx, p2.ID, user.ID)
|
||||
if got2.PreviewURL != "" {
|
||||
t.Fatalf("variants generated before restart")
|
||||
}
|
||||
closeDB()
|
||||
|
||||
st2, close2 := openStack(t, path, blobs)
|
||||
defer close2()
|
||||
worker2 := photos.NewWorker(st2.photos, 20*time.Millisecond, 10, time.Minute)
|
||||
wctx2, cancel2 := context.WithCancel(ctx)
|
||||
defer cancel2()
|
||||
go worker2.Run(wctx2)
|
||||
|
||||
waitFor(t, "variants of second photo after restart", func() bool {
|
||||
got, err := st2.photos.Get(ctx, p2.ID, user.ID)
|
||||
return err == nil && got.ThumbURL != "" && got.PreviewURL != ""
|
||||
})
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, what string, cond func() bool) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if cond() {
|
||||
return
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("timed out waiting for %s", what)
|
||||
}
|
||||
Reference in New Issue
Block a user