feat(observability): configurable logging, request ids and error logs

- internal/logging: LOG_LEVEL/LOG_FORMAT parsing and slog handler
- request_id middleware (X-Request-ID or generated, echoed) + debug access log
- httpx registers errors; middleware logs internal causes with request_id
- health excluded, objects/sse debug-only, no query/body/secrets in logs
- new observability capability spec; README/.env.example docs
This commit is contained in:
iu
2026-10-11 13:40:11 +03:00
parent a5d0cee0eb
commit 6361e48b20
18 changed files with 998 additions and 5 deletions
+14 -4
View File
@@ -18,6 +18,7 @@ import (
"galo/internal/auth"
"galo/internal/config"
"galo/internal/db"
"galo/internal/logging"
"galo/internal/settings"
"galo/internal/sse"
"galo/internal/storage"
@@ -31,16 +32,25 @@ func main() {
loadDotEnv(".env")
loadDotEnv("../.env")
logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}))
slog.SetDefault(logger)
slog.Info("starting galo", "version", version)
cfg, err := config.Load()
if err != nil {
slog.Error("config error", "error", err)
os.Exit(1)
}
level, err := logging.ParseLevel(cfg.Log.Level)
if err != nil {
slog.Error("invalid log level", "error", err)
os.Exit(1)
}
handler, err := logging.NewHandler(cfg.Log.Format, level)
if err != nil {
slog.Error("invalid log format", "error", err)
os.Exit(1)
}
slog.SetDefault(slog.New(handler))
slog.Info("starting galo", "version", version)
switch cfg.Database.Backend {
case "sqlite":
slog.Info("metadata backend", "backend", "sqlite", "path", cfg.Database.SQLitePath)
+1 -1
View File
@@ -91,7 +91,7 @@ func (s *Server) StartWorker(ctx context.Context) {
func (s *Server) Router() *gin.Engine {
r := gin.New()
r.Use(gin.Recovery())
r.Use(gin.Recovery(), requestIDMiddleware(), accessLogMiddleware())
authHandler := auth.NewHandler(s.auth)
albumsHandler := albums.NewHandler(s.albums)
+77
View File
@@ -0,0 +1,77 @@
package app
import (
"errors"
"log/slog"
"time"
"github.com/gin-gonic/gin"
"galo/internal/auth"
"galo/internal/httpx"
"galo/internal/logging"
)
const ctxRequestID = "logging.request_id"
// requestIDMiddleware assigns a request identifier, echoes it in the response
// header and stores it in the request context for downstream logging.
func requestIDMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
id := c.GetHeader("X-Request-ID")
if !logging.ValidRequestID(id) {
id = logging.NewRequestID()
}
c.Set(ctxRequestID, id)
c.Header("X-Request-ID", id)
c.Request = c.Request.WithContext(logging.WithRequestID(c.Request.Context(), id))
c.Next()
}
}
// accessLogMiddleware writes one debug-level line per completed request and
// logs collected errors. The query string, Authorization header and bodies are
// never logged.
func accessLogMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
start := time.Now()
c.Next()
if route := c.FullPath(); shouldLogAccess(route) {
logRequest(c, start, route)
}
logErrors(c)
}
}
func shouldLogAccess(route string) bool {
return route != "" && route != "/api/health"
}
func logRequest(c *gin.Context, start time.Time, route string) {
slog.Default().LogAttrs(c.Request.Context(), slog.LevelDebug, "http request",
slog.String("method", c.Request.Method),
slog.String("route", route),
slog.Int("status", c.Writer.Status()),
slog.Duration("latency", time.Since(start)),
slog.Int("bytes", c.Writer.Size()),
slog.String("client_ip", c.ClientIP()),
slog.String("user_id", auth.UserID(c)),
slog.String("request_id", logging.RequestID(c.Request.Context())),
)
}
func logErrors(c *gin.Context) {
for _, ginErr := range c.Errors {
level := slog.LevelError
var appErr *httpx.AppError
if errors.As(ginErr.Err, &appErr) && appErr.Status < 500 {
level = slog.LevelDebug
}
slog.Default().LogAttrs(c.Request.Context(), level, "request error",
slog.String("error", ginErr.Err.Error()),
slog.String("route", c.FullPath()),
slog.String("request_id", logging.RequestID(c.Request.Context())),
)
}
}
+183
View File
@@ -0,0 +1,183 @@
package app
import (
"bytes"
"errors"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
"galo/internal/httpx"
)
func captureLog(t *testing.T, level slog.Level) *bytes.Buffer {
t.Helper()
buf := &bytes.Buffer{}
prev := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(buf, &slog.HandlerOptions{Level: level})))
t.Cleanup(func() { slog.SetDefault(prev) })
return buf
}
func observabilityEngine() *gin.Engine {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(requestIDMiddleware(), accessLogMiddleware())
ok := func(c *gin.Context) { c.Status(http.StatusOK) }
r.GET("/api/albums", ok)
r.GET("/api/health", ok)
r.GET("/api/objects/*key", ok)
r.GET("/api/sse/albums/:id", ok)
r.GET("/browse/:id", ok)
return r
}
func TestRequestID(t *testing.T) {
r := observabilityEngine()
t.Run("provided id is reused and echoed", func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/api/albums", nil)
req.Header.Set("X-Request-ID", "req-abc-123")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if got := w.Header().Get("X-Request-ID"); got != "req-abc-123" {
t.Fatalf("X-Request-ID = %q, want req-abc-123", got)
}
})
t.Run("generated when absent", func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/api/albums", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if got := w.Header().Get("X-Request-ID"); got == "" {
t.Fatal("X-Request-ID is empty, want generated value")
}
})
t.Run("invalid id replaced", func(t *testing.T) {
req := httptest.NewRequest(http.MethodGet, "/api/albums", nil)
req.Header.Set("X-Request-ID", "bad id\nwith spaces")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if got := w.Header().Get("X-Request-ID"); got == "bad id\nwith spaces" || got == "" {
t.Fatalf("X-Request-ID = %q, want a freshly generated value", got)
}
})
}
func TestAccessLog(t *testing.T) {
t.Run("written at debug, without query", func(t *testing.T) {
buf := captureLog(t, slog.LevelDebug)
req := httptest.NewRequest(http.MethodGet, "/api/albums?token=SUPERSECRET", nil)
w := httptest.NewRecorder()
observabilityEngine().ServeHTTP(w, req)
out := buf.String()
for _, want := range []string{"http request", "route=/api/albums", "request_id="} {
if !strings.Contains(out, want) {
t.Fatalf("log missing %q:\n%s", want, out)
}
}
if strings.Contains(out, "SUPERSECRET") || strings.Contains(out, "token=") {
t.Fatalf("query string leaked into log:\n%s", out)
}
})
t.Run("silent at info", func(t *testing.T) {
buf := captureLog(t, slog.LevelInfo)
req := httptest.NewRequest(http.MethodGet, "/api/albums", nil)
w := httptest.NewRecorder()
observabilityEngine().ServeHTTP(w, req)
if strings.Contains(buf.String(), "http request") {
t.Fatalf("access line written at info:\n%s", buf.String())
}
})
}
func errorEngine() *gin.Engine {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(requestIDMiddleware(), accessLogMiddleware())
r.GET("/api/boom", func(c *gin.Context) { httpx.RespondError(c, errors.New("boom")) })
r.GET("/api/bad", func(c *gin.Context) { httpx.RespondError(c, httpx.BadRequest("bad", "bad request")) })
r.GET("/api/fail", func(c *gin.Context) { httpx.RespondError(c, httpx.Internal("internal exploded")) })
return r
}
func do(r *gin.Engine, target string, body string) {
req := httptest.NewRequest(http.MethodGet, target, strings.NewReader(body))
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
}
func TestErrorLogging(t *testing.T) {
t.Run("internal error at error level with cause", func(t *testing.T) {
buf := captureLog(t, slog.LevelDebug)
do(errorEngine(), "/api/boom", "")
out := buf.String()
if !strings.Contains(out, "level=ERROR") || !strings.Contains(out, "request error") ||
!strings.Contains(out, "error=boom") || !strings.Contains(out, "request_id=") {
t.Fatalf("internal error log wrong:\n%s", out)
}
})
t.Run("expected 4xx at debug", func(t *testing.T) {
buf := captureLog(t, slog.LevelDebug)
do(errorEngine(), "/api/bad", "")
out := buf.String()
if !strings.Contains(out, "level=DEBUG") || !strings.Contains(out, "request error") {
t.Fatalf("4xx should log at debug:\n%s", out)
}
})
t.Run("app error 5xx at error", func(t *testing.T) {
buf := captureLog(t, slog.LevelDebug)
do(errorEngine(), "/api/fail", "")
out := buf.String()
if !strings.Contains(out, "level=ERROR") || !strings.Contains(out, "internal exploded") {
t.Fatalf("5xx app error should log at error:\n%s", out)
}
})
t.Run("4xx hidden at info", func(t *testing.T) {
buf := captureLog(t, slog.LevelInfo)
do(errorEngine(), "/api/bad", "")
if strings.Contains(buf.String(), "request error") {
t.Fatalf("expected 4xx must be silent at info:\n%s", buf.String())
}
})
}
func TestLogPrivacy(t *testing.T) {
buf := captureLog(t, slog.LevelDebug)
do(errorEngine(), "/api/boom?token=SUPERSECRETTOKEN", "TOPSECRETBODY")
out := buf.String()
if strings.Contains(out, "SUPERSECRETTOKEN") || strings.Contains(out, "token=") {
t.Fatalf("query leaked into log:\n%s", out)
}
if strings.Contains(out, "TOPSECRETBODY") {
t.Fatalf("body leaked into log:\n%s", out)
}
}
func TestAccessLogFiltering(t *testing.T) {
buf := captureLog(t, slog.LevelDebug)
r := observabilityEngine()
for _, target := range []string{"/api/health", "/api/objects/abc/thumb.webp", "/api/sse/albums/a1", "/browse/42"} {
w := httptest.NewRecorder()
r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, target, nil))
}
out := buf.String()
if strings.Contains(out, "route=/api/health") {
t.Fatalf("/api/health must never be access-logged:\n%s", out)
}
for _, want := range []string{"route=/api/objects/*key", "route=/api/sse/albums/:id", "route=/browse/:id"} {
if !strings.Contains(out, want) {
t.Fatalf("expected %q logged at debug:\n%s", want, out)
}
}
}
+21
View File
@@ -5,6 +5,8 @@ import (
"os"
"strconv"
"time"
"galo/internal/logging"
)
type Postgres struct {
@@ -63,6 +65,11 @@ type Database struct {
SQLitePath string
}
type Log struct {
Level string
Format string
}
type Config struct {
Postgres Postgres
S3 S3
@@ -72,6 +79,7 @@ type Config struct {
Server Server
SSE SSE
Worker Worker
Log Log
}
func Load() (*Config, error) {
@@ -125,6 +133,15 @@ func Load() (*Config, error) {
return nil, fmt.Errorf("invalid WORKER_PENDING_TTL: %w", err)
}
logLevel := getEnv("LOG_LEVEL", "info")
if _, err := logging.ParseLevel(logLevel); err != nil {
return nil, err
}
logFormat := getEnv("LOG_FORMAT", "json")
if _, err := logging.ParseFormat(logFormat); err != nil {
return nil, err
}
cfg := &Config{
Postgres: Postgres{
Host: getEnv("PG_HOST", "pg.localnet"),
@@ -169,6 +186,10 @@ func Load() (*Config, error) {
Batch: workerBatch,
PendingTTL: workerPendingTTL,
},
Log: Log{
Level: logLevel,
Format: logFormat,
},
}
switch cfg.Database.Backend {
+44
View File
@@ -132,3 +132,47 @@ func TestLoadDatabaseBackend(t *testing.T) {
}
})
}
func TestLoadLog(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("LOG_LEVEL", "")
t.Setenv("LOG_FORMAT", "")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.Log.Level != "info" || cfg.Log.Format != "json" {
t.Fatalf("log = %+v, want info/json", cfg.Log)
}
})
t.Run("custom values", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("LOG_LEVEL", "debug")
t.Setenv("LOG_FORMAT", "text")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if cfg.Log.Level != "debug" || cfg.Log.Format != "text" {
t.Fatalf("log = %+v, want debug/text", cfg.Log)
}
})
t.Run("invalid level", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("LOG_LEVEL", "trace")
if _, err := Load(); err == nil {
t.Fatal("expected error for invalid LOG_LEVEL")
}
})
t.Run("invalid format", func(t *testing.T) {
setRequiredEnv(t)
t.Setenv("LOG_FORMAT", "xml")
if _, err := Load(); err == nil {
t.Fatal("expected error for invalid LOG_FORMAT")
}
})
}
+2
View File
@@ -53,6 +53,7 @@ func Internal(message string) *AppError {
}
func Abort(c *gin.Context, err error) {
_ = c.Error(err)
var appErr *AppError
if errors.As(err, &appErr) {
c.AbortWithStatusJSON(appErr.Status, ErrorResponse{ErrorBody{appErr.Code, appErr.Message}})
@@ -62,6 +63,7 @@ func Abort(c *gin.Context, err error) {
}
func RespondError(c *gin.Context, err error) {
_ = c.Error(err)
var appErr *AppError
if errors.As(err, &appErr) {
c.JSON(appErr.Status, ErrorResponse{ErrorBody{appErr.Code, appErr.Message}})
+68
View File
@@ -0,0 +1,68 @@
package httpx
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
)
func runHandler(t *testing.T, handler gin.HandlerFunc) (*gin.Context, *httptest.ResponseRecorder) {
t.Helper()
gin.SetMode(gin.TestMode)
w := httptest.NewRecorder()
c, _ := gin.CreateTestContext(w)
c.Request = httptest.NewRequest(http.MethodGet, "/", nil)
handler(c)
return c, w
}
func TestRespondErrorRegistersInternalError(t *testing.T) {
c, w := runHandler(t, func(c *gin.Context) {
RespondError(c, errors.New("boom"))
})
if len(c.Errors) != 1 {
t.Fatalf("c.Errors = %d, want 1", len(c.Errors))
}
if w.Code != http.StatusInternalServerError {
t.Fatalf("status = %d, want 500", w.Code)
}
if body := w.Body.String(); body != `{"error":{"code":"internal_error","message":"internal server error"}}` {
t.Fatalf("body = %s", body)
}
}
func TestRespondErrorRegistersAppError(t *testing.T) {
c, w := runHandler(t, func(c *gin.Context) {
RespondError(c, NotFound("photo_not_found", "photo not found"))
})
if len(c.Errors) != 1 {
t.Fatalf("c.Errors = %d, want 1", len(c.Errors))
}
if w.Code != http.StatusNotFound {
t.Fatalf("status = %d, want 404", w.Code)
}
}
func TestAbortRegistersInternalError(t *testing.T) {
c, w := runHandler(t, func(c *gin.Context) {
Abort(c, errors.New("kaboom"))
})
if len(c.Errors) != 1 {
t.Fatalf("c.Errors = %d, want 1", len(c.Errors))
}
if w.Code != http.StatusInternalServerError {
t.Fatalf("status = %d, want 500", w.Code)
}
}
// Ensure the package-level recovery of AppError still works for logging levels.
func TestInternalIsAppError500(t *testing.T) {
err := Internal("oops")
var appErr *AppError
if !errors.As(err, &appErr) || appErr.Status != http.StatusInternalServerError {
t.Fatalf("Internal() = %#v", err)
}
}
+95
View File
@@ -0,0 +1,95 @@
// Package logging configures the process-wide structured logger from
// environment-driven settings.
package logging
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"log/slog"
"os"
"strings"
)
// ParseLevel maps a textual level name to a slog.Level.
func ParseLevel(name string) (slog.Level, error) {
switch strings.ToLower(strings.TrimSpace(name)) {
case "debug":
return slog.LevelDebug, nil
case "info":
return slog.LevelInfo, nil
case "warn", "warning":
return slog.LevelWarn, nil
case "error":
return slog.LevelError, nil
default:
return 0, fmt.Errorf("invalid log level %q (want debug, info, warn or error)", name)
}
}
// ParseFormat normalizes a format name. An empty name defaults to JSON.
func ParseFormat(name string) (string, error) {
switch strings.ToLower(strings.TrimSpace(name)) {
case "", "json":
return "json", nil
case "text":
return "text", nil
default:
return "", fmt.Errorf("invalid log format %q (want json or text)", name)
}
}
// NewHandler builds a slog handler writing to stdout in the requested format.
// An empty format defaults to JSON.
func NewHandler(format string, level slog.Level) (slog.Handler, error) {
f, err := ParseFormat(format)
if err != nil {
return nil, err
}
opts := &slog.HandlerOptions{Level: level}
if f == "text" {
return slog.NewTextHandler(os.Stdout, opts), nil
}
return slog.NewJSONHandler(os.Stdout, opts), nil
}
type requestIDKey struct{}
// WithRequestID stores a request identifier in the context.
func WithRequestID(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, requestIDKey{}, id)
}
// RequestID returns the request identifier stored in the context, or "".
func RequestID(ctx context.Context) string {
if id, ok := ctx.Value(requestIDKey{}).(string); ok {
return id
}
return ""
}
// NewRequestID returns a fresh random request identifier.
func NewRequestID() string {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "unknown"
}
return hex.EncodeToString(b[:])
}
// ValidRequestID reports whether an incoming identifier is safe to reuse.
func ValidRequestID(id string) bool {
if id == "" || len(id) > 128 {
return false
}
for _, r := range id {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9':
case r == '-', r == '_', r == '.':
default:
return false
}
}
return true
}
+95
View File
@@ -0,0 +1,95 @@
package logging
import (
"log/slog"
"strings"
"testing"
)
func TestParseLevel(t *testing.T) {
cases := []struct {
in string
want slog.Level
ok bool
}{
{"debug", slog.LevelDebug, true},
{"DEBUG", slog.LevelDebug, true},
{" info ", slog.LevelInfo, true},
{"warn", slog.LevelWarn, true},
{"warning", slog.LevelWarn, true},
{"error", slog.LevelError, true},
{"trace", 0, false},
{"", 0, false},
}
for _, c := range cases {
got, err := ParseLevel(c.in)
if c.ok && err != nil {
t.Errorf("ParseLevel(%q) unexpected error: %v", c.in, err)
}
if !c.ok && err == nil {
t.Errorf("ParseLevel(%q) expected error, got %v", c.in, got)
}
if c.ok && got != c.want {
t.Errorf("ParseLevel(%q) = %v, want %v", c.in, got, c.want)
}
}
}
func TestParseFormat(t *testing.T) {
cases := []struct {
in string
want string
ok bool
}{
{"", "json", true},
{"json", "json", true},
{"TEXT", "text", true},
{" text ", "text", true},
{"xml", "", false},
}
for _, c := range cases {
got, err := ParseFormat(c.in)
if c.ok && err != nil {
t.Errorf("ParseFormat(%q) unexpected error: %v", c.in, err)
}
if !c.ok && err == nil {
t.Errorf("ParseFormat(%q) expected error, got %q", c.in, got)
}
if c.ok && got != c.want {
t.Errorf("ParseFormat(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestRequestIDHelpers(t *testing.T) {
if !ValidRequestID("req-abc_123.4") {
t.Error("ValidRequestID(valid) = false")
}
for _, bad := range []string{"", "has space", "has\nnewline", "semi;colon", strings.Repeat("a", 129)} {
if ValidRequestID(bad) {
t.Errorf("ValidRequestID(%q) = true, want false", bad)
}
}
id := NewRequestID()
if id == "" || !ValidRequestID(id) {
t.Fatalf("NewRequestID() = %q, want a valid non-empty id", id)
}
if NewRequestID() == id {
t.Fatal("NewRequestID() returned the same value twice")
}
}
func TestNewHandler(t *testing.T) {
for _, format := range []string{"", "json", "JSON", "text"} {
h, err := NewHandler(format, slog.LevelInfo)
if err != nil {
t.Errorf("NewHandler(%q) unexpected error: %v", format, err)
}
if h == nil {
t.Errorf("NewHandler(%q) returned nil handler", format)
}
}
if _, err := NewHandler("xml", slog.LevelInfo); err == nil {
t.Error("NewHandler(\"xml\") expected error, got nil")
}
}