diff --git a/deploy/nginx.conf b/deploy/nginx.conf index f73980c..8c2a594 100644 --- a/deploy/nginx.conf +++ b/deploy/nginx.conf @@ -16,7 +16,7 @@ http { proxy_connect_timeout 1s; proxy_read_timeout 6s; - proxy_next_upstream error timeout http_500 http_502 http_503 non_idempotent; + proxy_next_upstream error timeout http_500 http_502 non_idempotent; proxy_next_upstream_tries 2; } @@ -26,7 +26,7 @@ http { proxy_connect_timeout 1s; proxy_read_timeout 6s; - proxy_next_upstream error timeout http_500 http_502 http_503; + proxy_next_upstream error timeout http_500 http_502; proxy_next_upstream_tries 2; } } diff --git a/evidence/failure-go.txt b/evidence/failure-go.txt new file mode 100644 index 0000000..f6fc73a --- /dev/null +++ b/evidence/failure-go.txt @@ -0,0 +1,12 @@ +> make failure-go +go test -tags=failure ./student-tests/... -v -args -base-url=http://localhost:8080 +=== RUN TestRedisDown +--- PASS: TestRedisDown (3.04s) +=== RUN TestDatabaseDown +--- PASS: TestDatabaseDown (12.51s) +=== RUN TestS3Down +--- PASS: TestS3Down (55.01s) +=== RUN TestSessionExpiry +--- PASS: TestSessionExpiry (6.40s) +PASS +ok mtusi2026-messenger/student-tests 76.980s \ No newline at end of file diff --git a/internal/app/app.go b/internal/app/app.go index b48c130..80a3c9f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -115,6 +115,7 @@ func Run() error { cfg.Server.MaxInflightRequests, cfg.S3.MaxUploadBytes, pool, + s3Client, authService, userService, chatService, diff --git a/internal/app/router.go b/internal/app/router.go index 8170224..86bed0e 100644 --- a/internal/app/router.go +++ b/internal/app/router.go @@ -18,6 +18,7 @@ func NewRouter( maxInflightRequests int, maxUploadBytes int, db handler.Pinger, + storage handler.Pinger, authService *service.AuthService, userService *service.UserService, chatService *service.ChatService, @@ -26,7 +27,7 @@ func NewRouter( ) http.Handler { router := chi.NewRouter() - healthHandler := handler.NewHealthHandler(log, db) + healthHandler := handler.NewHealthHandler(log, db, storage) authHandler := handler.NewAuthHandler(authService) userHandler := handler.NewUserHandler(userService) chatHandler := handler.NewChatHandler(chatService) diff --git a/internal/handler/health.go b/internal/handler/health.go index ee8a2b3..9ffc69a 100644 --- a/internal/handler/health.go +++ b/internal/handler/health.go @@ -18,12 +18,12 @@ type Pinger interface { } type HealthHandler struct { - log *slog.Logger - db Pinger + log *slog.Logger + dependencies []Pinger } -func NewHealthHandler(log *slog.Logger, db Pinger) *HealthHandler { - return &HealthHandler{log: log, db: db} +func NewHealthHandler(log *slog.Logger, dependencies ...Pinger) *HealthHandler { + return &HealthHandler{log: log, dependencies: dependencies} } func (h *HealthHandler) Live(w http.ResponseWriter, r *http.Request) { @@ -39,16 +39,18 @@ func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), readyPingTimeout) defer cancel() - if err := h.db.Ping(ctx); err != nil { - h.log.ErrorContext(r.Context(), "readiness check failed", "error", err) - httpx.WriteError( - w, - http.StatusServiceUnavailable, - dto.ErrorCodeUnavailable, - "service unavailable", - w.Header().Get("X-Request-Id"), - ) - return + for _, dependency := range h.dependencies { + if err := dependency.Ping(ctx); err != nil { + h.log.ErrorContext(r.Context(), "readiness check failed", "error", err) + httpx.WriteError( + w, + http.StatusServiceUnavailable, + dto.ErrorCodeUnavailable, + "service unavailable", + w.Header().Get("X-Request-Id"), + ) + return + } } w.Header().Set("Content-Type", "application/json") diff --git a/internal/httpx/response.go b/internal/httpx/response.go index f2eac20..23f2771 100644 --- a/internal/httpx/response.go +++ b/internal/httpx/response.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "mtusi2026-messenger/internal/dto" + s3store "mtusi2026-messenger/internal/s3" ) func WriteJSON(w http.ResponseWriter, status int, body any) { @@ -51,7 +52,7 @@ func WriteInternalError(w http.ResponseWriter, requestID string, err error) { } func isUnavailable(err error) bool { - if errors.Is(err, context.DeadlineExceeded) { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, s3store.ErrUnavailable) { return true } diff --git a/internal/s3/client.go b/internal/s3/client.go index d8a84c0..c51188c 100644 --- a/internal/s3/client.go +++ b/internal/s3/client.go @@ -5,6 +5,7 @@ import ( "errors" "io" "log/slog" + "net" "net/http" "time" @@ -19,6 +20,7 @@ import ( ) var ( + ErrUnavailable = errors.New("s3 unavailable") ErrLoadConfig = errors.New("load aws config") ErrHeadBucket = errors.New("head bucket") ErrCreateBucket = errors.New("create bucket") @@ -118,6 +120,15 @@ func (c *Client) EnsureBucket(ctx context.Context) error { return ErrCreateBucket } +func (c *Client) Ping(ctx context.Context) error { + _, err := c.api.HeadBucket(ctx, &awss3.HeadBucketInput{Bucket: aws.String(c.bucket)}) + if err != nil { + c.log.Error("head bucket", "bucket", c.bucket, "error", err) + return operationError(ErrHeadBucket, err) + } + return nil +} + func (c *Client) Put( ctx context.Context, key string, @@ -134,7 +145,7 @@ func (c *Client) Put( }) if err != nil { c.log.Error("put object", "bucket", c.bucket, "key", key, "error", err) - return ErrPutObject + return operationError(ErrPutObject, err) } return nil @@ -151,7 +162,7 @@ func (c *Client) Get(ctx context.Context, key string) (*awss3.GetObjectOutput, e } if err != nil { c.log.Error("get object", "bucket", c.bucket, "key", key, "error", err) - return nil, ErrGetObject + return nil, operationError(ErrGetObject, err) } return out, nil @@ -228,6 +239,16 @@ func newS3Client( }), nil } +func operationError(operation, err error) error { + var networkError net.Error + var responseError *awshttp.ResponseError + if errors.Is(err, context.DeadlineExceeded) || errors.As(err, &networkError) || + (errors.As(err, &responseError) && responseError.HTTPStatusCode() >= 500) { + return ErrUnavailable + } + return operation +} + func isNotFound(err error) bool { var responseErr *awshttp.ResponseError if errors.As(err, &responseErr) && responseErr.HTTPStatusCode() == http.StatusNotFound { diff --git a/student-tests/attachment_helpers_test.go b/student-tests/attachment_helpers_test.go index 4fd6fd3..3da7613 100644 --- a/student-tests/attachment_helpers_test.go +++ b/student-tests/attachment_helpers_test.go @@ -1,4 +1,4 @@ -//go:build contract || reconcile +//go:build contract || reconcile || failure package studenttests_test diff --git a/student-tests/dependency_failure_test.go b/student-tests/dependency_failure_test.go index b0f2363..59d7ba1 100644 --- a/student-tests/dependency_failure_test.go +++ b/student-tests/dependency_failure_test.go @@ -3,10 +3,14 @@ package studenttests_test import ( + "bytes" + "io" "net/http" "os/exec" "testing" "time" + + "mtusi2026-messenger/internal/dto" ) func compose(t *testing.T, args ...string) { @@ -87,3 +91,66 @@ func TestDatabaseDown(t *testing.T) { requireStatus(t, resp, http.StatusUnauthorized) resp.Body.Close() } + +func TestS3Down(t *testing.T) { + user := createUser(t) + chat := createChat(t, user, user.ID) + original := makePNG(t, 32, 32) + att := uploadPNG(t, user, chat.ID) + + stopService(t, "s3") + for _, path := range []string{"/health/ready", "/api/v1/attachments/" + att.ID.String() + "/content"} { + resp := doRequest(t, http.MethodGet, path, user.Token, nil) + requireStatus(t, resp, http.StatusServiceUnavailable) + requireErrorCode(t, resp, dto.ErrorCodeUnavailable) + } + resp := uploadAttachment(t, user, chat.ID, "outage.png", "image/png", original) + requireStatus(t, resp, http.StatusServiceUnavailable) + requireErrorCode(t, resp, dto.ErrorCodeUnavailable) + resp = doRequest(t, http.MethodGet, "/health/live", "", nil) + requireStatus(t, resp, http.StatusOK) + resp.Body.Close() + + compose(t, "start", "s3") + waitReady(t) + deadline := time.Now().Add(60 * time.Second) + downloaded, uploaded := false, false + for time.Now().Before(deadline) && (!downloaded || !uploaded) { + if !downloaded { + req, err := http.NewRequest(http.MethodGet, *baseURL+"/api/v1/attachments/"+att.ID.String()+"/content", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+user.Token) + response, err := httpClient.Do(req) + if err == nil { + data, readErr := io.ReadAll(response.Body) + response.Body.Close() + if response.StatusCode == http.StatusOK && readErr == nil { + if !bytes.Equal(data, original) { + t.Fatal("original changed across S3 outage") + } + downloaded = true + } else if response.StatusCode != http.StatusServiceUnavailable && response.StatusCode != http.StatusBadGateway && response.StatusCode != http.StatusGatewayTimeout { + t.Fatalf("unexpected download recovery status: %d", response.StatusCode) + } + } + } + if !uploaded { + response := uploadAttachment(t, user, chat.ID, "recovery.png", "image/png", original) + status := response.StatusCode + response.Body.Close() + if status == http.StatusCreated { + uploaded = true + } else if status != http.StatusServiceUnavailable { + t.Fatalf("unexpected upload recovery status: %d", status) + } + } + if !downloaded || !uploaded { + time.Sleep(500 * time.Millisecond) + } + } + if !downloaded || !uploaded { + t.Fatalf("S3 did not recover in 60s: download=%t upload=%t", downloaded, uploaded) + } +} diff --git a/student.mk b/student.mk index 8da8ad4..6b2f745 100644 --- a/student.mk +++ b/student.mk @@ -14,7 +14,7 @@ help-go: @echo "contract-go: contract tests" @echo "race-go: concurrency tests (-race)" @echo "load-go: load test" - @echo "failure-go: dependency failure tests (stops redis/postgres containers)" + @echo "failure-go: dependency failure tests (stops redis/postgres/s3 containers)" @echo "reconcile: attachment audit; DRY_RUN=false to repair" contract-go: