fix(s3): handle storage outages and check bucket readiness
This commit is contained in:
+2
-2
@@ -16,7 +16,7 @@ http {
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 6s;
|
||||
proxy_next_upstream error timeout http_500 http_502 http_503 non_idempotent;
|
||||
proxy_next_upstream error timeout http_500 http_502 non_idempotent;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ http {
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 6s;
|
||||
proxy_next_upstream error timeout http_500 http_502 http_503;
|
||||
proxy_next_upstream error timeout http_500 http_502;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
> make failure-go
|
||||
go test -tags=failure ./student-tests/... -v -args -base-url=http://localhost:8080
|
||||
=== RUN TestRedisDown
|
||||
--- PASS: TestRedisDown (3.04s)
|
||||
=== RUN TestDatabaseDown
|
||||
--- PASS: TestDatabaseDown (12.51s)
|
||||
=== RUN TestS3Down
|
||||
--- PASS: TestS3Down (55.01s)
|
||||
=== RUN TestSessionExpiry
|
||||
--- PASS: TestSessionExpiry (6.40s)
|
||||
PASS
|
||||
ok mtusi2026-messenger/student-tests 76.980s
|
||||
@@ -115,6 +115,7 @@ func Run() error {
|
||||
cfg.Server.MaxInflightRequests,
|
||||
cfg.S3.MaxUploadBytes,
|
||||
pool,
|
||||
s3Client,
|
||||
authService,
|
||||
userService,
|
||||
chatService,
|
||||
|
||||
@@ -18,6 +18,7 @@ func NewRouter(
|
||||
maxInflightRequests int,
|
||||
maxUploadBytes int,
|
||||
db handler.Pinger,
|
||||
storage handler.Pinger,
|
||||
authService *service.AuthService,
|
||||
userService *service.UserService,
|
||||
chatService *service.ChatService,
|
||||
@@ -26,7 +27,7 @@ func NewRouter(
|
||||
) http.Handler {
|
||||
router := chi.NewRouter()
|
||||
|
||||
healthHandler := handler.NewHealthHandler(log, db)
|
||||
healthHandler := handler.NewHealthHandler(log, db, storage)
|
||||
authHandler := handler.NewAuthHandler(authService)
|
||||
userHandler := handler.NewUserHandler(userService)
|
||||
chatHandler := handler.NewChatHandler(chatService)
|
||||
|
||||
+16
-14
@@ -18,12 +18,12 @@ type Pinger interface {
|
||||
}
|
||||
|
||||
type HealthHandler struct {
|
||||
log *slog.Logger
|
||||
db Pinger
|
||||
log *slog.Logger
|
||||
dependencies []Pinger
|
||||
}
|
||||
|
||||
func NewHealthHandler(log *slog.Logger, db Pinger) *HealthHandler {
|
||||
return &HealthHandler{log: log, db: db}
|
||||
func NewHealthHandler(log *slog.Logger, dependencies ...Pinger) *HealthHandler {
|
||||
return &HealthHandler{log: log, dependencies: dependencies}
|
||||
}
|
||||
|
||||
func (h *HealthHandler) Live(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -39,16 +39,18 @@ func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), readyPingTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := h.db.Ping(ctx); err != nil {
|
||||
h.log.ErrorContext(r.Context(), "readiness check failed", "error", err)
|
||||
httpx.WriteError(
|
||||
w,
|
||||
http.StatusServiceUnavailable,
|
||||
dto.ErrorCodeUnavailable,
|
||||
"service unavailable",
|
||||
w.Header().Get("X-Request-Id"),
|
||||
)
|
||||
return
|
||||
for _, dependency := range h.dependencies {
|
||||
if err := dependency.Ping(ctx); err != nil {
|
||||
h.log.ErrorContext(r.Context(), "readiness check failed", "error", err)
|
||||
httpx.WriteError(
|
||||
w,
|
||||
http.StatusServiceUnavailable,
|
||||
dto.ErrorCodeUnavailable,
|
||||
"service unavailable",
|
||||
w.Header().Get("X-Request-Id"),
|
||||
)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"mtusi2026-messenger/internal/dto"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
func WriteJSON(w http.ResponseWriter, status int, body any) {
|
||||
@@ -51,7 +52,7 @@ func WriteInternalError(w http.ResponseWriter, requestID string, err error) {
|
||||
}
|
||||
|
||||
func isUnavailable(err error) bool {
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, s3store.ErrUnavailable) {
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
+23
-2
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrUnavailable = errors.New("s3 unavailable")
|
||||
ErrLoadConfig = errors.New("load aws config")
|
||||
ErrHeadBucket = errors.New("head bucket")
|
||||
ErrCreateBucket = errors.New("create bucket")
|
||||
@@ -118,6 +120,15 @@ func (c *Client) EnsureBucket(ctx context.Context) error {
|
||||
return ErrCreateBucket
|
||||
}
|
||||
|
||||
func (c *Client) Ping(ctx context.Context) error {
|
||||
_, err := c.api.HeadBucket(ctx, &awss3.HeadBucketInput{Bucket: aws.String(c.bucket)})
|
||||
if err != nil {
|
||||
c.log.Error("head bucket", "bucket", c.bucket, "error", err)
|
||||
return operationError(ErrHeadBucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) Put(
|
||||
ctx context.Context,
|
||||
key string,
|
||||
@@ -134,7 +145,7 @@ func (c *Client) Put(
|
||||
})
|
||||
if err != nil {
|
||||
c.log.Error("put object", "bucket", c.bucket, "key", key, "error", err)
|
||||
return ErrPutObject
|
||||
return operationError(ErrPutObject, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -151,7 +162,7 @@ func (c *Client) Get(ctx context.Context, key string) (*awss3.GetObjectOutput, e
|
||||
}
|
||||
if err != nil {
|
||||
c.log.Error("get object", "bucket", c.bucket, "key", key, "error", err)
|
||||
return nil, ErrGetObject
|
||||
return nil, operationError(ErrGetObject, err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
@@ -228,6 +239,16 @@ func newS3Client(
|
||||
}), nil
|
||||
}
|
||||
|
||||
func operationError(operation, err error) error {
|
||||
var networkError net.Error
|
||||
var responseError *awshttp.ResponseError
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.As(err, &networkError) ||
|
||||
(errors.As(err, &responseError) && responseError.HTTPStatusCode() >= 500) {
|
||||
return ErrUnavailable
|
||||
}
|
||||
return operation
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
var responseErr *awshttp.ResponseError
|
||||
if errors.As(err, &responseErr) && responseErr.HTTPStatusCode() == http.StatusNotFound {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build contract || reconcile
|
||||
//go:build contract || reconcile || failure
|
||||
|
||||
package studenttests_test
|
||||
|
||||
|
||||
@@ -3,10 +3,14 @@
|
||||
package studenttests_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"mtusi2026-messenger/internal/dto"
|
||||
)
|
||||
|
||||
func compose(t *testing.T, args ...string) {
|
||||
@@ -87,3 +91,66 @@ func TestDatabaseDown(t *testing.T) {
|
||||
requireStatus(t, resp, http.StatusUnauthorized)
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
func TestS3Down(t *testing.T) {
|
||||
user := createUser(t)
|
||||
chat := createChat(t, user, user.ID)
|
||||
original := makePNG(t, 32, 32)
|
||||
att := uploadPNG(t, user, chat.ID)
|
||||
|
||||
stopService(t, "s3")
|
||||
for _, path := range []string{"/health/ready", "/api/v1/attachments/" + att.ID.String() + "/content"} {
|
||||
resp := doRequest(t, http.MethodGet, path, user.Token, nil)
|
||||
requireStatus(t, resp, http.StatusServiceUnavailable)
|
||||
requireErrorCode(t, resp, dto.ErrorCodeUnavailable)
|
||||
}
|
||||
resp := uploadAttachment(t, user, chat.ID, "outage.png", "image/png", original)
|
||||
requireStatus(t, resp, http.StatusServiceUnavailable)
|
||||
requireErrorCode(t, resp, dto.ErrorCodeUnavailable)
|
||||
resp = doRequest(t, http.MethodGet, "/health/live", "", nil)
|
||||
requireStatus(t, resp, http.StatusOK)
|
||||
resp.Body.Close()
|
||||
|
||||
compose(t, "start", "s3")
|
||||
waitReady(t)
|
||||
deadline := time.Now().Add(60 * time.Second)
|
||||
downloaded, uploaded := false, false
|
||||
for time.Now().Before(deadline) && (!downloaded || !uploaded) {
|
||||
if !downloaded {
|
||||
req, err := http.NewRequest(http.MethodGet, *baseURL+"/api/v1/attachments/"+att.ID.String()+"/content", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+user.Token)
|
||||
response, err := httpClient.Do(req)
|
||||
if err == nil {
|
||||
data, readErr := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if response.StatusCode == http.StatusOK && readErr == nil {
|
||||
if !bytes.Equal(data, original) {
|
||||
t.Fatal("original changed across S3 outage")
|
||||
}
|
||||
downloaded = true
|
||||
} else if response.StatusCode != http.StatusServiceUnavailable && response.StatusCode != http.StatusBadGateway && response.StatusCode != http.StatusGatewayTimeout {
|
||||
t.Fatalf("unexpected download recovery status: %d", response.StatusCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !uploaded {
|
||||
response := uploadAttachment(t, user, chat.ID, "recovery.png", "image/png", original)
|
||||
status := response.StatusCode
|
||||
response.Body.Close()
|
||||
if status == http.StatusCreated {
|
||||
uploaded = true
|
||||
} else if status != http.StatusServiceUnavailable {
|
||||
t.Fatalf("unexpected upload recovery status: %d", status)
|
||||
}
|
||||
}
|
||||
if !downloaded || !uploaded {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
if !downloaded || !uploaded {
|
||||
t.Fatalf("S3 did not recover in 60s: download=%t upload=%t", downloaded, uploaded)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ help-go:
|
||||
@echo "contract-go: contract tests"
|
||||
@echo "race-go: concurrency tests (-race)"
|
||||
@echo "load-go: load test"
|
||||
@echo "failure-go: dependency failure tests (stops redis/postgres containers)"
|
||||
@echo "failure-go: dependency failure tests (stops redis/postgres/s3 containers)"
|
||||
@echo "reconcile: attachment audit; DRY_RUN=false to repair"
|
||||
|
||||
contract-go:
|
||||
|
||||
Reference in New Issue
Block a user