fix(s3): handle storage outages and check bucket readiness

This commit is contained in:
kk10
2026-11-05 11:04:41 +03:00
parent 592594ae94
commit d467cab01a
10 changed files with 127 additions and 22 deletions
+2 -2
View File
@@ -16,7 +16,7 @@ http {
proxy_connect_timeout 1s;
proxy_read_timeout 6s;
proxy_next_upstream error timeout http_500 http_502 http_503 non_idempotent;
proxy_next_upstream error timeout http_500 http_502 non_idempotent;
proxy_next_upstream_tries 2;
}
@@ -26,7 +26,7 @@ http {
proxy_connect_timeout 1s;
proxy_read_timeout 6s;
proxy_next_upstream error timeout http_500 http_502 http_503;
proxy_next_upstream error timeout http_500 http_502;
proxy_next_upstream_tries 2;
}
}
+12
View File
@@ -0,0 +1,12 @@
> make failure-go
go test -tags=failure ./student-tests/... -v -args -base-url=http://localhost:8080
=== RUN TestRedisDown
--- PASS: TestRedisDown (3.04s)
=== RUN TestDatabaseDown
--- PASS: TestDatabaseDown (12.51s)
=== RUN TestS3Down
--- PASS: TestS3Down (55.01s)
=== RUN TestSessionExpiry
--- PASS: TestSessionExpiry (6.40s)
PASS
ok mtusi2026-messenger/student-tests 76.980s
+1
View File
@@ -115,6 +115,7 @@ func Run() error {
cfg.Server.MaxInflightRequests,
cfg.S3.MaxUploadBytes,
pool,
s3Client,
authService,
userService,
chatService,
+2 -1
View File
@@ -18,6 +18,7 @@ func NewRouter(
maxInflightRequests int,
maxUploadBytes int,
db handler.Pinger,
storage handler.Pinger,
authService *service.AuthService,
userService *service.UserService,
chatService *service.ChatService,
@@ -26,7 +27,7 @@ func NewRouter(
) http.Handler {
router := chi.NewRouter()
healthHandler := handler.NewHealthHandler(log, db)
healthHandler := handler.NewHealthHandler(log, db, storage)
authHandler := handler.NewAuthHandler(authService)
userHandler := handler.NewUserHandler(userService)
chatHandler := handler.NewChatHandler(chatService)
+16 -14
View File
@@ -18,12 +18,12 @@ type Pinger interface {
}
type HealthHandler struct {
log *slog.Logger
db Pinger
log *slog.Logger
dependencies []Pinger
}
func NewHealthHandler(log *slog.Logger, db Pinger) *HealthHandler {
return &HealthHandler{log: log, db: db}
func NewHealthHandler(log *slog.Logger, dependencies ...Pinger) *HealthHandler {
return &HealthHandler{log: log, dependencies: dependencies}
}
func (h *HealthHandler) Live(w http.ResponseWriter, r *http.Request) {
@@ -39,16 +39,18 @@ func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), readyPingTimeout)
defer cancel()
if err := h.db.Ping(ctx); err != nil {
h.log.ErrorContext(r.Context(), "readiness check failed", "error", err)
httpx.WriteError(
w,
http.StatusServiceUnavailable,
dto.ErrorCodeUnavailable,
"service unavailable",
w.Header().Get("X-Request-Id"),
)
return
for _, dependency := range h.dependencies {
if err := dependency.Ping(ctx); err != nil {
h.log.ErrorContext(r.Context(), "readiness check failed", "error", err)
httpx.WriteError(
w,
http.StatusServiceUnavailable,
dto.ErrorCodeUnavailable,
"service unavailable",
w.Header().Get("X-Request-Id"),
)
return
}
}
w.Header().Set("Content-Type", "application/json")
+2 -1
View File
@@ -11,6 +11,7 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"mtusi2026-messenger/internal/dto"
s3store "mtusi2026-messenger/internal/s3"
)
func WriteJSON(w http.ResponseWriter, status int, body any) {
@@ -51,7 +52,7 @@ func WriteInternalError(w http.ResponseWriter, requestID string, err error) {
}
func isUnavailable(err error) bool {
if errors.Is(err, context.DeadlineExceeded) {
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, s3store.ErrUnavailable) {
return true
}
+23 -2
View File
@@ -5,6 +5,7 @@ import (
"errors"
"io"
"log/slog"
"net"
"net/http"
"time"
@@ -19,6 +20,7 @@ import (
)
var (
ErrUnavailable = errors.New("s3 unavailable")
ErrLoadConfig = errors.New("load aws config")
ErrHeadBucket = errors.New("head bucket")
ErrCreateBucket = errors.New("create bucket")
@@ -118,6 +120,15 @@ func (c *Client) EnsureBucket(ctx context.Context) error {
return ErrCreateBucket
}
func (c *Client) Ping(ctx context.Context) error {
_, err := c.api.HeadBucket(ctx, &awss3.HeadBucketInput{Bucket: aws.String(c.bucket)})
if err != nil {
c.log.Error("head bucket", "bucket", c.bucket, "error", err)
return operationError(ErrHeadBucket, err)
}
return nil
}
func (c *Client) Put(
ctx context.Context,
key string,
@@ -134,7 +145,7 @@ func (c *Client) Put(
})
if err != nil {
c.log.Error("put object", "bucket", c.bucket, "key", key, "error", err)
return ErrPutObject
return operationError(ErrPutObject, err)
}
return nil
@@ -151,7 +162,7 @@ func (c *Client) Get(ctx context.Context, key string) (*awss3.GetObjectOutput, e
}
if err != nil {
c.log.Error("get object", "bucket", c.bucket, "key", key, "error", err)
return nil, ErrGetObject
return nil, operationError(ErrGetObject, err)
}
return out, nil
@@ -228,6 +239,16 @@ func newS3Client(
}), nil
}
func operationError(operation, err error) error {
var networkError net.Error
var responseError *awshttp.ResponseError
if errors.Is(err, context.DeadlineExceeded) || errors.As(err, &networkError) ||
(errors.As(err, &responseError) && responseError.HTTPStatusCode() >= 500) {
return ErrUnavailable
}
return operation
}
func isNotFound(err error) bool {
var responseErr *awshttp.ResponseError
if errors.As(err, &responseErr) && responseErr.HTTPStatusCode() == http.StatusNotFound {
+1 -1
View File
@@ -1,4 +1,4 @@
//go:build contract || reconcile
//go:build contract || reconcile || failure
package studenttests_test
+67
View File
@@ -3,10 +3,14 @@
package studenttests_test
import (
"bytes"
"io"
"net/http"
"os/exec"
"testing"
"time"
"mtusi2026-messenger/internal/dto"
)
func compose(t *testing.T, args ...string) {
@@ -87,3 +91,66 @@ func TestDatabaseDown(t *testing.T) {
requireStatus(t, resp, http.StatusUnauthorized)
resp.Body.Close()
}
func TestS3Down(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
original := makePNG(t, 32, 32)
att := uploadPNG(t, user, chat.ID)
stopService(t, "s3")
for _, path := range []string{"/health/ready", "/api/v1/attachments/" + att.ID.String() + "/content"} {
resp := doRequest(t, http.MethodGet, path, user.Token, nil)
requireStatus(t, resp, http.StatusServiceUnavailable)
requireErrorCode(t, resp, dto.ErrorCodeUnavailable)
}
resp := uploadAttachment(t, user, chat.ID, "outage.png", "image/png", original)
requireStatus(t, resp, http.StatusServiceUnavailable)
requireErrorCode(t, resp, dto.ErrorCodeUnavailable)
resp = doRequest(t, http.MethodGet, "/health/live", "", nil)
requireStatus(t, resp, http.StatusOK)
resp.Body.Close()
compose(t, "start", "s3")
waitReady(t)
deadline := time.Now().Add(60 * time.Second)
downloaded, uploaded := false, false
for time.Now().Before(deadline) && (!downloaded || !uploaded) {
if !downloaded {
req, err := http.NewRequest(http.MethodGet, *baseURL+"/api/v1/attachments/"+att.ID.String()+"/content", nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer "+user.Token)
response, err := httpClient.Do(req)
if err == nil {
data, readErr := io.ReadAll(response.Body)
response.Body.Close()
if response.StatusCode == http.StatusOK && readErr == nil {
if !bytes.Equal(data, original) {
t.Fatal("original changed across S3 outage")
}
downloaded = true
} else if response.StatusCode != http.StatusServiceUnavailable && response.StatusCode != http.StatusBadGateway && response.StatusCode != http.StatusGatewayTimeout {
t.Fatalf("unexpected download recovery status: %d", response.StatusCode)
}
}
}
if !uploaded {
response := uploadAttachment(t, user, chat.ID, "recovery.png", "image/png", original)
status := response.StatusCode
response.Body.Close()
if status == http.StatusCreated {
uploaded = true
} else if status != http.StatusServiceUnavailable {
t.Fatalf("unexpected upload recovery status: %d", status)
}
}
if !downloaded || !uploaded {
time.Sleep(500 * time.Millisecond)
}
}
if !downloaded || !uploaded {
t.Fatalf("S3 did not recover in 60s: download=%t upload=%t", downloaded, uploaded)
}
}
+1 -1
View File
@@ -14,7 +14,7 @@ help-go:
@echo "contract-go: contract tests"
@echo "race-go: concurrency tests (-race)"
@echo "load-go: load test"
@echo "failure-go: dependency failure tests (stops redis/postgres containers)"
@echo "failure-go: dependency failure tests (stops redis/postgres/s3 containers)"
@echo "reconcile: attachment audit; DRY_RUN=false to repair"
contract-go: