diff --git a/Dockerfile b/Dockerfile index a4d4c426..a33beebc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,4 +19,4 @@ HEALTHCHECK --interval=5s --timeout=3s --retries=3 \ CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" ENV PYTHONUNBUFFERED=1 -CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "10"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 265bbfcb..d5599496 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,6 +7,14 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} volumes: - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + reservations: + cpus: '0.10' + memory: 128M healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d messenger"] interval: 3s @@ -18,6 +26,11 @@ services: command: redis-server --save 60 1 --loglevel warning volumes: - redisdata:/data + deploy: + resources: + limits: + cpus: '0.25' + memory: 256M healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 3s @@ -40,11 +53,18 @@ services: api-1: build: . + stop_grace_period: 15s environment: - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger - REDIS_URL=redis://redis:6379/0 - SESSION_TTL_SECONDS=3600 - INSTANCE_ID=api-replica-1 + - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M depends_on: postgres: condition: service_healthy @@ -60,11 +80,18 @@ services: api-2: build: . + stop_grace_period: 15s environment: - DATABASE_URL=postgresql://postgres:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}@postgres:5432/messenger - REDIS_URL=redis://redis:6379/0 - SESSION_TTL_SECONDS=3600 - INSTANCE_ID=api-replica-2 + - SECRET_HMAC_KEY=${SECRET_HMAC_KEY} + deploy: + resources: + limits: + cpus: '0.50' + memory: 256M depends_on: postgres: condition: service_healthy @@ -84,6 +111,11 @@ services: - "8080:8080" volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M depends_on: - api-1 - api-2 diff --git a/evidence/load-rolling.json b/evidence/load-rolling.json new file mode 100644 index 00000000..5e2f4c14 --- /dev/null +++ b/evidence/load-rolling.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 60.0, + "elapsed_seconds": 60.172, + "concurrency": 4, + "successful_requests": 2801, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 46.55, + "successful_latency_ms": { + "p50": 47.0, + "p95": 188.0, + "p99": 875.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/nginx.conf b/nginx.conf index 05ce4124..bf626811 100644 --- a/nginx.conf +++ b/nginx.conf @@ -2,8 +2,8 @@ events { worker_connections 1024; } http { upstream messenger_api { - server api-1:8080; - server api-2:8080; + server api-1:8080 max_fails=1 fail_timeout=5s; + server api-2:8080 max_fails=1 fail_timeout=5s; } server { @@ -11,6 +11,15 @@ http { location / { proxy_pass http://messenger_api; + + proxy_connect_timeout 2s; + proxy_send_timeout 5s; + proxy_read_timeout 5s; + + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..173f1dba --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu + +wait_ready() { + svc=$1 + timeout=60 + echo "Waiting for $svc readiness (up to ${timeout}s)..." + while [ "$timeout" -gt 0 ]; do + if docker compose exec -T "$svc" python -c \ + "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready')" 2>/dev/null; then + echo "$svc is ready!" + return 0 + fi + sleep 1 + timeout=$((timeout - 1)) + done + echo "$svc failed to become ready" >&2 + return 1 +} + +echo "Starting Rolling Update" + +echo "1. Rebuilding and replacing api-1..." +docker compose up -d --no-deps --build --force-recreate api-1 +wait_ready api-1 +echo "Reloading nginx to pick up new api-1 IP..." +docker compose exec -T nginx nginx -s reload + +echo "2. Rebuilding and replacing api-2..." +docker compose up -d --no-deps --build --force-recreate api-2 +wait_ready api-2 +echo "Reloading nginx to pick up new api-2 IP..." +docker compose exec -T nginx nginx -s reload + +echo "Rolling Update Completed Successfully" \ No newline at end of file