diff --git a/.env.example b/.env.example index d99d26a..1a6e0b9 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,10 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +API_REPLICAS=2 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +CLEANUP_GRACE_SECONDS=600 +CLEANUP_INTERVAL_SECONDS=300 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..66fc655 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,244 @@ +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: postgres + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-10} + GOMEMLIMIT: 150MiB + SHUTDOWN_TIMEOUT_SECONDS: 10 + DRAIN_SECONDS: 3 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + +services: + postgres: + image: postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - backend + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "--appendonly", "yes", "--save", "60", "1"] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860 + command: + - server + - -dir=/data + - -ip=s3 + - -master.volumeSizeLimitMB=128 + - -volume.max=30 + - -s3 + - -s3.port=9000 + - -s3.config=/run/secrets/s3_config + secrets: + - s3_config + volumes: + - s3_data:/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/dir/assign | grep -q fid"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - storage + restart: unless-stopped + + s3-init: + image: mtusi-messenger:lab4 + command: ["s3-init"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - storage + restart: "no" + + janitor: + image: mtusi-messenger:lab4 + command: ["janitor"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-600} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + networks: + - backend + - storage + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab4 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: "no" + + api: + image: mtusi-messenger:lab4 + environment: *api-env + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.5" + memory: 192M + networks: + - backend + - storage + restart: unless-stopped + + lb: + image: haproxy:3.0-alpine@sha256:cf3af0facf7eb22b9fee463daf0fbc07ead740b8150696dce724264faf7b4318 + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - api + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + pg_data: + valkey_data: + s3_data: + +networks: + backend: + driver: bridge + storage: + driver: bridge diff --git a/lb/haproxy.cfg b/lb/haproxy.cfg new file mode 100644 index 0000000..d63fed5 --- /dev/null +++ b/lb/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 2000 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + hold nx 1s + hold obsolete 1s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + +defaults + mode http + log global + option httplog + option redispatch 1 + retries 3 + retry-on conn-failure empty-response + timeout connect 1s + timeout client 20s + timeout server 15s + timeout http-request 10s + timeout queue 5s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr none on-marked-down shutdown-sessions + server-template api 1-8 api:8080 diff --git a/scripts/mksecrets.sh b/scripts/mksecrets.sh new file mode 100755 index 0000000..7da59df --- /dev/null +++ b/scripts/mksecrets.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +mkdir -p "$ROOT/secrets" +cd "$ROOT/secrets" +gen() { + [ -f "$1" ] || openssl rand -hex "$2" | tr -d '\n' > "$1" +} +if [ ! -f postgres_password ]; then + openssl rand -base64 24 | tr -d '\n' > postgres_password +fi +PASS=$(cat postgres_password) +ENC=$(printf '%s' "$PASS" | python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read(), safe=""))') +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$ENC" > database_url +printf 'redis://valkey:6379/0\n' > cache_url +gen s3_admin_access_key_id 8 +gen s3_admin_secret_access_key 20 +gen s3_access_key_id 8 +gen s3_secret_access_key 20 +BUCKET=$(grep -E '^S3_BUCKET=' "$ROOT/.env" 2>/dev/null | cut -d= -f2 || true) +BUCKET=${BUCKET:-messenger-attachments} +printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}\n' \ + "$(cat s3_admin_access_key_id)" "$(cat s3_admin_secret_access_key)" "$(cat s3_access_key_id)" "$(cat s3_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" > s3_config.json +chmod 644 ./* +echo "secrets ready in $ROOT/secrets" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..4c5c492 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +docker compose --env-file .env restart postgres valkey s3 +docker compose --env-file .env up -d --wait postgres valkey s3 +docker compose --env-file .env restart api lb janitor diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..60805ff --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +dc="docker compose --env-file .env" +old=$($dc ps -q api) +n=$(echo "$old" | wc -w) +$dc build migrate +$dc up -d --no-deps migrate janitor +$dc up -d --no-deps --no-recreate --scale api=$((n * 2)) api +$dc up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api +for c in $old; do + docker stop -t 10 "$c" >/dev/null + docker rm "$c" >/dev/null +done +$dc up -d --wait --no-deps --no-recreate --scale api=$n api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..1283ec1 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +id=$(docker compose --env-file .env ps -q api | while read -r c; do + case "$c" in "$TARGET_INSTANCE_ID"*) echo "$c";; esac +done) +[ -n "$id" ] || exit 1 +docker stop -t 10 "$id" >/dev/null