From c5e3ac3affc7edf0ced008194c3919d7d9635e49 Mon Sep 17 00:00:00 2001 From: Dmitrii Grishko Date: Sat, 14 Nov 2026 20:08:27 +0300 Subject: [PATCH] feat: complete lab 7 grade 3 security controls --- Makefile | 19 +- compose.yml | 249 +++++++++++++++--- contracts/trust-matrix.md | 32 +-- evidence/grade3-argon2.md | 83 ++++++ evidence/grade3-dependencies.md | 35 +++ evidence/grade3-expiry.md | 67 +++++ evidence/grade3-http-security.md | 11 + evidence/grade3-https.md | 23 ++ evidence/grade3-login.md | 21 ++ evidence/grade3-mtls.md | 12 + evidence/grade3-openbao.md | 14 + evidence/grade3-other-replica.md | 47 ++++ evidence/grade3-resilience.md | 139 ++++++++++ evidence/grade3-runtime.md | 11 + evidence/grade3-signals.md | 13 + infra/openbao/server.hcl | 9 + infra/otel/collector.yaml | 20 +- .../provisioning/datasources/messenger.yaml | 19 +- infra/otel/loki.yaml | 5 + infra/otel/prometheus-web.yaml | 5 + infra/otel/prometheus.yaml | 5 + infra/otel/tempo.yaml | 9 + infra/postgres-hba.conf | 2 + infra/postgres-roles.sql | 33 +++ infra/rabbitmq/bootstrap.sh | 15 ++ infra/rabbitmq/rabbitmq.conf | 10 + scripts/bao_operator.py | 55 ++++ scripts/bootstrap_tls.sh | 27 ++ scripts/check_bao_roles.py | 45 ++++ scripts/check_dependency_roles.sh | 26 ++ scripts/check_grade3_http.py | 46 ++++ scripts/check_grade3_runtime.py | 36 +++ scripts/check_grade3_signals.py | 61 +++++ scripts/check_grade3_tls.py | 67 +++++ scripts/check_other_replica.sh | 10 + scripts/check_session_expiry.py | 33 +++ scripts/collect_grade3.py | 101 +++++++ scripts/configure_bao.py | 68 +++++ scripts/init_env.py | 87 +++--- scripts/provision_dependencies.sh | 13 + scripts/restart.sh | 8 + scripts/stop-one.sh | 3 + security/THREAT_MODEL.md | 16 +- src/backend/commands/bao.go | 111 ++++++++ src/backend/commands/run.go | 5 + src/backend/repositories/redis.go | 10 +- src/backend/repositories/s3_admin.go | 65 +++-- src/backend/tests/test_fixture_test.go | 9 +- src/backend/workers/pipeline.go | 9 +- 49 files changed, 1670 insertions(+), 149 deletions(-) create mode 100644 evidence/grade3-argon2.md create mode 100644 evidence/grade3-dependencies.md create mode 100644 evidence/grade3-expiry.md create mode 100644 evidence/grade3-http-security.md create mode 100644 evidence/grade3-https.md create mode 100644 evidence/grade3-login.md create mode 100644 evidence/grade3-mtls.md create mode 100644 evidence/grade3-openbao.md create mode 100644 evidence/grade3-other-replica.md create mode 100644 evidence/grade3-resilience.md create mode 100644 evidence/grade3-runtime.md create mode 100644 evidence/grade3-signals.md create mode 100644 infra/openbao/server.hcl create mode 100644 infra/otel/prometheus-web.yaml create mode 100644 infra/postgres-hba.conf create mode 100644 infra/postgres-roles.sql create mode 100644 infra/rabbitmq/bootstrap.sh create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 scripts/bao_operator.py create mode 100644 scripts/check_bao_roles.py create mode 100755 scripts/check_dependency_roles.sh create mode 100644 scripts/check_grade3_http.py create mode 100644 scripts/check_grade3_runtime.py create mode 100644 scripts/check_grade3_signals.py create mode 100644 scripts/check_grade3_tls.py create mode 100755 scripts/check_other_replica.sh create mode 100644 scripts/check_session_expiry.py create mode 100644 scripts/collect_grade3.py create mode 100644 scripts/configure_bao.py create mode 100755 scripts/provision_dependencies.sh create mode 100644 src/backend/commands/bao.go diff --git a/Makefile b/Makefile index 5541ad6..b6a1a0f 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,13 @@ COMPOSE = docker compose --project-name highload-lab7 --env-file .env -f compose .PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status .PHONY: app-migrate app-s3-bootstrap worker-start worker-stop queue-status attachments-cleanup .PHONY: jobs-failed jobs-replay jobs-upgrade jobs-cleanup-staging -.PHONY: test-backend test-pipeline +.PHONY: test-backend test-pipeline app-provision .PHONY: help check validate test load test-persistence test-failover test-security help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "app-provision: prepare dependencies and OpenBao before first startup" @echo "app-up: build and start the stand; app-status: inspect it; app-down: stop it" check: @@ -51,15 +52,19 @@ app-init: $(PYTHON) scripts/init_env.py ./scripts/bootstrap_tls.sh +app-provision: app-build + $(COMPOSE) up -d openbao + ./scripts/provision_dependencies.sh + app-build: app-init $(COMPOSE) build app-up: app-init - $(COMPOSE) up -d --build --wait + $(COMPOSE) up -d --build --wait --wait-timeout 120 ./scripts/wait_ingress.sh app-start: app-init - $(COMPOSE) up -d --no-build --wait + $(COMPOSE) up -d --no-build --wait --wait-timeout 120 ./scripts/wait_ingress.sh app-stop: @@ -96,19 +101,19 @@ attachments-cleanup: $(COMPOSE) run --rm --no-deps upload-cleanup ./messenger attachments-cleanup $(CLEANUP_ARGS) jobs-failed: - $(COMPOSE) exec -T dispatcher ./messenger jobs list-failed + $(COMPOSE) run --rm -T job-admin ./messenger jobs list-failed jobs-replay: @test -n "$$JOB_ID" && test -n "$$REASON" || { echo "Set JOB_ID=... REASON='...'"; exit 2; } - $(COMPOSE) exec -T dispatcher ./messenger jobs replay "$$JOB_ID" "$$REASON" + $(COMPOSE) run --rm -T job-admin ./messenger jobs replay "$$JOB_ID" "$$REASON" jobs-upgrade: @test -n "$$ATTACHMENT_ID" && test -n "$$REASON" || { echo "Set ATTACHMENT_ID=... REASON='...'"; exit 2; } - $(COMPOSE) exec -T dispatcher ./messenger jobs upgrade "$$ATTACHMENT_ID" 2 "$$REASON" + $(COMPOSE) run --rm -T job-admin ./messenger jobs upgrade "$$ATTACHMENT_ID" 2 "$$REASON" jobs-cleanup-staging: @test -n "$$JOB_ID" || { echo "Set JOB_ID=..."; exit 2; } - $(COMPOSE) exec -T dispatcher ./messenger jobs cleanup-staging "$$JOB_ID" + $(COMPOSE) run --rm -T job-admin ./messenger jobs cleanup-staging "$$JOB_ID" test-backend: app-init $(COMPOSE) --profile test run --rm --build backend-tests diff --git a/compose.yml b/compose.yml index dd8f66c..308e9dd 100644 --- a/compose.yml +++ b/compose.yml @@ -1,3 +1,8 @@ +x-bao-environment: &bao-environment + BAO_ADDR: https://openbao:8200 + SSL_CERT_FILE: /run/tls/ca.crt +x-secret-tmpfs: &secret-tmpfs /run/runtime-secrets:rw,noexec,nosuid,size=1m,mode=1777 + name: highload-lab7 services: @@ -9,6 +14,13 @@ services: command: [ "redis-server", + "--port", "0", + "--tls-port", "6379", + "--tls-cert-file", "/run/tls/redis.crt", + "--tls-key-file", "/run/tls/redis.key", + "--tls-ca-cert-file", "/run/tls/ca.crt", + "--tls-auth-clients", "no", + "--aclfile", "/run/secrets/redis_acl", "--appendonly", "yes", "--appendfsync", @@ -18,8 +30,13 @@ services: ] volumes: - redis_data:/data + - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro + - ./secrets/tls/redis.crt:/run/tls/redis.crt:ro + - ./secrets/tls/redis.key:/run/tls/redis.key:ro + - ./secrets/redis_acl:/run/secrets/redis_acl:ro + - ./secrets/redis_password:/run/secrets/redis_password:ro healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: ["CMD-SHELL", "REDISCLI_AUTH=$$(cat /run/secrets/redis_password) redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger ping | grep -qx PONG"] interval: 3s timeout: 3s retries: 20 @@ -34,11 +51,36 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} volumes: - postgres_data:/var/lib/postgresql/data + - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro + - ./secrets/tls/db.crt:/run/tls/db.crt:ro + - ./secrets/tls/db.key:/run/tls/db.key:ro + - ./secrets/database_api_password:/run/secrets/database_api_password:ro + - ./secrets/database_worker_password:/run/secrets/database_worker_password:ro + - ./secrets/database_dispatcher_password:/run/secrets/database_dispatcher_password:ro + - ./secrets/database_cleanup_password:/run/secrets/database_cleanup_password:ro + - ./infra/postgres-hba.conf:/run/tls/pg_hba.conf:ro + entrypoint: ["/bin/sh", "-ec", "install -o postgres -g postgres -m 600 /run/tls/db.key /tmp/db.key; exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/run/tls/db.crt -c ssl_key_file=/tmp/db.key -c hba_file=/run/tls/pg_hba.conf"] healthcheck: test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] interval: 3s timeout: 3s retries: 20 + openbao: + image: openbao/openbao:2.4.4 + user: openbao + networks: [bao, edge] + ports: ["127.0.0.1:8200:8200"] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + mem_limit: 256m + cpus: 0.5 + entrypoint: ["bao"] + command: ["server", "-config=/openbao/config/server.hcl"] + volumes: + - openbao_data:/openbao/file + - ./infra/openbao/server.hcl:/openbao/config/server.hcl:ro + - ./secrets/tls/openbao.crt:/run/tls/bao.crt:ro + - ./secrets/tls/openbao.key:/run/tls/bao.key:ro migrate: cpus: 0.5 mem_limit: 128m @@ -46,9 +88,9 @@ services: build: ./src environment: DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url} - DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable volumes: - - ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro + - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro + - ./secrets/database_tls_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro command: ["./messenger", "migrate", "up"] depends_on: db: @@ -69,10 +111,14 @@ services: MINIO_API_CORS_ALLOW_ORIGIN: ${S3_CORS_ORIGINS:-http://127.0.0.1:5173,http://localhost:5173} volumes: - s3_data:/data + - ./secrets/tls/s3.crt:/run/minio-certs/public.crt:ro + - ./secrets/tls/s3.key:/run/minio-certs/private.key:ro + - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro - ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro - ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro + command: ["server", "/data", "--certs-dir", "/run/minio-certs", "--console-address", "127.0.0.1:9001"] healthcheck: - test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/ready"] + test: ["CMD", "curl", "--cacert", "/run/tls/ca.crt", "-f", "https://s3:9000/minio/health/ready"] interval: 3s timeout: 2s retries: 30 @@ -82,15 +128,29 @@ services: networks: [data] build: ./src environment: &s3-environment + SSL_CERT_FILE: /run/tls/ca.crt + S3_CLEANUP_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_cleanup_secret_access_key + S3_CLEANUP_ACCESS_KEY_ID_FILE: /run/secrets/s3_cleanup_access_key_id + S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key + S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id + S3_API_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + S3_API_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key - S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} - S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:9000} + S3_ENDPOINT: https://s3:9000 + S3_PUBLIC_ENDPOINT: https://localhost:9000 S3_REGION: ${S3_REGION:-us-east-1} S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id} S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key} volumes: + - ./secrets/s3_cleanup_secret_access_key:/run/secrets/s3_cleanup_secret_access_key:ro + - ./secrets/s3_cleanup_access_key_id:/run/secrets/s3_cleanup_access_key_id:ro + - ./secrets/s3_worker_secret_access_key:/run/secrets/s3_worker_secret_access_key:ro + - ./secrets/s3_worker_access_key_id:/run/secrets/s3_worker_access_key_id:ro + - ./secrets/s3_api_secret_access_key:/run/secrets/s3_api_secret_access_key:ro + - ./secrets/s3_api_access_key_id:/run/secrets/s3_api_access_key_id:ro + - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro - ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro - ./secrets/s3_secret_access_key:/run/secrets/s3_root_secret_access_key:ro - ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro @@ -100,19 +160,23 @@ services: s3: condition: service_healthy api: + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + read_only: true cpus: 0.5 mem_limit: 512m - networks: [proxy, data, telemetry] + networks: [proxy, data, ingest, bao] build: ./src deploy: replicas: ${API_REPLICAS:-2} stop_grace_period: 15s environment: &api-environment - <<: *s3-environment - OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT-http://otel-collector:4318} + <<: [*s3-environment, *bao-environment] + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/otel/client.crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/otel/client.key S3_ROOT_ACCESS_KEY_ID_FILE: "" S3_ROOT_SECRET_ACCESS_KEY_FILE: "" - BROKER_URL_FILE: /run/secrets/broker_url JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000} @@ -121,6 +185,7 @@ services: # Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit. GOMAXPROCS: ${API_GOMAXPROCS:-1} GOMEMLIMIT: 384MiB + BAO_ROLE: api TLS_CA_FILE: /run/tls/ca.crt TLS_CERT_FILE: /run/tls/api.crt TLS_KEY_FILE: /run/tls/api.key @@ -132,18 +197,17 @@ services: CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} REDIS_ADDR: redis:6379 - DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable tmpfs: + - *secret-tmpfs - /tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777 volumes: - - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro + - &ca ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro - ./secrets/tls/api.crt:/run/tls/api.crt:ro - ./secrets/tls/api.key:/run/tls/api.key:ro - - &s3-api-id ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro - - &s3-api-key ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro - - &broker-url ./secrets/broker_url:/run/secrets/broker_url:ro - - &db-url ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro - - &cache-url ./secrets/cache_url:${CACHE_URL_FILE:-/run/secrets/cache_url}:ro + - ./secrets/tls/bao-api.crt:/run/bao/client.crt:ro + - ./secrets/tls/bao-api.key:/run/bao/client.key:ro + - ./secrets/tls/otel-api.crt:/run/otel/client.crt:ro + - ./secrets/tls/otel-api.key:/run/otel/client.key:ro depends_on: s3-bootstrap: condition: service_completed_successfully @@ -172,25 +236,45 @@ services: export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)" exec docker-entrypoint.sh rabbitmq-server volumes: + - ./infra/rabbitmq/bootstrap.sh:/bootstrap.sh:ro + - ./secrets/broker_dispatcher_password:/run/secrets/broker_dispatcher_password:ro + - ./secrets/broker_worker_password:/run/secrets/broker_worker_password:ro - rabbitmq_data:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + - ./secrets/tls/lab7-ca.crt:/run/tls/ca.crt:ro + - ./secrets/tls/broker.crt:/run/tls/broker.crt:ro + - ./secrets/tls/broker.key:/run/tls/broker.key:ro - ./secrets/broker_user:/run/secrets/broker_user:ro - ./secrets/broker_password:/run/secrets/broker_password:ro healthcheck: - test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_port_connectivity"] + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running"] interval: 5s timeout: 10s start_period: 20s retries: 30 worker: + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + read_only: true tmpfs: + - *secret-tmpfs - /tmp:rw,noexec,nosuid,size=64m,mode=1777 build: ./src restart: unless-stopped cpus: 1.0 mem_limit: 512m - networks: [data, telemetry] - environment: *api-environment - volumes: [*s3-api-id, *s3-api-key, *broker-url, *db-url, *cache-url] + networks: [data, ingest, bao] + environment: + <<: *api-environment + BAO_ROLE: worker + BROKER_URL_FILE: /run/secrets/broker_url + CACHE_URL_FILE: "" + volumes: + - *ca + - ./secrets/tls/bao-worker.crt:/run/bao/client.crt:ro + - ./secrets/tls/bao-worker.key:/run/bao/client.key:ro + - ./secrets/tls/otel-worker.crt:/run/otel/client.crt:ro + - ./secrets/tls/otel-worker.key:/run/otel/client.key:ro command: ["./messenger", "worker"] healthcheck: disable: true @@ -202,13 +286,28 @@ services: broker: condition: service_healthy dispatcher: + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + read_only: true + tmpfs: [*secret-tmpfs] build: ./src restart: unless-stopped cpus: 0.25 mem_limit: 128m - networks: [data, telemetry] - environment: *api-environment - volumes: [*s3-api-id, *s3-api-key, *broker-url, *db-url, *cache-url] + networks: [data, ingest, bao] + environment: + <<: *api-environment + BAO_ROLE: dispatcher + BROKER_URL_FILE: /run/secrets/broker_url + CACHE_URL_FILE: "" + S3_ACCESS_KEY_ID_FILE: "" + S3_SECRET_ACCESS_KEY_FILE: "" + volumes: + - *ca + - ./secrets/tls/bao-dispatcher.crt:/run/bao/client.crt:ro + - ./secrets/tls/bao-dispatcher.key:/run/bao/client.key:ro + - ./secrets/tls/otel-dispatcher.crt:/run/otel/client.crt:ro + - ./secrets/tls/otel-dispatcher.key:/run/otel/client.key:ro command: ["./messenger", "dispatcher"] healthcheck: disable: true @@ -218,21 +317,26 @@ services: broker: condition: service_healthy upload-cleanup: + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + read_only: true + tmpfs: [*secret-tmpfs] restart: unless-stopped cpus: 0.25 mem_limit: 128m - networks: [data] + networks: [data, bao] build: ./src command: ["./messenger", "attachments-cleanup", "--loop"] environment: - <<: *s3-environment + <<: [*s3-environment, *bao-environment] + BAO_ROLE: cleanup S3_ROOT_ACCESS_KEY_ID_FILE: "" S3_ROOT_SECRET_ACCESS_KEY_FILE: "" DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url} volumes: - - ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro - - ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro - - ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro + - *ca + - ./secrets/tls/bao-cleanup.crt:/run/bao/client.crt:ro + - ./secrets/tls/bao-cleanup.key:/run/bao/client.key:ro healthcheck: disable: true depends_on: @@ -258,6 +362,24 @@ services: depends_on: api: condition: service_started + job-admin: + profiles: ["operator"] + build: ./src + networks: [data] + cpus: 0.25 + mem_limit: 128m + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: + <<: *s3-environment + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + volumes: + - *ca + - ./secrets/database_tls_url:/run/secrets/database_url:ro + - ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro + - ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro backend-tests: cpus: 2.0 mem_limit: 1g @@ -268,10 +390,13 @@ services: target: builder environment: <<: *s3-environment - S3_PUBLIC_ENDPOINT: http://s3:9000 - TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable + S3_PUBLIC_ENDPOINT: https://s3:9000 + TEST_DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt TEST_REDIS_ADDR: redis:6379 + CACHE_URL_FILE: /run/secrets/cache_url volumes: + - *ca + - ./secrets/cache_tls_url:/run/secrets/cache_url:ro - ./secrets/s3_api_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro - ./secrets/s3_api_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro - ./secrets/s3_access_key_id:/run/secrets/s3_root_access_key_id:ro @@ -285,52 +410,102 @@ services: condition: service_healthy command: ["go", "test", "-race", "-count=1", "-v", "./backend/..."] otel-collector: + cpus: 0.5 image: otel/opentelemetry-collector-contrib:0.120.0 - networks: [telemetry] + networks: [telemetry, ingest] mem_limit: 192m command: ["--config=/etc/otelcol-contrib/config.yaml"] volumes: + - ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro + - ./secrets/tls/collector-exporter.crt:/run/otel/client.crt:ro + - ./secrets/tls/collector-exporter.key:/run/otel/client.key:ro + - *ca + - ./secrets/tls/otel-collector.crt:/run/otel/server.crt:ro + - ./secrets/tls/otel-collector.key:/run/otel/server.key:ro + - ./secrets/tls/telemetry-ingest-ca.crt:/run/otel/ingest-ca.crt:ro - ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro prometheus: + cpus: 0.5 + mem_limit: 384m image: prom/prometheus:v3.2.1 networks: [telemetry] - command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"] + command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d", "--web.config.file=/etc/prometheus/web.yaml"] volumes: + - ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro + - ./secrets/tls/prometheus-scraper.crt:/run/otel/client.crt:ro + - ./secrets/tls/prometheus-scraper.key:/run/otel/client.key:ro + - *ca + - ./secrets/tls/prometheus.crt:/run/otel/server.crt:ro + - ./secrets/tls/prometheus.key:/run/otel/server.key:ro + - ./infra/otel/prometheus-web.yaml:/etc/prometheus/web.yaml:ro - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro - ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro - prometheus_data:/prometheus loki: + cpus: 0.5 + mem_limit: 384m image: grafana/loki:3.4.2 networks: [telemetry] command: ["-config.file=/etc/loki/lab.yaml"] volumes: + - ./secrets/tls/loki-client-ca.crt:/run/otel/loki-client-ca.crt:ro + - *ca + - ./secrets/tls/loki.crt:/run/otel/server.crt:ro + - ./secrets/tls/loki.key:/run/otel/server.key:ro - ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro - loki_data:/loki tempo: + cpus: 0.5 + mem_limit: 384m image: grafana/tempo:2.7.1 networks: [telemetry] command: ["-config.file=/etc/tempo/lab.yaml"] volumes: + - ./secrets/tls/telemetry-read-ca.crt:/run/otel/read-ca.crt:ro + - ./secrets/tls/telemetry-export-ca.crt:/run/otel/export-ca.crt:ro + - *ca + - ./secrets/tls/tempo.crt:/run/otel/server.crt:ro + - ./secrets/tls/tempo.key:/run/otel/server.key:ro - ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro - tempo_data:/var/tempo grafana: + entrypoint: + - /bin/sh + - -ec + - | + export GRAFANA_READER_CERT="$$(cat /run/otel/client.crt)" + export GRAFANA_READER_KEY="$$(cat /run/otel/client.key)" + exec /run.sh + cpus: 0.5 + mem_limit: 384m image: grafana/grafana:11.5.2 networks: [telemetry, edge] environment: - GF_AUTH_ANONYMOUS_ENABLED: "true" - GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + SSL_CERT_FILE: /run/tls/ca.crt + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /run/otel/server.crt + GF_SERVER_CERT_KEY: /run/otel/server.key + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password ports: - "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000" volumes: + - ./secrets/tls/grafana-reader.crt:/run/otel/client.crt:ro + - ./secrets/tls/grafana-reader.key:/run/otel/client.key:ro + - *ca + - ./secrets/tls/grafana.crt:/run/otel/server.crt:ro + - ./secrets/tls/grafana.key:/run/otel/server.key:ro - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro + - ./secrets/grafana_admin_password:/run/secrets/grafana_admin_password:ro - grafana_data:/var/lib/grafana volumes: + openbao_data: prometheus_data: loki_data: tempo_data: @@ -341,6 +516,8 @@ volumes: s3_data: networks: + bao: + internal: true edge: proxy: internal: true @@ -348,3 +525,5 @@ networks: internal: true telemetry: internal: true + ingest: + internal: true diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 00b3b59..36048bd 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,24 +1,24 @@ # Матрица доверия -Адреса и порты приведены для текущего `compose.yml` с настройками по умолчанию. Столбец «Цель» описывает конфигурацию, которую предстоит реализовать. Для новых серверных сертификатов планируется локальный CA и DNS SAN имени сервиса. Сроки и механизмы выдачи зависимых credentials уточняются при реализации. Текущие пароли зависимостей статические и создаются локально. +Все сетевые серверы предъявляют leaf со своим DNS SAN, подписанный локальным `lab7-ca`. Leaf действуют 30 дней, CA — 365 дней. Для доступа с хоста сертификаты HAProxy, MinIO, Grafana и OpenBao также содержат `localhost` и `127.0.0.1`. Пароли PostgreSQL, Redis, S3 и RabbitMQ статические, до смены оператором. Параметры действующего стенда приведены ниже. -| Связь | Текущее состояние | Цель и способ проверки | +| Связь | TLS, identity и права | Проверка | | --- | --- | --- | -| Клиент → HAProxy | HTTPS `localhost:8443`, локальный `lab7-ca`, SAN `localhost` и `127.0.0.1`, leaf 30 дней. Bearer-сессия по умолчанию 3600 с. | Выполнены `make test test-security CA_FILE=secrets/tls/lab7-ca.crt` и `python3 scripts/check_login_rate.py`. Дополнить отказом при неверном имени и запросом после истечения сессии. | -| HAProxy → API | mTLS `api:8080` на обеих репликах. HAProxy проверяет CA и имя `api`, API — цепочку клиента и URI SAN `urn:lab7:load-balancer`. Leaf 30 дней. Прямой порт API не опубликован. | Выполнен `python3 scripts/check_ingress_mtls.py`: верный LB принят, отсутствие cert, чужой CA и чужой SAN отклонены. На «4» проверить замену leaf и отказ для отозванной identity на новых соединениях. | -| API, worker, dispatcher, cleanup и migrate → PostgreSQL | `db:5432`, plaintext, общая роль. | `sslmode=verify-full`, отдельный владелец схемы для миграций и роли по фактическим операциям. Проверить вход с правильным CA, отказ с чужим CA и запрет DDL от runtime-роли. | -| API, worker и служебные задачи → S3 | HTTP `s3:9000`. У bootstrap есть root, приложения делят одну ограниченную identity. | HTTPS и отдельные политики для staging, originals и variants по операциям каждой роли. Проверить штатную загрузку, чтение нужного объекта и `AccessDenied` на чужой префикс. | -| Dispatcher и worker → RabbitMQ | AMQP `broker:5672` с общей учётной записью. | AMQPS с проверкой `broker`, разрешения producer и consumer раздельны, настройка topology вынесена в bootstrap. Проверить доставку и запрет публикации от consumer. На «4» добавить проверку клиентского SAN. | -| API, worker и dispatcher → OTel Collector | HTTP OTLP `otel-collector:4318` в сети `telemetry`. | HTTPS с проверкой имени и авторизация ingest. Проверить поступление сигналов и отказ при запросе без разрешённой identity. На «4» — mTLS с разными identities. | -| Collector → Loki/Tempo и Prometheus → Collector | HTTP `loki:3100/otlp`, `tempo:4318`; Prometheus забирает метрики с `otel-collector:8889`. | TLS и разрешения на ingest и scrape по роли. Проверить поступление логов, трейсов и метрик, отказ с чужим CA или credentials и запрет запросов на чтение от ingest-роли. | -| Grafana → источники | HTTP `prometheus:9090`, `loki:3100`, `tempo:3200`, в сети `telemetry`. | HTTPS и identities только для чтения. Проверить запросы из Grafana и отказ при попытке записи. | -| API/worker и фоновые роли → OpenBao | Сервис отсутствует. | `https://openbao:8200`, cert auth, persistent storage, отдельные KV-пути и токены с плановым TTL 15 минут. Проверить чтение своего пути и запрет чужого, записи и изменения политик. Проверить новый вход при sealed OpenBao. | -| Оператор → OpenBao, Grafana, S3 | OpenBao отсутствует. Grafana опубликована на `127.0.0.1:3000` по HTTP с anonymous viewer. S3 API опубликован на `127.0.0.1:9000` по HTTP. | Локальный административный доступ по HTTPS с отдельными credentials. Убрать анонимный доступ. Проверить отказ без административной роли и ограничения опубликованных портов. | -| API → Redis | `redis:6379` без TLS и пароля, хранение сессий и счётчиков входа. | TLS и ACL для ключей сессий и ограничения входа. Проверить вход и logout, отказ с чужим CA и запрет административных команд от прикладной роли. | -| Replication/кластерные каналы | Не применимо. API-реплики не обмениваются данными напрямую, экземпляры хранилищ одиночные. | Проверять восстановление при остановке каждой API-реплики по HTTPS. | +| Клиент → HAProxy | HTTPS `localhost:8443` с проверкой CA и SAN. Bearer-сессия по умолчанию 3600 с. | `make test test-security CA_FILE=secrets/tls/lab7-ca.crt`, `scripts/check_grade3_http.py`, `scripts/check_login_rate.py` и `scripts/check_session_expiry.py`. | +| HAProxy → API | mTLS `api:8080`. HAProxy проверяет CA и `api`. API проверяет clientAuth и URI SAN `urn:lab7:load-balancer`. Порт API не опубликован. | `python3 scripts/check_ingress_mtls.py` проверяет разрешённый LB и три отказа на каждой реплике. | +| API, worker, dispatcher, cleanup и migrate → PostgreSQL | `db:5432`, `sslmode=verify-full`, hostssl + SCRAM. Владелец запускает миграции, остальные используют отдельные login-роли и права на таблицы и последовательности из `infra/postgres-roles.sql`. | TLS probe с правильным и чужим CA, HTTPS smoke проверяет работу ролей API и worker. | +| API, worker и cleanup → S3 | `https://s3:9000`, приватный bucket. У bootstrap root. API принимает staging и читает опубликованные файлы, worker читает staging и публикует оригиналы и варианты, cleanup удаляет по разрешённым префиксам. Права находятся в `s3_admin.go`. Direct upload использует подписанную форму через локальный `https://localhost:9000`. | TLS probe и smoke с побайтной сверкой оригинала и thumbnail. | +| Dispatcher и worker → RabbitMQ | AMQPS `broker:5671`. Dispatcher публикует через default exchange, worker читает `attachments.v1`. Обе роли могут объявлять только эту durable очередь. API пишет намерение в БД. | TLS probe и smoke с переходом queued → ready. `infra/rabbitmq/bootstrap.sh` задаёт ACL. | +| API, worker и dispatcher → OTel Collector | HTTPS `otel-collector:4318`. Три client cert от выделенного ingest CA, который используется только для приложения. | TLS probe и проверка доставки свежих сигналов. | +| Collector → Loki/Tempo, Prometheus → Collector | mTLS `loki:3100/otlp`, `tempo:4318`, scrape `otel-collector:8889`. Клиенты имеют сертификаты отдельного export CA. | `scripts/check_grade3_tls.py` и `scripts/check_grade3_signals.py`. | +| Grafana → источники | mTLS `prometheus:9090`, `loki:3100`, `tempo:3200`. Grafana имеет ключ reader CA. Loki принимает reader и exporter CA на общем HTTP listener, поэтому граница контроля там ограничена этими двумя группами. | TLS probe с reader cert и запросы к каждому хранилищу в `scripts/check_grade3_signals.py`. | +| API, worker, dispatcher, cleanup → OpenBao | `https://openbao:8200`. Cert auth закрепляет отдельный leaf каждой роли. Токен до 900 с читает только `messenger/data/`. Сервисы получают credentials при запуске и держат их на tmpfs. | `python3 scripts/check_bao_roles.py`: свой путь разрешён, чужое чтение, чужой вход и запись запрещены. | +| Оператор → OpenBao, Grafana и S3 | Локальные HTTPS-порты 8200, 3000, 9000. OpenBao — операторский токен, Grafana — login, MinIO — отдельные root credentials для bootstrap. | Проверка публикаций контейнеров и TLS. | +| API → Redis | `rediss://redis:6379`, имя сервера проверяется. Отдельный пользователь messenger ограничен ключами `messenger:*` и операциями сессий и счётчиков. Plaintext listener отключён. | TLS probe, smoke, проверка expiry и общего ограничения входа. | +| Replication/кластерные каналы | Не применимо. Реплики API не обмениваются данными напрямую, хранилища одиночные. | Проверка восстановления после остановки API. | ## Health, scrape и bootstrap -HAProxy проверяет `GET /health/ready` каждой реплики через её mTLS listener. Скрипт `scripts/wait_ingress.sh` проверяет готовность через HTTPS с доверенным CA. Служебные проверки Postgres и RabbitMQ запускаются внутри контейнеров, Redis проверяется локальным CLI, MinIO — локальным HTTP-запросом. При переводе зависимостей на TLS проверки должны сохранять те же границы. Связь Prometheus с Collector учитывается отдельно в строке про scrape. +HAProxy проверяет `GET /health/ready` каждой реплики через её mTLS listener. Скрипт `scripts/wait_ingress.sh` проверяет готовность через HTTPS с доверенным CA. Служебные проверки Postgres и RabbitMQ запускаются внутри контейнеров, Redis проверяется локальным CLI, MinIO — HTTPS-запросом с проверкой CA. Redis healthcheck также использует TLS и ACL. Связь Prometheus с Collector учитывается отдельно в строке про scrape. -Bootstrap миграций и S3 выполняется одноразовыми контейнерами. Запланировано использовать для них отдельные учётные данные и TLS с проверкой сервера. Прикладные контейнеры должны получать только свои scoped credentials. Закрытые ключи, unseal shares и административные токены остаются у оператора вне Git и образов. Для отзыва и ротации на «4» проверяются новые соединения после смены credentials, включая отказ отозванной identity. +Bootstrap миграций и S3 выполняется одноразовыми контейнерами. Оператор запускает их с учётными данными владельцев и проверкой TLS. Прикладные контейнеры монтируют свои ключи cert auth и получают ограниченные credentials из OpenBao. Закрытые ключи, unseal shares и административные токены остаются у оператора вне Git и образов. Для отзыва и ротации на «4» проверяются новые соединения после смены credentials, включая отказ отозванной identity. diff --git a/evidence/grade3-argon2.md b/evidence/grade3-argon2.md new file mode 100644 index 0000000..a0a5a47 --- /dev/null +++ b/evidence/grade3-argon2.md @@ -0,0 +1,83 @@ +# Стоимость проверки хеша Argon2id на лимитах API + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`docker compose --profile test build backend-tests` + +Код завершения: 0. + +```text + Image highload-lab7-backend-tests Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.66kB done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 816B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/golang:1.22.2-bookworm +#3 DONE 1.1s + +#4 [internal] load .dockerignore +#4 transferring context: 327B done +#4 DONE 0.0s + +#5 [builder 1/7] FROM docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f +#5 resolve docker.io/library/golang:1.22.2-bookworm@sha256:d0902bacefdde1cf45528c098d14e55d78c107def8a22d148eabd71582d7a99f 0.0s done +#5 DONE 0.0s + +#6 [internal] load build context +#6 transferring context: 13.70kB done +#6 DONE 0.0s + +#7 [builder 4/7] COPY go.mod go.sum ./ +#7 CACHED + +#8 [builder 2/7] RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* +#8 CACHED + +#9 [builder 5/7] RUN go mod download +#9 CACHED + +#10 [builder 6/7] COPY backend ./backend +#10 CACHED + +#11 [builder 3/7] WORKDIR /src +#11 CACHED + +#12 [builder 7/7] RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./backend +#12 CACHED + +#13 exporting to image +#13 exporting layers +#13 exporting layers 6.7s done +#13 exporting manifest sha256:cc55619214f59b9b27ad25a2e23d1abce9f64566439be3b766526869522398f8 done +#13 exporting config sha256:60158d0274ef38fabce1b1b41aef1dce15861f92bfe46cdd75183cc143df20f5 done +#13 exporting attestation manifest sha256:facefdb8b90c9ec469b34788e90e8b0d663bf8c722e0e92d10cbc231bc2e6bc2 done +#13 exporting manifest list sha256:a0ef01b3efc503922fd45a76b39ec30f5d34f2085c9a228eeb3277e39df87eab done +#13 naming to docker.io/library/highload-lab7-backend-tests:latest done +#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest +#13 unpacking to docker.io/library/highload-lab7-backend-tests:latest 1.2s done +#13 DONE 7.9s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image highload-lab7-backend-tests Built +``` + +`docker run --rm --network none --cpus 0.5 --memory 512m -e GOMAXPROCS=1 highload-lab7-backend-tests go test ./backend/services -run '^$' -bench '^BenchmarkPasswordVerification$' -benchtime=3x -count=3` + +Код завершения: 0. + +```text +goos: linux +goarch: amd64 +pkg: messenger/src/backend/services +cpu: AMD Ryzen 9 PRO 7940HS w/ Radeon 780M Graphics +BenchmarkPasswordVerification 3 194744721 ns/op 67115832 B/op 74 allocs/op +BenchmarkPasswordVerification 3 176595711 ns/op 67115832 B/op 74 allocs/op +BenchmarkPasswordVerification 3 196042348 ns/op 67115832 B/op 74 allocs/op +PASS +ok messenger/src/backend/services 3.606s +``` diff --git a/evidence/grade3-dependencies.md b/evidence/grade3-dependencies.md new file mode 100644 index 0000000..1a862cf --- /dev/null +++ b/evidence/grade3-dependencies.md @@ -0,0 +1,35 @@ +# TLS хранилищ, брокера и телеметрии + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`python3 scripts/check_grade3_tls.py` + +Код завершения: 0. + +```text +PASS s3:9000: verified service SAN; foreign CA rejected +PASS broker:5671: verified service SAN; foreign CA rejected +PASS redis:6379: verified service SAN; foreign CA rejected +PASS db:5432: verified service SAN; foreign CA rejected +PASS otel-collector:4318: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS otel-collector:8889: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS prometheus:9090: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS loki:3100: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS tempo:4318: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS tempo:3200: verified service SAN; foreign CA rejected; foreign client identity rejected +PASS grafana:3000: verified service SAN; foreign CA rejected +PASS openbao:8200: verified service SAN; foreign CA rejected +``` + +`./scripts/check_dependency_roles.sh` + +Код завершения: 0. + +```text +PASS PostgreSQL: worker can update attachments; user table and schema creation denied +PASS RabbitMQ: producer can publish and consumer can read attachments.v1 +PASS Redis: application role cannot read server configuration +``` + +Итог: все команды завершились успешно. diff --git a/evidence/grade3-expiry.md b/evidence/grade3-expiry.md new file mode 100644 index 0000000..f0fd8c4 --- /dev/null +++ b/evidence/grade3-expiry.md @@ -0,0 +1,67 @@ +# Истечение сессии на стенде с коротким TTL + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`sleep 60` + +Код завершения: 0. + +```text + +``` + +`env SESSION_TTL_SECONDS=5 docker compose up -d --no-deps api` + +Код завершения: 0. + +```text + Container highload-lab7-api-1 Recreate + Container highload-lab7-api-2 Recreate + Container highload-lab7-api-1 Recreated + Container highload-lab7-api-2 Recreated + Container highload-lab7-api-1 Starting + Container highload-lab7-api-1 Started + Container highload-lab7-api-2 Starting + Container highload-lab7-api-2 Started +``` + +`./scripts/wait_ingress.sh` + +Код завершения: 0. + +```text + +``` + +`python3 scripts/check_session_expiry.py --ca-file secrets/tls/lab7-ca.crt` + +Код завершения: 0. + +```text +PASS: valid session accepted before expiry and rejected after expiry +``` + +`docker compose up -d --no-deps api` + +Код завершения: 0. + +```text + Container highload-lab7-api-1 Recreate + Container highload-lab7-api-2 Recreate + Container highload-lab7-api-2 Recreated + Container highload-lab7-api-1 Recreated + Container highload-lab7-api-1 Starting + Container highload-lab7-api-1 Started + Container highload-lab7-api-2 Starting + Container highload-lab7-api-2 Started +``` + +`./scripts/wait_ingress.sh` + +Код завершения: 0. + +```text + +``` + +Итог: все команды завершились успешно. diff --git a/evidence/grade3-http-security.md b/evidence/grade3-http-security.md new file mode 100644 index 0000000..d7a7747 --- /dev/null +++ b/evidence/grade3-http-security.md @@ -0,0 +1,11 @@ +# Лимиты, MIME, чужие вложения и неверный TLS hostname + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`python3 scripts/check_grade3_http.py --ca-file secrets/tls/lab7-ca.crt` + +Код завершения: 0. + +```text +PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname +``` diff --git a/evidence/grade3-https.md b/evidence/grade3-https.md new file mode 100644 index 0000000..b52535d --- /dev/null +++ b/evidence/grade3-https.md @@ -0,0 +1,23 @@ +# Бизнес-API, проверка CA, logout и вложения по HTTPS + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`make check` + +Код завершения: 0. + +```text +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +``` + +`make test test-security CA_FILE=secrets/tls/lab7-ca.crt` + +Код завершения: 0. + +```text +python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt" +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +python3 tests/security.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt" +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. +``` diff --git a/evidence/grade3-login.md b/evidence/grade3-login.md new file mode 100644 index 0000000..91b5b03 --- /dev/null +++ b/evidence/grade3-login.md @@ -0,0 +1,21 @@ +# Общий лимит входа по адресу клиента + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`sleep 60` + +Код завершения: 0. + +```text + +``` + +`python3 scripts/check_login_rate.py` + +Код завершения: 0. + +```text +PASS: request 13 returned 429 with Retry-After across 2 API replicas +``` + +Итог: все команды завершились успешно. diff --git a/evidence/grade3-mtls.md b/evidence/grade3-mtls.md new file mode 100644 index 0000000..d0e34dc --- /dev/null +++ b/evidence/grade3-mtls.md @@ -0,0 +1,12 @@ +# Допуск только роли LB на каждой реплике + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`python3 scripts/check_ingress_mtls.py` + +Код завершения: 0. + +```text +PASS d50f910b8693: LB permitted; absent, foreign CA and disallowed SAN rejected +PASS a96f9b981475: LB permitted; absent, foreign CA and disallowed SAN rejected +``` diff --git a/evidence/grade3-openbao.md b/evidence/grade3-openbao.md new file mode 100644 index 0000000..ff8b45a --- /dev/null +++ b/evidence/grade3-openbao.md @@ -0,0 +1,14 @@ +# Сервисная аутентификация и scoped policy OpenBao + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`python3 scripts/check_bao_roles.py` + +Код завершения: 0. + +```text +PASS api: scoped read, cross-role login/read and write denied, TTL <= 900 s +PASS worker: scoped read, cross-role login/read and write denied, TTL <= 900 s +PASS dispatcher: scoped read, cross-role login/read and write denied, TTL <= 900 s +PASS cleanup: scoped read, cross-role login/read and write denied, TTL <= 900 s +``` diff --git a/evidence/grade3-other-replica.md b/evidence/grade3-other-replica.md new file mode 100644 index 0000000..2adfc32 --- /dev/null +++ b/evidence/grade3-other-replica.md @@ -0,0 +1,47 @@ +# Работа HTTPS API при остановленной второй реплике + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`sleep 60` + +Код завершения: 0. + +```text + +``` + +`./scripts/check_other_replica.sh` + +Код завершения: 0. + +```text +Stopped API instance 50b06551b60d +python3 tests/smoke.py --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt" +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. + Container highload-lab7-redis-1 Waiting + Container highload-lab7-db-1 Waiting + Container highload-lab7-s3-1 Waiting + Container highload-lab7-s3-1 Healthy + Container highload-lab7-redis-1 Healthy + Container highload-lab7-s3-bootstrap-1 Starting + Container highload-lab7-db-1 Healthy + Container highload-lab7-migrate-1 Starting + Container highload-lab7-s3-bootstrap-1 Started + Container highload-lab7-s3-bootstrap-1 Waiting + Container highload-lab7-migrate-1 Started + Container highload-lab7-migrate-1 Waiting + Container highload-lab7-migrate-1 Exited + Container highload-lab7-s3-bootstrap-1 Exited + Container highload-lab7-api-2 Starting + Container highload-lab7-api-2 Started +``` + +`./scripts/wait_ingress.sh` + +Код завершения: 0. + +```text + +``` + +Итог: все команды завершились успешно. diff --git a/evidence/grade3-resilience.md b/evidence/grade3-resilience.md new file mode 100644 index 0000000..d2deafb --- /dev/null +++ b/evidence/grade3-resilience.md @@ -0,0 +1,139 @@ +# Сохранность после перезапуска хранилищ и отказ API + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`sleep 60` + +Код завершения: 0. + +```text + +``` + +`make test-persistence ACTION_SCRIPT=scripts/restart.sh CA_FILE=secrets/tls/lab7-ca.crt` + +Код завершения: 0. + +```text +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +`make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=secrets/tls/lab7-ca.crt` + +Код завершения: 0. + +```text +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 2, "first_recovery_seconds": 1.538} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +`docker compose start api` + +Код завершения: 0. + +```text + Container highload-lab7-s3-1 Waiting + Container highload-lab7-db-1 Waiting + Container highload-lab7-redis-1 Waiting + Container highload-lab7-s3-1 Healthy + Container highload-lab7-s3-bootstrap-1 Starting + Container highload-lab7-db-1 Healthy + Container highload-lab7-migrate-1 Starting + Container highload-lab7-redis-1 Healthy + Container highload-lab7-s3-bootstrap-1 Started + Container highload-lab7-s3-bootstrap-1 Waiting + Container highload-lab7-migrate-1 Started + Container highload-lab7-migrate-1 Waiting + Container highload-lab7-migrate-1 Exited + Container highload-lab7-s3-bootstrap-1 Exited + Container highload-lab7-api-1 Starting + Container highload-lab7-api-1 Started +``` + +`./scripts/wait_ingress.sh` + +Код завершения: 0. + +```text + +``` + +`sleep 60` + +Код завершения: 0. + +```text + +``` + +`make test-failover ACTION_SCRIPT=scripts/stop-one.sh CA_FILE=secrets/tls/lab7-ca.crt` + +Код завершения: 0. + +```text +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:8443" --ca-file "secrets/tls/lab7-ca.crt" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 2, "first_recovery_seconds": 1.658} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +``` + +`docker compose start api` + +Код завершения: 0. + +```text + Container highload-lab7-db-1 Waiting + Container highload-lab7-s3-1 Waiting + Container highload-lab7-redis-1 Waiting + Container highload-lab7-redis-1 Healthy + Container highload-lab7-s3-1 Healthy + Container highload-lab7-s3-bootstrap-1 Starting + Container highload-lab7-db-1 Healthy + Container highload-lab7-migrate-1 Starting + Container highload-lab7-s3-bootstrap-1 Started + Container highload-lab7-s3-bootstrap-1 Waiting + Container highload-lab7-migrate-1 Started + Container highload-lab7-migrate-1 Waiting + Container highload-lab7-migrate-1 Exited + Container highload-lab7-s3-bootstrap-1 Exited + Container highload-lab7-api-1 Starting + Container highload-lab7-api-1 Started +``` + +`./scripts/wait_ingress.sh` + +Код завершения: 0. + +```text + +``` + +Состояния контейнеров (restart): + +```text +/highload-lab7-api-1 2026-11-14T16:56:19.970448431Z +/highload-lab7-api-2 2026-11-14T16:56:20.135569481Z +/highload-lab7-broker-1 2026-11-14T16:50:40.248667479Z +/highload-lab7-db-1 2026-11-14T16:50:39.231870058Z +/highload-lab7-redis-1 2026-11-14T16:50:39.21876659Z +/highload-lab7-s3-1 2026-11-14T16:50:39.377104743Z +/highload-lab7-api-1 2026-11-14T16:57:33.129069552Z +/highload-lab7-api-2 2026-11-14T16:57:33.110951095Z +/highload-lab7-broker-1 2026-11-14T16:57:25.694563006Z +/highload-lab7-db-1 2026-11-14T16:57:26.301816228Z +/highload-lab7-redis-1 2026-11-14T16:57:24.787203563Z +/highload-lab7-s3-1 2026-11-14T16:57:24.914493761Z +``` + +Состояния контейнеров (failover): + +```text +/highload-lab7-api-1 exited +/highload-lab7-api-1 exited +``` + +Итог: все команды завершились успешно. diff --git a/evidence/grade3-runtime.md b/evidence/grade3-runtime.md new file mode 100644 index 0000000..f2bc46b --- /dev/null +++ b/evidence/grade3-runtime.md @@ -0,0 +1,11 @@ +# Изоляция, порты, лимиты и локальное размещение секретов + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`python3 scripts/check_grade3_runtime.py` + +Код завершения: 0. + +```text +PASS: local-only ports, runtime secrets on tmpfs, non-root app roles with limits and dropped capabilities +``` diff --git a/evidence/grade3-signals.md b/evidence/grade3-signals.md new file mode 100644 index 0000000..a90bf87 --- /dev/null +++ b/evidence/grade3-signals.md @@ -0,0 +1,13 @@ +# Доставка логов, метрик и трейсов по защищённым связям + +Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA. + +`python3 scripts/check_grade3_signals.py` + +Код завершения: 0. + +```text +PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS +``` + +Итог: все команды завершились успешно. diff --git a/infra/openbao/server.hcl b/infra/openbao/server.hcl new file mode 100644 index 0000000..579ed70 --- /dev/null +++ b/infra/openbao/server.hcl @@ -0,0 +1,9 @@ +api_addr = "https://openbao:8200" +storage "file" { + path = "/openbao/file" +} +listener "tcp" { + address = "0.0.0.0:8200" + tls_cert_file = "/run/tls/bao.crt" + tls_key_file = "/run/tls/bao.key" +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml index 4c64e0d..beef897 100644 --- a/infra/otel/collector.yaml +++ b/infra/otel/collector.yaml @@ -3,6 +3,10 @@ receivers: protocols: http: endpoint: 0.0.0.0:4318 + tls: + cert_file: /run/otel/server.crt + key_file: /run/otel/server.key + client_ca_file: /run/otel/ingest-ca.crt processors: memory_limiter: check_interval: 1s @@ -15,16 +19,28 @@ processors: exporters: prometheus: endpoint: 0.0.0.0:8889 + tls: + cert_file: /run/otel/server.crt + key_file: /run/otel/server.key + client_ca_file: /run/otel/export-ca.crt resource_to_telemetry_conversion: enabled: true otlphttp/loki: - endpoint: http://loki:3100/otlp + endpoint: https://loki:3100/otlp + tls: + ca_file: /run/tls/ca.crt + cert_file: /run/otel/client.crt + key_file: /run/otel/client.key sending_queue: queue_size: 64 retry_on_failure: max_elapsed_time: 30s otlphttp/tempo: - endpoint: http://tempo:4318 + endpoint: https://tempo:4318 + tls: + ca_file: /run/tls/ca.crt + cert_file: /run/otel/client.crt + key_file: /run/otel/client.key sending_queue: queue_size: 64 retry_on_failure: diff --git a/infra/otel/grafana/provisioning/datasources/messenger.yaml b/infra/otel/grafana/provisioning/datasources/messenger.yaml index aa2d1ee..bce23f0 100644 --- a/infra/otel/grafana/provisioning/datasources/messenger.yaml +++ b/infra/otel/grafana/provisioning/datasources/messenger.yaml @@ -4,14 +4,23 @@ datasources: uid: lab6-prometheus type: prometheus access: proxy - url: http://prometheus:9090 + url: https://prometheus:9090 isDefault: true + secureJsonData: + tlsClientCert: ${GRAFANA_READER_CERT} + tlsClientKey: ${GRAFANA_READER_KEY} + jsonData: + tlsAuth: true - name: Loki uid: lab6-loki type: loki access: proxy - url: http://loki:3100 + url: https://loki:3100 + secureJsonData: + tlsClientCert: ${GRAFANA_READER_CERT} + tlsClientKey: ${GRAFANA_READER_KEY} jsonData: + tlsAuth: true derivedFields: - name: Trace matcherType: label @@ -22,8 +31,12 @@ datasources: uid: lab6-tempo type: tempo access: proxy - url: http://tempo:3200 + url: https://tempo:3200 + secureJsonData: + tlsClientCert: ${GRAFANA_READER_CERT} + tlsClientKey: ${GRAFANA_READER_KEY} jsonData: + tlsAuth: true tracesToLogsV2: datasourceUid: lab6-loki spanStartTimeShift: '-5s' diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml index bf3a731..451dd4e 100644 --- a/infra/otel/loki.yaml +++ b/infra/otel/loki.yaml @@ -1,6 +1,11 @@ auth_enabled: false server: http_listen_port: 3100 + http_tls_config: + cert_file: /run/otel/server.crt + key_file: /run/otel/server.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/otel/loki-client-ca.crt common: path_prefix: /loki replication_factor: 1 diff --git a/infra/otel/prometheus-web.yaml b/infra/otel/prometheus-web.yaml new file mode 100644 index 0000000..6ada53d --- /dev/null +++ b/infra/otel/prometheus-web.yaml @@ -0,0 +1,5 @@ +tls_server_config: + cert_file: /run/otel/server.crt + key_file: /run/otel/server.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/otel/read-ca.crt diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml index be8489e..daa4275 100644 --- a/infra/otel/prometheus.yaml +++ b/infra/otel/prometheus.yaml @@ -4,5 +4,10 @@ rule_files: - /etc/prometheus/alerts.yaml scrape_configs: - job_name: otel-collector + scheme: https + tls_config: + ca_file: /run/tls/ca.crt + cert_file: /run/otel/client.crt + key_file: /run/otel/client.key static_configs: - targets: [otel-collector:8889] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml index 2a267d7..a591dab 100644 --- a/infra/otel/tempo.yaml +++ b/infra/otel/tempo.yaml @@ -1,11 +1,20 @@ server: http_listen_port: 3200 + http_tls_config: + cert_file: /run/otel/server.crt + key_file: /run/otel/server.key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/otel/read-ca.crt distributor: receivers: otlp: protocols: http: endpoint: 0.0.0.0:4318 + tls: + cert_file: /run/otel/server.crt + key_file: /run/otel/server.key + client_ca_file: /run/otel/export-ca.crt storage: trace: backend: local diff --git a/infra/postgres-hba.conf b/infra/postgres-hba.conf new file mode 100644 index 0000000..7645c21 --- /dev/null +++ b/infra/postgres-hba.conf @@ -0,0 +1,2 @@ +local all all trust +hostssl all all all scram-sha-256 diff --git a/infra/postgres-roles.sql b/infra/postgres-roles.sql new file mode 100644 index 0000000..e522011 --- /dev/null +++ b/infra/postgres-roles.sql @@ -0,0 +1,33 @@ +\getenv api_password api_password +\getenv worker_password worker_password +\getenv dispatcher_password dispatcher_password +\getenv cleanup_password cleanup_password +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +SELECT format('CREATE ROLE %I LOGIN', 'messenger_' || role) +FROM unnest(ARRAY['api', 'worker', 'dispatcher', 'cleanup']) AS role +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'messenger_' || role) +\gexec +ALTER ROLE messenger_api PASSWORD :'api_password'; +ALTER ROLE messenger_worker PASSWORD :'worker_password'; +ALTER ROLE messenger_dispatcher PASSWORD :'dispatcher_password'; +ALTER ROLE messenger_cleanup PASSWORD :'cleanup_password'; + +GRANT SELECT, INSERT ON users, chats, chat_members, messages, message_attachments TO messenger_api; +GRANT SELECT, INSERT, UPDATE ON attachments, direct_uploads TO messenger_api; +GRANT SELECT, INSERT, DELETE ON upload_attempts, message_idempotency TO messenger_api; +GRANT UPDATE ON message_idempotency TO messenger_api; +GRANT UPDATE ON upload_attempts TO messenger_api; +GRANT DELETE ON direct_uploads TO messenger_cleanup; +GRANT INSERT ON attachment_jobs TO messenger_api; +GRANT SELECT, UPDATE ON attachments, attachment_jobs TO messenger_worker; +GRANT SELECT, INSERT, UPDATE ON attachment_job_attempts, attachment_job_failures TO messenger_worker; +GRANT INSERT ON attachment_publications TO messenger_worker; +GRANT SELECT ON attachments, attachment_jobs, attachment_job_failures TO messenger_dispatcher; +GRANT UPDATE (published_at) ON attachment_jobs TO messenger_dispatcher; +GRANT SELECT, UPDATE ON upload_attempts, direct_uploads TO messenger_cleanup; +GRANT DELETE ON upload_attempts TO messenger_cleanup; +GRANT SELECT ON attachments TO messenger_cleanup; +GRANT USAGE ON SEQUENCE attachment_publications_id_seq TO messenger_worker; +GRANT SELECT ON attachment_jobs, attachment_job_failures TO messenger_cleanup; +GRANT UPDATE (staging_deleted_at) ON attachment_jobs TO messenger_cleanup; +GRANT UPDATE (updated_at) ON attachments TO messenger_cleanup; diff --git a/infra/rabbitmq/bootstrap.sh b/infra/rabbitmq/bootstrap.sh new file mode 100644 index 0000000..5ad589c --- /dev/null +++ b/infra/rabbitmq/bootstrap.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +until rabbitmq-diagnostics -q check_running >/dev/null 2>&1; do sleep 1; done +for role in dispatcher worker; do + username="messenger_$role" + password=$(cat "/run/secrets/broker_${role}_password") + if rabbitmqctl -q list_users | cut -f1 | grep -qx "$username"; then + rabbitmqctl -q change_password "$username" "$password" + else + rabbitmqctl -q add_user "$username" "$password" + fi + if [ "$role" = dispatcher ]; then write='^amq\.default$'; read='^$' + else write='^$'; read='^attachments\.v1$'; fi + rabbitmqctl -q set_permissions "$username" '^attachments\.v1$' "$write" "$read" +done diff --git a/infra/rabbitmq/rabbitmq.conf b/infra/rabbitmq/rabbitmq.conf new file mode 100644 index 0000000..bd5a67a --- /dev/null +++ b/infra/rabbitmq/rabbitmq.conf @@ -0,0 +1,10 @@ +listeners.tcp = none +listeners.ssl.default = 5671 +ssl_options.cacertfile = /run/tls/ca.crt +ssl_options.certfile = /run/tls/broker.crt +ssl_options.keyfile = /run/tls/broker.key +ssl_options.verify = verify_none +ssl_options.fail_if_no_peer_cert = false +management.tcp.port = 15672 +management.tcp.ip = 127.0.0.1 +prometheus.tcp.ip = 127.0.0.1 diff --git a/scripts/bao_operator.py b/scripts/bao_operator.py new file mode 100644 index 0000000..9bde661 --- /dev/null +++ b/scripts/bao_operator.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Initialize or unseal the local OpenBao; keep operator material on the host.""" +import argparse +import json +import os +import ssl +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +SECRETS = Path(__file__).resolve().parents[1] / "secrets" +MATERIAL = SECRETS / "bao-operator.json" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("init", "unseal")) + action = parser.parse_args().action + context = ssl.create_default_context(cafile=str(SECRETS / "tls/lab7-ca.crt")) + + def request(path, data): + req = Request( + "https://localhost:8200/v1/sys/" + path, + method="PUT", + data=json.dumps(data).encode(), + headers={"Content-Type": "application/json"}, + ) + with urlopen(req, context=context, timeout=15) as response: + return json.load(response) + + if action == "init": + # Refuse to overwrite the operator's only copy of the material. + fd = os.open(MATERIAL, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as output: + try: + data = request("init", {"secret_shares": 3, "secret_threshold": 2}) + except HTTPError: + MATERIAL.unlink() + raise + json.dump(data, output) + output.flush() + os.fsync(output.fileno()) + print("Operator material saved in secrets/bao-operator.json") + return + + data = json.loads(MATERIAL.read_text()) + for share in data["keys_base64"]: + if not request("unseal", {"key": share})["sealed"]: + print("OpenBao unsealed") + return + raise SystemExit("OpenBao remains sealed") + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap_tls.sh b/scripts/bootstrap_tls.sh index c3f170a..00cb3af 100755 --- a/scripts/bootstrap_tls.sh +++ b/scripts/bootstrap_tls.sh @@ -36,6 +36,33 @@ make_ca untrusted-ca chmod 644 "$dir/lab7-ca.crt" make_leaf ingress lab7-ca serverAuth DNS:localhost,IP:127.0.0.1 make_leaf api lab7-ca serverAuth DNS:api +for service in db redis s3 broker; do + san="DNS:$service" + if [ "$service" = s3 ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi + make_leaf "$service" lab7-ca serverAuth "$san" +done +for service in otel-collector loki tempo prometheus grafana; do + san="DNS:$service" + if [ "$service" = grafana ]; then san="$san,DNS:localhost,IP:127.0.0.1"; fi + make_leaf "$service" lab7-ca serverAuth "$san" +done +make_ca telemetry-read-ca +make_ca telemetry-export-ca +chmod 644 "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt" +make_leaf grafana-reader telemetry-read-ca clientAuth URI:urn:lab7:grafana +make_leaf collector-exporter telemetry-export-ca clientAuth URI:urn:lab7:collector +make_leaf prometheus-scraper telemetry-export-ca clientAuth URI:urn:lab7:prometheus +cat "$dir/telemetry-read-ca.crt" "$dir/telemetry-export-ca.crt" > "$dir/loki-client-ca.crt" +chmod 644 "$dir/loki-client-ca.crt" +make_ca telemetry-ingest-ca +chmod 644 "$dir/telemetry-ingest-ca.crt" +for role in api worker dispatcher; do + make_leaf "otel-$role" telemetry-ingest-ca clientAuth "URI:urn:lab7:otel:$role" +done +make_leaf openbao lab7-ca serverAuth DNS:openbao,DNS:localhost,IP:127.0.0.1 +for role in api worker dispatcher cleanup; do + make_leaf "bao-$role" lab7-ca clientAuth "URI:urn:lab7:bao:$role" +done make_leaf lb lab7-ca clientAuth URI:urn:lab7:load-balancer make_leaf wrong-role lab7-ca clientAuth URI:urn:lab7:probe make_leaf wrong-ca untrusted-ca clientAuth URI:urn:lab7:load-balancer diff --git a/scripts/check_bao_roles.py b/scripts/check_bao_roles.py new file mode 100644 index 0000000..4bfe262 --- /dev/null +++ b/scripts/check_bao_roles.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Prove each runtime identity can read only its own OpenBao secret.""" +import json +import ssl +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +TLS = Path(__file__).resolve().parents[1] / "secrets/tls" +ROLES = ("api", "worker", "dispatcher", "cleanup") + + +def request(context, path, *, body=None, token="", status=200): + req = Request( + f"https://localhost:8200/v1/{path}", + data=None if body is None else json.dumps(body).encode(), + headers={"X-Vault-Token": token, "Content-Type": "application/json"}, + ) + try: + with urlopen(req, context=context, timeout=5) as response: + assert response.status == status, (path, response.status, status) + return json.load(response) + except HTTPError as error: + assert error.code == status, (path, error.code, status) + + +def main(): + for role in ROLES: + context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt")) + context.load_cert_chain(TLS / f"bao-{role}.crt", TLS / f"bao-{role}.key") + auth = request(context, "auth/cert/login", body={"name": role})["auth"] + assert set(auth["policies"]) == {role} + assert 0 < auth["lease_duration"] <= 900 + token = auth["client_token"] + request(context, f"messenger/data/{role}", token=token) + request(context, f"messenger/data/{role}", token=token, body={"data": {}}, status=403) + request(context, f"sys/policies/acl/{role}", token=token, body={"policy": ""}, status=403) + for other in set(ROLES) - {role}: + request(context, f"messenger/data/{other}", token=token, status=403) + request(context, "auth/cert/login", body={"name": other}, status=400) + print(f"PASS {role}: scoped read, cross-role login/read and write denied, TTL <= 900 s") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_dependency_roles.sh b/scripts/check_dependency_roles.sh new file mode 100755 index 0000000..fda5252 --- /dev/null +++ b/scripts/check_dependency_roles.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." + +docker compose exec -T db sh -ec ' + export PGPASSWORD=$(cat /run/secrets/database_worker_password) + export PGCHANNELBINDING=disable PGSSLMODE=verify-full PGSSLROOTCERT=/run/tls/ca.crt + psql -X -h db -U messenger_worker "$POSTGRES_DB" -Atc " + SELECT has_table_privilege(current_user, '\''attachments'\'', '\''UPDATE'\'') + AND NOT has_table_privilege(current_user, '\''users'\'', '\''SELECT'\'') + AND NOT has_schema_privilege(current_user, '\''public'\'', '\''CREATE'\''); + " | grep -qx t +' +printf '%s\n' 'PASS PostgreSQL: worker can update attachments; user table and schema creation denied' +docker compose exec -T broker rabbitmqctl -q list_permissions -p / | + awk -F '\t' ' + $1=="messenger_dispatcher" && $2=="^attachments\\.v1$" && $3=="^amq\\.default$" && $4=="^$" { p=1 } + $1=="messenger_worker" && $2=="^attachments\\.v1$" && $3=="^$" && $4=="^attachments\\.v1$" { c=1 } + END {exit !(p && c)} + ' +printf '%s\n' 'PASS RabbitMQ: producer can publish and consumer can read attachments.v1' +docker compose exec -T redis sh -ec ' + export REDISCLI_AUTH=$(cat /run/secrets/redis_password) + redis-cli --tls --cacert /run/tls/ca.crt -h redis --user messenger CONFIG GET dir | grep -q NOPERM +' +printf '%s\n' 'PASS Redis: application role cannot read server configuration' diff --git a/scripts/check_grade3_http.py b/scripts/check_grade3_http.py new file mode 100644 index 0000000..7aa56de --- /dev/null +++ b/scripts/check_grade3_http.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Negative HTTP cases beyond the supplied smoke.""" +import ssl +import socket +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, run # noqa: E402 + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, _, eve, chat = client.bootstrap() + upload = f"/api/v1/chats/{chat['id']}/attachments?filename=probe.png" + headers = {"Content-Type": "image/png"} + client.call("POST", upload, 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers=headers) + client.call("POST", upload, 415, actor=alice, raw=b"not an image", headers=headers) + attachment, _ = client.attachment(alice, chat) + path = "/api/v1/attachments/" + attachment["id"] + spoofed = {"X-User-Id": alice["id"]} + for suffix in ("", "/content"): + client.call("GET", path + suffix, 404, actor=eve, headers=spoofed) + other = client.call("POST", "/api/v1/chats", 201, actor=eve, + body={"title": "ACL probe", "member_ids": [eve["id"], alice["id"]]}, schema="Chat")["json"] + client.call("POST", f"/api/v1/chats/{other['id']}/messages", 404, actor=eve, + body={"text": "", "attachment_ids": [attachment["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex, **spoofed}) + context = ssl.create_default_context(cafile=args.ca_file) + from urllib.parse import urlsplit + endpoint = urlsplit(client.base) + with socket.create_connection((endpoint.hostname, endpoint.port or 443), timeout=5) as connection: + try: + with context.wrap_socket(connection, server_hostname="wrong.lab.invalid"): + raise AssertionError("TLS accepted a wrong hostname") + except ssl.SSLCertVerificationError as error: + if error.verify_code != 62: + raise + print("PASS: upload size/MIME, forged identity, cross-chat attachment, wrong TLS hostname") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/check_grade3_runtime.py b/scripts/check_grade3_runtime.py new file mode 100644 index 0000000..7a1ad72 --- /dev/null +++ b/scripts/check_grade3_runtime.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Inspect runtime constraints without printing inspect output or secrets.""" +import json +import subprocess + +APP_ROLES = {"api", "worker", "dispatcher", "upload-cleanup"} +ALLOWED_PORTS = {("haproxy", "8443/tcp"), ("grafana", "3000/tcp"), + ("openbao", "8200/tcp"), ("s3", "9000/tcp")} + + +def main(): + ids = subprocess.check_output(["docker", "compose", "ps", "-q"], text=True).split() + assert ids, "No running Compose services" + services = json.loads(subprocess.check_output(["docker", "inspect", *ids], text=True)) + for service in services: + name = service["Config"]["Labels"]["com.docker.compose.service"] + config = service["HostConfig"] + assert not config["Privileged"] and config["NetworkMode"] != "host", name + mounts = [mount["Destination"] for mount in service["Mounts"]] + assert not any("docker.sock" in path for path in mounts), name + for port, bindings in (config.get("PortBindings") or {}).items(): + assert (name, port) in ALLOWED_PORTS, (name, port) + assert all(binding["HostIp"] == "127.0.0.1" for binding in bindings), name + if name in APP_ROLES: + assert service["Config"]["User"] == "app:app", name + assert config["CapDrop"] == ["ALL"] and not config["CapAdd"], name + assert config["Memory"] > 0 and config["NanoCpus"] > 0, name + assert config["ReadonlyRootfs"] and "/run/runtime-secrets" in config["Tmpfs"], name + assert not any(path.startswith("/run/secrets") for path in mounts), name + assert not any("BAO_TOKEN=" in field for field in service["Config"]["Env"]), name + assert APP_ROLES <= {svc["Config"]["Labels"]["com.docker.compose.service"] for svc in services} + print("PASS: local-only ports, runtime secrets on tmpfs, non-root app roles with limits and dropped capabilities") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_grade3_signals.py b/scripts/check_grade3_signals.py new file mode 100644 index 0000000..99a7c96 --- /dev/null +++ b/scripts/check_grade3_signals.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Confirm metrics, logs and traces reach their HTTPS stores.""" +import json +import ssl +import subprocess +import time +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen +from urllib.error import HTTPError +import http.client + +TLS = Path(__file__).resolve().parents[1] / "secrets/tls" + + +def query(service, port, path, params): + context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt")) + context.load_cert_chain(TLS / "grafana-reader.crt", TLS / "grafana-reader.key") + container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip() + address = subprocess.check_output(["docker", "inspect", "-f", + '{{(index .NetworkSettings.Networks "highload-lab7_telemetry").IPAddress}}', container], + text=True).strip() + connection = http.client.HTTPSConnection(service, port, context=context, timeout=5) + create_connection = connection._create_connection + connection._create_connection = lambda _, *args: create_connection((address, port), *args) + try: + connection.request("GET", f"{path}?{urlencode(params)}") + response = connection.getresponse() + assert response.status == 200, (service, response.status) + return json.load(response) + finally: + connection.close() + + +def main(): + try: + with urlopen("https://localhost:8443/api/v1/users/me", context=ssl.create_default_context( + cafile=str(TLS / "lab7-ca.crt")), timeout=5): + raise AssertionError("Anonymous request should be rejected") + except HTTPError as error: + assert error.code == 401 + since = int(time.time()) - 10 + for _ in range(30): + now = int(time.time()) + metrics = query("prometheus", 9090, "/api/v1/query", { + "query": 'up{job="otel-collector"} == 1'})["data"]["result"] + logs = query("loki", 3100, "/loki/api/v1/query_range", { + "query": '{service_name="messenger-api"}', "start": since * 10**9, + "end": now * 10**9, "limit": 1})["data"]["result"] + traces = query("tempo", 3200, "/api/search", { + "q": '{ resource.service.name = "messenger-api" }', + "start": since, "end": now, "limit": 1}).get("traces") + if metrics and logs and traces: + print("PASS: Prometheus scrape up; fresh messenger API log and trace found through mTLS") + return + time.sleep(2) + raise SystemExit("Telemetry signal missing within 60 seconds") + + +if __name__ == "__main__": + main() diff --git a/scripts/check_grade3_tls.py b/scripts/check_grade3_tls.py new file mode 100644 index 0000000..f6136b7 --- /dev/null +++ b/scripts/check_grade3_tls.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Verify trusted TLS on data-plane endpoints, from the Linux Docker host.""" +import json +import socket +import ssl +import subprocess +from pathlib import Path + +TLS = Path(__file__).resolve().parents[1] / "secrets/tls" +ENDPOINTS = ( + ("s3", 9000, None, False), + ("broker", 5671, None, False), + ("redis", 6379, None, False), + ("db", 5432, None, True), + ("otel-collector", 4318, "otel-api", False), + ("otel-collector", 8889, "prometheus-scraper", False), + ("prometheus", 9090, "grafana-reader", False), + ("loki", 3100, "collector-exporter", False), + ("loki", 3100, "grafana-reader", False), + ("tempo", 4318, "collector-exporter", False), + ("tempo", 3200, "grafana-reader", False), + ("grafana", 3000, None, False), + ("openbao", 8200, None, False), +) + + +def probe(address, service, port, identity, pg, ca): + context = ssl.create_default_context(cafile=str(TLS / f"{ca}.crt")) + if identity: + context.load_cert_chain(TLS / f"{identity}.crt", TLS / f"{identity}.key") + with socket.create_connection((address, port), timeout=5) as conn: + if pg: + conn.sendall(bytes.fromhex("0000000804d2162f")) + assert conn.recv(1) == b"S", "PostgreSQL did not accept TLS negotiation" + with context.wrap_socket(conn, server_hostname=service) as encrypted: + assert encrypted.version() in {"TLSv1.2", "TLSv1.3"} + if identity: + encrypted.sendall(b"GET / HTTP/1.0\r\n\r\n") + encrypted.recv(1) + + +def main(): + for service, port, identity, pg in ENDPOINTS: + container = subprocess.check_output(["docker", "compose", "ps", "-q", service], text=True).strip() + networks = json.loads(subprocess.check_output( + ["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", container], text=True)) + address = next(network["IPAddress"] for network in networks.values() if network["IPAddress"]) + probe(address, service, port, identity, pg, "lab7-ca") + try: + probe(address, service, port, identity, pg, "untrusted-ca") + except ssl.SSLCertVerificationError: + pass + else: + raise AssertionError(f"{service}:{port} accepted a foreign CA") + if identity: + try: + probe(address, service, port, "wrong-role", pg, "lab7-ca") + except ssl.SSLError: + pass + else: + raise AssertionError(f"{service}:{port} accepted a foreign client identity") + print(f"PASS {service}:{port}: verified service SAN; foreign CA rejected" + + ("; foreign client identity rejected" if identity else "")) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_other_replica.sh b/scripts/check_other_replica.sh new file mode 100755 index 0000000..4581f3e --- /dev/null +++ b/scripts/check_other_replica.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +replica=$(docker compose ps -q api | xargs docker inspect -f '{{.Name}} {{.Config.Hostname}}' | + awk '$1=="/highload-lab7-api-2" {print $2}') +test -n "$replica" +trap 'docker compose start api >/dev/null' EXIT +TARGET_INSTANCE_ID="$replica" ./scripts/stop-one.sh +./scripts/wait_ingress.sh +make test CA_FILE=secrets/tls/lab7-ca.crt diff --git a/scripts/check_session_expiry.py b/scripts/check_session_expiry.py new file mode 100644 index 0000000..d26834b --- /dev/null +++ b/scripts/check_session_expiry.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Use a stand started with SESSION_TTL_SECONDS between 3 and 15.""" +import base64 +import datetime as dt +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run, timestamp # noqa: E402 + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + actor = client.user("expiry") + registration = actor["registration"] + credentials = base64.b64encode( + f"{registration['username']}:{registration['password']}".encode() + ).decode() + session = client.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": "Basic " + credentials}, schema="Session")["json"] + actor["token"] = session["token"] + client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + remaining = (timestamp(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(0 < remaining <= 15, "Start rehearsal API with a short SESSION_TTL_SECONDS") + time.sleep(remaining + 0.3) + client.call("GET", "/api/v1/users/me", 401, actor=actor) + print("PASS: valid session accepted before expiry and rejected after expiry") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/collect_grade3.py b/scripts/collect_grade3.py new file mode 100644 index 0000000..39d9622 --- /dev/null +++ b/scripts/collect_grade3.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Run level-3 checks on the local rehearsal stand; write sanitized Markdown.""" +import argparse +import os +import shlex +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CA = "secrets/tls/lab7-ca.crt" +OUTPUT = ROOT / "evidence" +VALIDATION = {"CA_FILE": str(ROOT / CA)} +CHECKS = { + "grade3-https": ("Бизнес-API, проверка CA, logout и вложения по HTTPS", [ + ["make", "check"], ["make", "test", "test-security", f"CA_FILE={CA}"], + ]), + "grade3-mtls": ("Допуск только роли LB на каждой реплике", [ + ["python3", "scripts/check_ingress_mtls.py"], + ]), + "grade3-dependencies": ("TLS хранилищ, брокера и телеметрии", [ + ["python3", "scripts/check_grade3_tls.py"], + ["./scripts/check_dependency_roles.sh"], + ]), + "grade3-signals": ("Доставка логов, метрик и трейсов по защищённым связям", [ + ["python3", "scripts/check_grade3_signals.py"], + ]), + "grade3-openbao": ("Сервисная аутентификация и scoped policy OpenBao", [ + ["python3", "scripts/check_bao_roles.py"], + ]), + "grade3-runtime": ("Изоляция, порты, лимиты и локальное размещение секретов", [ + ["python3", "scripts/check_grade3_runtime.py"], + ]), + "grade3-http-security": ("Лимиты, MIME, чужие вложения и неверный TLS hostname", [ + ["python3", "scripts/check_grade3_http.py", "--ca-file", CA], + ]), + "grade3-resilience": ("Сохранность после перезапуска хранилищ и отказ API", [ + ["sleep", "60"], + ["make", "test-persistence", "ACTION_SCRIPT=scripts/restart.sh", f"CA_FILE={CA}"], + ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"], + ["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"], + ["sleep", "60"], + ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh", f"CA_FILE={CA}"], + ["docker", "compose", "start", "api"], ["./scripts/wait_ingress.sh"], + ]), + "grade3-other-replica": ("Работа HTTPS API при остановленной второй реплике", [ + ["sleep", "60"], ["./scripts/check_other_replica.sh"], ["./scripts/wait_ingress.sh"], + ]), + "grade3-login": ("Общий лимит входа по адресу клиента", [ + ["sleep", "60"], ["python3", "scripts/check_login_rate.py"], + ]), + "grade3-expiry": ("Истечение сессии на стенде с коротким TTL", [ + ["sleep", "60"], + ["env", "SESSION_TTL_SECONDS=5", "docker", "compose", "up", "-d", "--no-deps", "api"], + ["./scripts/wait_ingress.sh"], + ["python3", "scripts/check_session_expiry.py", "--ca-file", CA], + ["docker", "compose", "up", "-d", "--no-deps", "api"], ["./scripts/wait_ingress.sh"], + ]), + "grade3-argon2": ("Стоимость проверки хеша Argon2id на лимитах API", [ + ["docker", "compose", "--profile", "test", "build", "backend-tests"], + ["docker", "run", "--rm", "--network", "none", "--cpus", "0.5", "--memory", "512m", + "-e", "GOMAXPROCS=1", "highload-lab7-backend-tests", "go", "test", "./backend/services", + "-run", "^$", "-bench", "^BenchmarkPasswordVerification$", "-benchtime=3x", "-count=3"], + ]), +} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--only", nargs="+", choices=CHECKS) + selected = parser.parse_args().only or CHECKS + OUTPUT.mkdir(exist_ok=True) + if "grade3-resilience" in selected: + for name in ("restart", "failover"): + path = ROOT / ".memory" / f"{name}-containers.txt" + path.parent.mkdir(exist_ok=True) + path.write_text("") + VALIDATION[f"{name.upper()}_EVIDENCE_FILE"] = str(path) + passed = True + for slug in selected: + title, commands = CHECKS[slug] + report = [f"# {title}", "", "Предпосылки: подготовленный, разблокированный учебный стенд; Docker Compose, Python и локальный CA.", ""] + for command in commands: + result = subprocess.run(command, cwd=ROOT, env=os.environ | VALIDATION, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + passed &= result.returncode == 0 + report.extend([f"`{shlex.join(command)}`", "", f"Код завершения: {result.returncode}.", "", + "```text", "\n".join(line.rstrip() for line in result.stdout.splitlines()).rstrip(), "```", ""]) + if slug == "grade3-resilience": + for name in ("restart", "failover"): + rows = Path(VALIDATION[f"{name.upper()}_EVIDENCE_FILE"]).read_text().strip() + report.extend([f"Состояния контейнеров ({name}):", "", "```text", rows, "```", ""]) + report.extend(["Итог: " + ("все команды завершились успешно." if all( + line == "Код завершения: 0." for line in report if line.startswith("Код завершения:") + ) else "есть команды с ошибками."), ""]) + (OUTPUT / f"{slug}.md").write_text("\n".join(report), encoding="utf-8") + print(f"Recorded {slug}", flush=True) + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/scripts/configure_bao.py b/scripts/configure_bao.py new file mode 100644 index 0000000..62025f1 --- /dev/null +++ b/scripts/configure_bao.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Configure an initialized, unsealed local OpenBao using an operator token.""" +import argparse +import getpass +import json +import ssl +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" +TLS = SECRETS / "tls" + + +def main(): + context = ssl.create_default_context(cafile=str(TLS / "lab7-ca.crt")) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--operator-file", type=Path) + args = parser.parse_args() + token = (json.loads(args.operator_file.read_text())["root_token"] + if args.operator_file else getpass.getpass("OpenBao operator token: ")) + + def request(path, body=None): + data = None if body is None else json.dumps(body).encode() + req = Request( + "https://localhost:8200/v1/" + path, + data=data, + headers={"Content-Type": "application/json", "X-Vault-Token": token}, + ) + try: + with urlopen(req, context=context, timeout=5) as response: + return json.load(response) if response.status != 204 else {} + except HTTPError as error: + raise SystemExit(f"OpenBao {path}: HTTP {error.code}") + + if "cert/" not in request("sys/auth"): + request("sys/auth/cert", {"type": "cert"}) + if "messenger/" not in request("sys/mounts"): + request("sys/mounts/messenger", {"type": "kv", "options": {"version": "2"}}) + + for role in ("api", "worker", "dispatcher", "cleanup"): + request(f"sys/policies/acl/{role}", { + "policy": f'path "messenger/data/{role}" {{ capabilities = ["read"] }}', + }) + request(f"auth/cert/certs/{role}", { + "certificate": (TLS / f"bao-{role}.crt").read_text(), + "token_policies": [role], + "token_no_default_policy": True, + "token_ttl": "15m", + "token_max_ttl": "15m", + }) + + names = {"database_url": f"database_{role}_url"} + if role == "api": + names["cache_url"] = "cache_tls_url" + if role in ("worker", "dispatcher"): + names["broker_url"] = f"broker_{role}_url" + if role != "dispatcher": + for key in ("access_key_id", "secret_access_key"): + names[f"s3_{key}"] = f"s3_{role}_{key}" + values = {key: (SECRETS / name).read_text().strip() for key, name in names.items()} + request(f"messenger/data/{role}", {"data": values}) + print(f"Configured OpenBao role {role}") + + +if __name__ == "__main__": + main() diff --git a/scripts/init_env.py b/scripts/init_env.py index 1fc9ccb..278fd68 100644 --- a/scripts/init_env.py +++ b/scripts/init_env.py @@ -26,9 +26,9 @@ def main(): POSTGRES_USER="messenger", POSTGRES_PASSWORD=secrets.token_hex(24), ) - defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="http://s3:9000", - OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads", - S3_PUBLIC_ENDPOINT="http://127.0.0.1:9000", + defaults.update(API_HTTPS_PORT="8443", S3_ENDPOINT="https://s3:9000", + OTEL_EXPORTER_OTLP_ENDPOINT="https://otel-collector:4318", UPLOAD_CLEANUP_GRACE_SECONDS="600", UPLOAD_TMP_DIR="/tmp/uploads", + S3_PUBLIC_ENDPOINT="https://localhost:9000", S3_CORS_ORIGINS="http://127.0.0.1:5173,http://localhost:5173") values = defaults | existing for name, lower, upper in ( @@ -63,60 +63,45 @@ def main(): os.chmod(path, 0o600) secret_dir = ROOT / "secrets" secret_dir.mkdir(mode=0o700, exist_ok=True) - database_url = "postgres://{}:{}@db:5432/{}?sslmode=disable".format( + def create_secret(name, value): + target = secret_dir / name + if not target.exists(): + target.write_text(value + "\n", encoding="utf-8") + target.chmod(0o644) + return target.read_text(encoding="utf-8").strip() + + create_secret("grafana_admin_password", secrets.token_hex(24)) + database_url = "postgres://{}:{}@db:5432/{}?sslmode=verify-full&sslrootcert=/run/tls/ca.crt".format( quote(values["POSTGRES_USER"], safe=""), quote(values["POSTGRES_PASSWORD"], safe=""), quote(values["POSTGRES_DB"], safe=""), ) - for filename,content in (("broker_user","messenger"),("broker_password",secrets.token_hex(32))): - target=secret_dir/filename - if not target.exists(): - target.write_text(content+"\n",encoding="utf-8") - os.chmod(target,0o644) - broker_url="amqp://{}:{}@broker:5672/%2F".format( - quote((secret_dir/"broker_user").read_text().strip(),safe=""), - quote((secret_dir/"broker_password").read_text().strip(),safe="")) - # Migrate only the generated legacy Redis URL; preserve custom AMQP settings. - broker_file=secret_dir/"broker_url" - if broker_file.exists() and broker_file.read_text().strip()=="redis://broker:6379/0": - broker_file.write_text(broker_url+"\n",encoding="utf-8") + create_secret("broker_user", "messenger") + create_secret("broker_password", secrets.token_hex(32)) + redis_password = create_secret("redis_password", secrets.token_hex(32)) + cache_url = f"rediss://messenger:{redis_password}@redis:6379/0" + create_secret("redis_acl", "user default off\nuser messenger on >" + redis_password + + " ~messenger:* +ping +get +set +del +incr +expire +ttl +eval +evalsha +hello +client|setinfo") for name, content in ( - ("DATABASE_URL_FILE", database_url), - ("CACHE_URL_FILE", "redis://redis:6379/0"), - ("BROKER_URL_FILE", broker_url), - ("S3_ACCESS_KEY_ID_FILE", secrets.token_hex(12)), - ("S3_SECRET_ACCESS_KEY_FILE", secrets.token_hex(32)), + ("database_tls_url", database_url), + ("cache_tls_url", cache_url), ): - container_path = Path(values[name]) - if container_path.parent != Path("/run/secrets") or not container_path.name: - raise SystemExit(f"{name} must name a file directly inside /run/secrets") - target = ( - secret_dir - / { - "DATABASE_URL_FILE": "database_url", - "CACHE_URL_FILE": "cache_url", - "BROKER_URL_FILE": "broker_url", - "S3_ACCESS_KEY_ID_FILE": "s3_access_key_id", - "S3_SECRET_ACCESS_KEY_FILE": "s3_secret_access_key", - }[name] - ) - if not target.exists(): - # Containers run as an unprivileged user; the private host directory - # protects the files, while read permission lets the API consume them. - with target.open("x", encoding="utf-8") as file: - file.write(content + "\n") - os.chmod(target, 0o644) - # Existing S3 files remain the root credentials; API gets a new, separate identity. - for filename, content in ( - ("s3_api_access_key_id", secrets.token_hex(12)), - ("s3_api_secret_access_key", secrets.token_hex(32)), - ): - target = secret_dir / filename - if not target.exists(): - with target.open("x", encoding="utf-8") as file: - file.write(content + "\n") - os.chmod(target, 0o644) - print("Local .env and connection files ready; existing values and files preserved.") + create_secret(name, content) + create_secret("s3_access_key_id", secrets.token_hex(12)) + create_secret("s3_secret_access_key", secrets.token_hex(32)) + database = quote(values["POSTGRES_DB"], safe="") + for role in ("api", "worker", "dispatcher", "cleanup"): + password = create_secret(f"database_{role}_password", secrets.token_hex(24)) + create_secret(f"database_{role}_url", + f"postgres://messenger_{role}:{password}@db:5432/{database}" + "?sslmode=verify-full&sslrootcert=/run/tls/ca.crt") + if role in ("worker", "dispatcher"): + password = create_secret(f"broker_{role}_password", secrets.token_hex(24)) + create_secret(f"broker_{role}_url", f"amqps://messenger_{role}:{password}@broker:5671/%2F") + if role != "dispatcher": + create_secret(f"s3_{role}_access_key_id", secrets.token_hex(12)) + create_secret(f"s3_{role}_secret_access_key", secrets.token_hex(32)) + print("Local connection files ready; existing values and files preserved.") if __name__ == "__main__": diff --git a/scripts/provision_dependencies.sh b/scripts/provision_dependencies.sh new file mode 100755 index 0000000..9294717 --- /dev/null +++ b/scripts/provision_dependencies.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --wait db redis broker s3 +docker compose run --rm migrate +docker compose exec -T db sh -ec ' + for role in api worker dispatcher cleanup; do + export "${role}_password=$(cat "/run/secrets/database_${role}_password")" + done + psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" "$POSTGRES_DB" -f - +' < infra/postgres-roles.sql +docker compose exec -T broker sh /bootstrap.sh +docker compose run --rm s3-bootstrap diff --git a/scripts/restart.sh b/scripts/restart.sh index dcdc8d0..23e0c69 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,15 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." +if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then + docker compose ps -q db redis s3 broker api | xargs docker inspect \ + -f '{{.Name}} {{.State.StartedAt}}' > "$RESTART_EVIDENCE_FILE" +fi docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart db redis s3 broker >/dev/null docker compose --project-name highload-lab7 --env-file .env -f compose.yml up -d --wait db redis s3 broker >/dev/null docker compose --project-name highload-lab7 --env-file .env -f compose.yml restart api upload-cleanup worker dispatcher >/dev/null docker compose --project-name highload-lab7 --env-file .env -f compose.yml up -d --wait >/dev/null +if [ -n "${RESTART_EVIDENCE_FILE:-}" ]; then + docker compose ps -q db redis s3 broker api | xargs docker inspect \ + -f '{{.Name}} {{.State.StartedAt}}' >> "$RESTART_EVIDENCE_FILE" +fi diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 031c012..baff315 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -8,6 +8,9 @@ for container in $(docker compose --project-name highload-lab7 --env-file .env - if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then # Only containers from this Compose project's api service are eligible. docker stop --timeout 8 "$container" >/dev/null + if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then + docker inspect -f '{{.Name}} {{.State.Status}}' "$container" >> "$FAILOVER_EVIDENCE_FILE" + fi printf 'Stopped API instance %s\n' "$hostname" exit 0 fi diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 5bd9954..a1b3228 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -4,9 +4,9 @@ - Пароли и их Argon2id-хеши, Bearer-токены и записи сессий. Владение действующим токеном даёт права соответствующего пользователя. - Сообщения, состав чатов, метаданные и байты вложений, включая staging и производные изображения. PostgreSQL хранит бизнес-состояние, S3 — файлы, RabbitMQ — доставку фоновых заданий, Redis — сессии и счётчики входа. -- Ключи CA и сервисов, учётные данные хранилищ и будущие токены OpenBao. Отдельно защищаются телеметрия и её доступность, поскольку поддельные сигналы могут скрыть отказ, а утечка логов — раскрыть секреты. +- Ключи CA и сервисов, учётные данные хранилищ и токены OpenBao. Отдельно защищаются телеметрия и её доступность, поскольку поддельные сигналы могут скрыть отказ, а утечка логов — раскрыть секреты. -Внешний клиент может формировать произвольные HTTP-запросы к ingress. Аутентифицированный посторонний знает или угадывает идентификаторы ресурсов в чужом чате. Участник чата может загружать специально подготовленные файлы. При компрометации контейнера атакующий получает доступ к его файлам, учетным данным и доступным ему сетям. Оператор управляет хостом, Docker, сертификатами и хранилищами. Компрометация хоста, Docker daemon или ключа CA выходит за границы защиты приложения. Стенд развёрнут на одном хосте и проверяет отказ отдельной API-реплики. +Внешний клиент может формировать произвольные HTTP-запросы к ingress. Аутентифицированный посторонний знает или угадывает идентификаторы ресурсов в чужом чате. Участник чата может загружать специально подготовленные файлы. При компрометации контейнера атакующий получает доступ к его файлам, учётным данным и доступным ему сетям. Оператор управляет хостом, Docker, сертификатами и хранилищами. Компрометация хоста, Docker daemon или ключа CA выходит за границы защиты приложения. Стенд развёрнут на одном хосте и проверяет отказ отдельной API-реплики. ## Границы доверия @@ -14,18 +14,18 @@ | Угроза и путь | Защищаемое свойство | Контроль и проверка | Остаточное ограничение | | --- | --- | --- | --- | -| Подмена HTTPS-узла и перехват Basic или Bearer | Клиент передаёт credentials нужному серверу | HTTPS с проверкой CA и SAN. `make test-security CA_FILE=secrets/tls/lab7-ca.crt` подтвердил отказ без доверия к CA. Проверка с доверенным CA и ошибочным именем запланирована. | Токен, украденный на клиенте, пригоден до logout или expiry. | +| Подмена HTTPS-узла и перехват Basic или Bearer | Клиент передаёт credentials нужному серверу | HTTPS с проверкой CA и SAN. `make test-security CA_FILE=secrets/tls/lab7-ca.crt` подтвердил отказ без доверия к CA. Неверное имя проверяется в `scripts/check_grade3_http.py`. | Токен, украденный на клиенте, пригоден до logout или expiry. | | Прямой запрос к API из соседнего контейнера или предъявление сертификата другого сервиса | API принимает только HAProxy | API не публикует порт. `python3 scripts/check_ingress_mtls.py` проверил обе реплики с сертификатом LB, без сертификата, с чужим CA и с чужим URI SAN от своего CA. | Кража ключа LB позволяет имитировать его до смены доверия. Пользовательские ACL продолжают действовать. | -| Чтение чужих сообщений или файлов, подмена `X-User-Id`, добавление чужого attachment ID | Ресурсы доступны участникам соответствующего чата | Bearer определяет пользователя, членство проверяется API. Публичный smoke прошёл по HTTPS. Дополнить явными запросами к metadata/content и отправкой сообщения с чужим вложением и поддельным заголовком. | Участник может скопировать легально полученные данные. | +| Чтение чужих сообщений или файлов, подмена `X-User-Id`, добавление чужого attachment ID | Ресурсы доступны участникам соответствующего чата | Bearer определяет пользователя, членство проверяется API. Публичный smoke прошёл по HTTPS. `scripts/check_grade3_http.py` проверяет metadata/content и сообщение с чужим вложением и поддельным заголовком. | Участник может скопировать легально полученные данные. | | Повтор использования токена после logout или истечения срока | Отозванная сессия отклоняется всеми API | В Redis хранится ключ на основе SHA-256 токена с TTL, logout удаляет запись. Smoke проверил logout. Для expiry создать сессию на отдельном стенде с коротким TTL и ожидать 401 с тем же токеном после истечения. | Захваченная активная сессия имеет права пользователя. | | Перебор пароля и нагрузка на Argon2id через чередование реплик и поддельный `X-Forwarded-For` | Ограничение входа действует независимо от реплики | HAProxy заменяет заголовок фактическим адресом, Redis атомарно ведёт общий счётчик. `python3 scripts/check_login_rate.py` подтвердил 429 и `Retry-After` на 13-й попытке. Пароли хешируются Argon2id с индивидуальной солью. | Распределённые источники имеют отдельные окна, пользователи за одним NAT делят лимит. | | Загрузка слишком большого файла, изображения с чрезмерным числом пикселей или подменённым MIME | Ограничены память, CPU и объём входных данных | API ограничивает upload 10 МиБ и проверяет формат и размеры, обработка выполняется отдельно в worker. Проверить 413 для превышения размера/пикселей и 415 для ложного MIME, затем загрузить штатный файл и дождаться ready. | Параллельные загрузки конкурируют за общие ресурсы. | -| Компрометация worker и выход за пределы его роли | Worker имеет доступ только к ресурсам обработки | Запланированы TLS, отдельные DB/S3/AMQP identities и scoped policy OpenBao. Негативные пробы: чужая таблица или схема, запрещённый prefix, публикация от consumer, чтение чужого пути OpenBao. | Сейчас роли используют общие credentials. После разделения компрометация всё равно затронет разрешённую область worker. | -| Поддельный OTLP ingest или чтение телеметрии соседом | Источник сигналов авторизован, хранилища доступны по ролям | Сейчас действует сетевая сегментация Docker. Запланировать TLS и ACL на ingest/read, проверку отказа для недоверенной identity и поиск синтетического секрета в логах, трейcах и метриках. | Скомпрометированное приложение может посылать ложные сигналы в пределах своей роли. | -| Утечка bootstrap credentials или остановка OpenBao | Сервис получает только свои секреты, отказ закрывает новый доступ | Запланировать cert auth, короткие токены и раздельные KV-пути. Проверить отказ чужого чтения и записи, отсутствие административных секретов в контейнерах и отказ нового запуска при sealed OpenBao. | Действующие подключения и уже загруженные credentials могут жить до закрытия соединения или отзыва в целевой системе. | +| Компрометация worker и выход за пределы его роли | Worker имеет доступ только к ресурсам обработки | Действуют отдельные DB/S3/AMQP identities и scoped policy OpenBao. TLS, обработка в своей роли и запрет чтения чужого пути проверяются отдельно. | Компрометация затронет разрешённую область worker. | +| Поддельный OTLP ingest или чтение телеметрии соседом | Источник сигналов авторизован, хранилища доступны по ролям | Действует сетевая сегментация и mTLS с отдельными CA для ingest, export и read. Проверяется доставка в три хранилища и отказ чужого CA. | Скомпрометированное приложение может посылать ложные сигналы в пределах своей роли. | +| Утечка bootstrap credentials или остановка OpenBao | Сервис получает только свои секреты, отказ закрывает новый доступ | Cert auth выдаёт отдельный токен на 15 минут для чтения своего KV-пути. Проверяются запрет чтения соседнего пути, записи и отсутствие административных секретов в контейнерах. | Действующие подключения и уже загруженные credentials могут жить до закрытия соединения или отзыва в целевой системе. | ## Lifecycle и инциденты Сейчас оператор выпускает локальный CA и leaf-сертификаты через `scripts/bootstrap_tls.sh`. CA действует 365 дней, leaf — 30 дней. Закрытый ключ CA остаётся у оператора, контейнеры получают отдельные файлы. Повторный запуск bootstrap сохраняет имеющиеся сертификаты. -Для OpenBao планируется persistent storage и ручное раздельное хранение unseal shares и административных credentials у оператора. После рестарта оператор выполняет unseal. API и worker будут заново получать scoped токены по своим клиентским сертификатам. При компрометации отзыв identity блокирует новую аутентификацию, отзыв токенов закрывает доступ к OpenBao, а смена выданных паролей и закрытие активных подключений ограничивают доступ к зависимостям. Проверки expiry, renewal и ротации выполняются на новых соединениях. Сканирование зависимостей и разбор применимости уязвимостей относятся к следующему уровню. +OpenBao использует файловый backend на томе. Bootstrap сохраняет операторский материал на хосте с правами 0600. При эксплуатации оператор должен хранить доли ключа и административные credentials раздельно. После рестарта оператор выполняет unseal. API и worker заново получают scoped токены по своим клиентским сертификатам. При компрометации отзыв identity блокирует новую аутентификацию, отзыв токенов закрывает доступ к OpenBao, а смена выданных паролей и закрытие активных подключений ограничивают доступ к зависимостям. Проверки expiry, renewal и ротации выполняются на новых соединениях. Сканирование зависимостей и разбор применимости уязвимостей относятся к следующему уровню. diff --git a/src/backend/commands/bao.go b/src/backend/commands/bao.go new file mode 100644 index 0000000..60e2024 --- /dev/null +++ b/src/backend/commands/bao.go @@ -0,0 +1,111 @@ +package commands + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "time" +) + +var secretFiles = map[string]string{ + "database_url": "DATABASE_URL_FILE", + "cache_url": "CACHE_URL_FILE", + "broker_url": "BROKER_URL_FILE", + "s3_access_key_id": "S3_ACCESS_KEY_ID_FILE", + "s3_secret_access_key": "S3_SECRET_ACCESS_KEY_FILE", +} + +func loadBaoSecrets() error { + role := os.Getenv("BAO_ROLE") + address, err := url.Parse(os.Getenv("BAO_ADDR")) + if err != nil || address.Scheme != "https" || address.Host == "" { + return errors.New("invalid OpenBao HTTPS address") + } + cert, err := tls.LoadX509KeyPair("/run/bao/client.crt", "/run/bao/client.key") + if err != nil { + return fmt.Errorf("load OpenBao service identity: %w", err) + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{cert}} + defer transport.CloseIdleConnections() + client := &http.Client{ + Transport: transport, + Timeout: 5 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, + } + request := func(method, path string, body []byte, token string, result any) error { + req, err := http.NewRequest(method, address.String()+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("X-Vault-Token", token) + } + response, err := client.Do(req) + if err != nil { + return errors.New("OpenBao connection failed") + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("OpenBao returned HTTP %d", response.StatusCode) + } + return json.NewDecoder(response.Body).Decode(result) + } + var login struct { + Auth struct { + Token string `json:"client_token"` + } + } + body, err := json.Marshal(map[string]string{"name": role}) + if err != nil { + return err + } + if err = request("POST", "/v1/auth/cert/login", body, "", &login); err != nil { + return err + } + if login.Auth.Token == "" { + return errors.New("OpenBao returned an empty token") + } + var secret struct { + Data struct{ Data map[string]string } + } + if err = request("GET", "/v1/messenger/data/"+url.PathEscape(role), nil, login.Auth.Token, &secret); err != nil { + return err + } + for key, env := range secretFiles { + if os.Getenv(env) == "" { + continue + } + value := secret.Data.Data[key] + if value == "" { + return fmt.Errorf("missing %s in OpenBao", key) + } + dir := "/run/runtime-secrets" + path := filepath.Join(dir, key) + tmp, err := os.CreateTemp(dir, ".secret-") + if err != nil { + return err + } + _, writeErr := tmp.WriteString(value) + closeErr := tmp.Close() + if err := errors.Join(writeErr, closeErr); err != nil { + os.Remove(tmp.Name()) + return err + } + if err := os.Rename(tmp.Name(), path); err != nil { + os.Remove(tmp.Name()) + return err + } + if err := os.Setenv(env, path); err != nil { + return err + } + } + return nil +} diff --git a/src/backend/commands/run.go b/src/backend/commands/run.go index fa4fe47..d010420 100644 --- a/src/backend/commands/run.go +++ b/src/backend/commands/run.go @@ -16,6 +16,11 @@ import ( // Run dispatches the existing executable modes; the binary and CLI stay the same. func Run(args []string) error { + if os.Getenv("BAO_ROLE") != "" && (len(args) == 0 || args[0] != "image-process") { + if err := loadBaoSecrets(); err != nil { + return err + } + } if len(args) == 0 { if environment := os.Getenv("APP_ENV"); environment != "" { log.SetPrefix("[" + environment + "] ") diff --git a/src/backend/repositories/redis.go b/src/backend/repositories/redis.go index 950069c..c0c933d 100644 --- a/src/backend/repositories/redis.go +++ b/src/backend/repositories/redis.go @@ -57,14 +57,16 @@ func (s *SessionStore) Ping(ctx context.Context) error { // Apply the expiry and increment atomically across replicas. var loginWindow = redis.NewScript(` -local count = redis.call('INCR', KEYS[1]) -if count == 1 then - redis.call('EXPIRE', KEYS[1], 60) +local ttl = redis.call('TTL', KEYS[1]) +if ttl < 0 then + redis.call('SET', KEYS[1], 1, 'EX', 60) + return 0 end +local count = redis.call('INCR', KEYS[1]) if count <= 12 then return 0 end -return math.max(1, redis.call('TTL', KEYS[1])) +return math.max(1, ttl) `) func (s *SessionStore) LoginRetryAfter(ctx context.Context, source string) (int, error) { diff --git a/src/backend/repositories/s3_admin.go b/src/backend/repositories/s3_admin.go index c8947f9..3cdd15b 100644 --- a/src/backend/repositories/s3_admin.go +++ b/src/backend/repositories/s3_admin.go @@ -40,17 +40,6 @@ func BootstrapS3(ctx context.Context) error { if err != nil { return err } - access, err := readS3Secret("S3_ACCESS_KEY_ID_FILE") - if err != nil { - return err - } - secret, err := readS3Secret("S3_SECRET_ACCESS_KEY_FILE") - if err != nil { - return err - } - if access == rootAccess { - return errors.New("API and root credentials must differ") - } client, err := minio.New(endpoint.Host, &minio.Options{Creds: credentials.NewStaticV4(rootAccess, rootSecret, ""), Secure: endpoint.Scheme == "https", Region: os.Getenv("S3_REGION")}) if err != nil { return err @@ -66,23 +55,61 @@ func BootstrapS3(ctx context.Context) error { if err != nil { return err } - policy := map[string]any{"Version": "2012-10-17", "Statement": []any{ - map[string]any{"Effect": "Allow", "Action": []string{"s3:GetBucketLocation", "s3:ListBucket"}, "Resource": []string{"arn:aws:s3:::" + store.bucket}}, - map[string]any{"Effect": "Allow", "Action": []string{"s3:GetObject", "s3:PutObject", "s3:DeleteObject"}, "Resource": []string{"arn:aws:s3:::" + store.bucket + "/originals/*", "arn:aws:s3:::" + store.bucket + "/staging/direct/*", "arn:aws:s3:::" + store.bucket + "/staging/jobs/*", "arn:aws:s3:::" + store.bucket + "/variants/*"}}, - }} - data, err := json.Marshal(policy) + for _, role := range []string{"api", "worker", "cleanup"} { + access, err := readS3Secret("S3_" + strings.ToUpper(role) + "_ACCESS_KEY_ID_FILE") + if err != nil { + return err + } + secret, err := readS3Secret("S3_" + strings.ToUpper(role) + "_SECRET_ACCESS_KEY_FILE") + if err != nil { + return err + } + if access == rootAccess { + return errors.New("application and root credentials must differ") + } + if err := setS3Role(ctx, admin, store.bucket, role, access, secret); err != nil { + return err + } + } + return nil +} + +func setS3Role(ctx context.Context, admin *madmin.AdminClient, bucket, role, access, secret string) error { + resource := "arn:aws:s3:::" + bucket + statement := func(actions, resources []string) map[string]any { + return map[string]any{"Effect": "Allow", "Action": actions, "Resource": resources} + } + statements := []any{statement([]string{"s3:GetBucketLocation", "s3:ListBucket"}, []string{resource})} + add := func(action string, prefixes ...string) { + resources := make([]string, 0, len(prefixes)) + for _, prefix := range prefixes { + resources = append(resources, resource+"/"+prefix+"/*") + } + statements = append(statements, statement([]string{"s3:" + action}, resources)) + } + switch role { + case "api": + add("GetObject", "originals", "variants", "staging/direct") + add("PutObject", "staging/jobs", "staging/direct", "originals") + add("DeleteObject", "staging/jobs", "staging/direct") + case "worker": + add("GetObject", "staging/jobs", "staging/direct", "originals", "variants") + add("PutObject", "originals", "variants") + add("DeleteObject", "staging/jobs", "staging/direct", "variants") + case "cleanup": + add("DeleteObject", "originals", "staging/jobs", "staging/direct") + } + data, err := json.Marshal(map[string]any{"Version": "2012-10-17", "Statement": statements}) if err != nil { return err } - const name = "messenger-attachments-api" + name := "messenger-attachments-" + role if err = admin.AddCannedPolicy(ctx, name, data); err != nil { return err } if err = admin.SetUserReq(ctx, access, madmin.AddOrUpdateUserReq{SecretKey: secret, Status: madmin.AccountEnabled}); err != nil { return err } - // Explicitly assign the managed policy; creating a user alone grants no S3 rights. - // This idempotent API is supported by our pinned MinIO server release. return admin.SetPolicy(ctx, name, access, false) } diff --git a/src/backend/tests/test_fixture_test.go b/src/backend/tests/test_fixture_test.go index 4f0ee29..d2ac57c 100644 --- a/src/backend/tests/test_fixture_test.go +++ b/src/backend/tests/test_fixture_test.go @@ -79,6 +79,13 @@ func newTestFixture(t *testing.T) *testFixture { t.Fatal(err) } redisClient := repositories.NewRedisClient(redisAddress) + if os.Getenv("CACHE_URL_FILE") != "" { + redisClient.Close() + redisClient, err = repositories.RedisClientFromEnvironment() + if err != nil { + t.Fatal(err) + } + } ddl, err = os.ReadFile("../repositories/migrations/000003_message_idempotency.up.sql") if err != nil { t.Fatal(err) @@ -136,7 +143,7 @@ func newTestFixture(t *testing.T) *testFixture { chats := repositories.NewChatStore(db) messages := repositories.NewMessageStore(db) userService := services.NewUserService(users) - sessionStore := repositories.NewSessionStore(redisClient, "test:"+schema+":session:") + sessionStore := repositories.NewSessionStore(redisClient, "messenger:test:"+schema+":session:") sessions := services.NewSessionService(userService, sessionStore, time.Hour) router := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore), userService, sessions, services.NewChatService(chats), services.NewMessageService(messages)) return &testFixture{db: db, users: users, chats: chats, messages: messages, sessionStore: sessionStore, sessions: sessions, router: router} diff --git a/src/backend/workers/pipeline.go b/src/backend/workers/pipeline.go index e6c1707..f54fa16 100644 --- a/src/backend/workers/pipeline.go +++ b/src/backend/workers/pipeline.go @@ -57,9 +57,12 @@ func RunPipeline(mode string, args []string) error { return err } timeout := time.Duration(seconds) * time.Second - objects, err := repositories.NewObjectStoreFromEnvironment() - if err != nil { - return err + var objects *repositories.ObjectStore + if mode != "dispatcher" { + objects, err = repositories.NewObjectStoreFromEnvironment() + if err != nil { + return err + } } var signals *telemetry.Signals if mode == "worker" || mode == "dispatcher" {