From b035b50dbd8c21c1f7c37dea10923b72e3e13b89 Mon Sep 17 00:00:00 2001 From: yasn0clown Date: Wed, 18 Nov 2026 01:02:09 +0300 Subject: [PATCH] =?UTF-8?q?=D0=BB=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BD=D0=B0=D1=8F=20=D1=80=D0=B0=D0=B1=D0=BE=D1=82?= =?UTF-8?q?=D0=B0=207?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 8 + .env.example | 96 +- .gitattributes | 15 + .github/pull_request_template.md | 49 +- .gitignore | 2 + Dockerfile | 37 + Makefile | 47 +- REPORT.md | 128 ++- compose.faults.yaml | 17 + compose.yaml | 874 ++++++++++++++ contracts/trust-matrix.md | 65 +- docs/lab3/REPORT.md | 231 ++++ docs/lab3/evidence/README.md | 25 + docs/lab3/evidence/SUMMARY.txt | 34 + docs/lab3/evidence/check.txt | 7 + docs/lab3/evidence/failover.txt | 66 ++ docs/lab3/evidence/failover_series.json | 294 +++++ docs/lab3/evidence/failover_series.txt | 17 + docs/lab3/evidence/failover_series_kill.json | 294 +++++ docs/lab3/evidence/instances.txt | 22 + docs/lab3/evidence/lb_errors.txt | 17 + docs/lab3/evidence/lb_log_sample.txt | 62 + docs/lab3/evidence/load.json | 17 + docs/lab3/evidence/load.txt | 28 + docs/lab3/evidence/migrations.txt | 26 + docs/lab3/evidence/persistence.txt | 24 + docs/lab3/evidence/rolling_back.json | 147 +++ docs/lab3/evidence/rolling_update.json | 147 +++ docs/lab3/evidence/rolling_update.txt | 29 + docs/lab3/evidence/shutdown_timeline.txt | 17 + docs/lab3/evidence/smoke.txt | 3 + docs/lab3/evidence/stand.txt | 33 + docs/lab3/evidence/student-tests.txt | 134 +++ docs/lab4/REPORT.md | 273 +++++ docs/lab4/evidence/README.md | 11 + docs/lab4/evidence/SUMMARY.txt | 46 + docs/lab4/evidence/check.txt | 7 + docs/lab4/evidence/consistency.txt | 75 ++ docs/lab4/evidence/failover.txt | 66 ++ docs/lab4/evidence/failover_series.json | 294 +++++ docs/lab4/evidence/failover_series.txt | 17 + docs/lab4/evidence/failover_series_kill.json | 298 +++++ docs/lab4/evidence/instances.txt | 22 + docs/lab4/evidence/lb_log_sample.txt | 62 + docs/lab4/evidence/load.json | 17 + docs/lab4/evidence/load.txt | 29 + docs/lab4/evidence/persistence.txt | 48 + docs/lab4/evidence/reconcile.txt | 19 + docs/lab4/evidence/rolling_back.json | 147 +++ docs/lab4/evidence/rolling_update.json | 147 +++ docs/lab4/evidence/rolling_update.txt | 29 + docs/lab4/evidence/s3_bootstrap.txt | 225 ++++ docs/lab4/evidence/s3_outage.json | 98 ++ docs/lab4/evidence/s3_outage.txt | 24 + docs/lab4/evidence/s3_private.txt | 67 ++ docs/lab4/evidence/sandbox/check.txt | 6 + docs/lab4/evidence/sandbox/consistency.txt | 17 + docs/lab4/evidence/sandbox/failover.txt | 12 + .../evidence/sandbox/failover_series.json | 268 +++++ docs/lab4/evidence/sandbox/load.json | 17 + docs/lab4/evidence/sandbox/persistence.txt | 7 + docs/lab4/evidence/sandbox/reconcile.txt | 17 + docs/lab4/evidence/sandbox/s3_bootstrap.txt | 45 + docs/lab4/evidence/sandbox/s3_outage.json | 98 ++ .../lab4/evidence/sandbox/s3_outage_hang.json | 98 ++ docs/lab4/evidence/sandbox/s3_private.txt | 54 + docs/lab4/evidence/sandbox/smoke.txt | 3 + docs/lab4/evidence/sandbox/student-tests.txt | 120 ++ .../evidence/sandbox/upload_memory_c10.json | 43 + .../sandbox/upload_memory_c20_limit4.json | 39 + .../sandbox/upload_memory_c20_nolimit.json | 39 + docs/lab4/evidence/smoke.txt | 3 + docs/lab4/evidence/stand.txt | 42 + docs/lab4/evidence/student-tests.txt | 168 +++ docs/lab4/evidence/upload_memory.txt | 108 ++ docs/lab4/evidence/upload_memory_c10.json | 51 + docs/lab4/evidence/upload_memory_c20.json | 52 + docs/lab5/REPORT.md | 257 +++++ docs/lab5/evidence/README.md | 5 + docs/lab5/evidence/SUMMARY.txt | 54 + docs/lab5/evidence/check.txt | 7 + docs/lab5/evidence/consistency.txt | 75 ++ docs/lab5/evidence/failover.txt | 66 ++ docs/lab5/evidence/failover_series.json | 294 +++++ docs/lab5/evidence/failover_series.txt | 17 + docs/lab5/evidence/failover_series_kill.json | 298 +++++ docs/lab5/evidence/instances.txt | 22 + docs/lab5/evidence/lb_log_sample.txt | 62 + docs/lab5/evidence/load.json | 17 + docs/lab5/evidence/load.txt | 31 + docs/lab5/evidence/local/README.md | 33 + docs/lab5/evidence/local/failover.txt | 4 + docs/lab5/evidence/local/imaging_memory.txt | 5 + docs/lab5/evidence/local/latency.json | 14 + docs/lab5/evidence/local/persistence.txt | 3 + .../local/pipeline_and_attachments.txt | 11 + docs/lab5/evidence/local/pipeline_faults.txt | 18 + .../evidence/local/pipeline_faults_events.txt | 71 ++ docs/lab5/evidence/local/smoke.txt | 1 + docs/lab5/evidence/local/student-tests.txt | 111 ++ docs/lab5/evidence/local/worker_fault_log.txt | 8 + docs/lab5/evidence/local/worker_outage.json | 80 ++ docs/lab5/evidence/local/worker_outage.txt | 11 + docs/lab5/evidence/persistence.txt | 53 + docs/lab5/evidence/pipeline.txt | 14 + docs/lab5/evidence/pipeline_faults.txt | 152 +++ docs/lab5/evidence/reconcile.txt | 20 + docs/lab5/evidence/rolling_back.json | 147 +++ docs/lab5/evidence/rolling_update.json | 147 +++ docs/lab5/evidence/rolling_update.txt | 29 + docs/lab5/evidence/s3_bootstrap.txt | 225 ++++ docs/lab5/evidence/s3_outage.json | 98 ++ docs/lab5/evidence/s3_outage.txt | 24 + docs/lab5/evidence/s3_private.txt | 67 ++ docs/lab5/evidence/smoke.txt | 3 + docs/lab5/evidence/stand.txt | 61 + docs/lab5/evidence/student-tests.txt | 185 +++ docs/lab5/evidence/upload_memory.txt | 161 +++ docs/lab5/evidence/upload_memory_c10.json | 78 ++ docs/lab5/evidence/upload_memory_c20.json | 78 ++ docs/lab5/evidence/worker_log_sample.txt | 3 + docs/lab5/evidence/worker_outage.json | 80 ++ docs/lab5/evidence/worker_outage.txt | 12 + docs/lab6/REPORT.md | 136 +++ docs/lab6/evidence/README.md | 11 + docs/lab6/evidence/alerts_db.txt | Bin 0 -> 662 bytes docs/lab6/evidence/alerts_worker.txt | Bin 0 -> 690 bytes docs/lab6/evidence/check.txt | Bin 0 -> 180 bytes docs/lab6/evidence/incident_db.jsonl | Bin 0 -> 3004 bytes docs/lab6/evidence/incident_worker.jsonl | Bin 0 -> 6762 bytes .../local-sandbox/alerts_db_lock_180s.txt | 6 + .../alerts_queue_lag_and_db_short.txt | 6 + .../evidence/local-sandbox/db_lock_span.txt | 6 + .../local-sandbox/incident_worker_stop.jsonl | 25 + .../evidence/local-sandbox/metric_names.txt | 32 + docs/lab6/evidence/local-sandbox/smoke.txt | 2 + .../local-sandbox/telemetry_outage.txt | 21 + .../local-sandbox/trace_retry_chain.txt | 77 ++ .../local-sandbox/trace_upload_chain.txt | 43 + docs/lab6/evidence/probe.txt | Bin 0 -> 796 bytes docs/lab6/evidence/smoke.txt | Bin 0 -> 174 bytes docs/lab6/evidence/stand.txt | Bin 0 -> 5366 bytes docs/lab6/evidence/telemetry_down.jsonl | Bin 0 -> 3060 bytes docs/runbooks.md | 45 + docs/security/SCAN_TRIAGE.md | 38 + evidence/bao_policy.json | 70 ++ evidence/hardening.txt | 34 + evidence/leak_check.json | 31 + evidence/login_budget.txt | 28 + evidence/rehash.txt | 1 + evidence/run/SUMMARY.txt | 34 + evidence/run/bao-policy.txt | 74 ++ evidence/run/bao-sealed.txt | 8 + evidence/run/check-contract.txt | 4 + evidence/run/compose-config.txt | 3 + evidence/run/expiry.txt | 50 + evidence/run/hardening.txt | 38 + evidence/run/leak-check.txt | 35 + evidence/run/login-budget.txt | 33 + evidence/run/ps.txt | 6 + evidence/run/rehash.txt | 5 + evidence/run/restore-1.txt | 4 + evidence/run/restore-2.txt | 4 + evidence/run/revoke-identity.txt | 8 + evidence/run/rotate-db-password.txt | 5 + evidence/run/rotate-lb-cert.txt | 19 + evidence/run/rotate-secret-id.txt | 5 + evidence/run/s3-check.txt | 67 ++ evidence/run/security-http.txt | 29 + evidence/run/smoke-after-lifecycle.txt | 4 + evidence/run/storage-tests.txt | 32 + evidence/run/student-tests.txt | 111 ++ evidence/run/test-failover-1.txt | 6 + evidence/run/test-failover-2.txt | 6 + evidence/run/test-persistence.txt | 5 + evidence/run/test-security.txt | 4 + evidence/run/test-smoke.txt | 4 + evidence/run/tls-audit.txt | 718 ++++++++++++ evidence/run/up.txt | 162 +++ evidence/sandbox/README.md | 38 + evidence/sandbox/bao_policy.json | 65 ++ evidence/sandbox/leak_check.json | 31 + evidence/sandbox/lifecycle.txt | 58 + evidence/sandbox/pip-audit-after.txt | 2 + evidence/sandbox/pip-audit-before.txt | 104 ++ evidence/sandbox/public-tests.txt | 6 + evidence/sandbox/security_http.txt | 26 + evidence/sandbox/student-tests.txt | 12 + evidence/sandbox/tls_audit.json | 672 +++++++++++ evidence/sandbox/tls_audit.txt | 95 ++ evidence/security_http.txt | 25 + evidence/tls_audit.json | 714 ++++++++++++ infra/README.md | 19 +- infra/grafana/dashboards/messenger.json | 944 +++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 10 + .../provisioning/datasources/datasources.yaml | 72 ++ infra/haproxy/broker-gate.cfg | 42 + infra/haproxy/haproxy.cfg | 108 ++ infra/haproxy/otel-gate.cfg | 57 + infra/haproxy/s3-gate.cfg | 30 + infra/loki/loki.yaml | 57 + infra/openbao/agent-api.hcl | 81 ++ infra/openbao/agent-worker.hcl | 72 ++ infra/openbao/policies/messenger-api.hcl | 8 + infra/openbao/policies/messenger-worker.hcl | 8 + infra/openbao/server.hcl | 25 + infra/otel/collector.yaml | 113 ++ infra/postgres/entrypoint.sh | 10 + infra/postgres/initdb/10-roles.sh | 14 + infra/postgres/pg_hba.conf | 10 + infra/prometheus/client.yml | 6 + infra/prometheus/prometheus.yml | 31 + infra/prometheus/rules/messenger.yml | 66 ++ infra/prometheus/web.yml | 7 + infra/redis/broker.sh | 23 + infra/redis/cache.sh | 21 + infra/tempo/tempo.yaml | 47 + migrations/001_init.sql | 66 ++ .../002_session_cache_invalidations.sql | 13 + migrations/003_idempotency_keys.sql | 40 + migrations/004_attachments.sql | 68 ++ migrations/005_async_processing.sql | 116 ++ migrations/006_security.sql | 49 + requirements.txt | 19 + run.ps1 | 230 ++++ scripts/bao.py | 382 ++++++ scripts/init_secrets.py | 116 ++ scripts/ops.py | 871 ++++++++++++++ scripts/pki.py | 298 +++++ scripts/restart.cmd | 9 + scripts/restart.sh | 9 + scripts/sql/explain_history.sql | 93 ++ scripts/stop-one.cmd | 9 + scripts/stop-one.sh | 9 + security/THREAT_MODEL.md | 60 +- src/__init__.py | 0 src/broker.py | 225 ++++ src/config.py | 255 +++++ src/db.py | 158 +++ src/health.py | 51 + src/images.py | 285 +++++ src/imaging.py | 274 +++++ src/jobs.py | 473 ++++++++ src/jobs_admin.py | 156 +++ src/main.py | 1020 +++++++++++++++++ src/migrate.py | 25 + src/models.py | 113 ++ src/objectstore.py | 229 ++++ src/outbox.py | 187 +++ src/reconcile.py | 299 +++++ src/runtime.py | 53 + src/s3_bootstrap.py | 264 +++++ src/security.py | 148 +++ src/server.py | 82 ++ src/sessions.py | 243 ++++ src/storage.py | 707 ++++++++++++ src/telemetry.py | 397 +++++++ src/tls.py | 193 ++++ src/worker.py | 417 +++++++ student-tests/common.py | 128 +++ student-tests/dependency_failure.py | 207 ++++ student-tests/failover_series.py | 164 +++ student-tests/fixtures/orient6.jpg | Bin 0 -> 4812 bytes student-tests/fixtures/sample.jpg | Bin 0 -> 13121 bytes student-tests/fixtures/sample.png | Bin 0 -> 111817 bytes student-tests/fixtures/sample_progressive.jpg | Bin 0 -> 8520 bytes student-tests/fixtures/small.png | Bin 0 -> 93 bytes student-tests/fixtures/transparent.png | Bin 0 -> 777 bytes student-tests/imagegen.py | 93 ++ student-tests/loadgen.py | 170 +++ student-tests/race.py | 203 ++++ student-tests/revocation_partition.py | 38 + student-tests/rolling_update_load.py | 96 ++ student-tests/run_all.py | 51 + student-tests/s3_outage.py | 144 +++ student-tests/security/argon2_bench.py | 41 + student-tests/security/hardening_audit.py | 147 +++ student-tests/security/leak_check.py | 122 ++ student-tests/security/login_flood.py | 102 ++ student-tests/security/mtls_probe.py | 25 + student-tests/security/security_http.py | 196 ++++ student-tests/security/tls_audit.py | 447 ++++++++ student-tests/test_api.py | 139 +++ student-tests/test_attachment_consistency.py | 311 +++++ student-tests/test_attachments.py | 310 +++++ student-tests/test_auth.py | 110 ++ student-tests/test_concurrency.py | 81 ++ student-tests/test_expiry.py | 50 + student-tests/test_idempotency.py | 173 +++ student-tests/test_pipeline.py | 179 +++ student-tests/test_pipeline_faults.py | 311 +++++ student-tests/test_session_cache.py | 149 +++ student-tests/test_storage_idempotency.py | 144 +++ student-tests/test_storage_rollback.py | 101 ++ student-tests/tlsenv.py | 35 + student-tests/upload_memory.py | 233 ++++ student-tests/wait_url.py | 19 + student-tests/worker_outage.py | 133 +++ 298 files changed, 28830 insertions(+), 109 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.faults.yaml create mode 100644 compose.yaml create mode 100644 docs/lab3/REPORT.md create mode 100644 docs/lab3/evidence/README.md create mode 100644 docs/lab3/evidence/SUMMARY.txt create mode 100644 docs/lab3/evidence/check.txt create mode 100644 docs/lab3/evidence/failover.txt create mode 100644 docs/lab3/evidence/failover_series.json create mode 100644 docs/lab3/evidence/failover_series.txt create mode 100644 docs/lab3/evidence/failover_series_kill.json create mode 100644 docs/lab3/evidence/instances.txt create mode 100644 docs/lab3/evidence/lb_errors.txt create mode 100644 docs/lab3/evidence/lb_log_sample.txt create mode 100644 docs/lab3/evidence/load.json create mode 100644 docs/lab3/evidence/load.txt create mode 100644 docs/lab3/evidence/migrations.txt create mode 100644 docs/lab3/evidence/persistence.txt create mode 100644 docs/lab3/evidence/rolling_back.json create mode 100644 docs/lab3/evidence/rolling_update.json create mode 100644 docs/lab3/evidence/rolling_update.txt create mode 100644 docs/lab3/evidence/shutdown_timeline.txt create mode 100644 docs/lab3/evidence/smoke.txt create mode 100644 docs/lab3/evidence/stand.txt create mode 100644 docs/lab3/evidence/student-tests.txt create mode 100644 docs/lab4/REPORT.md create mode 100644 docs/lab4/evidence/README.md create mode 100644 docs/lab4/evidence/SUMMARY.txt create mode 100644 docs/lab4/evidence/check.txt create mode 100644 docs/lab4/evidence/consistency.txt create mode 100644 docs/lab4/evidence/failover.txt create mode 100644 docs/lab4/evidence/failover_series.json create mode 100644 docs/lab4/evidence/failover_series.txt create mode 100644 docs/lab4/evidence/failover_series_kill.json create mode 100644 docs/lab4/evidence/instances.txt create mode 100644 docs/lab4/evidence/lb_log_sample.txt create mode 100644 docs/lab4/evidence/load.json create mode 100644 docs/lab4/evidence/load.txt create mode 100644 docs/lab4/evidence/persistence.txt create mode 100644 docs/lab4/evidence/reconcile.txt create mode 100644 docs/lab4/evidence/rolling_back.json create mode 100644 docs/lab4/evidence/rolling_update.json create mode 100644 docs/lab4/evidence/rolling_update.txt create mode 100644 docs/lab4/evidence/s3_bootstrap.txt create mode 100644 docs/lab4/evidence/s3_outage.json create mode 100644 docs/lab4/evidence/s3_outage.txt create mode 100644 docs/lab4/evidence/s3_private.txt create mode 100644 docs/lab4/evidence/sandbox/check.txt create mode 100644 docs/lab4/evidence/sandbox/consistency.txt create mode 100644 docs/lab4/evidence/sandbox/failover.txt create mode 100644 docs/lab4/evidence/sandbox/failover_series.json create mode 100644 docs/lab4/evidence/sandbox/load.json create mode 100644 docs/lab4/evidence/sandbox/persistence.txt create mode 100644 docs/lab4/evidence/sandbox/reconcile.txt create mode 100644 docs/lab4/evidence/sandbox/s3_bootstrap.txt create mode 100644 docs/lab4/evidence/sandbox/s3_outage.json create mode 100644 docs/lab4/evidence/sandbox/s3_outage_hang.json create mode 100644 docs/lab4/evidence/sandbox/s3_private.txt create mode 100644 docs/lab4/evidence/sandbox/smoke.txt create mode 100644 docs/lab4/evidence/sandbox/student-tests.txt create mode 100644 docs/lab4/evidence/sandbox/upload_memory_c10.json create mode 100644 docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json create mode 100644 docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json create mode 100644 docs/lab4/evidence/smoke.txt create mode 100644 docs/lab4/evidence/stand.txt create mode 100644 docs/lab4/evidence/student-tests.txt create mode 100644 docs/lab4/evidence/upload_memory.txt create mode 100644 docs/lab4/evidence/upload_memory_c10.json create mode 100644 docs/lab4/evidence/upload_memory_c20.json create mode 100644 docs/lab5/REPORT.md create mode 100644 docs/lab5/evidence/README.md create mode 100644 docs/lab5/evidence/SUMMARY.txt create mode 100644 docs/lab5/evidence/check.txt create mode 100644 docs/lab5/evidence/consistency.txt create mode 100644 docs/lab5/evidence/failover.txt create mode 100644 docs/lab5/evidence/failover_series.json create mode 100644 docs/lab5/evidence/failover_series.txt create mode 100644 docs/lab5/evidence/failover_series_kill.json create mode 100644 docs/lab5/evidence/instances.txt create mode 100644 docs/lab5/evidence/lb_log_sample.txt create mode 100644 docs/lab5/evidence/load.json create mode 100644 docs/lab5/evidence/load.txt create mode 100644 docs/lab5/evidence/local/README.md create mode 100644 docs/lab5/evidence/local/failover.txt create mode 100644 docs/lab5/evidence/local/imaging_memory.txt create mode 100644 docs/lab5/evidence/local/latency.json create mode 100644 docs/lab5/evidence/local/persistence.txt create mode 100644 docs/lab5/evidence/local/pipeline_and_attachments.txt create mode 100644 docs/lab5/evidence/local/pipeline_faults.txt create mode 100644 docs/lab5/evidence/local/pipeline_faults_events.txt create mode 100644 docs/lab5/evidence/local/smoke.txt create mode 100644 docs/lab5/evidence/local/student-tests.txt create mode 100644 docs/lab5/evidence/local/worker_fault_log.txt create mode 100644 docs/lab5/evidence/local/worker_outage.json create mode 100644 docs/lab5/evidence/local/worker_outage.txt create mode 100644 docs/lab5/evidence/persistence.txt create mode 100644 docs/lab5/evidence/pipeline.txt create mode 100644 docs/lab5/evidence/pipeline_faults.txt create mode 100644 docs/lab5/evidence/reconcile.txt create mode 100644 docs/lab5/evidence/rolling_back.json create mode 100644 docs/lab5/evidence/rolling_update.json create mode 100644 docs/lab5/evidence/rolling_update.txt create mode 100644 docs/lab5/evidence/s3_bootstrap.txt create mode 100644 docs/lab5/evidence/s3_outage.json create mode 100644 docs/lab5/evidence/s3_outage.txt create mode 100644 docs/lab5/evidence/s3_private.txt create mode 100644 docs/lab5/evidence/smoke.txt create mode 100644 docs/lab5/evidence/stand.txt create mode 100644 docs/lab5/evidence/student-tests.txt create mode 100644 docs/lab5/evidence/upload_memory.txt create mode 100644 docs/lab5/evidence/upload_memory_c10.json create mode 100644 docs/lab5/evidence/upload_memory_c20.json create mode 100644 docs/lab5/evidence/worker_log_sample.txt create mode 100644 docs/lab5/evidence/worker_outage.json create mode 100644 docs/lab5/evidence/worker_outage.txt create mode 100644 docs/lab6/REPORT.md create mode 100644 docs/lab6/evidence/README.md create mode 100644 docs/lab6/evidence/alerts_db.txt create mode 100644 docs/lab6/evidence/alerts_worker.txt create mode 100644 docs/lab6/evidence/check.txt create mode 100644 docs/lab6/evidence/incident_db.jsonl create mode 100644 docs/lab6/evidence/incident_worker.jsonl create mode 100644 docs/lab6/evidence/local-sandbox/alerts_db_lock_180s.txt create mode 100644 docs/lab6/evidence/local-sandbox/alerts_queue_lag_and_db_short.txt create mode 100644 docs/lab6/evidence/local-sandbox/db_lock_span.txt create mode 100644 docs/lab6/evidence/local-sandbox/incident_worker_stop.jsonl create mode 100644 docs/lab6/evidence/local-sandbox/metric_names.txt create mode 100644 docs/lab6/evidence/local-sandbox/smoke.txt create mode 100644 docs/lab6/evidence/local-sandbox/telemetry_outage.txt create mode 100644 docs/lab6/evidence/local-sandbox/trace_retry_chain.txt create mode 100644 docs/lab6/evidence/local-sandbox/trace_upload_chain.txt create mode 100644 docs/lab6/evidence/probe.txt create mode 100644 docs/lab6/evidence/smoke.txt create mode 100644 docs/lab6/evidence/stand.txt create mode 100644 docs/lab6/evidence/telemetry_down.jsonl create mode 100644 docs/runbooks.md create mode 100644 docs/security/SCAN_TRIAGE.md create mode 100644 evidence/bao_policy.json create mode 100644 evidence/hardening.txt create mode 100644 evidence/leak_check.json create mode 100644 evidence/login_budget.txt create mode 100644 evidence/rehash.txt create mode 100644 evidence/run/SUMMARY.txt create mode 100644 evidence/run/bao-policy.txt create mode 100644 evidence/run/bao-sealed.txt create mode 100644 evidence/run/check-contract.txt create mode 100644 evidence/run/compose-config.txt create mode 100644 evidence/run/expiry.txt create mode 100644 evidence/run/hardening.txt create mode 100644 evidence/run/leak-check.txt create mode 100644 evidence/run/login-budget.txt create mode 100644 evidence/run/ps.txt create mode 100644 evidence/run/rehash.txt create mode 100644 evidence/run/restore-1.txt create mode 100644 evidence/run/restore-2.txt create mode 100644 evidence/run/revoke-identity.txt create mode 100644 evidence/run/rotate-db-password.txt create mode 100644 evidence/run/rotate-lb-cert.txt create mode 100644 evidence/run/rotate-secret-id.txt create mode 100644 evidence/run/s3-check.txt create mode 100644 evidence/run/security-http.txt create mode 100644 evidence/run/smoke-after-lifecycle.txt create mode 100644 evidence/run/storage-tests.txt create mode 100644 evidence/run/student-tests.txt create mode 100644 evidence/run/test-failover-1.txt create mode 100644 evidence/run/test-failover-2.txt create mode 100644 evidence/run/test-persistence.txt create mode 100644 evidence/run/test-security.txt create mode 100644 evidence/run/test-smoke.txt create mode 100644 evidence/run/tls-audit.txt create mode 100644 evidence/run/up.txt create mode 100644 evidence/sandbox/README.md create mode 100644 evidence/sandbox/bao_policy.json create mode 100644 evidence/sandbox/leak_check.json create mode 100644 evidence/sandbox/lifecycle.txt create mode 100644 evidence/sandbox/pip-audit-after.txt create mode 100644 evidence/sandbox/pip-audit-before.txt create mode 100644 evidence/sandbox/public-tests.txt create mode 100644 evidence/sandbox/security_http.txt create mode 100644 evidence/sandbox/student-tests.txt create mode 100644 evidence/sandbox/tls_audit.json create mode 100644 evidence/sandbox/tls_audit.txt create mode 100644 evidence/security_http.txt create mode 100644 evidence/tls_audit.json create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/haproxy/broker-gate.cfg create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/haproxy/otel-gate.cfg create mode 100644 infra/haproxy/s3-gate.cfg create mode 100644 infra/loki/loki.yaml create mode 100644 infra/openbao/agent-api.hcl create mode 100644 infra/openbao/agent-worker.hcl create mode 100644 infra/openbao/policies/messenger-api.hcl create mode 100644 infra/openbao/policies/messenger-worker.hcl create mode 100644 infra/openbao/server.hcl create mode 100644 infra/otel/collector.yaml create mode 100644 infra/postgres/entrypoint.sh create mode 100644 infra/postgres/initdb/10-roles.sh create mode 100644 infra/postgres/pg_hba.conf create mode 100644 infra/prometheus/client.yml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/prometheus/rules/messenger.yml create mode 100644 infra/prometheus/web.yml create mode 100644 infra/redis/broker.sh create mode 100644 infra/redis/cache.sh create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_session_cache_invalidations.sql create mode 100644 migrations/003_idempotency_keys.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_async_processing.sql create mode 100644 migrations/006_security.sql create mode 100644 requirements.txt create mode 100644 run.ps1 create mode 100644 scripts/bao.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/ops.py create mode 100644 scripts/pki.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/sql/explain_history.sql create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/broker.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/health.py create mode 100644 src/images.py create mode 100644 src/imaging.py create mode 100644 src/jobs.py create mode 100644 src/jobs_admin.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/objectstore.py create mode 100644 src/outbox.py create mode 100644 src/reconcile.py create mode 100644 src/runtime.py create mode 100644 src/s3_bootstrap.py create mode 100644 src/security.py create mode 100644 src/server.py create mode 100644 src/sessions.py create mode 100644 src/storage.py create mode 100644 src/telemetry.py create mode 100644 src/tls.py create mode 100644 src/worker.py create mode 100644 student-tests/common.py create mode 100644 student-tests/dependency_failure.py create mode 100644 student-tests/failover_series.py create mode 100644 student-tests/fixtures/orient6.jpg create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/fixtures/sample.png create mode 100644 student-tests/fixtures/sample_progressive.jpg create mode 100644 student-tests/fixtures/small.png create mode 100644 student-tests/fixtures/transparent.png create mode 100644 student-tests/imagegen.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/race.py create mode 100644 student-tests/revocation_partition.py create mode 100644 student-tests/rolling_update_load.py create mode 100644 student-tests/run_all.py create mode 100644 student-tests/s3_outage.py create mode 100644 student-tests/security/argon2_bench.py create mode 100644 student-tests/security/hardening_audit.py create mode 100644 student-tests/security/leak_check.py create mode 100644 student-tests/security/login_flood.py create mode 100644 student-tests/security/mtls_probe.py create mode 100644 student-tests/security/security_http.py create mode 100644 student-tests/security/tls_audit.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachment_consistency.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_pipeline.py create mode 100644 student-tests/test_pipeline_faults.py create mode 100644 student-tests/test_session_cache.py create mode 100644 student-tests/test_storage_idempotency.py create mode 100644 student-tests/test_storage_rollback.py create mode 100644 student-tests/tlsenv.py create mode 100644 student-tests/upload_memory.py create mode 100644 student-tests/wait_url.py create mode 100644 student-tests/worker_outage.py diff --git a/.dockerignore b/.dockerignore index bc16e55..1d8a6a2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,11 @@ __pycache__ node_modules evidence .DS_Store +operator-only +student-tests +docs +infra +scripts +tests +contracts +*.md diff --git a/.env.example b/.env.example index 2e9f054..4a518f3 100644 --- a/.env.example +++ b/.env.example @@ -1,41 +1,67 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 +# lab7: локальные НЕсекретные настройки стенда. Скопируйте в .env (run.ps1/ops.py up делает это сам). +# Секретов здесь нет и быть не должно: их создают scripts/init_secrets.py (secrets/), +# scripts/pki.py (secrets/pki/) и scripts/bao.py (secrets/bao/, operator-only/). +# +# Соответствие логическим именам skeleton (.env.example skeleton -> реализация): +# APP_PORT=8080 -> API слушает 8443 (только HTTPS+mTLS); наружу — LB 127.0.0.1:${HTTPS_PORT} +# DATABASE_URL_FILE -> /run/bao/database_url (отрисовывает OpenBao Agent; роль messenger_api/worker) +# CACHE_URL_FILE -> /run/bao/cache_url (rediss://, ACL messenger_api) +# BROKER_URL_FILE -> /run/bao/broker_url (rediss:// через broker-gate, ACL api_producer/worker_consumer) +# S3_*_FILE -> /run/bao/s3_* (identity messenger_api / messenger_worker) +# TLS_CA_FILE/CERT/KEY -> /run/secrets/ca_crt, /run/secrets/_crt|_key (scripts/pki.py) +# BAO_ADDR/BAO_CACERT -> в конфигурации агентов infra/openbao/agent-*.hcl (https://openbao:8200) +# ARGON2_* -> ниже (базовая линия OWASP m=19 MiB, t=2, p=1) + +COMPOSE_PROJECT_NAME=messenger-lab7 APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +APP_VERSION=lab7 + +# Публичный HTTPS-вход (LB) и HTTPS Grafana — только 127.0.0.1 +HTTPS_PORT=8443 +GRAFANA_PORT=3000 + API_REPLICAS=2 +WORKER_REPLICAS=1 +SESSION_TTL_SECONDS=3600 +SHORT_SESSION_TTL_SECONDS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=https://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 -JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab -# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix. -PUBLIC_BASE_URL=https://localhost:8443 -TLS_CA_FILE=/run/secrets/ca.crt -TLS_CERT_FILE=/run/secrets/service.crt -TLS_KEY_FILE=/run/secrets/service.key -BAO_ADDR=https://openbao:8200 -BAO_CACERT=/run/secrets/bao-ca.crt -# No root token here. Bootstrap operator -> short-lived service auth -> scoped token. + +# Argon2id (не ниже m=19456 KiB, t=2, p=1 — иначе API не стартует). Обоснование — REPORT.md. ARGON2_MEMORY_KIB=19456 ARGON2_ITERATIONS=2 ARGON2_PARALLELISM=1 +# Одновременных хешей на реплику и ожидание слота (дальше 503 + Retry-After) +PASSWORD_HASH_CONCURRENCY=4 +KDF_QUEUE_TIMEOUT_SECONDS=2 +# Общий для всех реплик rate limit входа (PostgreSQL), окно в секундах +LOGIN_RATE_WINDOW_SECONDS=60 +LOGIN_RATE_PER_USER=10 +LOGIN_RATE_PER_IP=300 +# Лимит тела не-upload запросов (LB проверяет то же значение по Content-Length) +MAX_JSON_BODY_BYTES=65536 +MAX_UPLOAD_BYTES=10485760 +MAX_IMAGE_PIXELS=40000000 + +# Подсеть сети secrets (OpenBao + агенты). Это же значение — secret_id_bound_cidrs AppRole. +# Если Docker пишет "Pool overlaps with other one on this address space" — смените подсеть +# ДО `bao.py configure` (или перезапустите configure после смены). +BAO_SECRETS_SUBNET=172.30.250.0/24 +BAO_SECRETS_CIDR=172.30.250.0/24 + +# Версии образов (закреплены) +OPENBAO_VERSION=2.1.0 +SEAWEEDFS_VERSION=4.50 + +# Очередь и worker (как в lab5/lab6) +BROKER_STREAM=attachments.jobs +BROKER_GROUP=attachment-workers +WORKER_CONCURRENCY=2 +JOB_MAX_ATTEMPTS=5 +JOB_TIMEOUT_SECONDS=60 +WORKER_IMAGE_TIMEOUT_SECONDS=30 +WORKER_IMAGE_MEMORY_MB=1024 +STAGING_RETENTION_SECONDS=86400 + +# Наблюдаемость +OTEL_SDK_DISABLED=false +OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1a041c9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +# Скрипты и конфиги, которые выполняются внутри Linux-контейнеров или sh, — всегда LF, +# даже при checkout на Windows (иначе HAProxy/sh спотыкаются о CRLF). +*.sh text eol=lf +*.cfg text eol=lf +*.sql text eol=lf +*.py text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf +# Пакетные файлы Windows — CRLF. +*.cmd text eol=crlf +*.hcl text eol=lf +*.conf text eol=lf +*.yml text eol=lf +*.json text eol=lf +*.ps1 text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8aef0ff..866222d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,39 @@ ## Лабораторная 7 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Попов Константин Дмитриевич, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявляется -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: +Перенесено решение lab6 целиком: HAProxy, 2 реплики API, PostgreSQL, Redis-кэш сессий, SeaweedFS S3, Redis Streams + worker + outbox/retry/DLQ, OTel Collector → Prometheus/Loki/Tempo → Grafana. Бизнес-API не менялся, транспорт — HTTPS. Изменения lab7: +- Внешний TLS 1.2+ на LB (127.0.0.1:8443, HTTP-порта нет). Обязательный mTLS LB→API: API принимает только клиентский сертификат с identity lb и проверяет CRL (src/tls.py). Health-listener API доступен только на 127.0.0.1 внутри контейнера. +- TLS на всех внутренних связях: + - PostgreSQL — verify-full, только hostssl + SCRAM; раздельные роли messenger_api / messenger_worker, у API нет DDL, worker не видит users/sessions. + - Redis-кэш — mTLS + ACL. + - S3 — HTTPS, раздельные identities по префиксам. SeaweedFS слушает только loopback и выдан в сеть через TCP-passthrough s3-gate: закрыт неаутентифицированный gRPC S3-шлюза. + - Loki, Tempo, Prometheus и Grafana — mTLS. + - OpenBao — HTTPS. +- Уровень 4: mTLS к broker и Collector через HAProxy-gates с allowlist identity (api/worker, для метрик — prometheus) и CRL; Redis ACL producer/consumer. Доказано, что чужой сертификат того же CA (intruder, lb, prometheus) не получает доступа даже с верным паролем. +- OpenBao 2.1 не в dev-режиме: file storage, ручной init/unseal оператором. AppRole с политиками на роль; секреты в приложение доставляет OpenBao Agent в tmpfs. Root token, unseal keys и SecretID приложению недоступны. +- Пароли: Argon2id (m=19 MiB, t=2, p=1, замер на стенде), rehash при входе, хеш-«пустышка» для несуществующих username. +- Вход: общий для реплик rate limit в PostgreSQL (429 + Retry-After до KDF), семафор KDF (503 + Retry-After). +- Лимиты тела: 64 KiB для JSON (на LB и в API), 10 MiB для загрузки. +- Ротация без пересборки: пароль БД (новые соединения пула берут новый пароль), leaf-сертификат LB с отзывом через CRL, SecretID агента; отзыв identity AppRole; работа API при sealed OpenBao. +- Hardening: cap_drop: ALL, no-new-privileges, non-root, read_only + tmpfs, лимиты CPU/RAM/PIDs. Сети internal без egress; наружу только 127.0.0.1:8443 и 127.0.0.1:3000. +- Обновлены зависимости по pip-audit (72 уязвимости → 0): FastAPI/Starlette, Pillow 13 (декодеры только PNG/JPEG), OpenTelemetry 1.45, cryptography 50. +- Документы: модель угроз (12 угроз → контроли → отрицательные тесты), заполненная contracts/trust-matrix.md, docs/security/SCAN_TRIAGE.md. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Команды запуска из чистого clone (Windows/PowerShell) одной командой: powershell -ExecutionPolicy Bypass -File .\run.ps1 (Linux: make all). После рестарта контейнера OpenBao: python scripts\bao.py unseal. + +Результаты проверок и ссылка на REPORT.md: Подробно — REPORT.md, раздел «Доказательства критериев». Реальный вывод всех шагов — evidence/run/*.txt, сводка — evidence/run/SUMMARY.txt, журнал действий с контейнерами — evidence/ops.log. + +Ограничения / известные ошибки: см. REPORT.md «Ограничения». + +Источники и использование ИИ: REPORT.md. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.gitignore b/.gitignore index b5a7189..70c916e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ .env.* !.env.example secrets/ +# lab7: root token и unseal shares OpenBao, accessors SecretID — только у оператора +operator-only/ certs/private/ *.key *.p12 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..653394c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Lab7: API мессенджера и worker (один образ) — FastAPI + PostgreSQL + Redis + S3 + Redis Streams + OTel. +# lab7: сервер только HTTPS/mTLS (:8443), health — loopback 127.0.0.1:8081; non-root uid 10001. +# Секреты и ключи в образ не попадают (.dockerignore: secrets/, operator-only/, *.key, *.pem). +FROM python:3.12.7-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8443 \ + HEALTH_PORT=8081 \ + APP_ENV=lab \ + MALLOC_ARENA_MAX=2 \ + MALLOC_MMAP_THRESHOLD_=1048576 \ + MALLOC_TRIM_THRESHOLD_=1048576 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --disable-pip-version-check -r requirements.txt + +COPY src ./src +COPY migrations ./migrations + +# Явный uid/gid 10001: совпадает с владельцем tmpfs-volume секретов OpenBao Agent (compose.yaml). +# Код и зависимости принадлежат root и только читаются процессом. +RUN groupadd --gid 10001 appuser \ + && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin appuser + +USER 10001:10001 + +EXPOSE 8443 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).read()" || exit 1 + +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src.server"] diff --git a/Makefile b/Makefile index 5ad43ad..a9482a7 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,20 @@ PYTHON ?= python3 BASE_URL ?= https://localhost:8443 -CA_FILE ?= +CA_FILE ?= secrets/pki/ca.crt ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-security +.PHONY: help check validate test load test-persistence test-failover test-security \ + up down destroy wait ps restore tls-audit security-http leak-check hardening bao-policy rehash \ + rotate-db-password rotate-lb-cert rotate-secret-id revoke-identity bao-sealed login-budget scan \ + test-student test-storage test-expiry all help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "skeleton: check validate test load test-persistence test-failover test-security" + @echo "stand: up down destroy wait ps restore (Windows: powershell -File run.ps1)" + @echo "lab7: tls-audit security-http leak-check hardening bao-policy rehash" + @echo " rotate-db-password rotate-lb-cert rotate-secret-id revoke-identity bao-sealed login-budget scan" + @echo "all: make all = up + test + security + resilience + lifecycle" check: $(PYTHON) scripts/check_contract.py @@ -22,12 +29,42 @@ load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/restart.sh"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/stop-one.sh"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-security: $(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +# ---------------------------------------------------------------- стенд (scripts/ops.py) +up: + $(PYTHON) scripts/ops.py up +down: + $(PYTHON) scripts/ops.py down +# НАМЕРЕННОЕ удаление данных стенда (volumes) +destroy: + $(PYTHON) scripts/ops.py destroy +wait ps restore tls-audit security-http leak-check hardening bao-policy bao-sealed login-budget scan rotate-lb-cert: + $(PYTHON) scripts/ops.py $@ +rehash: + $(PYTHON) scripts/ops.py rehash-demo +rotate-db-password: + $(PYTHON) scripts/ops.py rotate-db-password api +rotate-secret-id: + $(PYTHON) scripts/ops.py rotate-secret-id api +revoke-identity: + $(PYTHON) scripts/ops.py revoke-identity worker +test-student: + CA_FILE="$(CA_FILE)" $(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)" +test-storage: + $(PYTHON) scripts/ops.py in-network storage +test-expiry: + $(PYTHON) scripts/ops.py in-network expiry + +all: check up test test-security tls-audit security-http bao-policy hardening rehash leak-check test-expiry + $(MAKE) test-persistence ACTION_SCRIPT=scripts/restart.sh + $(MAKE) test-failover ACTION_SCRIPT=scripts/stop-one.sh && $(MAKE) restore + $(MAKE) rotate-db-password rotate-lb-cert rotate-secret-id revoke-identity bao-sealed test diff --git a/REPORT.md b/REPORT.md index 9801db4..49ec90f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,133 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: OpenBao 2.1.0 (server, Agent, AppRole, KV v2, политики), HAProxy 2.8 (TLS termination, `verify required`, `crl-file`, `ssl_c_s_dn`), Redis 7.4 TLS/ACL, PostgreSQL 16 (`hostssl`, SCRAM, `sslmode=verify-full`), SeaweedFS 4.50 (S3 TLS, identities с префиксами), OTel Collector 0.115.1 (TLS exporters), dskit `http_tls_config` (Loki/Tempo), Prometheus `web.config.file`; Python: `argon2-cffi` (PHC, `check_needs_rehash`), `cryptography` (CA, CRL, разбор клиентского cert), `uvicorn` (собственный подкласс H11Protocol для проверки identity). Документация: OWASP Password Storage / File Upload Cheat Sheets, OpenBao AppRole / policies / production hardening, Docker security. Помощь ИИ (Claude): проектирование матрицы доверия, код `src/tls.py`, `scripts/pki.py`, `scripts/bao.py`, конфигурации, отрицательные тесты, отчёт. + +Минимум lab6 перенесён целиком (HAProxy, 2 реплики API, PostgreSQL, Redis-кэш, SeaweedFS S3, Redis Streams, worker, outbox, retry/DLQ, OTel → Collector → Prometheus/Loki/Tempo → Grafana). **Бизнес-API не менялся**, транспорт — HTTPS. Изменения поведения, видимые клиенту: `X-Instance-Id` — случайный alias (`api-xxxxxxxx`), `429 rate_limited` на входе, `413` для JSON > 64 KiB, `503 + Retry-After` при перегрузке KDF — всё в пределах кодов контракта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Desktop с Compose v2 (Linux-контейнеры), Python 3.9+ (стандартная библиотека; пакет `cryptography` на хосте не нужен — PKI создаётся в контейнере `tools` без сети), свободные `127.0.0.1:8443` и `127.0.0.1:3000`. Образы закреплены: `python:3.12.7-slim-bookworm`, `postgres:16.4-alpine`, `redis:7.4.1-alpine`, `haproxy:2.8.30-alpine`, `chrislusf/seaweedfs:4.50`, `openbao/openbao:2.1.0`, `otel/opentelemetry-collector-contrib:0.115.1`, `prom/prometheus:v2.55.1`, `grafana/loki:3.3.2`, `grafana/tempo:2.6.1`, `grafana/grafana:11.4.0`. + +**Windows / PowerShell — одна команда**: + +```powershell +powershell -ExecutionPolicy Bypass -File .\run.ps1 # развернуть + все проверки, итог в evidence\run\SUMMARY.txt +``` + +**Linux/macOS:** `make all` (или те же шаги через `python3 scripts/ops.py …`). + +Что делает `python scripts/ops.py up` (вызывается из `run.ps1`): + +1. `.env` из `.env.example` (только несекретные настройки). +2. `scripts/init_secrets.py` → `secrets/` (пароли ролей БД/Redis/broker, ключи S3 трёх identities, пароль Grafana). Значения не печатаются; повторный запуск ничего не перезаписывает. +3. `docker compose build api` — один образ для api/worker/migrate/s3-init/tools. +4. `scripts/pki.py init` → `secrets/pki/` (CA, 15 leaf, CRL). **Действие оператора**: `ca.key` хранить у себя, никуда не монтируется. +5. `docker compose up -d openbao` → `scripts/bao.py init` (**оператор**: 3 unseal shares, порог 2, root token → `operator-only/openbao-init.json`, 0600) → `unseal` → `configure` (kv-v2, AppRole, политики, секреты в KV, RoleID/SecretID → `secrets/bao/`). +6. `docker compose up -d` — `migrate` (администратор БД, миграция `006_security.sql`: роли и права), `s3-init` (бакет + проверка прав трёх identities), агенты отрисовывают секреты, затем api ×2, worker, LB, телеметрия. +7. Ожидание: N реплик healthy, UP в HAProxy по mTLS-проверкам, `https://127.0.0.1:8443/health/ready` = 200 с проверкой CA. + +Тесты: `make test BASE_URL=https://localhost:8443 CA_FILE=secrets/pki/ca.crt`, `make test-security …` (в `run.ps1` — шаги `test-smoke`, `test-security`). **После каждого рестарта контейнера OpenBao** он sealed: `python scripts/bao.py unseal` (hook `restart` делает это сам). Безопасная остановка: `python scripts/ops.py down` (volumes целы). Намеренное удаление учебных данных: `python scripts/ops.py destroy` (+ удалить `operator-only/openbao-init.json` и `secrets/bao/*`, т.к. они относятся к удалённому хранилищу OpenBao). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` + 127.0.0.1:8443 ──TLS──► lb ──mTLS (identity lb, CRL)──► api ×2 ──TLS verify-full──► postgres (messenger_api) + ├──mTLS + ACL──────────────► redis (кэш сессий) + ├──TLS, identity API──────► s3 (staging/ write) + ├──mTLS(api)──► broker-gate ─lo─► broker (ACL api_producer) + └──mTLS(api)──► otel-gate ─lo─► collector ─mTLS─► loki / tempo + worker ──TLS──► postgres (messenger_worker) / s3 (final/ write, List) ; ──mTLS(worker)──► broker-gate / otel-gate + prometheus ──mTLS(prometheus)──► otel-gate :8889/:8888 ; grafana (127.0.0.1:3000 HTTPS) ──mTLS──► prometheus/loki/tempo + bao-agent-api / bao-agent-worker ──TLS + AppRole──► openbao (file storage, ручной unseal) + └── tmpfs volume api-secrets / worker-secrets ──► /run/bao (read-only) в api / worker +``` + +Роль → сервис → health → volume → сеть/порт → credentials: + +| Роль | Сервис | Health/readiness | Volume | Сети, порт | Credentials | +| --- | --- | --- | --- | --- | --- | +| api ×2 | `api` | loopback `127.0.0.1:8081/health/ready` (Docker), LB — `/health/ready` по mTLS | нет (tmpfs `/tmp`, ro `/run/bao`) | ingress, backend, storage, queue, telemetry-ingest; `8443` не опубликован | TLS: `api_crt/key`; секреты — `/run/bao/*` от агента | +| LB | `lb` | `127.0.0.1:8404/lb-health` | — | edge, ingress; **`127.0.0.1:8443`** | `lb_public_pem`, `lb_client_pem`, `ca_crt` | +| worker | `worker` | heartbeat-файл | — | backend, storage, queue, telemetry-ingest | `worker_crt/key`; `/run/bao/*` | +| postgres | `postgres` (uid 70) | `pg_isready` (unix socket) | `pgdata` | backend `5432` (только `hostssl`) | `postgres_crt/key`, `postgres_password`, пароли ролей (initdb) | +| redis (кэш) | `redis` | `redis-cli --tls … ping` (ACL `healthcheck`) | — (кэш без persistence) | backend `6379` TLS | `redis_crt/key`, `cache_api_password` | +| broker | `broker` + `broker-gate` | ping по loopback / `127.0.0.1:8405` | `brokerdata` (AOF fsync always) | queue `6380` (gate), Redis `127.0.0.1:6379` | `broker_pem`, `crl_pem`, пароли producer/consumer | +| s3 | `s3` + `s3-gate`, one-shot `s3-init` | порт 8334 + реальный TLS-путь в `s3-init`/`wait-s3` | `s3data` | storage `8333` (gate, TLS passthrough), SeaweedFS — только `127.0.0.1` | `s3_crt/key`, `s3_config` (identities) | +| миграции | one-shot `migrate` | exit 0 | — | backend | `database_admin_url` (администратор) | +| OpenBao | `openbao` | `bao status` (0/2 = процесс жив) | `baodata` | secrets `8200` HTTPS | `openbao_crt/key` | +| агенты | `bao-agent-api`, `bao-agent-worker` | файлы секретов отрисованы | `api-secrets`, `worker-secrets` (tmpfs) | secrets (подсеть = `secret_id_bound_cidrs`) | RoleID/SecretID своей роли | +| Collector | `otel-collector` + `otel-gate` | `127.0.0.1:8405/gate-health` | — | telemetry-ingest `4318` (gate), telemetry `8889/8888` (gate) | `otel_collector_*` | +| Prometheus / Loki / Tempo | как в lab6 | `promtool check healthy` по mTLS / — / — | `promdata`, `lokidata`, `tempodata` | telemetry (internal) | свои `*_crt/key` | +| Grafana | `grafana` | — | `grafanadata` | telemetry, ops; **`127.0.0.1:3000`** HTTPS | `grafana_admin_password`, `grafana_crt/key` | + +Ключевые решения: + +* **mTLS LB→API в приложении, а не sidecar-ом.** Uvicorn принимает только TLS с `CERT_REQUIRED`; подкласс `H11Protocol` (`src/tls.py`) после рукопожатия проверяет SAN/CN по `MTLS_ALLOWED_CLIENTS=lb` и serial по CRL (подпись CA, перечитывание по mtime ≤ 5 с, fail closed) и обрывает соединение до первого байта HTTP. Отказы пишутся в лог (`mtls.rejected` с причиной). Docker-healthcheck не умеет mTLS — для него минимальный listener на `127.0.0.1:8081` (только `/health/*`). +* **Gates для broker и Collector.** Redis и Collector умеют TLS/проверку CA, но не allowlist identity, поэтому TLS завершается в HAProxy-sidecar в том же сетевом namespace (`network_mode: service:…`); backend слушает только loopback. Gate проверяет CA + CRL + CN (`api|worker` / `prometheus`) и логирует отказ с CN. Плюс независимый второй фактор — Redis ACL (`default off`, producer/consumer). +* **Секреты только через OpenBao Agent.** Приложение читает файлы `/run/bao/*` (tmpfs-volume), а не ходит в OpenBao: синхронный путь не зависит от OpenBao, SecretID и токен приложению недоступны. Пароль БД перечитывается при каждом новом соединении пула (`RotatingConnection`), что даёт ротацию без рестарта. +* **S3 только через gate.** При ревизии найдено, что `weed server -s3` открывает неаутентифицированный gRPC S3-шлюза (порт S3 + 10000) на том же адресе, что и S3 (в lab6 — `0.0.0.0` в сети storage); выключить его флагом нельзя. Теперь весь SeaweedFS слушает `127.0.0.1` контейнера, а наружу выдан только порт S3 через `s3-gate` (TCP passthrough, TLS не снимается) — угроза T12. +* **Least privilege в хранилищах.** Роли БД разведены (worker не видит `users/sessions/messages`, у API нет DDL), S3-identities разведены по префиксам (у API нет List и записи в `final/`, поэтому полный обход бакета для сверки перенесён в worker: `ORPHAN_SWEEP_ENABLED=false` у API), ACL broker/кэша — по командам и ключам. +* **Argon2id.** Параметры из окружения с проверкой нижней границы OWASP (m ≥ 19 MiB, t ≥ 2, p ≥ 1 — иначе процесс не стартует). Замер на стенде: `ops.py login-budget`; в песочнице подготовки: hash/verify ≈ 24 мс, 19 MiB; при квоте 0.5 CPU ≈ 21 вход/с на реплику. Варианты m=64 MiB/t=2 (≈130 мс, 64 MiB) сокращают пропускную способность в 5 раз и при 4 слотах KDF требуют 256 MiB из 320M лимита реплики — поэтому оставлена базовая линия OWASP; PHC-строка хранит параметры, и `check_needs_rehash` пересчитывает старые хеши при успешном входе (compare-and-set в БД). +* **Login DoS.** Порядок: (1) общий rate limit в PostgreSQL (атомарный upsert счётчика окна; ключи — SHA-256 username/IP) → 429 + Retry-After **до** KDF; (2) семафор KDF 4/реплику + ожидание слота 2 с → 503 + Retry-After; (3) хеш-«пустышка» для несуществующих username. IP клиента берётся из `X-Forwarded-For`, который LB перезаписывает, — доверие обосновано тем, что к API подключается только identity `lb`. +* **Hardening.** Все сервисы: `cap_drop: ALL`, `no-new-privileges`, лимиты CPU/RAM/PIDs; почти все — non-root и `read_only` + tmpfs; все сети приложения `internal` (нет egress); опубликованы только `127.0.0.1:8443` и `127.0.0.1:3000`. Исключения ниже. + +Соответствие переменных `.env.example` skeleton → реализация — в шапке `.env.example`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Проверки на Docker-стенде собираются автоматически: `run.ps1` пишет вывод каждого шага в `evidence/run/<шаг>.txt` и сводку в `evidence/run/SUMMARY.txt`; `ops.py` дописывает каждое действие с контейнерами в `evidence/ops.log`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат (песочница) | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke по HTTPS | `make test BASE_URL=https://localhost:8443 CA_FILE=secrets/pki/ca.crt` | `PASS: lab7 public API scenario` | `evidence/sandbox/public-tests.txt`, `evidence/run/test-smoke.txt` | +| Внешний TLS: доверенный CA / недоверенный | `make test-security` | `PASS: trusted HTTPS works; untrusted CA rejected` | `public-tests.txt`, `run/test-security.txt` | +| **3:** Argon2id, salt+параметры, база ≥ 19 MiB/t2/p1, обоснование | `ops.py login-budget`, `rehash-demo`; код `src/security.py` | PHC `m=19456,t=2,p=1`; замер в `lifecycle.txt` п.7 | `evidence/sandbox/lifecycle.txt` | +| **3:** LB→API mTLS: нет cert / чужой CA / неверная identity → отказ; нет обхода | `ops.py tls-audit` (раздел lb→api) | нет cert, чужой CA (CN=lb), `intruder`, `worker`, `api` → отказ; `lb` → 200; plaintext → нет ответа | `evidence/sandbox/tls_audit.txt` | +| **3:** остальные связи TLS с проверкой сервера; раздельные creds | `ops.py tls-audit` | PG verify-full (plaintext/чужой CA/IP вместо имени → отказ), Redis TLS, S3 HTTPS, broker/collector gates, Loki/Tempo/Prometheus mTLS, OpenBao HTTPS | `tls_audit.txt`, `contracts/trust-matrix.md` | +| **3:** OpenBao не dev, persistent, сервисная auth, scoped policy, без root в app | `ops.py bao-policy`, `infra/openbao/*` | 12/12: свой путь читается; чужой путь, запись, политики, выпуск SecretID — deny; токен TTL ≤ 15 мин, не root | `evidence/sandbox/bao_policy.json` | +| **3:** контейнеры без privileged/host/docker.sock, non-root, caps, лимиты, только нужные порты | `ops.py hardening` | на Docker-стенде; в песочнице не применимо | `evidence/run/hardening.txt` | +| **3:** 429/Retry-After, лимит размера, посторонний, logout, недоверенный TLS | `ops.py security-http`, `in-network expiry` | 24/24: 404 постороннему, `X-User-Id` игнор, logout 204→401→401, 429 + Retry-After, 413 LB/API/upload, TLS ошибки проверки | `evidence/sandbox/security_http.txt` | +| **3:** секреты не в Git/образах/telemetry; локальный bootstrap сертификатов | `.gitignore`, `.dockerignore`, `ops.py leak-check`, `pki.py init` | канарейки и все рабочие секреты не найдены в Loki/Tempo/Prometheus (с положительным контролем) | `evidence/sandbox/leak_check.json` | +| **3:** перезапуск с сохранением данных и отказ реплики по HTTPS | `tests/resilience.py --mode persistence/failover` (hooks `scripts/restart.cmd|sh`, `stop-one.cmd|sh`) | на Docker-стенде | `evidence/run/test-persistence.txt`, `test-failover-*.txt`, `evidence/ops.log` | +| **4:** mTLS broker и Collector, разные identities, чужой cert того же CA без прав | `ops.py tls-audit` (broker, collector) | `intruder`/`lb`/`prometheus` **с верным паролем** → отказ на gate; `api` без AUTH → NOAUTH; ACL: producer не читает/не удаляет, consumer не пишет чужие ключи; `api`/`worker` → OTLP 200, `prometheus` на OTLP и `api` на метрики → отказ | `tls_audit.txt` | +| **4:** нет утечек credentials в hash/ошибках; rehash; бюджет login DoS; общий rate limit | `rehash-demo`, `security-http`, `login-budget` | `m=8192,t=1` → вход → `m=19456,t=2`; 429 с ответами от 2 реплик; поток 150/с: 136 до KDF, 4397 × 429, `/users/me` без ошибок | `lifecycle.txt` п.6–7, `security_http.txt` | +| **4:** ротация секрета и leaf без пересборки, отзыв старой identity, expiry/sealed | `rotate-db-password`, `rotate-lb-cert`, `rotate-secret-id`, `revoke-identity`, `bao-sealed` | старый пароль/сертификат/SecretID отклоняются на новом соединении, новые принимаются, smoke PASS; sealed → API работает, агент восстанавливается после unseal | `lifecycle.txt` п.1–5 | +| **4:** read_only/no-new-privileges/drop caps/tmpfs, исключения; scan с разбором | `ops.py hardening`, `ops.py scan` | pip-audit: 72 уязвимости → обновление → 0, все тесты повторены | `evidence/sandbox/pip-audit-*.txt`, `docs/security/SCAN_TRIAGE.md` | +| **4:** ≥ 6 угроз → контроли → тесты; worker ограничен CPU/RAM/время; TLS-аудит всей матрицы | `security/THREAT_MODEL.md` (12 угроз), `tls_audit.py` (101 проверка) | worker: подпроцесс RLIMIT_AS 1 GiB, timeout 30 с, контейнер 1 CPU/1280M; Pillow только PNG/JPEG-плагины | `THREAT_MODEL.md`, `tls_audit.json` | +| Регрессия lab2–6 | `run.ps1 -Action extra` | 91 тест (58 выполнено по HTTPS, 0 ошибок) + 12 white-box тестов хранилища | `evidence/sandbox/student-tests.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Конфигурация замеров: Windows 10, 32GB, Ryzen 5 7500f. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали (песочница) | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Ротация пароля `messenger_api` (`rotate-db-password`) | Новые соединения — новым паролем; старый отклоняется; API не перезапускается | агент перерисовал файл за 11 с (интервал был 15 с; в репозитории 5 с), старый пароль: `password authentication failed`, после разрыва всех сессий smoke PASS | Окно ≤ интервала рендера, когда НОВЫЕ соединения со старым паролем получают отказ (открытые работают); потерь нет | +| Ротация `lb-client` + CRL (`rotate-lb-cert`) | Старый принят до отзыва, отклонён после; новый работает | 200/200 → после revoke старый 000 (`certificate_revoked`), новый 200 | ≤ 5 с (перечитывание CRL); LB перезапускается за ~1 с | +| SecretID: ротация и отзыв роли | Старый SecretID бесполезен; после отзыва агент не логинится | old denied / new accepted; after revoke-role denied; recovered | Секреты уже отрисованы → api/worker работают до восстановления | +| OpenBao sealed | Синхронный путь не зависит от OpenBao | файлы на месте, API работает; агент: `Vault is sealed (retry…)`; после unseal перерисовал секрет сам | ≤ 1 мин после unseal (backoff агента) | +| Поток входов 30 с, 8 потоков, 1 IP | 429 до KDF, обычные запросы не падают | 150 попыток/с: 401 = 136, 429 = 4397; `/users/me` p50 9→39 мс, p95 13→112 мс, 0 ошибок | Мгновенно после окончания потока (окно 60 с для этого IP) | +| Рестарт всех хранилищ + OpenBao (`test-persistence`) | Данные и прежний токен живы | на Docker-стенде | см. `evidence/ops.log` | +| Остановка реплики (`test-failover`) ×2 | Запись восстанавливается ≤ 10 с, без дублей | на Docker-стенде | см. `evidence/run/test-failover-*.txt` | + +## Исключения hardening (обоснование) + +| Сервис | Исключение | Почему и чем компенсировано | +| --- | --- | --- | +| `s3` (SeaweedFS) | процесс uid 0 | образ рассчитан на root и пишет в `/data`; `cap_drop: ALL` (root без capabilities не обходит DAC чужих файлов), rootfs read-only, сеть `storage` internal, админ-порты на `127.0.0.1` | +| `grafana` | rootfs не read-only | совместимость Grafana 11 с read-only rootfs не проверена в подготовке; uid 472, `cap_drop: ALL`, no-new-privileges, доступ только `127.0.0.1` + пароль | +| `tempo-init` | uid 0 + `CAP_CHOWN`, без лимитов | one-shot `chown` volume, завершается до старта tempo, `network_mode: none` | +| `postgres`, `broker`, `prometheus`, `loki`, `tempo`, `openbao`, `grafana` | записываемые volumes | stateful-сервисы: данные только в своём named volume, rootfs read-only (кроме grafana) | +| `openbao` | `disable_mlock = true` | mlock требует `CAP_IPC_LOCK`; на учебном хосте swap не используется для секретов; в production — вернуть mlock | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +* Один Docker-host: отказ хоста, PostgreSQL, OpenBao, LB не выдерживается; HA не заявляется. +* Redis не связывает клиентский сертификат с ACL-пользователем: ключ `api` + украденный пароль `worker_consumer` дали бы права consumer (смягчено тем, что пароли ролей разведены политиками OpenBao). Loki/Tempo принимают любой сертификат стенда — защищены сегментацией сети `telemetry`. +* Окно ротации пароля БД (до 5 с) для новых соединений; двухпарольная схема (две роли с переключением) — следующий шаг, как и динамические DB credentials OpenBao. +* Все unseal shares и root token — у одного оператора рядом со стендом (документированный учебный компромисс); CA без промежуточного звена, без OCSP; leaf-сертификаты 30 дней с ручным `pki.py renew`. +* trivy-отчёт по ОС-пакетам образов получается только на стенде (`ops.py scan`); его разбор по методике `SCAN_TRIAGE.md` нужно внести после запуска. +* Совместимость Grafana с read-only rootfs и полный прогон в Docker в среде подготовки не проверены — это первое, что подтверждает `run.ps1`. diff --git a/compose.faults.yaml b/compose.faults.yaml new file mode 100644 index 0000000..91e8095 --- /dev/null +++ b/compose.faults.yaml @@ -0,0 +1,17 @@ +# lab5: стенд внедрения отказов worker'а (только учебный, не для обычной работы). +# docker compose -f compose.yaml -f compose.faults.yaml up -d --no-deps worker +# Включает маркеры в filename (fault-crash-after-effects, fault-crash-before-ack, +# fault-transient-until-3, fault-transient-until-replay, fault-permanent, fault-slow) +# и укорачивает аренду/backoff, чтобы сценарии проходили за секунды. +# Возврат к обычному worker: docker compose up -d --no-deps --force-recreate worker +# Используется автоматически: python scripts/ops.py pipeline-faults +services: + worker: + environment: + WORKER_FAULTS: "true" + JOB_TIMEOUT_SECONDS: "25" + JOB_LEASE_GRACE_SECONDS: "2" + JOB_CLAIM_IDLE_SECONDS: "30" + JOB_RETRY_BASE_SECONDS: "1" + JOB_RETRY_MAX_SECONDS: "4" + WORKER_IMAGE_TIMEOUT_SECONDS: "5" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..6c7cb9a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,874 @@ +# Лабораторная 7: стенд lab6 + TLS/mTLS на всех связях, OpenBao, минимальные привилегии. +# +# клиент --TLS--> lb :8443 --mTLS (lb)--> api x N (:8443, allowlist identity lb + CRL) +# api/worker --TLS verify-full--> postgres (роли messenger_api / messenger_worker) +# api --mTLS + ACL--> redis (кэш сессий) +# api/worker --mTLS--> broker-gate (allowlist api|worker + CRL) --loopback--> broker (ACL producer/consumer) +# api/worker --TLS (сквозной, через s3-gate passthrough)--> s3 (раздельные identities по префиксам) +# api/worker --mTLS--> otel-gate (allowlist api|worker) --loopback--> otel-collector --mTLS--> loki/tempo +# prometheus --mTLS--> otel-gate (allowlist prometheus); grafana --mTLS--> prometheus/loki/tempo +# bao-agent-{api,worker} --TLS + AppRole--> openbao (не dev, file storage, ручной unseal) +# -> tmpfs volume {api,worker}-secrets -> /run/bao (ro) в api/worker +# +# Сети (все internal, кроме edge и ops): egress в интернет нет ни у одного сервиса приложения. +# edge lb (публикует 127.0.0.1:8443) ingress lb, api +# backend api, worker, postgres, redis, migrate storage api, worker, s3, s3-init +# queue api, worker, broker(+gate) telemetry-ingest api, worker, otel-collector(+gate) +# telemetry otel-collector, prometheus, loki, tempo, grafana ops grafana (127.0.0.1:3000) +# secrets openbao, bao-agent-api, bao-agent-worker (подсеть BAO_SECRETS_SUBNET = secret_id_bound_cidrs) +# Наружу (только 127.0.0.1): lb 8443 (HTTPS), grafana 3000 (HTTPS + пароль admin). +name: messenger-lab7 + +# ---------------------------------------------------------------- общие фрагменты +x-hardened: &hardened + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + +x-queue-env: &queue-env + BROKER_URL_FILE: /run/bao/broker_url + BROKER_STREAM: ${BROKER_STREAM:-attachments.jobs} + BROKER_GROUP: ${BROKER_GROUP:-attachment-workers} + BROKER_TIMEOUT_SECONDS: ${BROKER_TIMEOUT_SECONDS:-2} + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_GRACE_SECONDS: ${JOB_LEASE_GRACE_SECONDS:-10} + JOB_CLAIM_IDLE_SECONDS: ${JOB_CLAIM_IDLE_SECONDS:-75} + JOB_MAX_DELIVERIES: ${JOB_MAX_DELIVERIES:-10} + JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2} + JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + RELAY_POLL_INTERVAL_SECONDS: ${RELAY_POLL_INTERVAL_SECONDS:-1} + RELAY_FAST_PATH_GRACE_SECONDS: ${RELAY_FAST_PATH_GRACE_SECONDS:-2} + +x-otel-env: &otel-env + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_CERTIFICATE: /run/secrets/ca_crt + # OTel SDK >= 1.4x задаёт CA на requests.Session, а requests предпочитает REQUESTS_CA_BUNDLE: + # фиксируем его на CA стенда, чтобы внешняя переменная не подменила доверие экспортёра. + REQUESTS_CA_BUNDLE: /run/secrets/ca_crt + OTEL_EXPORTER_OTLP_TIMEOUT: "5" + OTEL_RESOURCE_ATTRIBUTES: ${OTEL_RESOURCE_ATTRIBUTES:-deployment.environment.name=lab} + OTEL_SEMCONV_STABILITY_OPT_IN: http + OTEL_BSP_MAX_QUEUE_SIZE: "2048" + OTEL_BSP_MAX_EXPORT_BATCH_SIZE: "512" + OTEL_BSP_EXPORT_TIMEOUT: "5000" + OTEL_BLRP_MAX_QUEUE_SIZE: "2048" + OTEL_BLRP_EXPORT_TIMEOUT: "5000" + OTEL_METRIC_EXPORT_INTERVAL: "10000" + +x-s3-env: &s3-env + S3_ENDPOINT: https://s3:8333 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/bao/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/bao/s3_secret_access_key + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10} + S3_MAX_ATTEMPTS: ${S3_MAX_ATTEMPTS:-2} + S3_CIRCUIT_OPEN_SECONDS: ${S3_CIRCUIT_OPEN_SECONDS:-5} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + ATTACHMENT_GC_INTERVAL_SECONDS: ${ATTACHMENT_GC_INTERVAL_SECONDS:-300} + ATTACHMENT_GC_GRACE_SECONDS: ${ATTACHMENT_GC_GRACE_SECONDS:-900} + ORPHAN_SWEEP_INTERVAL_SECONDS: ${ORPHAN_SWEEP_INTERVAL_SECONDS:-3600} + +x-api-env: &api-env + APP_ENV: ${APP_ENV:-lab} + APP_VERSION: ${APP_VERSION:-lab7} + APP_PORT: "8443" + HEALTH_PORT: "8081" + OTEL_SERVICE_NAME: messenger-api + <<: [*queue-env, *otel-env, *s3-env] + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/api_crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/api_key + # lab7: HTTPS-сервер с обязательным клиентским сертификатом; allowlist и CRL. + TLS_CERT_FILE: /run/secrets/api_crt + TLS_KEY_FILE: /run/secrets/api_key + TLS_CA_FILE: /run/secrets/ca_crt + TLS_CLIENT_CERT_FILE: /run/secrets/api_crt + TLS_CLIENT_KEY_FILE: /run/secrets/api_key + MTLS_ALLOWED_CLIENTS: lb + MTLS_CRL_FILE: /run/secrets/crl_pem + # Секреты — файлы, отрисованные OpenBao Agent (tmpfs volume api-secrets). + DATABASE_URL_FILE: /run/bao/database_url + CACHE_URL_FILE: /run/bao/cache_url + SESSION_STORE: postgres + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-3} + DB_POOL_MAX_LIFETIME_SECONDS: ${DB_POOL_MAX_LIFETIME_SECONDS:-600} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.25} + CACHE_RETRY_AFTER_SECONDS: ${CACHE_RETRY_AFTER_SECONDS:-5} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + READY_DB_TIMEOUT_SECONDS: ${READY_DB_TIMEOUT_SECONDS:-1} + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + GRACEFUL_TIMEOUT_SECONDS: ${GRACEFUL_TIMEOUT_SECONDS:-10} + KEEP_ALIVE_SECONDS: ${KEEP_ALIVE_SECONDS:-30} + # lab7: Argon2id (базовая линия OWASP), бюджет KDF и общий rate limit входа. + ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456} + ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2} + ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-4} + KDF_QUEUE_TIMEOUT_SECONDS: ${KDF_QUEUE_TIMEOUT_SECONDS:-2} + LOGIN_RATE_WINDOW_SECONDS: ${LOGIN_RATE_WINDOW_SECONDS:-60} + LOGIN_RATE_PER_USER: ${LOGIN_RATE_PER_USER:-10} + LOGIN_RATE_PER_IP: ${LOGIN_RATE_PER_IP:-300} + MAX_JSON_BODY_BYTES: ${MAX_JSON_BODY_BYTES:-65536} + RUN_MIGRATIONS_ON_START: "false" + ORPHAN_SWEEP_ENABLED: "false" + S3_DOWNLOAD_DEADLINE_SECONDS: ${S3_DOWNLOAD_DEADLINE_SECONDS:-5} + S3_UPLOAD_DEADLINE_SECONDS: ${S3_UPLOAD_DEADLINE_SECONDS:-10} + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-4} + UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5} + UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-60} + IMAGE_DECODE_CONCURRENCY: ${IMAGE_DECODE_CONCURRENCY:-2} + +x-api-healthcheck: &api-healthcheck + # Loopback health listener (127.0.0.1:8081, только /health/*); LB проверяет по mTLS. + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2).read()"] + interval: 3s + timeout: 2s + retries: 2 + start_period: 20s + +x-api-limits: &api-limits + limits: + cpus: "0.50" + memory: 320M + pids: 128 + +x-api-secrets: &api-secrets + - ca_crt + - crl_pem + - api_crt + - api_key + +x-api-common: &api-common + <<: *hardened + image: messenger-api:${APP_VERSION:-lab7} + environment: *api-env + secrets: *api-secrets + volumes: + - api-secrets:/run/bao:ro + tmpfs: + - /tmp:size=16m,uid=10001,gid=10001 + networks: [ingress, backend, storage, queue, telemetry-ingest] + dns_opt: ["timeout:1", "attempts:1"] + +services: + # ------------------------------------------------------------------ PostgreSQL + postgres: + image: postgres:16.4-alpine + <<: *hardened + restart: unless-stopped + # Сразу uid 70 (postgres): entrypoint не нужен root/gosu, capabilities не нужны. + user: "70:70" + entrypoint: ["/bin/sh", "/opt/messenger/entrypoint.sh"] + command: + - postgres + - -c + - max_connections=100 + - -c + - ssl=on + - -c + - ssl_cert_file=/run/pgtls/server.crt + - -c + - ssl_key_file=/run/pgtls/server.key + - -c + - ssl_min_protocol_version=TLSv1.2 + - -c + - hba_file=/opt/messenger/pg_hba.conf + - -c + - password_encryption=scram-sha-256 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password, postgres_crt, postgres_key, db_api_password, db_worker_password] + volumes: + - pgdata:/var/lib/postgresql/data + - ./infra/postgres/entrypoint.sh:/opt/messenger/entrypoint.sh:ro + - ./infra/postgres/pg_hba.conf:/opt/messenger/pg_hba.conf:ro + - ./infra/postgres/initdb:/docker-entrypoint-initdb.d:ro + tmpfs: + - /run/pgtls:size=1m,uid=70,gid=70,mode=0700 + - /var/run/postgresql:size=1m,uid=70,gid=70 + - /tmp:size=16m,uid=70,gid=70 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h /var/run/postgresql"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: [backend] + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M, pids: 200} + + # ------------------------------------------------------------------ Redis: кэш сессий + redis: + image: redis:7.4.1-alpine + <<: *hardened + restart: unless-stopped + user: redis + entrypoint: ["/bin/sh", "/opt/messenger/cache.sh"] + secrets: [ca_crt, redis_crt, redis_key, cache_api_password] + volumes: + - ./infra/redis/cache.sh:/opt/messenger/cache.sh:ro + tmpfs: ["/tmp:size=1m", "/data:size=1m"] + healthcheck: + test: ["CMD-SHELL", "redis-cli --tls --cacert /run/secrets/ca_crt --cert /run/secrets/redis_crt --key /run/secrets/redis_key -h 127.0.0.1 --user healthcheck --pass x --no-auth-warning ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + networks: [backend] + deploy: + resources: + limits: {cpus: "0.25", memory: 96M, pids: 64} + + # ------------------------------------------------------------------ broker + mTLS gate + broker: + image: redis:7.4.1-alpine + <<: *hardened + restart: unless-stopped + user: redis + entrypoint: ["/bin/sh", "/opt/messenger/broker.sh"] + secrets: [broker_api_password, broker_worker_password] + volumes: + - brokerdata:/data + - ./infra/redis/broker.sh:/opt/messenger/broker.sh:ro + tmpfs: ["/tmp:size=1m"] + healthcheck: + test: ["CMD-SHELL", "redis-cli -h 127.0.0.1 --user healthcheck --pass x --no-auth-warning ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + networks: [queue] + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.50", memory: 320M, pids: 64} + + broker-gate: + image: haproxy:2.8.30-alpine + <<: *hardened + restart: unless-stopped + # Тот же сетевой namespace, что у broker: Redis слушает только его 127.0.0.1:6379. + network_mode: "service:broker" + volumes: + - ./infra/haproxy/broker-gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + secrets: [ca_crt, crl_pem, broker_pem] + tmpfs: ["/tmp:size=1m"] + depends_on: + broker: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8405/gate-health"] + interval: 3s + timeout: 2s + retries: 10 + deploy: + resources: + limits: {cpus: "0.25", memory: 64M, pids: 64} + + # ------------------------------------------------------------------ S3 (SeaweedFS) + s3: + image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION:-4.50} + <<: *hardened + restart: unless-stopped + entrypoint: ["/usr/bin/weed"] + cap_add: [DAC_OVERRIDE, CHOWN, FOWNER] + + + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -s3 + # lab7: S3 API и его неаутентифицированный gRPC (8334+10000) — только loopback; + # в сеть storage порт S3 выдаёт s3-gate (TCP passthrough, TLS сквозной). + - -s3.port=8334 + - -s3.ip.bind=127.0.0.1 + - -s3.config=/run/secrets/s3_config + - -s3.iam=false + - -s3.port.iceberg=0 + - -s3.port.lance=0 + - -s3.autoCreateBucket=false + # lab7: S3 API только по HTTPS (сертификат s3, SAN DNS:s3). + - -s3.cert.file=/run/secrets/s3_crt + - -s3.key.file=/run/secrets/s3_key + - -s3.concurrentUploadLimitMB=128 + - -master.volumeSizeLimitMB=256 + - -master.electionTimeout=2s + - -master.telemetry=false + - -volume.max=0 + environment: + GOMEMLIMIT: 400MiB + secrets: [s3_config, s3_crt, s3_key] + volumes: + - s3data:/data + tmpfs: ["/tmp:size=64m"] + healthcheck: + # Порт S3 открыт (TLS проверяют s3-init и `ops.py wait-s3` с CA стенда). + test: ["CMD-SHELL", "nc -z 127.0.0.1 8334 || wget -q -O /dev/null https://127.0.0.1:8334/healthz"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + networks: [storage] + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M, pids: 256} + + s3-gate: + image: haproxy:2.8.30-alpine + <<: *hardened + restart: unless-stopped + network_mode: "service:s3" + volumes: + - ./infra/haproxy/s3-gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + tmpfs: ["/tmp:size=1m"] + depends_on: + s3: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8405/gate-health"] + interval: 3s + timeout: 2s + retries: 10 + deploy: + resources: + limits: {cpus: "0.25", memory: 64M, pids: 64} + + s3-init: + build: . + image: messenger-api:${APP_VERSION:-lab7} + <<: *hardened + command: ["python", "-m", "src.s3_bootstrap"] + restart: "no" + environment: + S3_ENDPOINT: https://s3:8333 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + TLS_CA_FILE: /run/secrets/ca_crt + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id + S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key + secrets: [ca_crt, s3_admin_access_key_id, s3_admin_secret_access_key, s3_access_key_id, + s3_secret_access_key, s3_worker_access_key_id, s3_worker_secret_access_key] + tmpfs: ["/tmp:size=8m"] + depends_on: + s3-gate: + condition: service_healthy + networks: [storage] + deploy: + resources: + limits: {cpus: "0.50", memory: 192M, pids: 64} + + # ------------------------------------------------------------------ миграции (администратор БД) + migrate: + build: . + image: messenger-api:${APP_VERSION:-lab7} + <<: *hardened + command: ["python", "-m", "src.migrate"] + restart: "no" + environment: + DATABASE_URL_FILE: /run/secrets/database_admin_url + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: "60000" + MIGRATE_WAIT_SECONDS: "60" + OTEL_SDK_DISABLED: "true" + secrets: [database_admin_url, ca_crt] + tmpfs: ["/tmp:size=8m"] + depends_on: + postgres: + condition: service_healthy + networks: [backend] + deploy: + resources: + limits: {cpus: "0.50", memory: 256M, pids: 64} + + # ------------------------------------------------------------------ OpenBao + agents + openbao: + image: openbao/openbao:${OPENBAO_VERSION:-2.1.0} + <<: *hardened + restart: unless-stopped + user: openbao + entrypoint: ["/bin/bao"] + command: ["server", "-config=/openbao/config/server.hcl"] + secrets: [ca_crt, openbao_crt, openbao_key] + volumes: + - baodata:/openbao/file + - ./infra/openbao/server.hcl:/openbao/config/server.hcl:ro + tmpfs: ["/tmp:size=8m"] + healthcheck: + # 0 = unsealed, 2 = sealed (процесс жив, ждёт оператора), 1 = ошибка. + test: ["CMD-SHELL", "bao status -address=https://127.0.0.1:8200 -ca-cert=/run/secrets/ca_crt -tls-server-name=openbao >/dev/null 2>&1; test $$? -ne 1"] + interval: 5s + timeout: 3s + retries: 20 + networks: [secrets] + deploy: + resources: + limits: {cpus: "0.50", memory: 256M, pids: 128} + + bao-agent-api: + image: openbao/openbao:${OPENBAO_VERSION:-2.1.0} + <<: *hardened + restart: unless-stopped + user: "10001:10001" + entrypoint: ["/bin/bao"] + command: ["agent", "-config=/openbao/agent/agent.hcl"] + secrets: + - ca_crt + - {source: bao_api_role_id, target: bao_role_id} + - {source: bao_api_secret_id, target: bao_secret_id} + volumes: + - ./infra/openbao/agent-api.hcl:/openbao/agent/agent.hcl:ro + - api-secrets:/run/bao + tmpfs: ["/tmp:size=8m"] + healthcheck: + test: ["CMD-SHELL", "test -s /run/bao/database_url && test -s /run/bao/cache_url && test -s /run/bao/broker_url && test -s /run/bao/s3_secret_access_key"] + interval: 3s + timeout: 2s + retries: 60 + depends_on: + openbao: + condition: service_started + networks: [secrets] + deploy: + resources: + limits: {cpus: "0.25", memory: 128M, pids: 64} + + bao-agent-worker: + image: openbao/openbao:${OPENBAO_VERSION:-2.1.0} + <<: *hardened + restart: unless-stopped + user: "10001:10001" + entrypoint: ["/bin/bao"] + command: ["agent", "-config=/openbao/agent/agent.hcl"] + secrets: + - ca_crt + - {source: bao_worker_role_id, target: bao_role_id} + - {source: bao_worker_secret_id, target: bao_secret_id} + volumes: + - ./infra/openbao/agent-worker.hcl:/openbao/agent/agent.hcl:ro + - worker-secrets:/run/bao + tmpfs: ["/tmp:size=8m"] + healthcheck: + test: ["CMD-SHELL", "test -s /run/bao/database_url && test -s /run/bao/broker_url && test -s /run/bao/s3_secret_access_key"] + interval: 3s + timeout: 2s + retries: 60 + depends_on: + openbao: + condition: service_started + networks: [secrets] + deploy: + resources: + limits: {cpus: "0.25", memory: 128M, pids: 64} + + # ------------------------------------------------------------------ API и worker + api: + build: . + <<: *api-common + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + bao-agent-api: + condition: service_healthy + # Только сеть ingress (LB) и зависимости; порт 8443 — mTLS, наружу не публикуется. + expose: ["8443"] + healthcheck: *api-healthcheck + stop_signal: SIGTERM + stop_grace_period: 20s + deploy: + replicas: ${API_REPLICAS:-2} + resources: *api-limits + + worker: + build: . + image: messenger-api:${APP_VERSION:-lab7} + <<: *hardened + command: ["python", "-m", "src.worker"] + restart: unless-stopped + environment: + <<: [*queue-env, *otel-env, *s3-env] + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /run/secrets/worker_crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /run/secrets/worker_key + APP_VERSION: ${APP_VERSION:-lab7} + APP_ENV: ${APP_ENV:-lab} + INSTANCE_ID_MODE: hostname + TLS_CA_FILE: /run/secrets/ca_crt + TLS_CLIENT_CERT_FILE: /run/secrets/worker_crt + TLS_CLIENT_KEY_FILE: /run/secrets/worker_key + DATABASE_URL_FILE: /run/bao/database_url + DB_POOL_MIN: "1" + DB_POOL_MAX: ${WORKER_DB_POOL_MAX:-6} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-3} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + # Upload-worker: CPU/RAM/время одного изображения ограничены (подпроцесс + RLIMIT_AS + timeout). + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_IMAGE_TIMEOUT_SECONDS: ${WORKER_IMAGE_TIMEOUT_SECONDS:-30} + WORKER_IMAGE_MEMORY_MB: ${WORKER_IMAGE_MEMORY_MB:-1024} + WORKER_FAULTS: "false" + secrets: [ca_crt, worker_crt, worker_key] + volumes: + - worker-secrets:/run/bao:ro + depends_on: + postgres: + condition: service_healthy + broker-gate: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + bao-agent-worker: + condition: service_healthy + healthcheck: + test: ["CMD", "python", "-c", "import os,time,sys; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 90 else 1)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 20s + stop_signal: SIGTERM + stop_grace_period: 75s + tmpfs: + - /tmp:size=16m,uid=10001,gid=10001 + networks: [backend, storage, queue, telemetry-ingest] + dns_opt: ["timeout:1", "attempts:1"] + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: {cpus: "1.00", memory: 1280M, pids: 128} + + # Экземпляр с коротким TTL — только для автотеста истечения сессии (профиль ttl). + api-short-ttl: + <<: *api-common + profiles: [ttl] + restart: "no" + environment: + <<: *api-env + SESSION_TTL_SECONDS: ${SHORT_SESSION_TTL_SECONDS:-2} + depends_on: + api: + condition: service_healthy + expose: ["8443"] + healthcheck: *api-healthcheck + deploy: + resources: *api-limits + + # ------------------------------------------------------------------ LB + lb: + image: haproxy:2.8.30-alpine + <<: *hardened + restart: unless-stopped + ports: + - "127.0.0.1:${HTTPS_PORT:-8443}:8443" + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + secrets: [ca_crt, lb_public_pem, lb_client_pem] + tmpfs: ["/tmp:size=1m"] + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 2s + retries: 3 + stop_grace_period: 5s + networks: [edge, ingress] + deploy: + resources: + limits: {cpus: "0.50", memory: 128M, pids: 64} + + # ------------------------------------------------------------------ наблюдаемость + otel-collector: + image: otel/opentelemetry-collector-contrib:0.115.1 + <<: *hardened + restart: unless-stopped + command: ["--config=/etc/otelcol/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + secrets: [ca_crt, otel_collector_crt, otel_collector_key] + environment: + GOMEMLIMIT: 200MiB + networks: [telemetry-ingest, telemetry] + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.50", memory: 384M, pids: 128} + + otel-gate: + image: haproxy:2.8.30-alpine + <<: *hardened + restart: unless-stopped + network_mode: "service:otel-collector" + volumes: + - ./infra/haproxy/otel-gate.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + secrets: [ca_crt, crl_pem, otel_collector_pem] + tmpfs: ["/tmp:size=1m"] + depends_on: [otel-collector] + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8405/gate-health"] + interval: 5s + timeout: 2s + retries: 10 + deploy: + resources: + limits: {cpus: "0.25", memory: 64M, pids: 64} + + prometheus: + image: prom/prometheus:v2.55.1 + <<: *hardened + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.config.file=/etc/prometheus/web.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/web.yml:/etc/prometheus/web.yml:ro + - ./infra/prometheus/client.yml:/etc/prometheus/client.yml:ro + - ./infra/prometheus/rules:/etc/prometheus/rules:ro + - promdata:/prometheus + secrets: [ca_crt, prometheus_crt, prometheus_key] + healthcheck: + test: ["CMD", "promtool", "check", "healthy", "--url=https://127.0.0.1:9090", "--http.config.file=/etc/prometheus/client.yml"] + interval: 5s + timeout: 3s + retries: 20 + networks: [telemetry] + deploy: + resources: + limits: {cpus: "0.50", memory: 512M, pids: 128} + + loki: + image: grafana/loki:3.3.2 + <<: *hardened + restart: unless-stopped + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + secrets: [ca_crt, loki_crt, loki_key] + tmpfs: ["/tmp:size=16m"] + networks: [telemetry] + deploy: + resources: + limits: {cpus: "0.50", memory: 512M, pids: 128} + + tempo-init: + image: grafana/tempo:2.6.1 + user: root + entrypoint: ["chown", "10001:10001", "/var/tempo"] + restart: "no" + cap_drop: [ALL] + cap_add: [CHOWN] + security_opt: ["no-new-privileges:true"] + volumes: + - tempodata:/var/tempo + network_mode: none + + tempo: + image: grafana/tempo:2.6.1 + <<: *hardened + restart: unless-stopped + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + secrets: [ca_crt, tempo_crt, tempo_key] + tmpfs: ["/tmp:size=16m"] + depends_on: + tempo-init: + condition: service_completed_successfully + networks: [telemetry] + deploy: + resources: + limits: {cpus: "0.50", memory: 512M, pids: 128} + + grafana: + image: grafana/grafana:11.4.0 + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: unless-stopped + environment: + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /run/secrets/grafana_crt + GF_SERVER_CERT_KEY: /run/secrets/grafana_key + GF_SERVER_ROOT_URL: https://localhost:${GRAFANA_PORT:-3000}/ + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + GF_SECURITY_COOKIE_SECURE: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_PLUGINS_PLUGIN_ADMIN_ENABLED: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/messenger.json + secrets: [grafana_admin_password, ca_crt, grafana_crt, grafana_key] + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + depends_on: + prometheus: + condition: service_healthy + networks: [telemetry, ops] + deploy: + resources: + limits: {cpus: "0.50", memory: 256M, pids: 128} + + # ------------------------------------------------------------------ инструменты оператора + # PKI без Python-зависимостей на хосте: docker compose --profile tools run --rm tools scripts/pki.py init + tools: + build: . + image: messenger-api:${APP_VERSION:-lab7} + profiles: [tools] + user: "0:0" + entrypoint: ["python"] + working_dir: /work + network_mode: none + cap_drop: [ALL] + cap_add: [CHOWN, DAC_OVERRIDE, FOWNER] + security_opt: ["no-new-privileges:true"] + volumes: + - ./scripts:/work/scripts:ro + - ./secrets:/work/secrets + + # TLS/ACL-аудит всей матрицы (student-tests/security/tls_audit.py). Подключается к каждой + # внутренней сети как «посторонний клиент»; получает только тестовые материалы оператора. + audit: + image: messenger-api:${APP_VERSION:-lab7} + profiles: [audit] + <<: *hardened + entrypoint: ["python"] + working_dir: /app + environment: + AUDIT_SECRETS: /run/secrets + OTEL_SDK_DISABLED: "true" + secrets: [ca_crt, crl_pem, intruder_crt, intruder_key, lb_client_pem, api_crt, api_key, worker_crt, worker_key, + prometheus_crt, prometheus_key, grafana_crt, grafana_key, redis_crt, redis_key, + cache_api_password, broker_api_password, broker_worker_password, + db_api_password, db_worker_password, s3_access_key_id, s3_secret_access_key, + s3_worker_access_key_id, s3_worker_secret_access_key] + volumes: + - ./student-tests:/app/student-tests:ro + tmpfs: ["/tmp:size=16m,uid=10001,gid=10001"] + networks: [ingress, backend, storage, queue, telemetry-ingest, telemetry, secrets] + +volumes: + pgdata: + s3data: + brokerdata: + baodata: + promdata: + lokidata: + tempodata: + grafanadata: + # Секреты, отрисованные OpenBao Agent: только память (tmpfs), владелец uid 10001, + # переживают рестарт контейнеров api/worker, но не рестарт Docker (агент отрисует заново). + api-secrets: + driver: local + driver_opts: {type: tmpfs, device: tmpfs, o: "size=1m,uid=10001,gid=10001,mode=0700"} + worker-secrets: + driver: local + driver_opts: {type: tmpfs, device: tmpfs, o: "size=1m,uid=10001,gid=10001,mode=0700"} + +networks: + edge: {} + ops: {} + ingress: {internal: true} + backend: {internal: true} + storage: {internal: true} + queue: {internal: true} + telemetry-ingest: {internal: true} + telemetry: {internal: true} + secrets: + internal: true + ipam: + config: + - subnet: ${BAO_SECRETS_SUBNET:-172.30.250.0/24} + +secrets: + # PKI (scripts/pki.py): ca.key НЕ монтируется никуда. + ca_crt: {file: ./secrets/pki/ca.crt} + crl_pem: {file: ./secrets/pki/crl.pem} + lb_public_pem: {file: ./secrets/pki/lb-public.pem} + lb_client_pem: {file: ./secrets/pki/lb-client.pem} + api_crt: {file: ./secrets/pki/api.crt} + api_key: {file: ./secrets/pki/api.key} + worker_crt: {file: ./secrets/pki/worker.crt} + worker_key: {file: ./secrets/pki/worker.key} + postgres_crt: {file: ./secrets/pki/postgres.crt} + postgres_key: {file: ./secrets/pki/postgres.key} + redis_crt: {file: ./secrets/pki/redis.crt} + redis_key: {file: ./secrets/pki/redis.key} + broker_pem: {file: ./secrets/pki/broker.pem} + s3_crt: {file: ./secrets/pki/s3.crt} + s3_key: {file: ./secrets/pki/s3.key} + otel_collector_crt: {file: ./secrets/pki/otel-collector.crt} + otel_collector_key: {file: ./secrets/pki/otel-collector.key} + otel_collector_pem: {file: ./secrets/pki/otel-collector.pem} + loki_crt: {file: ./secrets/pki/loki.crt} + loki_key: {file: ./secrets/pki/loki.key} + tempo_crt: {file: ./secrets/pki/tempo.crt} + tempo_key: {file: ./secrets/pki/tempo.key} + prometheus_crt: {file: ./secrets/pki/prometheus.crt} + prometheus_key: {file: ./secrets/pki/prometheus.key} + grafana_crt: {file: ./secrets/pki/grafana.crt} + grafana_key: {file: ./secrets/pki/grafana.key} + openbao_crt: {file: ./secrets/pki/openbao.crt} + openbao_key: {file: ./secrets/pki/openbao.key} + intruder_crt: {file: ./secrets/pki/intruder.crt} + intruder_key: {file: ./secrets/pki/intruder.key} + # Учётные данные (scripts/init_secrets.py). Приложению — только через OpenBao. + postgres_password: {file: ./secrets/postgres_password} + database_admin_url: {file: ./secrets/database_admin_url} + db_api_password: {file: ./secrets/db_api_password} + db_worker_password: {file: ./secrets/db_worker_password} + cache_api_password: {file: ./secrets/cache_api_password} + broker_api_password: {file: ./secrets/broker_api_password} + broker_worker_password: {file: ./secrets/broker_worker_password} + s3_config: {file: ./secrets/s3_config.json} + s3_admin_access_key_id: {file: ./secrets/s3_admin_access_key_id} + s3_admin_secret_access_key: {file: ./secrets/s3_admin_secret_access_key} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_worker_access_key_id: {file: ./secrets/s3_worker_access_key_id} + s3_worker_secret_access_key: {file: ./secrets/s3_worker_secret_access_key} + grafana_admin_password: {file: ./secrets/grafana_admin_password} + # AppRole (scripts/bao.py): только для агентов своей роли. + bao_api_role_id: {file: ./secrets/bao/api_role_id} + bao_api_secret_id: {file: ./secrets/bao/api_secret_id} + bao_worker_role_id: {file: ./secrets/bao/worker_role_id} + bao_worker_secret_id: {file: ./secrets/bao/worker_secret_id} diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..852dd9b 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,48 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия — решение lab7 (целевая оценка 4) -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +Это обязательная матрица покрытия. Для каждой строки указаны реальный порт, endpoint, CA, service identity, проверка SAN/ACL, срок жизни credentials и команда положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. + +**Общее для всех строк.** Один локальный CA стенда (`scripts/pki.py`, EC P-256, 10 лет; `ca.key` есть только у оператора в `secrets/pki/` и не монтируется ни в один контейнер). Leaf-сертификаты — 30 дней (`PKI_LEAF_DAYS`), CRL — 30 дней, перевыпуск `pki.py renew`. Минимальная версия TLS — 1.2 везде, где продукт позволяет её задать. Identity = CN (и SAN DNS) сертификата сервиса: `lb`, `api`, `worker`, `prometheus`, `grafana`, `otel-collector`, ... Сертификат `intruder` подписан тем же CA и не входит ни в один allowlist — им доказывается «доверенный CA ≠ право». Сводный автоматический тест всей матрицы: `python scripts/ops.py tls-audit` (101 проверка, `evidence/tls_audit.json`). | Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Клиент → ingress/LB | **TLS 1.2+** на `127.0.0.1:8443` (`lb-public`, SAN `localhost`, `127.0.0.1`, `::1`, `lb`), HTTP-порта нет; Bearer-сессия (256 бит, SHA-256 в БД, TTL 3600 с) | Expiry/отзыв сессии проверяются в PostgreSQL (кэш не продлевает); **общий для реплик** rate limit входа в PostgreSQL (10 попыток/мин на username, 300/мин на IP) → `429` + `Retry-After` до Argon2id; лимит тела 64 KiB (LB) / 10 MiB upload (API); внешние `X-Forwarded-For`, `traceparent`, `tracestate`, `baggage` удаляются | `infra/haproxy/haproxy.cfg` (frontend public). `make test-security`, `ops.py security-http` (24 проверки: пустой trust store, чужое имя, plaintext, 429 с ответами от ≥2 реплик, 413), `ops.py in-network expiry` | +| LB → API (все реплики) | **mTLS**: API слушает только HTTPS `api:8443` с `CERT_REQUIRED` (CA стенда), после рукопожатия — allowlist identity **`lb`** + CRL (`src/tls.py`); LB проверяет сервер (`verify required`, `verifyhost api`). Порт API не публикуется; health — только `127.0.0.1:8081` внутри контейнера | Ротация leaf `lb-client` без пересборки (`pki.py rotate` → `restart lb`), отзыв старого через CRL — API перечитывает CRL ≤ 5 с, `no-ssl-reuse` исключает обход через TLS resumption; просроченный/битый CRL → отказ всем | `infra/haproxy/haproxy.cfg` (server-template ... `ssl verify required crt lb_client_pem verifyhost api no-ssl-reuse`), `src/tls.py`, `src/server.py`. Отрицательные: нет cert, чужой CA с CN=lb, `intruder`, `worker`, `api` как клиент, plaintext, порт 8081 → отказ; `lb` → 200 (`tls-audit`). Ротация/отзыв: `ops.py rotate-lb-cert` | +| API/worker → PostgreSQL | **TLS `sslmode=verify-full`** (CA + SAN `postgres`), `pg_hba`: только `hostssl` + SCRAM, `hostnossl … reject`; раздельные роли **`messenger_api`** (DML своих таблиц, без DDL/TRUNCATE) и **`messenger_worker`** (только таблицы конвейера вложений; нет доступа к `users`, `sessions`, `messages`); миграции — администратор `messenger` только в one-shot `migrate` | Ротация пароля роли без рестарта: `ALTER ROLE` + OpenBao KV → агент перерисовывает `database_url` (≤ 5 с) → пул открывает новые соединения новым паролем (`src/db.py RotatingConnection`, `max_lifetime` 600 с). mTLS к БД не включён (допустимое усиление) | `infra/postgres/pg_hba.conf`, `entrypoint.sh`, `initdb/10-roles.sh`, `migrations/006_security.sql`. Отрицательные: plaintext, чужой CA, IP вместо имени, `worker` читает `users`/`sessions`, `api` делает DDL/TRUNCATE → отказ (`tls-audit`). Ротация: `ops.py rotate-db-password api` | +| API/worker → S3 | **TLS** (`https://s3:8333`, SAN `s3`, boto3 `verify=CA`), TLS завершает сам SeaweedFS; он слушает только `127.0.0.1:8334` своего namespace, порт в сеть выдаёт `s3-gate` (TCP passthrough) — так недоступен неаутентифицированный gRPC S3-шлюза (`8334+10000`), которым можно переписать identities; раздельные identities SeaweedFS: **API** — `Write staging/*`, `Read staging/* final/* originals/*`, без List; **worker** — `Write final/* staging/* originals/*`, `Read`, `List` бакета; admin-ключ — только one-shot `s3-init`; анонимный доступ — 403 | Отказ на чужой префикс (API → `final/` = AccessDenied, API LIST = AccessDenied, worker вне префиксов = AccessDenied); ключи хранятся в OpenBao KV по ролям | `secrets/s3_config.json` (`scripts/init_secrets.py`), `src/s3_bootstrap.py`, `infra/haproxy/s3-gate.cfg`. `tls-audit` (8 проверок прав + TLS/имя/plaintext + gRPC/master/filer/8334 недоступны), `ops.py s3-check` | +| API/worker → broker | **TLS** через `broker-gate` (`broker:6380`, SAN `broker`); Redis слушает только `127.0.0.1:6379` своего namespace; ACL: `default off`, producer **`api_producer`** (XADD, создание группы), consumer **`worker_consumer`** (XREADGROUP/XACK/XAUTOCLAIM/XPENDING/XTRIM/XADD), ни у кого нет `@dangerous/@admin` | **mTLS**: gate требует клиентский сертификат CA стенда + CRL и пропускает только CN **`api`** и **`worker`** (Redis 7.4 не умеет allowlist SAN — отсюда sidecar с эквивалентной проверкой, разрешённой заданием). `intruder`, `lb`, `prometheus` с верным паролем ACL → отказ на gate; `api` без AUTH → `NOAUTH` | `infra/haproxy/broker-gate.cfg`, `infra/redis/broker.sh`, `compose.yaml` (`network_mode: service:broker`). `tls-audit`: 5 отрицательных identity, 7 ACL-отказов, loopback-порт 6379 недоступен, plaintext → отказ | +| API/worker → Collector | **TLS** через `otel-gate` (`otel-collector:4318`); Collector слушает только `127.0.0.1:14318/18889/18888/13133`; сеть `telemetry-ingest` — только api, worker, collector | **mTLS с раздельными identities**: gate пропускает на OTLP только CN **`api`** и **`worker`** (у каждого свой сертификат, `OTEL_EXPORTER_OTLP_CLIENT_*`); метрики `:8889/:8888` — только CN **`prometheus`**. Перекрёстные роли отклоняются | `infra/haproxy/otel-gate.cfg`, `infra/otel/collector.yaml`. `tls-audit`: нет cert / чужой CA / `intruder` / `prometheus` / `lb` → отказ, `api`/`worker` → 200; `api` на `:8889` → отказ | +| Collector → telemetry backends | **mTLS** Collector → Tempo (OTLP gRPC `tempo:4317`) и → Loki (`https://loki:3100/otlp`): CA + имя сервера, клиентский cert `otel-collector`; Prometheus скрейпит Collector по mTLS (identity `prometheus`) | Loki/Tempo принимают любой клиентский сертификат CA стенда (allowlist identity у них нет) — **least privilege обеспечен сегментацией**: сеть `telemetry` (internal) содержит только collector, prometheus, loki, tempo, grafana; api/worker туда не подключены (`hardening`: из api/worker `loki/tempo/prometheus` → `no-dns`). Ротация — как любой leaf (`pki.py rotate otel-collector`) | `infra/loki/loki.yaml`, `infra/tempo/tempo.yaml` (`http_tls_config`, `RequireAndVerifyClientCert`, внутренний gRPC на 127.0.0.1). `tls-audit`: нет cert / чужой CA / plaintext → отказ | +| Grafana → data sources | **mTLS** к Prometheus (`--web.config.file`, `RequireAndVerifyClientCert`), Loki, Tempo: проверка сервера (CA + `serverName`), клиентский cert `grafana` | Отрицательные проверки доступа к backend без cert / с чужим CA / plaintext; Prometheus без публикации порта на хост | `infra/grafana/provisioning/datasources/datasources.yaml` (`tlsAuth`, `tlsAuthWithCACert`, `$__file{…}`), `infra/prometheus/web.yml`. `tls-audit` (12 проверок) | +| API/worker/agent → OpenBao | **TLS** `https://openbao:8200` (SAN `openbao`), не dev-mode, file storage на volume; приложение **не ходит в OpenBao само** — OpenBao Agent своей роли: AppRole (RoleID + SecretID), политика только `kv/data/messenger/` (read) | Токен агента: TTL 15 мин, max 1 ч, renew агентом; SecretID: TTL 7 дней, `secret_id_bound_cidrs`/`token_bound_cidrs` = подсеть сети `secrets`; ротация SecretID с уничтожением старого; отзыв identity (`revoke-role`: все SecretID и токены роли); sealed/недоступен — уже отрисованные секреты продолжают работать, агент повторяет с backoff | `infra/openbao/*.hcl`, `scripts/bao.py`. `ops.py bao-policy` (12 allow/deny), `rotate-secret-id`, `revoke-identity`, `bao-sealed`; `tls-audit`: имя, plaintext, KV без токена → 403 | +| Оператор → OpenBao/Grafana/S3 admin | OpenBao и S3-admin **не публикуются**: оператор работает через `docker exec` (граница доверия — доступ к Docker хоста), root token/unseal — в `operator-only/` (0600, вне Git/образов); Grafana — **HTTPS** `127.0.0.1:3000` + пароль admin, анонимный вход и регистрация выключены, cookie Secure | Разделение: root token только у оператора (init/configure/rotation), у приложения — AppRole с read-only политикой; admin-ключ S3 — только у one-shot `s3-init`; статистика HAProxy — только `127.0.0.1` внутри контейнера lb | `scripts/bao.py`, `compose.yaml` (`ports` только lb и grafana). `ops.py hardening`: опубликованы только `127.0.0.1:8443` и `127.0.0.1:3000`; `security-http`: Grafana без пароля → 401 | +| API → Redis (кэш сессий) | **TLS** (`port 0`, `tls-port 6379`) + ACL: `default off`, `messenger_api` — только ключи `sess:*`, без `@dangerous/@admin`; в ACL-файле SHA-256 пароля | **mTLS** (`tls-auth-clients yes`, клиентский cert `api`); ротация пароля — новый пароль в `secrets/cache_api_password` + `kv patch` + `restart redis`, агент перерисует `cache_url` | `infra/redis/cache.sh`. `tls-audit`: `SET sess:*` ok; другой ключ, `FLUSHALL`, `CONFIG`, `KEYS` → NOPERM; без AUTH → NOAUTH; без cert / чужой CA / plaintext → отказ | +| Replication/кластерные каналы | **N/A**: один экземпляр PostgreSQL, Redis, broker, SeaweedFS (`weed server` — master/volume/filer в одном процессе, внутренние порты на `127.0.0.1` контейнера), OpenBao без HA. Межузловых каналов нет | N/A — отказ узла хранилища не выдерживается (см. REPORT.md, ограничения) | `compose.yaml` (`-ip.bind=127.0.0.1`), `ops.py s3-check` (admin-порты SeaweedFS недоступны из сети) | -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. +## Каналы, которые тоже не должны быть обходом -## Обязательные отрицательные проверки +| Канал | Как устроен | Почему не обход | +| --- | --- | --- | +| Health реплики API | `127.0.0.1:8081` внутри контейнера, только `GET /health/live|ready` (`src/health.py`) | Слушает loopback: из сети недоступен (`tls-audit`: refused). Бизнес-маршрутов нет | +| Health LB / gates | `127.0.0.1:8404` (stats, `/lb-health`), `127.0.0.1:8405` (gates) | Только loopback контейнера, порты не публикуются; `ops.py` читает stats через `docker compose exec lb` | +| Loopback hop gate → broker / collector / s3 | `network_mode: service:broker` / `service:otel-collector` / `service:s3`: gate и сервис в одном сетевом namespace, backend слушает `127.0.0.1` (у S3 — без снятия TLS) | Plaintext только внутри namespace одного сервиса; снаружи backend-порты недоступны (`tls-audit`: 6379, 14318, 18889, 18888, 13133 → refused) | +| Bootstrap | `pki.py` в контейнере `tools` с `network_mode: none`; `bao.py` — `docker exec` в openbao; `s3-init`, `migrate` — one-shot | Секреты передаются через env процесса `docker exec` (не argv хоста) или stdin, не печатаются; one-shot сервисы не остаются запущенными | +| Scrape метрик | Prometheus → otel-gate `:8889/:8888` (mTLS, CN `prometheus`) | Другие identity (api и т.д.) отклоняются gate | +| Egress | все сети приложения `internal: true` | api/worker не могут выйти в интернет (`hardening`: `1.1.1.1:443` unreachable) | -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. +## Обязательные отрицательные проверки — где доказано + +1. **HTTPS снаружи**: доверенный CA + имя → 200; пустой trust store и чужое имя → ошибка проверки сертификата, а не «сервер недоступен» — `make test-security`, `security-http`, `tls-audit` (client → lb). +2. **LB → API**: без cert, чужой CA (с CN `lb`), доверенный CA с identity `intruder`/`worker`/`api` → отказ; `lb` → 200 — `tls-audit` (lb → api). +3. **Broker и Collector (уровень 4)**: тот же набор + перекрёстные роли (`prometheus` на ingest, `api` на метрики) — `tls-audit`. +4. **ACL пользователей**: посторонний не читает metadata/content и не прикрепляет чужой файл, `X-User-Id` не меняет Bearer identity — `security-http`. +5. **OpenBao**: политика читает только свой путь, запись политики/чужого пути/выпуск SecretID запрещены, worker не получает секреты API, логин SecretID вне разрешённой сети отклоняется — `bao-policy`. +6. **Отзыв (уровень 4)** с новыми соединениями: CRL + `no-ssl-reuse` для LB-сертификата, пароль БД проверяется на новом соединении, SecretID — новым логином — `rotate-lb-cert`, `rotate-db-password`, `rotate-secret-id`, `revoke-identity`. ## Bootstrap и эксплуатация -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. - -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. - -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +* **Кто владеет чем.** Оператор (тот, у кого Docker на хосте): `secrets/pki/ca.key`, `operator-only/openbao-init.json` (root token + 3 unseal shares, порог 2), `operator-only/approle-accessors.json`, исходные пароли ролей в `secrets/`. В образы и Git ничего из этого не попадает (`.gitignore`, `.dockerignore`). +* **Первый секрет для доступа к OpenBao** — SecretID роли в `secrets/bao/_secret_id`, который монтируется **только** в контейнер агента этой роли. Он живёт 7 дней, работает только из подсети `secrets`, перевыпускается `ops.py rotate-secret-id` (старый уничтожается). Приложение SecretID не видит; токен агента не пишется на диск. +* **После рестарта OpenBao** он sealed: `python scripts/bao.py unseal` (hook `restart` делает это сам как действие оператора). Пока sealed, api/worker работают на уже отрисованных секретах (tmpfs-volume живёт, пока запущен агент) и даже перезапускаются (`ops.py bao-sealed`); новые ротации не доставляются до unseal. +* **Renewal.** Токены агентов продлевает агент (до `token_max_ttl` 1 ч, затем повторный логин SecretID). Сертификаты: `pki.py renew --days 7` перевыпускает истекающие leaf и CRL; потребителя перезапускают без пересборки (`docker compose restart `), файлы пишутся «на месте», поэтому bind-mount видит новое содержимое. +* **Компромисс учебного стенда.** Все unseal shares у одного оператора и рядом со стендом (в production — у разных людей, root token отзывается после настройки и создаётся `bao operator generate-root`). CA без промежуточного звена и без OCSP. diff --git a/docs/lab3/REPORT.md b/docs/lab3/REPORT.md new file mode 100644 index 0000000..641d022 --- /dev/null +++ b/docs/lab3/REPORT.md @@ -0,0 +1,231 @@ +# Отчёт — лабораторная 3 + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0; PostgreSQL 16, Redis 7, HAProxy 2.8 LTS. Документация: Docker Compose (services, deploy, depends_on, healthcheck), HAProxy 2.8 Configuration Manual (server-template, resolvers, http-check, retries/redispatch, http-error), PostgreSQL (INSERT … ON CONFLICT, deferrable constraints, advisory locks), Uvicorn (graceful shutdown). Помощь ИИ (Claude) использовалась при проектировании схемы идемпотентности и drain-фазы, написании скриптов стенда, тестов и формулировке отчёта; решения проверены тестами, приведёнными ниже. + +## Запуск из чистого clone + +**Версии.** API: сборка из `Dockerfile` на `python:3.12.7-slim-bookworm`; `postgres:16.4-alpine`; `redis:7.4.1-alpine`; `haproxy:2.8.30-alpine`. Зависимости Python закреплены в `requirements.txt`. Нужны Docker Desktop / Docker Engine с Compose v2 и Python 3.9+ на хосте (тесты и `scripts/ops.py` — только стандартная библиотека). Вариант оркестрации — **Compose + HAProxy** (Swarm не используется). + +```powershell +$env:PYTHONUTF8 = "1" +# 1. Локальные секреты (один раз; каталог secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py +copy .env.example .env # Linux/macOS: cp .env.example .env + +# 2. Проверка, сборка и запуск: postgres+redis -> migrate (однократно) -> api x2 -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Ожидание готовности: 2 реплики healthy, обе UP в HAProxy, /health/ready через LB = 200 +python scripts/ops.py wait +python scripts/ops.py ps # X-Instance-Id <-> контейнер <-> статус в LB +``` + +Время прогрева: первая сборка образа 1–2 мин; дальше стенд готов примерно через 20–30 с (healthcheck PostgreSQL, миграции, `start_period` API, `rise 2` в HAProxy). + +```powershell +# Публичные проверки (Linux/macOS: make check / make test / make test-persistence ... ) +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd # Linux/macOS: scripts/restart.sh +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd # Linux/macOS: scripts/stop-one.sh +python scripts/ops.py restore # вернуть остановленную реплику; затем повторить failover для другой + +# Собственные проверки уровня 4 +python student-tests/run_all.py --base-url http://localhost:8080 # API, auth, идемпотентность через реплики, race +python scripts/ops.py in-network storage # тесты с прямым доступом к БД (внутри compose-сети) +python scripts/ops.py in-network expiry # истечение сессии (api-short-ttl, TTL 2 с) +python student-tests/failover_series.py --output evidence/failover_series.json # каждая реплика по очереди, SIGTERM +python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json # то же, SIGKILL +python student-tests/rolling_update_load.py --output evidence/rolling_update.json # rolling update под нагрузкой +python tests/load.py --duration 30 --concurrency 8 --output evidence/load.json + +# Всё перечисленное одной командой с записью результатов в evidence/ (~8–10 мин): +python scripts/collect_evidence.py +``` + +**Rolling update.** Задать новый тег `APP_VERSION` (например, `lab3.1`) в `.env` или в окружении и выполнить `python scripts/ops.py rolling-update`: сборка образа, однократные миграции (`docker compose run --rm migrate`), затем для каждой старой реплики: +1 новая → healthy и UP в HAProxy → graceful stop старой → удаление. + +**Остановка.** Безопасная, данные сохраняются: `docker compose --env-file .env --profile ttl down` (volume `pgdata` остаётся). Намеренное удаление всех учебных данных — только вручную: `docker compose --env-file .env --profile ttl down -v`. Hooks и тесты никогда не удаляют volumes. + +**Действия оператора:** создать секреты до первого запуска; пароль PostgreSQL задаётся только при инициализации пустого volume, поэтому `init_secrets.py` не перезаписывает существующие файлы. После `stop-one`/тестов отказа вернуть реплику `python scripts/ops.py restore`. + +## Архитектура и решения + +``` + 127.0.0.1:8080 (единственная публичная точка входа API) +Клиент ──HTTP──► lb: HAProxy 2.8 ── roundrobin, активные проверки GET /health/ready + │ 127.0.0.1:8404 — страница состояния (UP/DOWN реплик), только чтение + │ обнаружение реплик: Docker DNS имени `api` -> server-template api1..api6 + ▼ + ┌───────── api (deploy.replicas = 2, без ports, без container_name) ─────────┐ + │ api-1: X-Instance-Id = hostname = короткий ID контейнера 0.5 CPU / 256M │ + │ api-2: ... 0.5 CPU / 256M │ + └──────┬──────────────────────────────────────────────────────────┬──────────┘ + ▼ ▼ + postgres 16 (1 CPU / 512M, volume pgdata) redis 7 (0.25 CPU / 96M) + источник истины: users, chats, messages, общий кэш проверок токена, + sessions, idempotency_keys, schema_migrations без persistence + ▲ + migrate (one-shot): python -m src.migrate — до старта реплик, один раз +``` + +Порядок старта задаётся `depends_on`: `postgres`, `redis` (healthy) → `migrate` (`service_completed_successfully`) → `api` (healthy) → `lb`. API-реплики не публикуют порты: `expose: 8080` только внутри сети `backend`. Наружу (на 127.0.0.1) опубликован только `lb`. + +### Почему HAProxy, а не Nginx + +В open-source Nginx нет активных health checks: он узнаёт об отказе только по ошибкам живых запросов (`max_fails`). HAProxy сам опрашивает `/health/ready` с заданным периодом. Поэтому время обнаружения отказа можно указать числом, а фаза осушения при SIGTERM действительно выводит реплику из ротации до остановки. `server-template` + `resolvers` синхронизируют список реплик с DNS-записями сервиса `api` каждые 2 с. Масштабирование и rolling update поэтому не требуют перезапуска или перечитывания конфига LB. Sticky sessions не используются (`balance roundrobin`, без cookie). + +### Что перенесено из lab2 и что изменено + +Перенесены без изменения семантики: модели и валидация, формат ошибок, keyset-пагинация, Argon2id, Basic → Bearer, сессии в PostgreSQL + кэш в Redis с надёжной инвалидацией (`session_cache_invalidations`), собственные тесты lab2. Схема lab2 уже рассчитана на несколько реплик: сессии лежат в общей БД, кэш — в общем Redis, а долг инвалидации перечитывается всеми репликами не реже раза в секунду. + +Изменено в lab3: + +1. **Переход 2 → 3 по контракту:** `X-Instance-Id` во всех ответах API, включая ошибки и health (middleware + `error_response`). Обязательный `Idempotency-Key` для `POST /chats/{id}/messages`. +2. **Общая идемпотентность** — таблица `idempotency_keys` (миграция `003`), подробности ниже. +3. **Миграции вынесены из реплик** в one-shot сервис `migrate` (`RUN_MIGRATIONS_ON_START=false`). Реплика готова только когда в `schema_migrations` есть последняя версия, которую знает её образ. +4. **Graceful shutdown с осушением** (`src/server.py`): по SIGTERM реплика сразу отвечает 503 на readiness, но ещё 3 с обслуживает запросы, затем штатно останавливается. +5. **Лимит одновременных хешей Argon2id** (`PASSWORD_HASH_CONCURRENCY=4`). У реплики лимит 256M, а threadpool FastAPI — до 40 потоков; без лимита всплеск входов мог бы занять 40 × 19 MiB и привести к OOM-kill. +6. **Readiness** использует отдельный короткий таймаут БД (`READY_DB_TIMEOUT_SECONDS=1`) меньше `timeout check` HAProxy. +7. Тесты lab2 адаптированы к переходу: при отправке сообщения добавляется `Idempotency-Key`, имя проекта сменилось на `messenger-lab3`. Тест истечения сессии и тесты хранилища теперь идут внутри compose-сети, без публикации портов в обход LB. + +### Идемпотентность отправки сообщения (`src/storage.py`, `migrations/003_idempotency_keys.sql`) + +- Ключ: `PRIMARY KEY (user_id, method, chat_id, idem_key)` — ровно область из контракта «пользователь + метод + путь чата + ключ». В строке хранятся SHA-256 канонического JSON тела (`sort_keys`, без пробелов: порядок ключей и форматирование не важны), `message_id`, `expires_at = now + IDEMPOTENCY_TTL_SECONDS (86400)`. +- **Одна транзакция READ COMMITTED:** + 1. `INSERT INTO idempotency_keys … ON CONFLICT DO UPDATE … WHERE expires_at <= now() RETURNING`. Конкурентная транзакция с тем же ключом **ждёт на уникальном индексе**, пока первая не сделает COMMIT/ROLLBACK. Это и даёт «ровно одно сообщение» при одновременных повторах через разные реплики: арбитр — PostgreSQL, а не память процесса. + 2. Ключ наш → `INSERT messages … WHERE EXISTS (членство)` с заранее выбранным id. FK `message_id → messages` объявлен `DEFERRABLE INITIALLY DEFERRED` и проверяется при COMMIT, поэтому `message_id NOT NULL` с первого шага. + 3. Ключ уже подтверждён → снова проверяется членство (и сессия — dependency `current_user`). Hash совпал → 201 и то же сообщение (+ информационный заголовок `Idempotency-Replayed: true`); не совпал → 409. +- Любая ошибка или обрыв до COMMIT откатывает и ключ, и сообщение: транзиентный 5xx не фиксируется. 404 и 400 ключ не «сжигают». Истёкший, но ещё не удалённый ключ можно занять заново. Фоновая очистка раз в 5 мин удаляет истёкшие ключи пачками под `pg_try_advisory_xact_lock`, то есть одной репликой за раз. +- Порядок проверок: аутентификация → ключ (нет/неверный → 400) → тело → доступ к чату (404). + +### Readiness, таймауты и остановка — согласованные значения + +| Параметр | Значение | Почему | +| --- | --- | --- | +| HAProxy `inter` / `fastinter` / `fall` / `rise` | 1 с / 0.5 с / 2 / 2 | DOWN после 2 неудачных проверок подряд, вторая — через 0.5 с | +| HAProxy `timeout check` | 1.5 с | больше `READY_DB_TIMEOUT_SECONDS` = 1 с | +| HAProxy `timeout connect`, `retries`, `option redispatch 1` | 1 с, 2, да | несостоявшееся соединение повторяется на другой реплике; запрос ещё не отправлен — безопасно и для POST | +| HAProxy `timeout server` | 10 с | > максимум запроса API: пул 3 с + `statement_timeout` 5 с | +| `resolvers` hold valid/obsolete | 2 с / 2 с | новые/исчезнувшие контейнеры попадают в LB за ≤ ~2–3 с | +| API `DRAIN_SECONDS` | 3 с | > худшее обнаружение DOWN при ответах 503 (≈ 1 + 0.5 с + задержка проверок) | +| API `GRACEFUL_TIMEOUT_SECONDS` | 10 с | дождаться запросов в обработке (≤ 8 с) | +| compose `stop_grace_period` (api) | 20 с | > 3 + 10 — SIGKILL не прилетает во время graceful stop | +| Docker healthcheck api | 3 с / timeout 2 с / retries 2 / start_period 20 с | для `depends_on` и `ops.py` (rolling update ждёт healthy) | +| Uvicorn keep-alive | 30 с | дольше, чем LB держит простаивающие соединения к серверу | + +**Максимальное время обнаружения отказа балансировщиком:** graceful stop (сразу 503) и падение процесса (соединение отклонено) — до ≈ 1.5–2 с; зависшая реплика без ответа — до 1 + 1.5 + 0.5 + 1.5 = **4.5 с**. Пока реплика не помечена DOWN, запрос к ней, не сумевший соединиться за 1 с, уходит на другую. **Измерено** (`evidence/failover_series*.json`, `evidence/shutdown_timeline.txt`): DOWN через 0.96–1.41 с после SIGTERM и через 1.10–1.28 с после SIGKILL. + +**Временная шкала graceful stop** (`evidence/shutdown_timeline.txt`): SIGTERM → +0.01 с readiness 503 → +1.30 с HAProxy DOWN (новые запросы больше не приходят) → +3.02 с закрыт listening-сокет → +3.12 с процесс завершён. LB выводит реплику из ротации на ~1.7 с раньше, чем она перестаёт принимать соединения, поэтому graceful stop не даёт ошибок клиенту. + +**Критичные зависимости readiness:** PostgreSQL (и наличие мигрированной схемы). Redis не критичен — это кэш с обходом (circuit breaker 5 с), как в lab2. Если ни одна реплика не готова, LB сам отвечает `503` в общем JSON-формате ошибок с тем же `X-Request-Id` (`http-error`, `evidence/lb_errors.txt`). `X-Instance-Id` в таком ответе нет — контракт это допускает. + +### Миграции и совместимость с работающей версией + +- `migrate` выполняется **один раз** до старта реплик. Повтор — no-op (`schema_migrations`); две одновременные копии сериализуются `pg_advisory_lock` (`evidence/migrations.txt`: копия B применила 3 миграции, копия A — `nothing applied`). Реплики миграции не запускают и не готовы, пока схема не мигрирована: на пустой БД readiness = 503 `Database schema is not migrated`, после `migrate` — 200 без рестарта. +- Rolling update применяет миграции **до** замены реплик, поэтому допускаются только **расширяющие** (expand) миграции, совместимые со старой версией. `003` только добавляет таблицу. Проверено: код lab2 (старая версия) проходит smoke lab2 на схеме lab3 одновременно с репликами lab3 (`evidence/migrations.txt`). +- Ограничение: смешанный парк «lab2 + lab3» во время обновления формально совместим по схеме, но реплики lab2 не требуют и не записывают `Idempotency-Key`. Поэтому гарантия идемпотентности действует только когда все реплики уже lab3. Удаляющие/переименовывающие миграции нужно делить на expand → переключение кода → contract отдельным релизом. + +### Соответствие `.env.example` и реальной конфигурации + +| Переменная | Где используется | +| --- | --- | +| `APP_HOST`, `APP_PORT` | внутри контейнера Uvicorn слушает `0.0.0.0:8080`; `APP_PORT` — публикуемый порт LB `127.0.0.1:${APP_PORT}` | +| `LB_STATS_PORT` | страница состояния HAProxy `127.0.0.1:8404` (`ops.py`, тесты отказов) | +| `APP_ENV`, `APP_VERSION` | `APP_VERSION` — тег образа `messenger-api:${APP_VERSION}` (rolling update) и строка в логе старта | +| `SESSION_TTL_SECONDS`, `SHORT_SESSION_TTL_SECONDS` | TTL сессий; второй — для `api-short-ttl` (профиль `ttl`) | +| `DATABASE_URL_FILE`, `CACHE_URL_FILE` | Compose secrets `/run/secrets/database_url`, `/run/secrets/cache_url` | +| `SESSION_STORE` | `postgres` — источник истины сессий; Redis — общий кэш | +| `SESSION_CACHE_TTL_SECONDS` | TTL записи кэша и «надгробия» logout | +| `API_REPLICAS` | `deploy.replicas` сервиса `api` (2); ожидаемое число реплик в `ops.py` | +| `IDEMPOTENCY_TTL_SECONDS` | срок хранения результата ключа (≥ 86400, иначе API не стартует) | +| `DB_POOL_*`, `DB_*_TIMEOUT*` | пул (1..10 на реплику), ожидание соединения 3 с, connect 3 с, statement 5 с | +| `CACHE_TIMEOUT_SECONDS`, `CACHE_RETRY_AFTER_SECONDS` | таймаут Redis 250 мс и обход кэша на 5 с после ошибки | +| `READY_DB_TIMEOUT_SECONDS`, `DRAIN_SECONDS`, `GRACEFUL_TIMEOUT_SECONDS`, `KEEP_ALIVE_SECONDS` | readiness и остановка — таблица выше | +| `PASSWORD_HASH_CONCURRENCY` | одновременных хешей Argon2id на реплику | + +`max_connections=100` у PostgreSQL ≥ (2 реплики + 1 во время rolling update) × 10 + migrate + api-short-ttl + запас. + +### Инфраструктура + +| Роль | Service | Версия | Health/readiness | Volume | Сеть/порт | Credentials | Лимиты | +| --- | --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | `haproxy:2.8.30-alpine` | `wget /lb-health` (monitor-uri) | — (конфиг ro bind) | `backend`; **127.0.0.1:8080**, 127.0.0.1:8404 | — | 0.5 CPU / 128M | +| api × 2 | `api` | сборка `Dockerfile`, `python:3.12.7-slim-bookworm` | `/health/ready` (БД + схема + не в drain) | нет, `read_only` + tmpfs `/tmp` | `backend`, `expose 8080` (наружу нет) | secrets `database_url`, `cache_url` | 0.5 CPU / 256M | +| миграции | `migrate` | тот же Dockerfile | one-shot, exit 0 | — | `backend` | secret `database_url` | 0.5 CPU / 256M | +| postgres | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `pgdata` | `backend`, наружу нет | secret `postgres_password` | 1 CPU / 512M | +| redis | `redis` | `redis:7.4.1-alpine`, user `redis` | `redis-cli ping` | нет (кэш) | `backend`, наружу нет | secret `redis_password` | 0.25 CPU / 96M | +| тест TTL | `api-short-ttl` (профиль `ttl`) | образ api | `/health/ready` | — | `backend`, не в LB, портов нет | как api | 0.5 CPU / 256M | + +`container_name` нигде не задан. Секреты общие только там, где нужны: пароль PostgreSQL есть только у `postgres`, пароль Redis — у `redis`, строки подключения — у `api`/`migrate`. Bootstrap: схема создаётся сервисом `migrate` на пустом volume; повтор безопасен. Hooks: `scripts/restart.sh|.cmd` → `ops.py restart` (stop lb+api → restart postgres+redis → start api → healthy → start lb); `scripts/stop-one.sh|.cmd` → `ops.py stop-one` (одна реплика по `TARGET_INSTANCE_ID`, остаётся остановленной). Каждое действие пишется в `evidence/ops.log`. + +## Доказательства критериев + +**Стенд, на котором сняты приложенные результаты.** Windows 10, Ryzen 5 7500f, 32 GB RAM. PostgreSQL 16, Redis 7 и HAProxy **2.8.30**, Docker Desktop. Код приложения, миграции и **тот же `infra/haproxy/haproxy.cfg`** использованы как есть. Только `server-template` + DNS заменены на 6 статических слотов, реплики запущены как `python -m src.server` (2 процесса), CPU/RAM-лимитов нет. Compose-файл проверен `docker compose config`, конфиг HAProxy — `haproxy -c` той же версии 2.8.30, что в образе. `scripts/ops.py` прогнан против заглушки `docker`. + +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | `PASS: lab3 public API scenario` | `evidence/smoke.txt` | +| Контракт | `make check`, `make validate`, `docker compose config --quiet`, `haproxy -c` | все PASS / valid | `evidence/check.txt` | +| Стенд в `compose.yaml`, ≥ 2 реплики, одна точка входа, общая БД и хранилище сессий | `compose.yaml`; `python scripts/ops.py ps` | `api` × 2 без `ports`; публикуется только `lb`; одна `postgres`, один `redis` | `compose.yaml`, `evidence/stand.txt` (после `collect_evidence.py`) | +| Разные запросы одного пользователя — разными репликами без повторного входа | 20 × `GET /users/me` с одним токеном через LB | чередование двух `X-Instance-Id`, все 200 | `evidence/instances.txt` | +| Остановка любой одной реплики: ≤ 10 с, чтение/запись продолжаются, подтверждённые доступны | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` × 2 (вторая реплика после восстановления первой) + SIGKILL | PASS ×3; `failed_attempts: 0`; запись не прерывалась (первая запись после возврата hook — 3.3 с = длительность graceful hook; при SIGKILL — 0.11 с); остановлены разные реплики | `evidence/failover.txt` | +| Какой процесс реально остановлен | журнал hook | `SIGTERM nat-api-4` в прогоне 1, `SIGTERM nat-api-3` в прогоне 2 (в compose — `evidence/ops.log`: имя контейнера, IP, состояние после) | `evidence/failover.txt` | +| Повтор POST с тем же ключом не создаёт дубль, в т.ч. конкурентно и при смене реплики | smoke; `test_idempotency.py` (24 параллельных POST, Barrier); failover-тест повторяет неопределённые POST тем же ключом | 24 × 201, один `id`, ответы через **обе** реплики, в истории 1 сообщение; дублей нет ни в одном сценарии отказа | `evidence/student-tests.txt`, `evidence/failover*.txt` | +| Persistence: рестарт стенда с volumes и тем же токеном | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; PID до/после: postgres 1469→2901, redis 1479→2911, api 2846→2917 и 2866→2925, HAProxy перезапущен | `evidence/persistence.txt` | +| Миграции отдельно и согласованно | сервис `migrate`; readiness проверяет схему | реплики не мигрируют; повтор no-op; гонка двух `migrate` безопасна | `evidence/migrations.txt` | + +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Health/readiness, timeouts и SIGTERM согласованы; LB исключает недоступный backend; период проверок и макс. время обнаружения | `infra/haproxy/haproxy.cfg`, `src/server.py`; замер шкалы остановки | проверки каждые 1 с (0.5 с после первой неудачи), fall 2; обнаружение ≤ 1.5–2 с (503/RST), ≤ 4.5 с (зависание). Замер: 503 на readiness через 0.01 с, DOWN через 1.30 с, приём прекращён через 3.02 с | таблица выше, `evidence/shutdown_timeline.txt`, `evidence/lb_log_sample.txt` | +| Последовательно остановить каждую реплику (восстанавливая предыдущую); RPS, ошибки, время восстановления | `failover_series.py` (4 писателя + 2 читателя, timeout клиента 2 с) | **SIGTERM:** 0 ошибок; DOWN в LB через 1.41 / 0.96 с; RPS 407 → 324 (одна реплика) → 354 после restore; p95 30–31 мс; реплика снова в ротации через 3.0 / 2.6 с после restore. **SIGKILL:** 6 / 3 оборванных запроса (502) в окне ≤ 0.21 с после kill, все записи повторены тем же ключом без дублей; DOWN через 1.28 / 1.10 с | `evidence/failover_series.json`, `evidence/failover_series_kill.json`, `evidence/failover_series.txt` | +| Общая идемпотентность параллельными запросами с одним ключом | `test_idempotency.py`, `test_storage_idempotency.py` | 24 параллельных → 1 сообщение через 2 реплики; 12 разных тел с одним ключом → один победитель, остальные 409; 16 потоков на уровне хранилища → одна вставка; сбой после вставки ключа и сообщения → ни ключа, ни сообщения, повтор успешен | `evidence/student-tests.txt` | +| CPU/RAM-лимиты; API без портов в обход LB; секреты/хранилища общие только где нужно; нет `container_name` | `compose.yaml` | лимиты у всех сервисов (api 0.5 CPU / 256M); у `api` только `expose`; `container_name` отсутствует; RSS реплики под нагрузкой 132 MB < 256M | `compose.yaml`, `evidence/load.txt` | +| Rolling update под нагрузкой | `rolling_update_load.py` → `ops.py rolling-update` (на нативном стенде — тот же алгоритм) | замена 2 реплик за 11.8 с, **0 ошибок**, RPS 388 → 381 во время → 385 после, p95 31 → 34 мс; после — только новые реплики; 5709 подтверждённых сообщений на месте, дублей нет | `evidence/rolling_update.json`, `evidence/rolling_update.txt` | +| Миграции однократно и совместимы с работающей версией | сервис `migrate`; код lab2 на схеме lab3 | см. раздел «Миграции»; ограничение смешанного парка описано | `evidence/migrations.txt` | +| Возможности lab2 сохранены | `run_all.py` (47 тестов + race), `test_expiry` на экземпляре с TTL 2 с | все PASS (1 skip — expiry в общем прогоне, запускается отдельно и проходит) | `evidence/student-tests.txt` | + +## Эксперименты и отказы + +**Гипотеза.** Если всё состояние, от которого зависит корректность (данные, сессии, ключи идемпотентности), лежит в общем PostgreSQL, а LB активно проверяет готовность и реплика по SIGTERM сначала выходит из ротации, то: (1) graceful-остановка любой реплики и rolling update проходят без ошибок для клиента; (2) падение реплики даёт только кратковременные ошибки запросов, которые были в обработке, и клиент безопасно повторяет их с тем же ключом; (3) дублей и потерь подтверждённых сообщений нет. Подтверждено измерениями ниже. + +**Методика.** Генератор `student-tests/loadgen.py`: 4 писателя (POST с `Idempotency-Key`; при обрыве, timeout или 5xx — повтор с тем же ключом и телом до 201) и 2 читателя (история, `/users/me`); timeout клиента 2 с; опрос страницы HAProxy каждые 100 мс. Каждый раунд: 5 с прогрева → действие → 15 с без реплики → restore → 3 с. После раунда — полный обход истории и сверка со всеми подтверждёнными ответами. Стенд — см. «Доказательства»: клиент на той же машине, без лимитов. Повторов для статистики нет: разброс RPS между раундами ±5–8 %. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| SIGTERM реплики 1 под нагрузкой | 0 ошибок, LB выводит до остановки | 0 ошибок; DOWN через 1.41 с; реплика отвечала ещё 1.34 с (до DOWN) | hook 3.3 с; RPS 407 → 324 → 354; возврат в ротацию 3.0 с после restore; потерь нет | +| SIGTERM реплики 2 | то же | 0 ошибок; DOWN через 0.96 с | hook 3.4 с; RPS 393 → 304 → 348; возврат 2.6 с; потерь нет | +| SIGKILL реплики 1 | ошибки только у запросов в обработке, повтор без дублей | 6 × 502 в окне 0.21 с, 4 повтора записи тем же ключом; DOWN через 1.28 с | новые запросы успешны сразу (redispatch); возврат 2.8 с; дублей нет | +| SIGKILL реплики 2 | то же | 3 × 502 в окне 0.14 с, 1 повтор записи; DOWN через 1.10 с | возврат 2.7 с; дублей нет | +| Публичный failover (×2 graceful, ×1 kill) | PASS, ≤ 10 с | PASS; 0 failed attempts | первая запись после hook: 3.3 с / 3.4 с / 0.11 с | +| Rolling update 2 реплик под нагрузкой | 0 ошибок, после — только новые | 0 ошибок; `[api1,api2] → [api1,api2,api3] → [api2,api3] → … → [api3,api4]` | 11.8 с; RPS не просел (388 / 381 / 385) | +| Рестарт API + PostgreSQL + Redis + LB | данные и сессия сохраняются | PASS, тот же токен | потерь нет | +| Все реплики недоступны | LB отвечает 503 в формате контракта | JSON 503 с `request_id` = `X-Request-Id` | — | + +**Нагрузка** (`tests/load.py`, closed-loop POST, без retry): 30 с, concurrency 8 → 10 955 успешных, 0 ошибок, **364 RPS**, p50 19.9 мс, p95 42.8 мс, p99 53.4 мс (только успешные); набор данных — один чат, ~11 тыс. новых сообщений. RSS каждой реплики 132 MB, 14 потоков. Во время отказа одной реплики RPS падает примерно на 20 %: при 2 vCPU на машину одна реплика (один процесс Uvicorn) упирается в CPU. Это пропускная способность HTTP-подтверждений в данном профиле, а не максимум системы; с лимитами 0.5 CPU на реплику в Docker цифры будут ниже. + +## Ограничения и следующие шаги + +- **Отказы, которые выдерживаются:** остановка или падение любой одной API-реплики, rolling update реплик, рестарт всего стенда с volumes. **Не выдерживаются:** отказ `lb`, `postgres` (единственная копия, без репликации) и Docker-host — это SPOF. Отказ `redis` не влияет на корректность (обход кэша, как в lab2). +- При падении (SIGKILL) запросы, которые реплика обрабатывала в этот момент, завершаются ошибкой 502. LB не повторяет уже отправленные POST — по контракту это делает клиент с тем же `Idempotency-Key`. Для `POST /users` и `POST /chats` повтор после неопределённого результата может создать второй объект (контракт этого не запрещает и прозрачных повторов не требует). +- HAProxy держит до 6 слотов реплик (`server-template api 1-6`). Больше — поменять число в конфиге. +- Страница состояния HAProxy на 127.0.0.1:8404 без аутентификации, только чтение и только localhost. +- Приложенные результаты сняты на нативном эквиваленте стенда (см. «Доказательства»), без CPU/RAM-лимитов и без Docker DNS. Перед защитой их нужно повторить `python scripts/collect_evidence.py` на Docker; конкретные цифры на ноутбуке будут другими. +- Время обнаружения на Docker дополнительно включает DNS-синхронизацию (до ~2 с на появление новой реплики в LB). На отказ это не влияет: остановленную реплику исключают активные проверки раньше, чем DNS. +- Идемпотентность гарантирована, только когда все реплики — lab3 (см. «Миграции»). +- HTTP без TLS — допустимо только на локальном учебном стенде. Пароль Redis передаётся `redis-server` аргументом внутри контейнера (как в lab2). + +## Вопросы защиты — кратко + +- **Отказ процесса vs отказ узла.** Процесс (контейнер реплики) падает, остальные реплики на том же host живы, общие PostgreSQL/Redis/LB целы — это и проверяет лаба. При отказе узла пропадает всё, что на нём: здесь это весь стенд, включая LB и единственную БД. Защита от него — несколько host, репликация БД, LB с переносом адреса (трек 2 уровня 5, не заявлен). +- **Почему depends_on/healthcheck — не балансировщик.** `depends_on` управляет только порядком старта, а Docker healthcheck лишь меняет статус контейнера. Ни один из них не перенаправляет трафик и не выводит больную реплику из ротации. Это делает LB своими проверками (здесь — HAProxy `http-check` каждые 1 с). +- **Какие POST можно повторять после timeout.** Только те, что защищены ключом идемпотентности: здесь `POST /chats/{id}/messages` с тем же `Idempotency-Key` и тем же телом. Ключ защищает от дубля, если ответ потерялся после COMMIT: повтор вернёт исходное сообщение, а не создаст второе. `POST /users`, `POST /chats`, `POST /auth/sessions` после неопределённого результата повторять «вслепую» нельзя — создастся новый объект или новая сессия. +- **Сессии в локальной памяти при росте числа реплик.** Токен, выданный репликой A, неизвестен реплике B: каждый второй запрос получит 401, logout на одной реплике не отзовёт сессию на другой, рестарт реплики разлогинит её пользователей. «Чинить» это sticky sessions — значит терять сессии при отказе реплики. Поэтому сессии лежат в общем PostgreSQL (кэш — общий Redis), а ключи идемпотентности — в общей БД. diff --git a/docs/lab3/evidence/README.md b/docs/lab3/evidence/README.md new file mode 100644 index 0000000..d074e56 --- /dev/null +++ b/docs/lab3/evidence/README.md @@ -0,0 +1,25 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Файлы этой работы (lab3) + +`python scripts/collect_evidence.py` на compose-стенде перезаписывает их результатами Docker; `SUMMARY.txt` — итог по шагам. + +| Файл | Что доказывает | +| --- | --- | +| `check.txt` | make check / validate, `docker compose config`, `haproxy -c` | +| `smoke.txt` | публичный smoke lab3 через LB | +| `instances.txt` | одна сессия обслуживается разными репликами (X-Instance-Id) | +| `student-tests.txt` | собственные тесты: lab2 + идемпотентность через реплики + хранилище + expiry | +| `failover.txt` | публичный failover для обеих реплик и SIGKILL; какой процесс/контейнер остановлен | +| `failover_series*.json/.txt` | уровень 4: каждая реплика по очереди под нагрузкой — RPS, ошибки, обнаружение, восстановление | +| `rolling_update*.json/.txt` | уровень 4: rolling update под нагрузкой | +| `persistence.txt` | рестарт API + PostgreSQL + Redis + LB с тем же токеном | +| `migrations.txt` | миграции однократно, readiness ждёт схему, совместимость со старой версией | +| `shutdown_timeline.txt` | SIGTERM → readiness 503 → DOWN в LB → выход процесса | +| `lb_errors.txt`, `lb_log_sample.txt` | ответы и журнал балансировщика | +| `load.json`, `load.txt` | нагрузка 30 с, RSS реплик | +| `ops.log` | журнал scripts/ops.py (реальные контейнеры — после прогона на Docker) | diff --git a/docs/lab3/evidence/SUMMARY.txt b/docs/lab3/evidence/SUMMARY.txt new file mode 100644 index 0000000..0a23b4e --- /dev/null +++ b/docs/lab3/evidence/SUMMARY.txt @@ -0,0 +1,34 @@ +# 2026-10-26 19:35:57 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Rolling update lab3 -> lab3.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab3/evidence/check.txt b/docs/lab3/evidence/check.txt new file mode 100644 index 0000000..a01561a --- /dev/null +++ b/docs/lab3/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-10-26 19:29:57 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-10-26 19:29:57 UTC | Compose config | exit 0 | 0.2s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab3/evidence/failover.txt b/docs/lab3/evidence/failover.txt new file mode 100644 index 0000000..6375f76 --- /dev/null +++ b/docs/lab3/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-10-26 19:30:28 UTC | Failover #1: реплики до | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP +# 2026-10-26 19:30:44 UTC | Failover #1: make test-failover | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 4.672} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:30:44 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 exited unhealthy messenger-api:lab3 - - + +# 2026-10-26 19:30:51 UTC | Failover #1: restore | exit 0 | 6.4s +$ python scripts/ops.py restore +2026-10-26T19:30:51.183Z restore: started ['messenger-lab3-api-2']; 2 replicas healthy and UP in LB after 6.3s + +# 2026-10-26 19:30:51 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:07 UTC | Failover #2: make test-failover | exit 0 | 16.2s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.094} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:31:07 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 exited unhealthy messenger-api:lab3 - - + +# 2026-10-26 19:31:13 UTC | Failover #2: restore | exit 0 | 6.0s +$ python scripts/ops.py restore +2026-10-26T19:31:13.832Z restore: started ['messenger-lab3-api-2']; 2 replicas healthy and UP in LB after 5.9s + +# 2026-10-26 19:31:14 UTC | Failover #3: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:30 UTC | Failover #3: make test-failover | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.766} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:31:30 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 exited unhealthy messenger-api:lab3 - - +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:31:37 UTC | Failover #3: restore | exit 0 | 6.4s +$ python scripts/ops.py restore +2026-10-26T19:31:37.160Z restore: started ['messenger-lab3-api-1']; 2 replicas healthy and UP in LB after 6.3s + +# остановленные публичным тестом реплики: ['a21409fffeec', 'a21409fffeec', 'c4c5ae5b0949'] diff --git a/docs/lab3/evidence/failover_series.json b/docs/lab3/evidence/failover_series.json new file mode 100644 index 0000000..571c5c3 --- /dev/null +++ b/docs/lab3/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "rounds": [ + { + "target_instance": "a21409fffeec", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.09, + "lb_detect_down_seconds": 1.77, + "target_last_served_after_stop_seconds": 1.67, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.7, + "lb_back_up_seconds": 1.39, + "target_back_in_rotation_seconds": 6.69, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1322, + "ok": 1322, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 264.4, + "write_ok_rps": 159.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1059, + "ok": 1059, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 105.9, + "write_ok_rps": 64.0, + "latency_ms_ok": { + "p50": 47.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 439, + "ok": 439, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 87.8, + "write_ok_rps": 53.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "c4c5ae5b0949" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.0, + "requests": 1618, + "ok": 1618, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 161.8, + "write_ok_rps": 96.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 6.77, + [ + "api1" + ] + ], + [ + 21.39, + [ + "api1", + "api2" + ] + ], + [ + 21.77, + [ + "api1" + ] + ], + [ + 24.28, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2674, + "stored_messages": 2674, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c4c5ae5b0949", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.03, + "lb_detect_down_seconds": 0.97, + "target_last_served_after_stop_seconds": 0.91, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.56, + "lb_back_up_seconds": 0.67, + "target_back_in_rotation_seconds": 6.56, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1353, + "ok": 1353, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 269.7, + "write_ok_rps": 158.3, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 990, + "ok": 990, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 99.0, + "write_ok_rps": 59.2, + "latency_ms_ok": { + "p50": 62.0, + "p95": 156.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.03, + "requests": 410, + "ok": 410, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 81.8, + "write_ok_rps": 49.1, + "latency_ms_ok": { + "p50": 93.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec" + ] + }, + "after_restore": { + "from_s": 20.03, + "to_s": 29.86, + "requests": 1602, + "ok": 1601, + "errors": 1, + "error_ratio": 0.0006, + "ok_rps": 162.9, + "write_ok_rps": 98.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 187.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 5.98, + [ + "api2" + ] + ], + [ + 20.7, + [ + "api1", + "api2" + ] + ], + [ + 20.94, + [ + "api2" + ] + ], + [ + 23.47, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2604, + "stored_messages": 2604, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab3/evidence/failover_series.txt b/docs/lab3/evidence/failover_series.txt new file mode 100644 index 0000000..e4f0e74 --- /dev/null +++ b/docs/lab3/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-10-26 19:32:39 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 61.9s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['a21409fffeec', 'c4c5ae5b0949'] +-- stopping a21409fffeec (mode=stop) under load + hook 4.09s | LB DOWN after 1.77 s | errors 0 (-) | RPS before 264.4, outage 105.9, after restore 161.8 | p95 outage 110.0 ms | back in rotation 6.69 s | data ok=True +-- stopping c4c5ae5b0949 (mode=stop) under load + hook 4.03s | LB DOWN after 0.97 s | errors 0 (-) | RPS before 269.7, outage 99.0, after restore 162.9 | p95 outage 156.0 ms | back in rotation 6.56 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-10-26 19:33:42 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 63.1s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['a21409fffeec', 'c4c5ae5b0949'] +-- stopping a21409fffeec (mode=kill) under load + hook 1.01s | LB DOWN after 2.53 s | errors 0 (-) | RPS before 235.6, outage 76.1, after restore 146.5 | p95 outage 187.0 ms | back in rotation 7.06 s | data ok=True +-- stopping c4c5ae5b0949 (mode=kill) under load + hook 0.92s | LB DOWN after 2.38 s | errors 0 (-) | RPS before 270.1, outage 78.3, after restore 149.7 | p95 outage 187.0 ms | back in rotation 7.19 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab3/evidence/failover_series_kill.json b/docs/lab3/evidence/failover_series_kill.json new file mode 100644 index 0000000..f54d5b8 --- /dev/null +++ b/docs/lab3/evidence/failover_series_kill.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "rounds": [ + { + "target_instance": "a21409fffeec", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.01, + "lb_detect_down_seconds": 2.53, + "target_last_served_after_stop_seconds": 0.5, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.09, + "lb_back_up_seconds": 1.45, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1178, + "ok": 1178, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 235.6, + "write_ok_rps": 137.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 110.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 761, + "ok": 761, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 76.1, + "write_ok_rps": 46.0, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 1015.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.01, + "requests": 454, + "ok": 454, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 90.5, + "write_ok_rps": 53.8, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "c4c5ae5b0949" + ] + }, + "after_restore": { + "from_s": 20.01, + "to_s": 30.44, + "requests": 1527, + "ok": 1527, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 146.5, + "write_ok_rps": 85.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 7.53, + [ + "api1" + ] + ], + [ + 21.47, + [ + "api1", + "api2" + ] + ], + [ + 21.94, + [ + "api1" + ] + ], + [ + 24.48, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2318, + "stored_messages": 2318, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c4c5ae5b0949", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 0.92, + "lb_detect_down_seconds": 2.38, + "target_last_served_after_stop_seconds": 0.44, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.25, + "lb_back_up_seconds": 1.45, + "target_back_in_rotation_seconds": 7.19, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1355, + "ok": 1355, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 270.1, + "write_ok_rps": 156.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 783, + "ok": 783, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 78.3, + "write_ok_rps": 47.5, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 1016.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.02, + "requests": 439, + "ok": 439, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 87.8, + "write_ok_rps": 52.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 125.0, + "p99": 203.0 + }, + "instances": [ + "a21409fffeec" + ] + }, + "after_restore": { + "from_s": 20.02, + "to_s": 30.56, + "requests": 1579, + "ok": 1579, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 149.7, + "write_ok_rps": 89.7, + "latency_ms_ok": { + "p50": 16.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 7.39, + [ + "api2" + ] + ], + [ + 21.47, + [ + "api1", + "api2" + ] + ], + [ + 21.58, + [ + "api2" + ] + ], + [ + 24.11, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2474, + "stored_messages": 2474, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab3/evidence/instances.txt b/docs/lab3/evidence/instances.txt new file mode 100644 index 0000000..788d114 --- /dev/null +++ b/docs/lab3/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-10-26 19:29:59 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C08C-1F90-6ADFAA37167-088A-0008 + 2 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C09C-1F90-6ADFAA37189-088B-0008 + 3 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0A0-1F90-6ADFAA371A4-088C-0008 + 4 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0A4-1F90-6ADFAA371B1-088D-0008 + 5 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0A6-1F90-6ADFAA371B5-088E-0008 + 6 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0AA-1F90-6ADFAA371B8-088F-0008 + 7 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0B2-1F90-6ADFAA371D0-0890-0008 + 8 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0BE-1F90-6ADFAA371E0-0891-0008 + 9 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0C6-1F90-6ADFAA371E3-0892-0008 +10 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0CE-1F90-6ADFAA371FE-0893-0008 +11 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0D6-1F90-6ADFAA37202-0894-0008 +12 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0E4-1F90-6ADFAA3721D-0895-0008 +13 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C0EC-1F90-6ADFAA37220-0896-0008 +14 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C0FC-1F90-6ADFAA3723C-0897-0008 +15 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C106-1F90-6ADFAA3724B-0898-0008 +16 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C10E-1F90-6ADFAA3724D-0899-0008 +17 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C11A-1F90-6ADFAA37268-089A-0008 +18 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C126-1F90-6ADFAA3726B-089C-0008 +19 200 X-Instance-Id=c4c5ae5b0949 X-Request-Id=AC120001C130-1F90-6ADFAA37286-089D-0008 +20 200 X-Instance-Id=a21409fffeec X-Request-Id=AC120001C140-1F90-6ADFAA3728A-089E-0008 +разных реплик: 2 diff --git a/docs/lab3/evidence/lb_errors.txt b/docs/lab3/evidence/lb_errors.txt new file mode 100644 index 0000000..4a1f868 --- /dev/null +++ b/docs/lab3/evidence/lb_errors.txt @@ -0,0 +1,17 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker) +# все реплики остановлены -> ответ формирует сам HAProxy (http-error): общий JSON-формат ошибок, X-Request-Id совпадает с request_id; X-Instance-Id нет (контракт это допускает) +$ curl -si http://127.0.0.1:8080/api/v1/users/me +HTTP/1.1 503 Service Unavailable +x-request-id: 7F000001CD38-1F90-6ADCA7141E5-336F-0B75 +content-length: 128 +content-type: application/json + +{"error":{"code":"unavailable","message":"No API instance is available","request_id":"7F000001CD38-1F90-6ADCA7141E5-336F-0B75"}} + +$ curl -si -H "X-Request-Id: demo-123" http://127.0.0.1:8080/health/ready +HTTP/1.1 503 Service Unavailable +x-request-id: demo-123 +content-length: 97 +content-type: application/json + +{"error":{"code":"unavailable","message":"No API instance is available","request_id":"demo-123"}} diff --git a/docs/lab3/evidence/lb_log_sample.txt b/docs/lab3/evidence/lb_log_sample.txt new file mode 100644 index 0000000..a05fe96 --- /dev/null +++ b/docs/lab3/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-10-26 19:35:57 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.2s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.18.0.1:34156 [26/Oct/2026:19:35:54.564] public api/api1 0/0/0/7/7 201 401 ---- 8/8/7/1/0 0/0 rid=AC120001856C-1F90-6ADFAB9A234-1128-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34116 [26/Oct/2026:19:35:54.563] public api/api1 0/0/0/10/10 201 401 ---- 7/7/6/0/0 0/0 rid=AC1200018544-1F90-6ADFAB9A233-1126-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34168 [26/Oct/2026:19:35:54.578] public api/api1 0/0/0/5/5 201 401 ---- 8/8/7/0/0 0/0 rid=AC1200018578-1F90-6ADFAB9A242-1129-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34064 [26/Oct/2026:19:35:54.536] public api/api2 0/0/0/58/59 201 401 ---- 8/8/7/7/0 0/0 rid=AC1200018510-1F90-6ADFAB9A218-111F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34040 [26/Oct/2026:19:35:54.536] public api/api2 0/0/0/62/62 201 401 ---- 7/7/6/6/0 0/0 rid=AC12000184F8-1F90-6ADFAB9A218-111D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34098 [26/Oct/2026:19:35:54.537] public api/api2 0/0/0/61/62 201 401 ---- 6/6/5/5/0 0/0 rid=AC1200018532-1F90-6ADFAB9A219-1121-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34100 [26/Oct/2026:19:35:54.538] public api/api2 0/0/0/63/63 201 401 ---- 6/6/5/4/0 0/0 rid=AC1200018534-1F90-6ADFAB9A21A-1123-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34162 [26/Oct/2026:19:35:54.578] public api/api2 0/0/0/27/28 201 401 ---- 5/5/4/3/0 0/0 rid=AC1200018572-1F90-6ADFAB9A242-112A-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34186 [26/Oct/2026:19:35:54.600] public api/api1 0/0/0/7/7 201 401 ---- 6/6/3/0/0 0/0 rid=AC120001858A-1F90-6ADFAB9A258-112C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34128 [26/Oct/2026:19:35:54.563] public api/api2 0/0/0/44/44 201 401 ---- 5/5/2/2/0 0/0 rid=AC1200018550-1F90-6ADFAB9A233-1125-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34182 [26/Oct/2026:19:35:54.587] public api/api2 0/0/0/21/21 201 401 ---- 4/4/1/1/0 0/0 rid=AC1200018586-1F90-6ADFAB9A24B-112B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34142 [26/Oct/2026:19:35:54.564] public api/api2 0/0/0/45/45 201 401 ---- 3/3/0/0/0 0/0 rid=AC120001855E-1F90-6ADFAB9A234-1127-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34198 [26/Oct/2026:19:35:54.614] public api/api1 0/0/0/14/14 201 401 ---- 8/8/6/2/0 0/0 rid=AC1200018596-1F90-6ADFAB9A25F-112E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34230 [26/Oct/2026:19:35:54.621] public api/api1 0/0/0/10/11 201 401 ---- 7/7/5/1/0 0/0 rid=AC12000185B6-1F90-6ADFAB9A26A-1130-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34242 [26/Oct/2026:19:35:54.622] public api/api1 0/0/0/9/9 201 401 ---- 6/6/4/0/0 0/0 rid=AC12000185C2-1F90-6ADFAB9A26E-1132-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34270 [26/Oct/2026:19:35:54.638] public api/api1 0/0/0/6/7 201 401 ---- 8/8/6/1/0 0/0 rid=AC12000185DE-1F90-6ADFAB9A27D-1134-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34266 [26/Oct/2026:19:35:54.640] public api/api1 0/0/0/6/6 201 401 ---- 7/7/5/0/0 0/0 rid=AC12000185DA-1F90-6ADFAB9A26E-1136-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34278 [26/Oct/2026:19:35:54.656] public api/api1 0/0/0/5/5 201 401 ---- 8/8/6/0/0 0/0 rid=AC12000185E6-1F90-6ADFAB9A27F-1138-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34302 [26/Oct/2026:19:35:54.686] public api/api1 0/0/0/6/6 201 401 ---- 8/8/7/0/0 0/0 rid=AC12000185FE-1F90-6ADFAB9A299-113A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34282 [26/Oct/2026:19:35:54.639] public api/api2 0/0/0/66/66 201 401 ---- 8/8/7/7/0 0/0 rid=AC12000185EA-1F90-6ADFAB9A27F-1135-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34254 [26/Oct/2026:19:35:54.622] public api/api2 0/0/0/83/84 201 401 ---- 7/7/6/6/0 0/0 rid=AC12000185CE-1F90-6ADFAB9A26E-1133-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34294 [26/Oct/2026:19:35:54.671] public api/api2 0/0/0/36/36 201 401 ---- 6/6/5/5/0 0/0 rid=AC12000185F6-1F90-6ADFAB9A28C-1139-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34338 [26/Oct/2026:19:35:54.715] public api/api1 0/0/0/5/5 201 401 ---- 8/8/5/0/0 0/0 rid=AC1200018622-1F90-6ADFAB9A2CB-113C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34346 [26/Oct/2026:19:35:54.734] public api/api1 0/0/1/6/7 201 401 ---- 8/8/7/0/0 0/0 rid=AC120001862A-1F90-6ADFAB9A2CC-113E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34360 [26/Oct/2026:19:35:54.745] public api/api1 0/0/0/5/5 201 401 ---- 8/8/7/0/0 0/0 rid=AC1200018638-1F90-6ADFAB9A2E9-1140-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34290 [26/Oct/2026:19:35:54.652] public api/api2 0/0/0/56/123 201 401 ---- 8/8/7/7/0 0/0 rid=AC12000185F2-1F90-6ADFAB9A28C-1137-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34192 [26/Oct/2026:19:35:54.609] public api/api2 0/0/0/171/172 201 401 ---- 8/8/7/6/0 0/0 rid=AC1200018590-1F90-6ADFAB9A25E-112D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34240 [26/Oct/2026:19:35:54.620] public api/api2 0/0/1/159/160 201 401 ---- 7/7/6/5/0 0/0 rid=AC12000185C0-1F90-6ADFAB9A26C-1131-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34214 [26/Oct/2026:19:35:54.620] public api/api2 0/0/1/160/161 201 401 ---- 6/6/5/4/0 0/0 rid=AC12000185A6-1F90-6ADFAB9A269-112F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34312 [26/Oct/2026:19:35:54.698] public api/api2 0/0/0/87/87 201 401 ---- 5/5/4/3/0 0/0 rid=AC1200018608-1F90-6ADFAB9A2BA-113B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34374 [26/Oct/2026:19:35:54.779] public api/api1 0/0/0/8/8 201 401 ---- 4/4/3/0/0 0/0 rid=AC1200018646-1F90-6ADFAB9A30B-1142-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34364 [26/Oct/2026:19:35:54.755] public api/api2 0/0/0/35/35 201 401 ---- 3/3/2/2/0 0/0 rid=AC120001863C-1F90-6ADFAB9A2F3-1141-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34326 [26/Oct/2026:19:35:54.734] public api/api2 0/0/0/59/59 201 401 ---- 2/2/1/1/0 0/0 rid=AC1200018616-1F90-6ADFAB9A2CB-113F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34354 [26/Oct/2026:19:35:54.725] public api/api2 0/0/0/68/68 201 401 ---- 1/1/0/0/0 0/0 rid=AC1200018632-1F90-6ADFAB9A2D5-113D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34384 [26/Oct/2026:19:35:54.801] public api/api1 0/0/0/14/14 201 401 ---- 8/8/7/3/0 0/0 rid=AC1200018650-1F90-6ADFAB9A31B-1144-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34426 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/15/16 201 401 ---- 7/7/6/2/0 0/0 rid=AC120001867A-1F90-6ADFAB9A322-1148-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34392 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/16/16 201 401 ---- 6/6/5/1/0 0/0 rid=AC1200018658-1F90-6ADFAB9A31D-1146-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34440 [26/Oct/2026:19:35:54.802] public api/api1 0/0/0/16/16 201 401 ---- 5/5/4/0/0 0/0 rid=AC1200018688-1F90-6ADFAB9A322-114A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34390 [26/Oct/2026:19:35:54.799] public api/api2 0/0/1/18/20 201 401 ---- 4/4/3/3/0 0/0 rid=AC1200018656-1F90-6ADFAB9A31C-1143-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34382 [26/Oct/2026:19:35:54.801] public api/api2 0/0/0/18/18 201 401 ---- 3/3/2/2/0 0/0 rid=AC120001864E-1F90-6ADFAB9A31B-1145-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34408 [26/Oct/2026:19:35:54.802] public api/api2 0/0/0/17/17 201 401 ---- 2/2/1/1/0 0/0 rid=AC1200018668-1F90-6ADFAB9A322-1147-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34422 [26/Oct/2026:19:35:54.802] public api/api2 0/0/0/18/18 201 401 ---- 1/1/0/0/0 0/0 rid=AC1200018676-1F90-6ADFAB9A322-1149-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34462 [26/Oct/2026:19:35:54.837] public api/api1 0/0/0/14/14 201 401 ---- 8/8/7/3/0 0/0 rid=AC120001869E-1F90-6ADFAB9A344-114E-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34508 [26/Oct/2026:19:35:54.838] public api/api1 0/0/0/16/16 201 401 ---- 7/7/6/2/0 0/0 rid=AC12000186CC-1F90-6ADFAB9A346-1152-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34450 [26/Oct/2026:19:35:54.834] public api/api1 0/0/0/20/20 201 401 ---- 6/6/5/1/0 0/0 rid=AC1200018692-1F90-6ADFAB9A33F-114C-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34490 [26/Oct/2026:19:35:54.838] public api/api1 0/0/0/16/17 201 401 ---- 5/5/4/0/0 0/0 rid=AC12000186BA-1F90-6ADFAB9A346-1150-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34534 [26/Oct/2026:19:35:54.865] public api/api1 0/0/0/10/10 201 401 ---- 8/8/7/1/0 0/0 rid=AC12000186E6-1F90-6ADFAB9A361-1156-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34530 [26/Oct/2026:19:35:54.865] public api/api1 0/0/0/12/12 201 401 ---- 7/7/6/0/0 0/0 rid=AC12000186E2-1F90-6ADFAB9A361-1155-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34446 [26/Oct/2026:19:35:54.834] public api/api2 0/0/0/59/59 201 401 ---- 8/8/6/6/0 0/0 rid=AC120001868E-1F90-6ADFAB9A337-114B-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34458 [26/Oct/2026:19:35:54.836] public api/api2 0/0/0/62/62 201 401 ---- 8/8/6/5/0 0/0 rid=AC120001869A-1F90-6ADFAB9A343-114D-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34538 [26/Oct/2026:19:35:54.897] public api/api1 0/0/1/8/9 201 401 ---- 8/8/7/1/0 0/0 rid=AC12000186EA-1F90-6ADFAB9A372-1158-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34478 [26/Oct/2026:19:35:54.837] public api/api2 0/0/0/71/71 201 401 ---- 7/7/6/5/0 0/0 rid=AC12000186AE-1F90-6ADFAB9A345-114F-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34570 [26/Oct/2026:19:35:54.904] public api/api1 0/0/0/9/9 201 401 ---- 8/8/5/0/0 0/0 rid=AC120001870A-1F90-6ADFAB9A388-115A-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34594 [26/Oct/2026:19:35:54.934] public api/api1 0/0/1/5/6 201 401 ---- 7/7/6/0/0 0/0 rid=AC1200018722-1F90-6ADFAB9A391-115D-0008 inst=c38982836a1d "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34498 [26/Oct/2026:19:35:54.838] public api/api2 0/0/0/71/136 201 401 ---- 6/6/5/5/0 0/0 rid=AC12000186C2-1F90-6ADFAB9A346-1151-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34522 [26/Oct/2026:19:35:54.856] public api/api2 0/0/0/121/121 201 401 ---- 5/5/4/4/0 0/0 rid=AC12000186DA-1F90-6ADFAB9A358-1153-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34536 [26/Oct/2026:19:35:54.882] public api/api2 0/0/0/96/96 201 401 ---- 4/4/3/3/0 0/0 rid=AC12000186E8-1F90-6ADFAB9A372-1157-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34528 [26/Oct/2026:19:35:54.865] public api/api2 0/0/0/114/115 201 401 ---- 3/3/2/2/0 0/0 rid=AC12000186E0-1F90-6ADFAB9A361-1154-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34556 [26/Oct/2026:19:35:54.899] public api/api2 0/0/0/86/86 201 401 ---- 2/2/1/1/0 0/0 rid=AC12000186FC-1F90-6ADFAB9A383-1159-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" +lb-1 | 172.18.0.1:34586 [26/Oct/2026:19:35:54.934] public api/api2 0/0/1/51/53 201 401 ---- 1/1/0/0/0 0/0 rid=AC120001871A-1F90-6ADFAB9A391-115C-0008 inst=ecaa2acc2a0a "POST /api/v1/chats/6afb6bc0-d1d7-463c-9bae-0d680f8b449c/messages" diff --git a/docs/lab3/evidence/load.json b/docs/lab3/evidence/load.json new file mode 100644 index 0000000..87063ba --- /dev/null +++ b/docs/lab3/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.093, + "concurrency": 8, + "successful_requests": 4367, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 145.117, + "successful_latency_ms": { + "p50": 32.0, + "p95": 157.0, + "p99": 203.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab3/evidence/load.txt b/docs/lab3/evidence/load.txt new file mode 100644 index 0000000..c7c17fb --- /dev/null +++ b/docs/lab3/evidence/load.txt @@ -0,0 +1,28 @@ +# 2026-10-26 19:35:55 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.4s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.093, + "concurrency": 8, + "successful_requests": 4367, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 145.117, + "successful_latency_ms": { + "p50": 32.0, + "p95": 157.0, + "p99": 203.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-10-26 19:35:57 UTC | CPU/RAM после нагрузки | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab3-api-6 19.92% 89.26MiB / 256MiB 34.87% +messenger-lab3-api-5 0.67% 89.33MiB / 256MiB 34.89% +messenger-lab3-lb-1 0.14% 14.34MiB / 128MiB 11.20% +messenger-lab3-postgres-1 11.42% 68.96MiB / 512MiB 13.47% +messenger-lab3-redis-1 2.36% 4.148MiB / 96MiB 4.32% + diff --git a/docs/lab3/evidence/migrations.txt b/docs/lab3/evidence/migrations.txt new file mode 100644 index 0000000..b07edae --- /dev/null +++ b/docs/lab3/evidence/migrations.txt @@ -0,0 +1,26 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker), те же src/ и migrations/ +## 1. Реплика не готова, пока схема не мигрирована; migrate однократный, повтор — no-op; две копии одновременно — advisory lock +$ # реплика против ПУСТОЙ БД, миграции ещё не выполнялись (RUN_MIGRATIONS_ON_START=false) +$ curl -s -w " %{http_code}" http://127.0.0.1:18099/health/ready +{"error":{"code":"unavailable","message":"Database schema is not migrated","request_id":"1f3109e3-a934-4c1b-bfbd-22350e35e05d"}} 503 +$ psql -c "select to_regclass('schema_migrations')" +NULL +$ python -m src.migrate # = сервис migrate (one-shot) +applied migrations: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +$ python -m src.migrate # повтор: no-op +schema is up to date; nothing applied +$ # две копии migrate одновременно (advisory lock) на ещё одной пустой БД +NOTICE: database "messenger_race" does not exist, skipping +copy A: schema is up to date; nothing applied +copy B: applied migrations: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +schema_migrations rows: 001_init, 002_session_cache_invalidations, 003_idempotency_keys +$ curl -s -w " %{http_code}" http://127.0.0.1:18099/health/ready # та же реплика, без рестарта +{"status":"ok"} 200 + +## 2. Совместимость миграции 003 со старой версией (rolling update): код lab2 работает на схеме lab3 одновременно с репликами lab3 +$ # старая версия: код lab2 (без Idempotency-Key/X-Instance-Id) против БД, мигрированной до 003 +migrations applied at startup: none +$ cd lab2 && python tests/smoke.py --base-url http://127.0.0.1:18098 # контракт lab2 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +$ cd lab3 && python tests/smoke.py --base-url http://127.0.0.1:8080 # одновременно реплики lab3 за LB +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab3/evidence/persistence.txt b/docs/lab3/evidence/persistence.txt new file mode 100644 index 0000000..6113558 --- /dev/null +++ b/docs/lab3/evidence/persistence.txt @@ -0,0 +1,24 @@ +# 2026-10-26 19:35:10 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab3-api-5 Up 31 seconds (healthy) 32 seconds ago +messenger-lab3-api-6 Up 21 seconds (healthy) 22 seconds ago +messenger-lab3-lb-1 Up 18 minutes (healthy) 44 minutes ago +messenger-lab3-migrate-1 Exited (0) 4 minutes ago 44 minutes ago +messenger-lab3-postgres-1 Up 18 minutes (healthy) 44 minutes ago +messenger-lab3-redis-1 Up 18 minutes (healthy) 44 minutes ago +# 2026-10-26 19:35:24 UTC | Persistence: make test-persistence | exit 0 | 14.1s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-10-26 19:35:24 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab3-api-5 Up 7 seconds (healthy) 46 seconds ago +messenger-lab3-api-6 Up 7 seconds (healthy) 36 seconds ago +messenger-lab3-lb-1 Up Less than a second (health: starting) 44 minutes ago +messenger-lab3-migrate-1 Exited (0) 5 minutes ago 44 minutes ago +messenger-lab3-postgres-1 Up 7 seconds (healthy) 44 minutes ago +messenger-lab3-redis-1 Up 7 seconds (healthy) 44 minutes ago + diff --git a/docs/lab3/evidence/rolling_back.json b/docs/lab3/evidence/rolling_back.json new file mode 100644 index 0000000..e79934d --- /dev/null +++ b/docs/lab3/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 27.69, + "replicas_before": [ + "6e2f0c643449", + "b6e350f043f3" + ], + "replicas_after": [ + "c38982836a1d", + "ecaa2acc2a0a" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1342, + "ok": 1342, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 267.5, + "write_ok_rps": 159.5, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 94.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3" + ] + }, + "during_update": { + "from_s": 5.02, + "to_s": 32.7, + "requests": 6993, + "ok": 6993, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 252.6, + "write_ok_rps": 149.3, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 140.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3", + "c38982836a1d", + "ecaa2acc2a0a" + ] + }, + "after": { + "from_s": 32.7, + "to_s": 40.77, + "requests": 2045, + "ok": 2045, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 253.6, + "write_ok_rps": 152.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 94.0 + }, + "instances": [ + "c38982836a1d", + "ecaa2acc2a0a" + ] + } + }, + "lb_timeline": [ + [ + 0.02, + [ + "api1", + "api3" + ] + ], + [ + 11.42, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.23, + [ + "api1", + "api3" + ] + ], + [ + 14.77, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.16, + [ + "api1", + "api2" + ] + ], + [ + 25.72, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.66, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6166, + "stored_messages": 6166, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab3/evidence/rolling_update.json b/docs/lab3/evidence/rolling_update.json new file mode 100644 index 0000000..e4d09f8 --- /dev/null +++ b/docs/lab3/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 28.38, + "replicas_before": [ + "a21409fffeec", + "c4c5ae5b0949" + ], + "replicas_after": [ + "6e2f0c643449", + "b6e350f043f3" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1161, + "ok": 1161, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 232.2, + "write_ok_rps": 136.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 187.0 + }, + "instances": [ + "a21409fffeec", + "c4c5ae5b0949" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 33.38, + "requests": 6950, + "ok": 6950, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 244.9, + "write_ok_rps": 145.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 156.0 + }, + "instances": [ + "6e2f0c643449", + "a21409fffeec", + "b6e350f043f3", + "c4c5ae5b0949" + ] + }, + "after": { + "from_s": 33.38, + "to_s": 41.41, + "requests": 2008, + "ok": 2008, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 250.0, + "write_ok_rps": 147.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 110.0 + }, + "instances": [ + "6e2f0c643449", + "b6e350f043f3" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 12.5, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.73, + [ + "api1", + "api2" + ] + ], + [ + 15.25, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.28, + [ + "api2", + "api3" + ] + ], + [ + 25.73, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.34, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 5981, + "stored_messages": 5981, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab3/evidence/rolling_update.txt b/docs/lab3/evidence/rolling_update.txt new file mode 100644 index 0000000..1fef464 --- /dev/null +++ b/docs/lab3/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-10-26 19:34:26 UTC | Rolling update lab3 -> lab3.1 под нагрузкой | exit 0 | 44.1s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['a21409fffeec', 'c4c5ae5b0949'] +2026-10-26T19:33:48.176Z rolling-update: start; replicas=2 old=['c4c5ae5b0949', 'a21409fffeec'] +2026-10-26T19:33:51.770Z rolling-update: images built in 3.6s (tag lab3.1) +2026-10-26T19:33:53.990Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-10-26T19:34:04.788Z rolling-update: step 1/2 new=6e2f0c643449 (messenger-api:lab3.1, ready+UP in 6.8s) replaced old=c4c5ae5b0949 (graceful stop 3.8s) +2026-10-26T19:34:15.884Z rolling-update: step 2/2 new=b6e350f043f3 (messenger-api:lab3.1, ready+UP in 6.7s) replaced old=a21409fffeec (graceful stop 4.3s) +2026-10-26T19:34:16.330Z rolling-update: done in 28.1s; replicas now ['6e2f0c643449', 'b6e350f043f3'] +update 28.4s rc=0 | replicas ['a21409fffeec', 'c4c5ae5b0949'] -> ['6e2f0c643449', 'b6e350f043f3'] | errors during update 0 (-) | RPS before 232.2, during 244.9, after 250.0 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-10-26 19:35:09 UTC | Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой | exit 0 | 43.4s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['6e2f0c643449', 'b6e350f043f3'] +2026-10-26T19:34:32.642Z rolling-update: start; replicas=2 old=['6e2f0c643449', 'b6e350f043f3'] +2026-10-26T19:34:35.497Z rolling-update: images built in 2.9s (tag lab3) +2026-10-26T19:34:37.818Z rolling-update: migrations once in 2.3s: schema is up to date; nothing applied +2026-10-26T19:34:48.605Z rolling-update: step 1/2 new=c38982836a1d (messenger-api:lab3, ready+UP in 7.0s) replaced old=6e2f0c643449 (graceful stop 3.7s) +2026-10-26T19:34:59.414Z rolling-update: step 2/2 new=ecaa2acc2a0a (messenger-api:lab3, ready+UP in 6.8s) replaced old=b6e350f043f3 (graceful stop 3.9s) +2026-10-26T19:34:59.817Z rolling-update: done in 27.4s; replicas now ['c38982836a1d', 'ecaa2acc2a0a'] +update 27.7s rc=0 | replicas ['6e2f0c643449', 'b6e350f043f3'] -> ['c38982836a1d', 'ecaa2acc2a0a'] | errors during update 0 (-) | RPS before 267.5, during 252.6, after 253.6 | p95 during 78.0 ms | data ok=True +PASS: rolling update under load + +# 2026-10-26 19:35:09 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c38982836a1d messenger-lab3-api-5 running healthy messenger-api:lab3 api2 UP +ecaa2acc2a0a messenger-lab3-api-6 running healthy messenger-api:lab3 api3 UP + diff --git a/docs/lab3/evidence/shutdown_timeline.txt b/docs/lab3/evidence/shutdown_timeline.txt new file mode 100644 index 0000000..7c3bddb --- /dev/null +++ b/docs/lab3/evidence/shutdown_timeline.txt @@ -0,0 +1,17 @@ +# 2026-10-24 12:39 UTC | НАТИВНЫЙ стенд (без Docker), те же src/ и migrations/ +# Настройки: HAProxy inter 1s / fastinter 500ms / fall 2 / rise 2 / timeout check 1.5s; API DRAIN_SECONDS=3, GRACEFUL_TIMEOUT_SECONDS=10 +# graceful stop timeline, t=0 — начало наблюдения; интервал опроса 50 мс + +0.02s (от SIGTERM -0.48s) LB api3: UP + +0.02s (от SIGTERM -0.48s) nat-api-3 /health/ready -> 200 + +0.50s (от SIGTERM -0.00s) SIGTERM -> nat-api-3 (pid 3196) + +0.51s (от SIGTERM +0.01s) nat-api-3 /health/ready -> 503 + +1.26s (от SIGTERM +0.76s) LB api3: UP 1/2 + +1.80s (от SIGTERM +1.30s) LB api3: DOWN + +3.52s (от SIGTERM +3.02s) nat-api-3 /health/ready -> connection refused + +3.62s (от SIGTERM +3.12s) nat-api-3 process exited + +# лог реплики: +2026-10-24 15:39:19,576 INFO messenger.server signal 15: instance nat-api-3 draining for 3.0s (readiness=503), then graceful stop +2026-10-24 15:39:22,580 INFO uvicorn.error Shutting down +2026-10-24 15:39:22,681 INFO messenger.api shutdown: instance=nat-api-3 closing pool +2026-10-24 15:39:22,682 INFO uvicorn.error Finished server process [3196] diff --git a/docs/lab3/evidence/smoke.txt b/docs/lab3/evidence/smoke.txt new file mode 100644 index 0000000..574b73d --- /dev/null +++ b/docs/lab3/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-10-26 19:30:00 UTC | Публичный smoke | exit 0 | 0.9s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab3/evidence/stand.txt b/docs/lab3/evidence/stand.txt new file mode 100644 index 0000000..20ae6e5 --- /dev/null +++ b/docs/lab3/evidence/stand.txt @@ -0,0 +1,33 @@ +# 2026-10-26 19:29:57 UTC | Стенд: контейнеры | exit 0 | 0.2s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab3-api-1 messenger-api:lab3 "python -m src.server" api 39 minutes ago Up 12 minutes (healthy) 8080/tcp +messenger-lab3-api-2 messenger-api:lab3 "python -m src.server" api 39 minutes ago Up 13 minutes (healthy) 8080/tcp +messenger-lab3-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 39 minutes ago Up 13 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab3-migrate-1 messenger-lab3-migrate "python -m src.migra…" migrate 39 minutes ago Exited (0) 13 minutes ago +messenger-lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 39 minutes ago Up 13 minutes (healthy) 5432/tcp +messenger-lab3-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 39 minutes ago Up 13 minutes (healthy) 6379/tcp +# 2026-10-26 19:29:57 UTC | Стенд: реплики и статус в LB | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c4c5ae5b0949 messenger-lab3-api-1 running healthy messenger-api:lab3 api1 UP +a21409fffeec messenger-lab3-api-2 running healthy messenger-api:lab3 api2 UP + +# 2026-10-26 19:29:59 UTC | Стенд: лимиты CPU/RAM | exit 0 | 1.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab3-lb-1 1.87% 12.87MiB / 128MiB 10.06% +messenger-lab3-api-2 16.79% 204.4MiB / 256MiB 79.86% +messenger-lab3-api-1 0.31% 222.9MiB / 256MiB 87.07% +messenger-lab3-postgres-1 2.23% 58.11MiB / 512MiB 11.35% +messenger-lab3-redis-1 0.22% 4.559MiB / 96MiB 4.75% + +# 2026-10-26 19:29:59 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.3s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp + diff --git a/docs/lab3/evidence/student-tests.txt b/docs/lab3/evidence/student-tests.txt new file mode 100644 index 0000000..2489d48 --- /dev/null +++ b/docs/lab3/evidence/student-tests.txt @@ -0,0 +1,134 @@ +# 2026-10-26 19:30:06 UTC | Собственные тесты (HTTP через LB) | exit 0 | 6.2s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests) ... ok +test_password_never_returned (test_api.ApiBehaviorTests) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests) ... ok +test_login_with_body_rejected (test_auth.AuthTests) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests) ... ok +test_malformed_basic_headers (test_auth.AuthTests) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 47 tests in 4.640s + +OK (skipped=13) + + 24 parallel POSTs, one key -> 1 message; replicas: ['a21409fffeec', 'c4c5ae5b0949'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-10-26 19:30:14 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 7.5s +$ python scripts/ops.py in-network storage + Container messenger-lab3-api-run-bdc116a8fd34 Creating + Container messenger-lab3-api-run-bdc116a8fd34 Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.223s + +OK + Container messenger-lab3-api-run-630b208e9588 Creating + Container messenger-lab3-api-run-630b208e9588 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.864s + +OK + +# 2026-10-26 19:30:27 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 13.7s +$ python scripts/ops.py in-network expiry + Container messenger-lab3-redis-1 Running + Container messenger-lab3-postgres-1 Running + Container messenger-lab3-api-1 Running + Container messenger-lab3-api-2 Running + Container messenger-lab3-api-short-ttl-1 Creating + Container messenger-lab3-api-short-ttl-1 Created + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Starting + Container messenger-lab3-migrate-1 Started + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-redis-1 Waiting + Container messenger-lab3-redis-1 Healthy + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-short-ttl-1 Starting + Container messenger-lab3-api-short-ttl-1 Started + Container messenger-lab3-api-run-a36baf72714e Creating + Container messenger-lab3-api-run-a36baf72714e Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab3-api-run-c445a7ef00cd Creating + Container messenger-lab3-api-run-c445a7ef00cd Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.620s + +OK + diff --git a/docs/lab4/REPORT.md b/docs/lab4/REPORT.md new file mode 100644 index 0000000..0e67451 --- /dev/null +++ b/docs/lab4/REPORT.md @@ -0,0 +1,273 @@ +# Отчёт — лабораторная 4 + +> Перенесено из корня репозитория lab4 при переходе на lab5 без изменений по существу. Пути `evidence/...` в этом файле указывают на `docs/lab4/evidence/`. Отчёт lab5 — [REPORT.md](../../REPORT.md). + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0, **boto3 1.35.99**, **Pillow 11.1.0**; PostgreSQL 16.4, Redis 7.4.1, HAProxy 2.8.30, **SeaweedFS 4.50**. Документация: Amazon S3 (PutObject, Content-MD5, ListObjectsV2), SeaweedFS wiki (Amazon S3 API, S3 credentials/identities, `weed server`), спецификация PNG (ISO/IEC 15948: чанки, CRC, Adam7), JPEG (ITU T.81: маркеры SOF/SOS/EOI), Pillow (`Image.open`, `draft`), glibc `mallopt` (MALLOC_ARENA_MAX, MALLOC_MMAP_THRESHOLD_). Помощь ИИ (Claude) — при проектировании схемы согласованности БД/S3, проверки изображений, тестов, экспериментов и при написании отчёта. + +Решения lab3 (балансировщик, реплики, идемпотентность, drain, rolling update) перенесены без изменений семантики; их отчёт и доказательства сохранены в [docs/lab3/REPORT.md](../lab3/REPORT.md). + +## Запуск из чистого clone + +**Версии.** API: сборка из `Dockerfile` на `python:3.12.7-slim-bookworm`; `postgres:16.4-alpine`; `redis:7.4.1-alpine`; `haproxy:2.8.30-alpine`; **`chrislusf/seaweedfs:4.50`** (тег задаётся `SEAWEEDFS_VERSION` в `.env`, не `latest`). Нужны Docker Engine / Docker Desktop с Compose v2 и Python 3.9+ на хосте (тесты и `scripts/ops.py` — только стандартная библиотека). Оркестрация — **Compose + HAProxy** (Swarm не используется). + +```sh +# 1. Секреты (один раз; каталог secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py # + 4 файла S3-ключей и secrets/s3_config.json +cp .env.example .env # Windows: copy .env.example .env + +# 2. Сборка и запуск: postgres+redis -> migrate; s3 -> s3-init (bucket) -> api x2 -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Готовность: 2 реплики healthy и UP в HAProxy; S3 реально пишет/читает +python scripts/ops.py wait +python scripts/ops.py wait-s3 +python scripts/ops.py ps +``` + +Прогрев: первая сборка 1–3 мин, далее стенд готов за ~30–40 с (healthcheck PostgreSQL и SeaweedFS, миграции, bootstrap бакета, `start_period` API, `rise 2` в HAProxy). + +```sh +# Публичные проверки +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh # Windows: scripts\restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh # Windows: scripts\stop-one.cmd +python scripts/ops.py restore # вернуть реплику, затем failover повторить для второй + +# Собственные проверки уровня 4 (lab4) +python student-tests/run_all.py --base-url http://localhost:8080 # lab2-3 + test_attachments.py (17 тестов) +python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы +python scripts/ops.py s3-check # приватный бакет вне API, права ключей, закрытые админ-порты +python scripts/ops.py reconcile # сверка БД<->S3, dry-run (--apply — исправить) +python student-tests/s3_outage.py --output evidence/s3_outage.json +python student-tests/upload_memory.py --concurrency 10 --docker --output evidence/upload_memory_c10.json + +# Всё перечисленное (и проверки lab3) одной командой с записью в evidence/: +python scripts/collect_evidence.py # ~12–15 мин; --quick ~7–8 мин +``` + +**Остановка.** Безопасная (данные PostgreSQL и S3 сохраняются): `docker compose --env-file .env --profile ttl --profile faults down`. Намеренное удаление всех учебных данных (volumes `pgdata`, `s3data`) — только вручную: `docker compose --env-file .env --profile ttl --profile faults down -v`. Hooks и тесты volumes не удаляют. + +**Действия оператора.** Создать секреты до первого запуска (пароль PostgreSQL задаётся только при инициализации пустого volume, поэтому `init_secrets.py` существующие файлы не перезаписывает; S3-ключи не связаны с данными и могут быть заменены — см. docstring скрипта). После `stop-one` вернуть реплику `python scripts/ops.py restore`. Если тег `chrislusf/seaweedfs:4.50` недоступен в вашем registry-зеркале, укажите соседний проверенный тег в `SEAWEEDFS_VERSION`. + +## Архитектура и решения + +``` + 127.0.0.1:8080 (единственная публичная точка входа) +Клиент ──HTTP──► lb: HAProxy 2.8 ── roundrobin, активные проверки /health/ready + │ POST .../attachments: timeout server 90 s, остальное 10 s + ▼ сеть backend + ┌──────── api x2 (read-only rootfs, tmpfs /tmp 16m, 0.5 CPU / 320M) ────────┐ + │ загрузка: поток -> лимит 10 MiB -> проверка PNG/JPEG -> [1] pending в БД │ + │ -> [2] S3 PUT (Content-MD5) -> [3] ready в БД -> 201 │ + │ скачивание: проверка членства -> поток из S3 (64 KiB) -> клиент │ + │ фон: сборщик pending/сирот (advisory lock, одна реплика за раз) │ + └──────┬───────────────────────┬────────────────────────────┬───────────────┘ + ▼ backend ▼ backend ▼ storage (internal: true) + postgres 16 (pgdata) redis 7 (кэш сессий) s3: SeaweedFS 4.50 (s3data) + users, chats, messages, только S3 API :8333 в сети storage; + sessions, idempotency_keys, master/filer/volume UI — 127.0.0.1 + attachments, message_attachments внутри контейнера; без анонимов + ▲ ▲ + migrate (one-shot) s3-init (one-shot, admin-ключи): + бакет + проверки прав и приватности +``` + +Порядок старта: `postgres`, `redis` (healthy) → `migrate`; `s3` (healthy) → `s3-init` (оба `service_completed_successfully`) → `api` (healthy) → `lb`. + +**Источник истины.** PostgreSQL — для всех метаданных, включая вложения (владелец, чат, имя, тип, размер, SHA-256, размеры в пикселях, ключ объекта, состояние). Байты оригинала — только в приватном бакете `messenger-attachments`. На API нет ни volume, ни каталога для файлов: rootfs read-only, `/tmp` — tmpfs 16 MiB (тело загрузки держится в памяти процесса до записи в S3, не на диске; `python scripts/ops.py api-fs`). + +### Выбор S3-сервера: SeaweedFS + +MinIO с осени 2025 г. не публикует community-образы и бинарники для свежих релизов, а консоль администратора у него встроена в тот же процесс. SeaweedFS (Apache-2.0): версия закрепляется тегом, S3-identities с правами на конкретный бакет задаются статическим JSON-конфигом (`-s3.config`), а все административные HTTP-интерфейсы (master 9333, filer 8888, volume 8080) привязываются к 127.0.0.1 внутри контейнера (`-ip.bind=127.0.0.1`), наружу в сеть `storage` выставлен только S3 (`-s3.ip.bind=0.0.0.0`). Встроенный IAM API отключён (`-s3.iam=false`), автосоздание бакетов — тоже (`-s3.autoCreateBucket=false`). + +| Роль | Сервис | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | haproxy:2.8.30-alpine | `/lb-health`; бэкенды по `/health/ready` | — | backend; 127.0.0.1:8080, :8404 | — | +| API ×2 | `api` | сборка `Dockerfile` | `/health/ready` (PostgreSQL + схема) | нет (read-only) | backend + storage; без ports | secrets: database_url, cache_url, s3_access_key_id, s3_secret_access_key | +| БД | `postgres` | postgres:16.4-alpine | `pg_isready` | `pgdata` | backend | postgres_password | +| Кэш сессий | `redis` | redis:7.4.1-alpine | `redis-cli ping` | нет (только кэш) | backend | redis_password | +| S3 | `s3` | chrislusf/seaweedfs:4.50 | `wget /healthz` (+ `ops.py wait-s3`) | `s3data` | storage (internal); только :8333 | secret s3_config (identities) | +| Миграции | `migrate` | сборка | one-shot | — | backend | database_url | +| Bucket bootstrap | `s3-init` | сборка | one-shot | — | storage | admin + api S3-ключи | +| Тестовые отказы | `api-fault-*` (профиль `faults`) | образ API | `/health/ready` | — | backend + storage | как у api | + +### Модель данных (`migrations/004_attachments.sql`) + +- `attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key UNIQUE, state pending|ready, created_at, updated_at)`. CHECK-ограничения повторяют контракт: имя 1–255, тип png/jpeg, размер 1..10 485 760, `width*height ≤ 40 000 000`, sha256 в нижнем регистре. +- `object_key = "originals/" + случайный UUID`. Он **не** совпадает с `id` вложения, не содержит `filename` и не раскрывается в API. +- `message_attachments(message_id, attachment_id, chat_id, position)`: порядок `attachment_ids` сохраняется. **Составные внешние ключи** `(message_id, chat_id) → messages(id, chat_id)` и `(attachment_id, chat_id) → attachments(id, chat_id)` на уровне БД запрещают привязать файл чужого чата. `ON DELETE RESTRICT` не даёт удалить вложение, на которое ссылается сообщение. +- `messages_text_len` расширен до 0..4000: `text=""` допустим только вместе с вложениями (проверяет API). Миграция только расширяющая — реплики lab3 работают на новой схеме (rolling update). + +### Загрузка, `201` и границы транзакций + +`POST /api/v1/chats/{id}/attachments?filename=…` (сырые bytes, `src/main.py`): + +1. Дешёвые проверки до чтения тела: аутентификация → `filename` (400) → `Content-Type` только `image/png|image/jpeg` (415) → `Content-Length > 10 MiB` → 413 без чтения → членство в чате (404) → слот загрузки (`MAX_CONCURRENT_UPLOADS=4` на реплику; ожидание ≤ 5 с, иначе 503 + `Retry-After`). +2. Тело читается потоком; как только счётчик превысил 10 MiB — 413 (и для chunked без `Content-Length`). Остаток тела (до 10 MiB) дочитывается в никуда, чтобы клиент получил 413, а не обрыв. Общий дедлайн приёма — `UPLOAD_READ_TIMEOUT_SECONDS=60` (медленный клиент → 400). Обрыв клиентом → ничего не записано. +3. Проверка содержимого (`src/images.py`, ≤ 2 одновременных на реплику): + - сигнатура PNG/JPEG; **сигнатура должна совпасть с заявленным Content-Type** (подмена MIME → 415); + - размеры из IHDR / SOFn **до** распаковки: > 40 Мп → 413 (pixel bomb 8000×6000 размером ~50 КБ отклоняется за миллисекунды); + - PNG: все CRC, порядок IHDR…IDAT…IEND, допустимые bit depth/color type, **потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»**: объём должен в точности совпасть с ожидаемым (с учётом Adam7). Так обнаруживаются обрезанные и испорченные данные, а zip-бомба обрывается, как только вывод превысил ожидание; память O(64 KiB) при любом размере картинки; + - JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI → обрезанный файл = 415). Baseline дополнительно декодируется Pillow в режиме `draft` (1/8 масштаба). Progressive Pillow не декодирует: libjpeg держит коэффициенты всего кадра, замер — **119 MiB** на 39.7 Мп против **4.9 MiB** у baseline; + - независимый парсер Pillow должен увидеть тот же формат и размеры. +4. `[1]` `INSERT attachments … state='pending'` (с проверкой членства в том же операторе) — **COMMIT до S3**. +5. `[2]` S3 `PutObject` с `Content-MD5` (S3 сам отклоняет повреждённую передачу — BadDigest; проверено) и `x-amz-meta-sha256`. +6. `[3]` `UPDATE … SET state='ready' WHERE id=… AND state='pending'` — **COMMIT** → `201 Attachment {status: ready, variants: [], error_code: null}`. + +**Когда честно вернуть 201:** только после шага `[3]`. Байты к этому моменту записаны в S3 (PUT подтверждён с проверкой MD5), а метаданные зафиксированы в PostgreSQL. Ответ 201 до любого из этих событий означал бы, что «готовое» вложение может не скачаться. + +**S3 PUT прошёл, а COMMIT `[3]` — нет.** Остаются строка `pending` (не видна в API, прикрепить нельзя) и объект. При ошибке БД API сразу компенсирует: удаляет объект и пытается удалить строку, клиент получает 503. Если процесс умер (компенсации нет), строку и объект уберёт сборщик после grace period. Обе ветки проверены внедрением отказа (`api-fault-db`, `api-fault-crash`). + +### Согласованность БД и S3: компенсация + сборщик (`src/reconcile.py`) + +Главный инвариант: **ключ объекта записан в БД раньше, чем объект появится в бакете.** Поэтому объект без строки может появиться только после того, как строку удалили. Ключ одноразовый, удалённая строка не возрождается. + +| Ситуация | Кто исправляет | Как | +| --- | --- | --- | +| Ошибка S3 в `[2]` | запрос | DELETE своей pending-строки, 503 | +| Ошибка БД в `[3]` | запрос | DELETE объекта (+ попытка DELETE строки), 503 | +| Процесс умер между `[1]`/`[2]`/`[3]`, обрыв, сеть | сборщик, проход A | pending старше grace: **сначала** `DELETE … WHERE state='pending' RETURNING object_key` (атомарный захват; если запрос успел `[3]`, строка уже ready и не трогается), **потом** DELETE объекта | +| Объект без строки (сбой удаления в A, ручная запись) | сборщик, проход B | листинг `originals/` страницами; нет строки **и** `LastModified` старше grace → повторная проверка → DELETE | +| ready-строка без объекта | только отчёт (C) | байты не восстановить; авто-удаление метаданных опасно — `missing_objects` в отчёте | + +- **Grace period** `ATTACHMENT_GC_GRACE_SECONDS=900` (минимум 300, конфиг меньше не примет). Это намного больше самой долгой загрузки: чтение ≤ 60 с + проверка + S3 PUT (connect 2 с + read 10 с) × 2 попытки ≈ 90 с. Если загрузка всё же переживёт grace, `[3]` вернёт 0 строк → API удалит свой объект и ответит 503. +- **Свежие и используемые вложения не удаляются:** сборщик трогает только `pending` старше grace. ready-строки не удаляет никогда, а на используемые ещё и стоит FK RESTRICT. Тесты: `test_fresh_pending_is_kept`, `test_ready_and_referenced_are_never_collected` (прямой `DELETE` получает ForeignKeyViolation), `test_pending_cannot_be_attached`. +- Фоновый проход A — каждые 300 с, полный обход бакета B — раз в час. Выполняет одна реплика за раз (`pg_try_advisory_lock`). Ручной запуск: `python scripts/ops.py reconcile` (**dry-run** по умолчанию, ничего не меняет) и `… reconcile --apply`. Повторный `--apply` — no-op (проверено). Стоимость обхода на стенде: 238 объектов — 0.036 с (листинг + один `SELECT … = ANY` на страницу из 1000 ключей). + +### Ограничение памяти и потоковая обработка + +- Лимит байт применяется **во время** чтения (на 10 MiB + 1 байт). В памяти одной загрузки — тело (≤ 10 MiB) + кратковременно куски до склейки. Проверка PNG — O(64 KiB), JPEG baseline — ≤ ~5 MiB. Тело уходит в S3 как `BytesIO` поверх тех же bytes, без копии. +- Одновременно на реплику ≤ 4 загрузки (`MAX_CONCURRENT_UPLOADS`). Остальные ждут слот ≤ 5 с **до чтения тела**, поэтому их байты не накапливаются в памяти. Скачивание идёт потоком из S3 кусками по 64 KiB (`StreamingResponse`), без буферизации файла. +- Найдено и исправлено замерами (см. «Эксперименты»): `zlib.unconsumed_tail` копировал остаток IDAT на каждом шаге (пик 2× файла и квадратичное копирование), а botocore склеивал заголовки с телом `bytes` в один буфер. В образе также настроен glibc malloc (`MALLOC_ARENA_MAX=2`, фиксированный порог mmap 1 MiB), чтобы большие буферы возвращались ОС сразу. + +### Безопасность бакета и ключей + +- **Bootstrap идемпотентен** (`src/s3_bootstrap.py`, сервис `s3-init`): `CreateBucket` → `BucketAlreadyOwnedByYou` считается успехом (`evidence/sandbox/s3_bootstrap.txt`: повторный запуск — `already exists (idempotent no-op)`). После создания бакета s3-init проверяет права ключей и приватность; при нарушении он завершается с ошибкой, и API не стартует. +- **Отдельные ключи API** (`messenger_api`) имеют только `Read/Write/List` на `messenger-attachments`. `CreateBucket`/`DeleteBucket` → AccessDenied. Admin-ключи (`bootstrap_admin`) смонтированы только в `s3-init`. Identity `anonymous` не объявлена. +- **Проверка приватности вне API** (`python scripts/ops.py s3-check`): из сети `storage`, без подписи, к S3 напрямую. Анонимный GET реального объекта последнего вложения → 403, анонимный листинг бакета и списка бакетов → 403. Тот же объект с ключами API → 200 и совпадающий SHA-256. +- **Админ-консоль не открыта в общей сети:** порты 9333/8888/8080/19333/18888 с адреса `s3` закрыты (проверка в том же скрипте). Сеть `storage` — `internal: true`. LB в неё не входит, порты S3 на хост не публикуются. +- **Случайный ключ и приватный бакет не заменяют ACL приложения:** доступ к `GET /attachments/{id}` и `/content` проверяется по членству в чате на каждый запрос (посторонний → 404 без раскрытия существования). Download идёт через API без redirect и без подписанных URL. Ответ содержит `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `Content-Disposition` с безопасно закодированным именем. + +### Недоступность S3 и readiness + +S3 **не входит** в readiness: если бы реплики при падении S3 становились not-ready, LB вывел бы обе, и вместе с файлами пропали бы чат и вход. Вместо этого файловые операции быстро отвечают `503 unavailable` + `Retry-After` (таймауты клиента S3: connect 2 с, read 10 с, ≤ 2 попытки), а всё остальное работает. Критичные зависимости readiness — по-прежнему PostgreSQL и мигрированная схема. + +Таймауты boto3 не покрывают DNS: в Docker остановленный контейнер `s3` пропадает из DNS, и разрешение имени висит секундами. В первом прогоне на Docker Desktop загрузка ждала 12–13 с, а скачивание превысило `timeout server` HAProxy (10 с) и получило **504 от LB** вместо 503. Исправление (`src/main.py`, `s3_call`; `src/objectstore.py`, `CircuitBreaker`): +- у каждого вызова S3 из запроса есть общий дедлайн: скачивание 5 с, загрузка 10 с; +- после отказа circuit breaker 5 с сразу отвечает 503, затем делает одну пробную попытку; +- у API в compose `dns_opt: timeout:1, attempts:1`. + +Проверено на худшем случае — S3, который принимает соединение и молчит: upload 503 за 10.0 с, download 503 за 5.0 с (`evidence/sandbox/s3_outage_hang.json`). При S3, который сразу отказывает: 1.8–2.3 с и 0.01 с (breaker открыт). + +**Особенность SeaweedFS после рестарта** (замер): `/healthz` отвечает сразу, но первые операции блокируются. Первое чтение после рестарта шло ~15 с, первая запись ~29 с: выборы Raft-лидера (`electionTimeout` 10 с), затем heartbeat volume-сервера. С `-master.electionTimeout=2s` первая запись — ~19 с. Поэтому restart-hook и `ops.py wait-s3` ждут **реальной** готовности: проба put/get/delete и чтение существующего объекта, каждая операция < 2 с (`s3_bootstrap.py --wait-ready`); на стенде готовность наступала через 14.5–17.1 с. Штатная остановка SeaweedFS длится ~22 с, поэтому `stop_grace_period: 10s` (как у Docker по умолчанию) и затем SIGKILL. Подтверждённые записи к этому моменту уже в page cache ОС; тест сохранности после такого рестарта проходит. + +### Что перенесено из lab3 и что изменено + +Перенесены без изменения семантики: всё API lab2–3, HAProxy-конфигурация (обнаружение реплик через DNS, drain, `retries`/`redispatch`), идемпотентность, миграции one-shot, rolling update, hooks и собственные тесты lab2–3 (47 HTTP-тестов + 12 тестов хранилища проходят на lab4). + +Изменено в lab4: + +1. Переход контракта 3 → 4: поля `attachment_ids` в запросе и ответе сообщения (отсутствие = `[]` и для отпечатка идемпотентности; порядок значим), три новых endpoint. +2. Сервисы `s3`, `s3-init`, volume `s3data`, сеть `storage`; секреты S3; профиль `faults`. +3. HAProxy: для `POST …/attachments` — `timeout server 90s` (`http-request set-timeout`), остальным запросам — прежние 10 с. +4. Лимит памяти API 256M → 320M (4 загрузки по 10 MiB, замер ниже); glibc malloc tuning в `Dockerfile`. +5. `scripts/ops.py`: restart включает S3 и ждёт его прогрева; новые `wait-s3`, `s3-check`, `reconcile`, `api-fs`, `in-network consistency`. `collect_evidence.py` дополнен проверками lab4. + +### Соответствие `.env.example` и реальной конфигурации + +Переменные lab3 — без изменений (см. [docs/lab3/REPORT.md](../lab3/REPORT.md)). Новые: + +| Переменная | Значение | Где используется | +| --- | --- | --- | +| `SEAWEEDFS_VERSION` | 4.50 | тег образа `s3` | +| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | http://s3:8333, us-east-1, messenger-attachments | api, s3-init, `src/config.py` | +| `S3_ACCESS_KEY_ID_FILE`, `S3_SECRET_ACCESS_KEY_FILE` | /run/secrets/… | ключи API (compose secrets) | +| `S3_CONNECT_TIMEOUT_SECONDS`, `S3_READ_TIMEOUT_SECONDS`, `S3_MAX_ATTEMPTS` | 2, 10, 2 | `src/objectstore.py` | +| `S3_DOWNLOAD_DEADLINE_SECONDS`, `S3_UPLOAD_DEADLINE_SECONDS`, `S3_CIRCUIT_OPEN_SECONDS` | 5, 10, 5 | общий бюджет вызова S3 (вкл. DNS), circuit breaker | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | 10485760, 40000000 | не выше контракта (иначе API не стартует) | +| `MAX_CONCURRENT_UPLOADS`, `UPLOAD_QUEUE_TIMEOUT_SECONDS` | 4, 5 | слоты загрузки на реплику | +| `UPLOAD_READ_TIMEOUT_SECONDS` | 60 | дедлайн приёма тела | +| `IMAGE_DECODE_CONCURRENCY` | 2 | разборов изображения на реплику | +| `ATTACHMENT_GC_INTERVAL_SECONDS`, `ATTACHMENT_GC_GRACE_SECONDS`, `ORPHAN_SWEEP_INTERVAL_SECONDS` | 300, 900, 3600 | `src/reconcile.py` | +| `FAULT_INJECTION` | пусто | только сервисы `api-fault-*` | + +## Доказательства критериев + +**Где получены результаты.** Код проверен на стенде из тех же компонентов, запущенных нативными процессами: PostgreSQL 16, Redis 7, SeaweedFS 4.50 (тот же бинарник, те же флаги), HAProxy 2.8.30 с этим `haproxy.cfg` (статические серверы вместо DNS-шаблона) и 2 реплики API. Среда: Windows 10, Ryzen 5 7500f, 32 GB, Python 3.12, без cgroup-лимитов. Результаты — в `evidence/`. + +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | PASS: файл → сообщение с attachment_ids → download Бобом, SHA-256 совпал, Ева 404 | evidence/sandbox/smoke.txt; evidence/smoke.txt | +| Возможности lab3 сохранены | `run_all.py`, `in-network storage`, failover, persistence | 47 HTTP + 12 DB-тестов lab2–3 OK; failover/persistence PASS | evidence/sandbox/student-tests.txt | +| S3 в общем стенде; метаданные в PostgreSQL, байты в приватном бакете | `compose.yaml`, `ops.py api-fs`, `s3-check` | API без volumes, read-only rootfs; объекты только в `s3data` | evidence/s3_private.txt | +| 201 ready после сохранения; оригинал побайтно | `test_png_roundtrip…`, `test_exactly_max_size_is_accepted` | 201, `status=ready`, `variants=[]`; SHA-256 и Content-Length совпадают (вкл. ровно 10 485 760 байт) | evidence/sandbox/student-tests.txt | +| Message с attachment_ids, посторонний без доступа | smoke, `test_message_with_attachments…` | порядок id сохранён; Ева: 404 на метаданные и bytes | там же | +| PNG/JPEG ≤10 MiB, ≤40 Мп; MIME по содержимому; опасные имена; лимиты до распаковки | `test_too_large_*`, `test_too_many_pixels`, `test_mime_spoofing_is_415`, `test_dangerous_filenames_are_400` | 413/413/415/400; 48 Мп файл 50 КБ → 413 без распаковки | там же | +| Upload → стоп обслужившей реплики → download через другую | `make test-failover` ×2 | PASS, 138–139 записей, 0 ошибок; файл скачан после остановки | evidence/sandbox/failover.txt | +| Restart стенда сохраняет метаданные и оригинал | `make test-persistence` | PASS; сменились PID PostgreSQL, SeaweedFS и обеих реплик; файл скачан со старым токеном | evidence/sandbox/persistence.txt | +| Недоступность S3, чужой чат, restart | `s3_outage.py`; `test_attachment_from_other_chat…`; persistence | 503 за 1.0–1.7 с, чат работает; 404; PASS | evidence/sandbox/s3_outage.json | + +### Уровень 4 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Тесты: большой файл, повреждённое изображение, подмена MIME, чужое attachment_id, файл другого чата; нет 500 | `student-tests/test_attachments.py` (17 тестов) | 413 (Content-Length и chunked), 415 (обрезанный PNG/JPEG, неверный CRC, неверный объём IDAT, мусор, пустое тело, GIF), 415 (6 подмен MIME), 404 (чужое/неизвестное id, файл другого чата, Ева в своём чате), 400 (12 опасных имён); каждый негативный ответ < 500 в общем формате | evidence/sandbox/student-tests.txt | +| Обрыв загрузки и отказ между S3/БД не оставляют вечных объектов; компенсация/сверка с grace | `test_attachment_consistency.py` (10 тестов) в сети стенда | обрыв → 0 строк, 0 объектов; ошибка S3 → 503 + нет строки; ошибка БД после PUT → 503 + объект удалён; **падение процесса после PUT** → pending + объект → сборщик удаляет оба; dry-run ничего не меняет; повтор no-op; свежие/ready/используемые не удаляются | evidence/sandbox/consistency.txt; evidence/consistency.txt | +| Лимиты/таймауты/потоковая обработка; ≥10 одновременных загрузок с замером пика | `upload_memory.py --concurrency 10/20` | 10×9.5 MiB ×3 волны: 30/30 201, пик RSS 156–166 MiB на реплику (база 88–95); при 20 одновременных пик тот же (159–162); без лимитера 262–291 MiB | evidence/sandbox/upload_memory_*.json | +| Bootstrap идемпотентен | повторный `s3-init` | `already exists (idempotent no-op)`, все проверки ok | evidence/sandbox/s3_bootstrap.txt | +| Отдельные S3 credentials с ограниченными правами | `s3_bootstrap` проверки | API-ключи: put/get своего бакета — ok; CreateBucket/DeleteBucket — AccessDenied | evidence/sandbox/s3_private.txt | +| Админ-консоль не в общей сети | `ops.py s3-check` | 9333/8888/8080/19333/18888 закрыты, S3 :8333 — единственный | там же | +| Private bucket вне API | `ops.py s3-check` | анонимный GET реального объекта → 403, листинг → 403; с ключами API тот же объект → 200, SHA-256 совпал | там же | + +## Эксперименты и отказы + +**Стенд замеров:** Windows 10, Ryzen 5 7500f, 32 GB; все процессы (2 реплики API, HAProxy, PostgreSQL, Redis, SeaweedFS, генератор нагрузки) на одном хосте. Реплика: 1 процесс Uvicorn, threadpool по умолчанию. Файлы — синтетический шум, PNG без сжатия, 9.5 MiB (несжимаемые, проверка проходит по полной программе). Память — VmRSS процесса реплики (опрос каждые 20 мс) и пик RSS, который считает ядро (VmHWM, обнуляется перед прогоном). Загрузки через LB (roundrobin по двум репликам), повторы — волнами. + +**Гипотеза:** пик памяти реплики ≈ база + `MAX_CONCURRENT_UPLOADS` × (~2 × размер файла) и не зависит от числа одновременных клиентов; без лимитера память растёт линейно с конкуренцией. + +| Прогон | Одновременно | Загрузок | 201 / SHA ok | p50 / p95, с | База → пик RSS, MiB (реплика 1 / 2) | +| --- | --- | --- | --- | --- | --- | +| до исправлений (zlib tail, bytes в botocore) | 10 | 30 | 30 / 30 | 2.01 / 2.29 | 124 → 237 / 124 → 244 | +| исправления кода, malloc по умолчанию | 10 | 30 | 30 / 30 | 1.20 / 1.34 | 124 → 187 / 124 → 187 | +| + MALLOC_ARENA_MAX=2, mmap ≥ 1 MiB (**итог**) | 10 | 30 | 30 / 30 | 1.32 / 2.07 | 95 → 163 / 95 → 156 (пик ядра VmHWM 166 / 165) | +| итог, вдвое больше клиентов | 20 | 40 | 40 / 40 | 2.16 / 3.08 | 115 → 162 / 97 → 159 | +| лимитер выключен (`MAX_CONCURRENT_UPLOADS=64`) | 20 | 40 | 40 / 40 | 2.90 / 3.03 | 88 → 262 / 89 → 291 | + +Вывод: гипотеза подтверждена. Прирост ~61–72 MiB ≈ 4 слота × 2 × 9.5 MiB. При удвоении клиентов пик не растёт (лишние ждут слот, их тело ещё не прочитано), растёт только задержка. Без лимитера пик +170–200 MiB при той же пропускной способности: при лимите 256 MiB контейнер был бы убит OOM. Поэтому лимит API — 320M, запас ~155 MiB. Микрозамеры (tracemalloc, 9.5 MiB PNG): проверка 19 MiB → 1.5 MiB и 0.17 → 0.06 с; `put_object(bytes)` 25 MiB → `BytesIO` 0.1 MiB. Ограничения: один хост и общий CPU для клиента и сервера; RSS не включает page cache; в Docker тот же скрипт (`--docker`) читает VmHWM процесса API внутри контейнера, `docker stats` и cgroup `memory.peak`. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| SIGTERM обслужившей реплики (public failover ×2) | записи идут через другую, файл скачивается | 138/139 записей, 0 ошибок, файл совпал | первая запись после hook ≤ 4.5 с (hook ждёт drain 3 с); потерь нет | +| Каждая реплика по очереди под нагрузкой (`failover_series.py`) | 0 ошибок, LB DOWN < 2 с | DOWN 1.30 / 1.53 с, 0 ошибок, RPS 318 → 239 / 283 → 243 | возврат в ротацию 2.9 / 3.1 с; данные целы | +| Рестарт API + PostgreSQL + Redis + S3 + LB | данные, сессия и файл сохранены | PASS, PID всех процессов сменились | ~38 с hook (из них прогрев S3 16 с); потерь нет | +| Остановка S3 | файлы → 503 быстро, остальное работает | upload 503 за 1.0–1.7 с, download 503 за 1.0 с, readiness 200, сообщения 201 | после старта S3 прогрев 17 с; старый файл побайтно цел | +| S3 «висит» (принимает соединение, не отвечает) | 503 в пределах дедлайна, не 504 | upload 503 за 10.0 с, download 503 за 5.0 с | прогрев 16 с; старый файл цел | +| Ошибка S3 PUT (`api-fault-s3`) | 503, строки нет | 503, 0 строк | сразу | +| Ошибка БД после PUT (`api-fault-db`) | 503, объект удалён | 503, 0 строк, число объектов не изменилось | сразу | +| Падение процесса после PUT (`api-fault-crash`) | pending + объект, убирает сборщик | 1 pending + объект → после grace (сдвинутого в тесте) 0 / 0 | ≤ grace + интервал (900 + 300 с по умолчанию) | +| Обрыв клиентом на 0.2 из 5 MiB | ничего не записано | 0 строк, объектов столько же | сразу | +| Нагрузка lab3 (`tests/load.py`, 20 с, c=4) | без регрессии | 276 RPS, 0 ошибок, p50/p95/p99 13/24/33 мс | — | + +## Ограничения и следующие шаги + +- Один Docker-host, один PostgreSQL, один процесс SeaweedFS без репликации: отказ host/БД/S3 не переживается. S3 недоступен → файловые операции 503 (остальное работает). Восстановление S3 занимает 15–30 с прогрева. +- Тело загрузки держится в памяти процесса (≤ 10 MiB × 4 слота). Это ограничено и измерено, но не «настоящий» стриминг в S3. Следующий шаг — multipart upload кусками по 5 MiB с проверкой по частям или presigned upload (трек 1). +- Progressive JPEG проверяется структурно (маркеры, EOI), без декодирования пикселей: экономия ~115 MiB на 40 Мп. Испорченные энтропийные данные внутри структурно целого progressive-файла не обнаруживаются (их увидит lab5 при создании thumbnail → `failed`). +- `missing_objects` (ready без байтов) сборщик только сообщает; решение о судьбе метаданных — за оператором. +- Ключи идемпотентности, созданные репликой lab3 до обновления, имели отпечаток без `attachment_ids`; повтор такого ключа на lab4 даст 409 (окно 24 ч после rolling update). +- SIGKILL SeaweedFS через 10 с при остановке: безопасно для подтверждённых записей при живом ядре ОС, не при потере питания. +- Docker-прогон (`collect_evidence.py`) выполняется на машине студента. Нативный прогон подтверждает код, HAProxy-конфиг и флаги SeaweedFS, но не compose-обвязку. + +## Вопросы защиты — кратко + +- **Почему файл в Docker volume одной API-реплики не равен объектному хранилищу?** Volume привязан к одному контейнеру и host. Другая реплика его не видит, поэтому после остановки загрузившей реплики файл недоступен (тест failover это и проверяет). У volume нет общего API доступа, ключей с правами и проверки целостности (Content-MD5), он не масштабируется независимо от API, а с read-only rootfs его пришлось бы ещё и открывать на запись. S3 — общий сервис с собственными ACL, его читают все реплики. +- **Когда можно честно вернуть 201?** Только после того, как S3 подтвердил PUT (с проверкой MD5) **и** COMMIT перевёл строку в `ready`. Раньше — нельзя: «готовое» вложение могло бы не скачаться. +- **S3 PUT прошёл, COMMIT — нет?** Остаются невидимая pending-строка и объект. Ошибку БД API компенсирует сам (удаляет объект, 503). Если процесс умер, сборщик после grace сначала атомарно удаляет pending-строку, потом объект. Объект без строки убирает проход по бакету. Строка всегда создаётся раньше объекта, поэтому «вечных» объектов нет. +- **Почему случайный ключ и приватный бакет не заменяют ACL приложения?** Приватный бакет защищает от анонимного доступа к S3, но клиенты в него и не ходят: все запросы идут через API с ключами API, которые читают **любой** объект бакета. Кто из пользователей что может скачать, знает только приложение (членство в чате). Случайный ключ — защита «неугадываемостью»: он может утечь (лог, ссылка), а доступ после выхода из чата или для чужого пользователя он не ограничивает. Поэтому каждый запрос проверяет членство и отвечает 404 постороннему. diff --git a/docs/lab4/evidence/README.md b/docs/lab4/evidence/README.md new file mode 100644 index 0000000..59eb259 --- /dev/null +++ b/docs/lab4/evidence/README.md @@ -0,0 +1,11 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Что лежит здесь (lab4) + +- `sandbox/` — результаты, полученные при разработке на стенде из тех же компонентов, запущенных нативными процессами (PostgreSQL 16, Redis 7, SeaweedFS 4.50, HAProxy 2.8.30, 2 реплики API; без Docker). Как получены — REPORT.md, раздел «Доказательства критериев». +- Остальные файлы создаёт `python scripts/collect_evidence.py` на compose-стенде (сводка — `SUMMARY.txt`). Токены и пароли в файлы не пишутся. +- Доказательства lab3 сохранены в `docs/lab3/evidence/`. diff --git a/docs/lab4/evidence/SUMMARY.txt b/docs/lab4/evidence/SUMMARY.txt new file mode 100644 index 0000000..c28d9f4 --- /dev/null +++ b/docs/lab4/evidence/SUMMARY.txt @@ -0,0 +1,46 @@ +# 2026-11-04 14:12:25 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Образ SeaweedFS (закреплённая версия) evidence/stand.txt +OK S3 bootstrap: лог первого запуска s3-init evidence/s3_bootstrap.txt +OK S3 bootstrap: повторный запуск (идемпотентность) evidence/s3_bootstrap.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB, включая test_attachments.py) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) evidence/consistency.txt +OK Сверка БД <-> S3 (dry-run) evidence/reconcile.txt +OK Приватный бакет вне API, права ключей, закрытые админ-порты evidence/s3_private.txt +OK На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes evidence/s3_private.txt +OK API: файлы в /tmp (tmpfs) после загрузок evidence/s3_private.txt +OK Пик памяти: 10 одновременных загрузок по 9.5 MiB evidence/upload_memory.txt +OK Пик памяти: 20 одновременных загрузок evidence/upload_memory.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Недоступность S3 и восстановление evidence/s3_outage.txt +OK Rolling update lab4 -> lab4.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB) evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Persistence: сверка БД <-> S3 после рестарта evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab4/evidence/check.txt b/docs/lab4/evidence/check.txt new file mode 100644 index 0000000..6abe771 --- /dev/null +++ b/docs/lab4/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-11-04 14:03:55 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-04 14:03:55 UTC | Compose config | exit 0 | 0.2s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab4/evidence/consistency.txt b/docs/lab4/evidence/consistency.txt new file mode 100644 index 0000000..fd9c005 --- /dev/null +++ b/docs/lab4/evidence/consistency.txt @@ -0,0 +1,75 @@ +# 2026-11-04 14:04:49 UTC | Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) | exit 0 | 15.8s +$ python scripts/ops.py in-network consistency + Container messenger-lab4-redis-1 Running + Container messenger-lab4-s3-1 Running + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-api-1 Running + Container messenger-lab4-api-2 Running + Container messenger-lab4-api-fault-s3-1 Creating + Container messenger-lab4-api-fault-db-1 Creating + Container messenger-lab4-api-fault-crash-1 Creating + Container messenger-lab4-api-fault-db-1 Created + Container messenger-lab4-api-fault-s3-1 Created + Container messenger-lab4-api-fault-crash-1 Created + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-redis-1 Waiting + Container messenger-lab4-redis-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-crash-1 Starting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-s3-1 Starting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-fault-db-1 Starting + Container messenger-lab4-api-fault-crash-1 Started + Container messenger-lab4-api-fault-s3-1 Started + Container messenger-lab4-api-fault-db-1 Started + Container messenger-lab4-api-run-46322fbc844a Creating + Container messenger-lab4-api-run-46322fbc844a Created +ready: http://api-fault-s3:8080/health/ready + Container messenger-lab4-api-run-ff8a88e550db Creating + Container messenger-lab4-api-run-ff8a88e550db Created +ready: http://api-fault-db:8080/health/ready + Container messenger-lab4-api-run-476fc515a8f0 Creating + Container messenger-lab4-api-run-476fc515a8f0 Created +ready: http://api-fault-crash:8080/health/ready + Container messenger-lab4-api-run-9b2dce8846fe Creating + Container messenger-lab4-api-run-9b2dce8846fe Created +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.224s + +OK diff --git a/docs/lab4/evidence/failover.txt b/docs/lab4/evidence/failover.txt new file mode 100644 index 0000000..65ce6a9 --- /dev/null +++ b/docs/lab4/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-11-04 14:05:30 UTC | Failover #1: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP +# 2026-11-04 14:05:46 UTC | Failover #1: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.4s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.187} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:05:46 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 exited unhealthy messenger-api:lab4 - - + +# 2026-11-04 14:05:52 UTC | Failover #1: restore | exit 0 | 6.3s +$ python scripts/ops.py restore +2026-11-04T14:05:52.981Z restore: started ['messenger-lab4-api-2']; 2 replicas healthy and UP in LB after 6.2s + +# 2026-11-04 14:05:53 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:09 UTC | Failover #2: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:06:09 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 exited unhealthy messenger-api:lab4 - - + +# 2026-11-04 14:06:16 UTC | Failover #2: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-04T14:06:16.514Z restore: started ['messenger-lab4-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-04 14:06:16 UTC | Failover #3: реплики до | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:33 UTC | Failover #3: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:06:33 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 exited unhealthy messenger-api:lab4 - - +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:06:40 UTC | Failover #3: restore | exit 0 | 6.7s +$ python scripts/ops.py restore +2026-11-04T14:06:40.028Z restore: started ['messenger-lab4-api-1']; 2 replicas healthy and UP in LB after 6.6s + +# остановленные публичным тестом реплики: ['4fc3ad11e268', '4fc3ad11e268', 'c9fab9c9e2c1'] diff --git a/docs/lab4/evidence/failover_series.json b/docs/lab4/evidence/failover_series.json new file mode 100644 index 0000000..9aad2e2 --- /dev/null +++ b/docs/lab4/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "rounds": [ + { + "target_instance": "4fc3ad11e268", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.12, + "lb_detect_down_seconds": 1.56, + "target_last_served_after_stop_seconds": 1.56, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.62, + "lb_back_up_seconds": 1.28, + "target_back_in_rotation_seconds": 7.62, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1155, + "ok": 1155, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 231.0, + "write_ok_rps": 137.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1010, + "ok": 1010, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 101.0, + "write_ok_rps": 61.4, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 588, + "ok": 588, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 117.6, + "write_ok_rps": 43.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 187.0, + "p99": 219.0 + }, + "instances": [ + "c9fab9c9e2c1" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.92, + "requests": 1531, + "ok": 1531, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 140.2, + "write_ok_rps": 82.4, + "latency_ms_ok": { + "p50": 31.0, + "p95": 110.0, + "p99": 187.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 6.56, + [ + "api1" + ] + ], + [ + 21.28, + [ + "api1", + "api2" + ] + ], + [ + 21.61, + [ + "api1" + ] + ], + [ + 24.17, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2426, + "stored_messages": 2426, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "c9fab9c9e2c1", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.83, + "lb_detect_down_seconds": 1.55, + "target_last_served_after_stop_seconds": 1.39, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.0, + "lb_back_up_seconds": 1.62, + "target_back_in_rotation_seconds": 6.99, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1015, + "ok": 1015, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 203.0, + "write_ok_rps": 121.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 172.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 833, + "ok": 833, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 83.3, + "write_ok_rps": 49.3, + "latency_ms_ok": { + "p50": 78.0, + "p95": 187.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 371, + "ok": 371, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 74.2, + "write_ok_rps": 44.2, + "latency_ms_ok": { + "p50": 93.0, + "p95": 187.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.27, + "requests": 1496, + "ok": 1496, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 145.7, + "write_ok_rps": 86.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.55, + [ + "api2" + ] + ], + [ + 21.62, + [ + "api1", + "api2" + ] + ], + [ + 22.58, + [ + "api2" + ] + ], + [ + 25.03, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2215, + "stored_messages": 2215, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/failover_series.txt b/docs/lab4/evidence/failover_series.txt new file mode 100644 index 0000000..1a84b38 --- /dev/null +++ b/docs/lab4/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-11-04 14:07:43 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 63.4s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['4fc3ad11e268', 'c9fab9c9e2c1'] +-- stopping 4fc3ad11e268 (mode=stop) under load + hook 4.12s | LB DOWN after 1.56 s | errors 0 (-) | RPS before 231.0, outage 101.0, after restore 140.2 | p95 outage 110.0 ms | back in rotation 7.62 s | data ok=True +-- stopping c9fab9c9e2c1 (mode=stop) under load + hook 4.83s | LB DOWN after 1.55 s | errors 0 (-) | RPS before 203.0, outage 83.3, after restore 145.7 | p95 outage 187.0 ms | back in rotation 6.99 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-11-04 14:08:46 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 62.7s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['4fc3ad11e268', 'c9fab9c9e2c1'] +-- stopping 4fc3ad11e268 (mode=kill) under load + hook 1.09s | LB DOWN after 1.97 s | errors 6 (502) | RPS before 244.2, outage 77.1, after restore 119.5 | p95 outage 187.0 ms | back in rotation 7.52 s | data ok=True +-- stopping c9fab9c9e2c1 (mode=kill) under load + hook 1.14s | LB DOWN after 2.59 s | errors 6 (502) | RPS before 186.8, outage 82.9, after restore 134.8 | p95 outage 188.0 ms | back in rotation 6.83 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab4/evidence/failover_series_kill.json b/docs/lab4/evidence/failover_series_kill.json new file mode 100644 index 0000000..96f1688 --- /dev/null +++ b/docs/lab4/evidence/failover_series_kill.json @@ -0,0 +1,298 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "rounds": [ + { + "target_instance": "4fc3ad11e268", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.09, + "lb_detect_down_seconds": 1.97, + "target_last_served_after_stop_seconds": 0.47, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.56, + "restore_hook_seconds": 7.53, + "lb_back_up_seconds": 2.0, + "target_back_in_rotation_seconds": 7.52, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1221, + "ok": 1221, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 244.2, + "write_ok_rps": 145.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 777, + "ok": 771, + "errors": 6, + "error_ratio": 0.0077, + "ok_rps": 77.1, + "write_ok_rps": 40.7, + "latency_ms_ok": { + "p50": 63.0, + "p95": 187.0, + "p99": 1015.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 565, + "ok": 565, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 113.0, + "write_ok_rps": 44.8, + "latency_ms_ok": { + "p50": 31.0, + "p95": 187.0, + "p99": 281.0 + }, + "instances": [ + "c9fab9c9e2c1" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.84, + "requests": 1296, + "ok": 1296, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 119.5, + "write_ok_rps": 68.1, + "latency_ms_ok": { + "p50": 31.0, + "p95": 172.0, + "p99": 203.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.97, + [ + "api1" + ] + ], + [ + 22.0, + [ + "api1", + "api2" + ] + ], + [ + 22.45, + [ + "api1" + ] + ], + [ + 24.97, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2100, + "stored_messages": 2100, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + }, + { + "target_instance": "c9fab9c9e2c1", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.14, + "lb_detect_down_seconds": 2.59, + "target_last_served_after_stop_seconds": 0.51, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.59, + "restore_hook_seconds": 6.83, + "lb_back_up_seconds": 1.61, + "target_back_in_rotation_seconds": 6.83, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 934, + "ok": 934, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 186.8, + "write_ok_rps": 111.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 835, + "ok": 829, + "errors": 6, + "error_ratio": 0.0072, + "ok_rps": 82.9, + "write_ok_rps": 40.8, + "latency_ms_ok": { + "p50": 32.0, + "p95": 188.0, + "p99": 1000.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 541, + "ok": 541, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 108.2, + "write_ok_rps": 37.6, + "latency_ms_ok": { + "p50": 31.0, + "p95": 203.0, + "p99": 282.0 + }, + "instances": [ + "4fc3ad11e268" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.16, + "requests": 1369, + "ok": 1369, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 134.8, + "write_ok_rps": 77.6, + "latency_ms_ok": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.59, + [ + "api2" + ] + ], + [ + 21.61, + [ + "api1", + "api2" + ] + ], + [ + 21.95, + [ + "api2" + ] + ], + [ + 24.53, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 1941, + "stored_messages": 1941, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/instances.txt b/docs/lab4/evidence/instances.txt new file mode 100644 index 0000000..df79b9b --- /dev/null +++ b/docs/lab4/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-11-04 14:04:00 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001993A-1F90-6AEB3B50162-0571-0008 + 2 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019944-1F90-6AEB3B50169-0572-0008 + 3 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001994A-1F90-6AEB3B5017C-0573-0008 + 4 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019958-1F90-6AEB3B50199-0574-0008 + 5 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001995E-1F90-6AEB3B5019C-0575-0008 + 6 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019960-1F90-6AEB3B501B7-0576-0008 + 7 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019966-1F90-6AEB3B501C6-0577-0008 + 8 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019974-1F90-6AEB3B501C9-0578-0008 + 9 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019978-1F90-6AEB3B501E5-0579-0008 +10 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019980-1F90-6AEB3B501F4-057A-0008 +11 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019984-1F90-6AEB3B50205-057B-0008 +12 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC1200019988-1F90-6AEB3B50209-057C-0008 +13 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC1200019998-1F90-6AEB3B5020C-057D-0008 +14 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC120001999C-1F90-6AEB3B50223-057E-0008 +15 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC120001999E-1F90-6AEB3B50226-057F-0008 +16 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199AE-1F90-6AEB3B50241-0580-0008 +17 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC12000199BA-1F90-6AEB3B50245-0581-0008 +18 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199CA-1F90-6AEB3B50260-0582-0008 +19 200 X-Instance-Id=4fc3ad11e268 X-Request-Id=AC12000199D2-1F90-6AEB3B50263-0584-0008 +20 200 X-Instance-Id=c9fab9c9e2c1 X-Request-Id=AC12000199DE-1F90-6AEB3B50266-0585-0008 +разных реплик: 2 diff --git a/docs/lab4/evidence/lb_log_sample.txt b/docs/lab4/evidence/lb_log_sample.txt new file mode 100644 index 0000000..234f9e0 --- /dev/null +++ b/docs/lab4/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-11-04 14:12:25 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.18.0.1:56696 [04/Nov/2026:14:12:22.325] public api/api2 0/0/0/11/11 201 421 ---- 8/8/7/2/0 0/0 rid=AC120001DD78-1F90-6AEB3D46145-0F07-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56712 [04/Nov/2026:14:12:22.328] public api/api2 0/0/0/10/11 201 421 ---- 8/8/6/1/0 0/0 rid=AC120001DD88-1F90-6AEB3D46148-0F09-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56732 [04/Nov/2026:14:12:22.330] public api/api2 0/0/0/9/9 201 421 ---- 7/7/5/0/0 0/0 rid=AC120001DD9C-1F90-6AEB3D4614A-0F0B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56746 [04/Nov/2026:14:12:22.340] public api/api2 0/0/0/6/6 201 421 ---- 8/8/6/0/0 0/0 rid=AC120001DDAA-1F90-6AEB3D46154-0F0D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56734 [04/Nov/2026:14:12:22.349] public api/api2 0/0/1/6/7 201 421 ---- 8/8/6/0/0 0/0 rid=AC120001DD9E-1F90-6AEB3D46152-0F0F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56764 [04/Nov/2026:14:12:22.365] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDBC-1F90-6AEB3D4615C-0F11-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56666 [04/Nov/2026:14:12:22.308] public api/api1 0/0/0/88/89 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD5A-1F90-6AEB3D46134-0F04-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56788 [04/Nov/2026:14:12:22.398] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDD4-1F90-6AEB3D4618E-0F13-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56688 [04/Nov/2026:14:12:22.324] public api/api1 0/0/0/84/85 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD70-1F90-6AEB3D46144-0F06-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56802 [04/Nov/2026:14:12:22.410] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDE2-1F90-6AEB3D4619A-0F15-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56698 [04/Nov/2026:14:12:22.325] public api/api1 0/0/0/99/100 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DD7A-1F90-6AEB3D46145-0F08-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56808 [04/Nov/2026:14:12:22.427] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DDE8-1F90-6AEB3D461AB-0F17-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56748 [04/Nov/2026:14:12:22.341] public api/api1 0/0/0/156/157 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DDAC-1F90-6AEB3D46155-0F0E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56768 [04/Nov/2026:14:12:22.358] public api/api1 0/0/0/140/140 201 421 ---- 7/7/6/6/0 0/0 rid=AC120001DDC0-1F90-6AEB3D46166-0F10-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56784 [04/Nov/2026:14:12:22.395] public api/api1 0/0/0/103/104 201 421 ---- 6/6/5/5/0 0/0 rid=AC120001DDD0-1F90-6AEB3D46174-0F12-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56678 [04/Nov/2026:14:12:22.334] public api/api1 0/0/1/164/166 201 421 ---- 8/8/4/4/0 0/0 rid=AC120001DD66-1F90-6AEB3D4613D-0F0C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56728 [04/Nov/2026:14:12:22.328] public api/api1 0/0/0/171/172 201 421 ---- 7/7/5/3/0 0/0 rid=AC120001DD98-1F90-6AEB3D46148-0F0A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56800 [04/Nov/2026:14:12:22.406] public api/api1 0/0/0/96/96 201 421 ---- 8/8/5/3/0 0/0 rid=AC120001DDE0-1F90-6AEB3D46196-0F14-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56806 [04/Nov/2026:14:12:22.418] public api/api1 0/0/0/88/88 201 421 ---- 8/8/5/3/0 0/0 rid=AC120001DDE6-1F90-6AEB3D461A2-0F16-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56846 [04/Nov/2026:14:12:22.501] public api/api2 0/0/0/10/10 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DE0E-1F90-6AEB3D461F5-0F1A-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56856 [04/Nov/2026:14:12:22.501] public api/api2 0/0/0/12/12 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DE18-1F90-6AEB3D461F5-0F1C-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56872 [04/Nov/2026:14:12:22.502] public api/api1 0/0/0/12/12 201 421 ---- 8/8/3/2/0 0/0 rid=AC120001DE28-1F90-6AEB3D461F6-0F1D-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56824 [04/Nov/2026:14:12:22.434] public api/api1 0/0/0/84/84 201 421 ---- 8/8/3/2/0 0/0 rid=AC120001DDF8-1F90-6AEB3D461B2-0F18-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56840 [04/Nov/2026:14:12:22.501] public api/api1 0/0/0/17/18 201 421 ---- 7/7/2/1/0 0/0 rid=AC120001DE08-1F90-6AEB3D461F5-0F1B-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56898 [04/Nov/2026:14:12:22.512] public api/api2 0/0/0/6/6 201 421 ---- 6/6/1/0/0 0/0 rid=AC120001DE42-1F90-6AEB3D46200-0F1E-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56860 [04/Nov/2026:14:12:22.518] public api/api2 0/0/1/11/12 201 421 ---- 8/8/5/2/0 0/0 rid=AC120001DE1C-1F90-6AEB3D461F6-0F20-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56926 [04/Nov/2026:14:12:22.521] public api/api2 0/0/0/12/12 201 421 ---- 8/8/5/1/0 0/0 rid=AC120001DE5E-1F90-6AEB3D46209-0F24-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56888 [04/Nov/2026:14:12:22.519] public api/api2 0/0/0/14/14 201 421 ---- 7/7/4/0/0 0/0 rid=AC120001DE38-1F90-6AEB3D461FD-0F22-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56874 [04/Nov/2026:14:12:22.519] public api/api1 0/0/0/15/16 201 421 ---- 6/6/5/3/0 0/0 rid=AC120001DE2A-1F90-6AEB3D461F8-0F21-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56916 [04/Nov/2026:14:12:22.520] public api/api1 0/0/0/14/14 201 421 ---- 6/6/4/3/0 0/0 rid=AC120001DE54-1F90-6AEB3D46208-0F23-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56904 [04/Nov/2026:14:12:22.516] public api/api1 0/0/0/19/20 201 421 ---- 7/7/5/3/0 0/0 rid=AC120001DE48-1F90-6AEB3D46204-0F1F-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56956 [04/Nov/2026:14:12:22.536] public api/api2 0/0/0/12/12 201 421 ---- 8/8/7/3/0 0/0 rid=AC120001DE7C-1F90-6AEB3D46218-0F2B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56942 [04/Nov/2026:14:12:22.535] public api/api2 0/0/0/15/16 201 421 ---- 8/8/6/2/0 0/0 rid=AC120001DE6E-1F90-6AEB3D46217-0F29-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56930 [04/Nov/2026:14:12:22.534] public api/api2 0/0/0/17/17 201 421 ---- 7/7/5/1/0 0/0 rid=AC120001DE62-1F90-6AEB3D46209-0F27-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56984 [04/Nov/2026:14:12:22.538] public api/api2 0/0/0/15/15 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DE98-1F90-6AEB3D4621A-0F2D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57024 [04/Nov/2026:14:12:22.564] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/1/0 0/0 rid=AC120001DEC0-1F90-6AEB3D4622A-0F30-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56998 [04/Nov/2026:14:12:22.565] public api/api2 0/0/0/8/8 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DEA6-1F90-6AEB3D46227-0F2F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57044 [04/Nov/2026:14:12:22.596] public api/api2 0/0/0/6/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DED4-1F90-6AEB3D4623E-0F33-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56934 [04/Nov/2026:14:12:22.533] public api/api1 0/0/0/80/80 201 421 ---- 8/8/7/7/0 0/0 rid=AC120001DE66-1F90-6AEB3D46215-0F26-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57066 [04/Nov/2026:14:12:22.615] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DEEA-1F90-6AEB3D46267-0F35-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56952 [04/Nov/2026:14:12:22.535] public api/api1 0/0/0/88/88 201 421 ---- 8/8/6/6/0 0/0 rid=AC120001DE78-1F90-6AEB3D46217-0F2A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56900 [04/Nov/2026:14:12:22.534] public api/api1 0/0/1/88/90 201 421 ---- 7/7/5/5/0 0/0 rid=AC120001DE44-1F90-6AEB3D46202-0F28-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:56970 [04/Nov/2026:14:12:22.537] public api/api1 0/0/0/87/87 201 421 ---- 6/6/4/4/0 0/0 rid=AC120001DE8A-1F90-6AEB3D46219-0F2C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57018 [04/Nov/2026:14:12:22.565] public api/api1 0/0/0/62/63 201 421 ---- 8/8/6/5/0 0/0 rid=AC120001DEBA-1F90-6AEB3D46229-0F31-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57036 [04/Nov/2026:14:12:22.572] public api/api1 0/0/0/55/55 201 421 ---- 7/7/5/4/0 0/0 rid=AC120001DECC-1F90-6AEB3D4623C-0F32-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57014 [04/Nov/2026:14:12:22.553] public api/api1 0/0/0/74/75 201 421 ---- 6/6/4/3/0 0/0 rid=AC120001DEB6-1F90-6AEB3D46229-0F2E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57052 [04/Nov/2026:14:12:22.604] public api/api1 0/0/0/23/24 201 421 ---- 5/5/3/2/0 0/0 rid=AC120001DEDC-1F90-6AEB3D4625C-0F34-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57114 [04/Nov/2026:14:12:22.626] public api/api2 0/0/0/7/7 201 421 ---- 8/8/5/2/0 0/0 rid=AC120001DF1A-1F90-6AEB3D46272-0F37-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57138 [04/Nov/2026:14:12:22.629] public api/api2 0/0/0/9/9 201 421 ---- 8/8/4/1/0 0/0 rid=AC120001DF32-1F90-6AEB3D46275-0F39-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57142 [04/Nov/2026:14:12:22.630] public api/api2 0/0/0/10/10 201 421 ---- 8/8/4/0/0 0/0 rid=AC120001DF36-1F90-6AEB3D46276-0F3B-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57156 [04/Nov/2026:14:12:22.641] public api/api2 0/0/0/5/5 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DF44-1F90-6AEB3D46276-0F3D-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57170 [04/Nov/2026:14:12:22.648] public api/api2 0/0/0/4/4 201 421 ---- 8/8/5/0/0 0/0 rid=AC120001DF52-1F90-6AEB3D46288-0F3F-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57168 [04/Nov/2026:14:12:22.657] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/0/0 0/0 rid=AC120001DF50-1F90-6AEB3D46282-0F41-0008 inst=7d965c2245c9 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57096 [04/Nov/2026:14:12:22.625] public api/api1 0/0/0/86/86 201 421 ---- 7/7/6/6/0 0/0 rid=AC120001DF08-1F90-6AEB3D46271-0F36-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57108 [04/Nov/2026:14:12:22.626] public api/api1 0/0/0/89/89 201 421 ---- 6/6/5/5/0 0/0 rid=AC120001DF14-1F90-6AEB3D46272-0F38-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57160 [04/Nov/2026:14:12:22.656] public api/api1 0/0/0/61/61 201 421 ---- 5/5/4/4/0 0/0 rid=AC120001DF48-1F90-6AEB3D4627B-0F42-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57122 [04/Nov/2026:14:12:22.630] public api/api1 0/0/0/91/91 201 421 ---- 4/4/3/3/0 0/0 rid=AC120001DF22-1F90-6AEB3D46276-0F3A-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57082 [04/Nov/2026:14:12:22.642] public api/api1 0/0/0/81/81 201 421 ---- 3/3/2/2/0 0/0 rid=AC120001DEFA-1F90-6AEB3D4626F-0F3E-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57164 [04/Nov/2026:14:12:22.640] public api/api1 0/0/0/83/83 201 421 ---- 2/2/1/1/0 0/0 rid=AC120001DF4C-1F90-6AEB3D46280-0F3C-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" +lb-1 | 172.18.0.1:57180 [04/Nov/2026:14:12:22.654] public api/api1 0/0/0/69/70 201 421 ---- 1/1/0/0/0 0/0 rid=AC120001DF5C-1F90-6AEB3D4628E-0F40-0008 inst=4749f3b4b256 "POST /api/v1/chats/698779cf-d855-4184-b4be-60607d5fd0c8/messages" diff --git a/docs/lab4/evidence/load.json b/docs/lab4/evidence/load.json new file mode 100644 index 0000000..281ec48 --- /dev/null +++ b/docs/lab4/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.156, + "concurrency": 8, + "successful_requests": 3821, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 126.708, + "successful_latency_ms": { + "p50": 31.0, + "p95": 187.0, + "p99": 266.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab4/evidence/load.txt b/docs/lab4/evidence/load.txt new file mode 100644 index 0000000..ef62856 --- /dev/null +++ b/docs/lab4/evidence/load.txt @@ -0,0 +1,29 @@ +# 2026-11-04 14:12:22 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.6s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.156, + "concurrency": 8, + "successful_requests": 3821, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 126.708, + "successful_latency_ms": { + "p50": 31.0, + "p95": 187.0, + "p99": 266.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-11-04 14:12:24 UTC | CPU/RAM после нагрузки | exit 0 | 2.1s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab4-api-6 17.48% 88.79MiB / 320MiB 27.75% +messenger-lab4-api-5 17.67% 89.4MiB / 320MiB 27.94% +messenger-lab4-lb-1 2.18% 14.14MiB / 128MiB 11.05% +messenger-lab4-postgres-1 0.14% 68.93MiB / 512MiB 13.46% +messenger-lab4-redis-1 2.43% 4.82MiB / 96MiB 5.02% +messenger-lab4-s3-1 0.31% 97.91MiB / 512MiB 19.12% + diff --git a/docs/lab4/evidence/persistence.txt b/docs/lab4/evidence/persistence.txt new file mode 100644 index 0000000..e453f74 --- /dev/null +++ b/docs/lab4/evidence/persistence.txt @@ -0,0 +1,48 @@ +# 2026-11-04 14:11:03 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab4-api-5 Up 32 seconds (healthy) 32 seconds ago +messenger-lab4-api-6 Up 20 seconds (healthy) 21 seconds ago +messenger-lab4-lb-1 Up 11 minutes (healthy) 13 minutes ago +messenger-lab4-migrate-1 Exited (0) 6 minutes ago 13 minutes ago +messenger-lab4-postgres-1 Up 12 minutes (healthy) 13 minutes ago +messenger-lab4-redis-1 Up 12 minutes (healthy) 13 minutes ago +messenger-lab4-s3-1 Up About a minute (healthy) 13 minutes ago +messenger-lab4-s3-init-1 Exited (0) 6 minutes ago 13 minutes ago +# 2026-11-04 14:11:49 UTC | Persistence: make test-persistence (API + PostgreSQL + Redis + S3 + LB) | exit 0 | 45.6s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-04 14:11:49 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.2s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab4-api-5 Up 7 seconds (healthy) About a minute ago +messenger-lab4-api-6 Up 7 seconds (healthy) About a minute ago +messenger-lab4-lb-1 Up 1 second (health: starting) 14 minutes ago +messenger-lab4-migrate-1 Exited (0) 7 minutes ago 14 minutes ago +messenger-lab4-postgres-1 Up 38 seconds (healthy) 14 minutes ago +messenger-lab4-redis-1 Up 39 seconds (healthy) 14 minutes ago +messenger-lab4-s3-1 Up 28 seconds (healthy) 14 minutes ago +messenger-lab4-s3-init-1 Exited (0) 7 minutes ago 14 minutes ago + +# 2026-11-04 14:11:52 UTC | Persistence: сверка БД <-> S3 после рестарта | exit 0 | 2.4s +$ python scripts/ops.py reconcile + Container messenger-lab4-api-run-e029350f1930 Creating + Container messenger-lab4-api-run-e029350f1930 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 182, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.062, + "ready_rows": 182, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.073 +} + diff --git a/docs/lab4/evidence/reconcile.txt b/docs/lab4/evidence/reconcile.txt new file mode 100644 index 0000000..2147d08 --- /dev/null +++ b/docs/lab4/evidence/reconcile.txt @@ -0,0 +1,19 @@ +# 2026-11-04 14:04:52 UTC | Сверка БД <-> S3 (dry-run) | exit 0 | 2.2s +$ python scripts/ops.py reconcile + Container messenger-lab4-api-run-fb929425df7f Creating + Container messenger-lab4-api-run-fb929425df7f Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 107, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.014, + "ready_rows": 107, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.025 +} diff --git a/docs/lab4/evidence/rolling_back.json b/docs/lab4/evidence/rolling_back.json new file mode 100644 index 0000000..0f4c8bb --- /dev/null +++ b/docs/lab4/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 28.28, + "replicas_before": [ + "1c20a55eebeb", + "ee285aa5856f" + ], + "replicas_after": [ + "4749f3b4b256", + "7d965c2245c9" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1026, + "ok": 1026, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 205.2, + "write_ok_rps": 123.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 157.0 + }, + "instances": [ + "1c20a55eebeb", + "ee285aa5856f" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 33.28, + "requests": 5808, + "ok": 5808, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 205.4, + "write_ok_rps": 122.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 172.0 + }, + "instances": [ + "1c20a55eebeb", + "4749f3b4b256", + "7d965c2245c9", + "ee285aa5856f" + ] + }, + "after": { + "from_s": 33.28, + "to_s": 41.34, + "requests": 1492, + "ok": 1492, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 185.1, + "write_ok_rps": 111.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "4749f3b4b256", + "7d965c2245c9" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api3" + ] + ], + [ + 11.78, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.7, + [ + "api1", + "api3" + ] + ], + [ + 15.26, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 18.2, + [ + "api1", + "api2" + ] + ], + [ + 25.8, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 29.51, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 4970, + "stored_messages": 4970, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab4/evidence/rolling_update.json b/docs/lab4/evidence/rolling_update.json new file mode 100644 index 0000000..7c74c98 --- /dev/null +++ b/docs/lab4/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 29.38, + "replicas_before": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "replicas_after": [ + "1c20a55eebeb", + "ee285aa5856f" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1180, + "ok": 1180, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 236.0, + "write_ok_rps": 140.6, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 34.38, + "requests": 6657, + "ok": 6657, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 226.6, + "write_ok_rps": 133.9, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 157.0 + }, + "instances": [ + "1c20a55eebeb", + "4fc3ad11e268", + "c9fab9c9e2c1", + "ee285aa5856f" + ] + }, + "after": { + "from_s": 34.38, + "to_s": 42.45, + "requests": 1674, + "ok": 1674, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 207.2, + "write_ok_rps": 121.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 110.0 + }, + "instances": [ + "1c20a55eebeb", + "ee285aa5856f" + ] + } + }, + "lb_timeline": [ + [ + 0.03, + [ + "api1", + "api2" + ] + ], + [ + 12.44, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 13.16, + [ + "api1", + "api2" + ] + ], + [ + 15.72, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.44, + [ + "api2", + "api3" + ] + ], + [ + 26.94, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 31.47, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 5617, + "stored_messages": 5617, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab4/evidence/rolling_update.txt b/docs/lab4/evidence/rolling_update.txt new file mode 100644 index 0000000..03b1ccc --- /dev/null +++ b/docs/lab4/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-11-04 14:10:19 UTC | Rolling update lab4 -> lab4.1 под нагрузкой | exit 0 | 44.8s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['4fc3ad11e268', 'c9fab9c9e2c1'] +2026-11-04T14:09:41.527Z rolling-update: start; replicas=2 old=['c9fab9c9e2c1', '4fc3ad11e268'] +2026-11-04T14:09:45.307Z rolling-update: images built in 3.8s (tag lab4.1) +2026-11-04T14:09:47.489Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-11-04T14:09:59.096Z rolling-update: step 1/2 new=1c20a55eebeb (messenger-api:lab4.1, ready+UP in 7.0s) replaced old=c9fab9c9e2c1 (graceful stop 4.4s) +2026-11-04T14:10:09.967Z rolling-update: step 2/2 new=ee285aa5856f (messenger-api:lab4.1, ready+UP in 6.6s) replaced old=4fc3ad11e268 (graceful stop 4.0s) +2026-11-04T14:10:10.389Z rolling-update: done in 29.1s; replicas now ['1c20a55eebeb', 'ee285aa5856f'] +update 29.4s rc=0 | replicas ['4fc3ad11e268', 'c9fab9c9e2c1'] -> ['1c20a55eebeb', 'ee285aa5856f'] | errors during update 0 (-) | RPS before 236.0, during 226.6, after 207.2 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-11-04 14:11:03 UTC | Rolling update обратно lab4.1 -> lab4 (откат) под нагрузкой | exit 0 | 43.8s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['1c20a55eebeb', 'ee285aa5856f'] +2026-11-04T14:10:26.009Z rolling-update: start; replicas=2 old=['1c20a55eebeb', 'ee285aa5856f'] +2026-11-04T14:10:28.537Z rolling-update: images built in 2.5s (tag lab4) +2026-11-04T14:10:30.771Z rolling-update: migrations once in 2.2s: schema is up to date; nothing applied +2026-11-04T14:10:42.303Z rolling-update: step 1/2 new=4749f3b4b256 (messenger-api:lab4, ready+UP in 7.2s) replaced old=1c20a55eebeb (graceful stop 4.1s) +2026-11-04T14:10:53.630Z rolling-update: step 2/2 new=7d965c2245c9 (messenger-api:lab4, ready+UP in 7.0s) replaced old=ee285aa5856f (graceful stop 4.2s) +2026-11-04T14:10:54.052Z rolling-update: done in 28.0s; replicas now ['4749f3b4b256', '7d965c2245c9'] +update 28.3s rc=0 | replicas ['1c20a55eebeb', 'ee285aa5856f'] -> ['4749f3b4b256', '7d965c2245c9'] | errors during update 0 (-) | RPS before 205.2, during 205.4, after 185.1 | p95 during 94.0 ms | data ok=True +PASS: rolling update under load + +# 2026-11-04 14:11:03 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4749f3b4b256 messenger-lab4-api-5 running healthy messenger-api:lab4 api2 UP +7d965c2245c9 messenger-lab4-api-6 running healthy messenger-api:lab4 api3 UP + diff --git a/docs/lab4/evidence/s3_bootstrap.txt b/docs/lab4/evidence/s3_bootstrap.txt new file mode 100644 index 0000000..52e17d5 --- /dev/null +++ b/docs/lab4/evidence/s3_bootstrap.txt @@ -0,0 +1,225 @@ +# 2026-11-04 14:03:57 UTC | S3 bootstrap: лог первого запуска s3-init | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color s3-init +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +# 2026-11-04 14:04:00 UTC | S3 bootstrap: повторный запуск (идемпотентность) | exit 0 | 2.3s +$ docker compose --env-file .env run --rm -T s3-init + Container messenger-lab4-s3-1 Running + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-s3-init-run-8fa89d92841c Creating + Container messenger-lab4-s3-init-run-8fa89d92841c Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} + diff --git a/docs/lab4/evidence/s3_outage.json b/docs/lab4/evidence/s3_outage.json new file mode 100644 index 0000000..465a6c6 --- /dev/null +++ b/docs/lab4/evidence/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 20.34, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 6.53s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "6.53s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 6.31s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "6.31s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 3.66s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "3.66s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "20.34s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/s3_outage.txt b/docs/lab4/evidence/s3_outage.txt new file mode 100644 index 0000000..bcb31b0 --- /dev/null +++ b/docs/lab4/evidence/s3_outage.txt @@ -0,0 +1,24 @@ +# 2026-11-04 14:09:34 UTC | Недоступность S3 и восстановление | exit 0 | 48.7s +$ python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json +PASS baseline upload 201 — 201 +$ docker compose --env-file .env stop s3 +PASS readiness stays 200 while S3 is down (S3 is not a readiness dependency) — 200 +PASS text message 201 during S3 outage — 201 +PASS history 200 during S3 outage — 200 +PASS attachment metadata 200 (PostgreSQL only) — 200 +PASS message referencing existing attachment 201 — 201 +PASS upload #1 -> 503 unavailable — 503 unavailable in 6.53s +PASS upload #1 failed fast (<= 15s) — 6.53s +PASS upload #1 has Retry-After +PASS upload #2 -> 503 unavailable — 503 unavailable in 6.31s +PASS upload #2 failed fast (<= 15s) — 6.31s +PASS upload #2 has Retry-After +PASS download -> 503 during outage — 503 in 3.66s +PASS download failed fast — 3.66s +PASS session still valid — 200 +$ docker compose --env-file .env start s3 +$ C:\Users\mrend\AppData\Local\Programs\Python\Python312\python.exe scripts/ops.py wait-s3 +PASS S3 read/write warm-up after start — 20.34s +PASS old file downloads byte-for-byte after recovery — 200 0.02s +PASS new upload 201 after recovery (same token, no API restart) — 201 +PASS diff --git a/docs/lab4/evidence/s3_private.txt b/docs/lab4/evidence/s3_private.txt new file mode 100644 index 0000000..4a729d3 --- /dev/null +++ b/docs/lab4/evidence/s3_private.txt @@ -0,0 +1,67 @@ +# 2026-11-04 14:04:54 UTC | Приватный бакет вне API, права ключей, закрытые админ-порты | exit 0 | 2.3s +$ python scripts/ops.py s3-check + Container messenger-lab4-api-run-33922d9bc56f Creating + Container messenger-lab4-api-run-33922d9bc56f Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} +# 2026-11-04 14:04:54 UTC | На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes | exit 0 | 0.5s +$ python scripts/ops.py api-fs +messenger-lab4-api-1 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/database_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] +messenger-lab4-api-2 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/database_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] + +# 2026-11-04 14:04:55 UTC | API: файлы в /tmp (tmpfs) после загрузок | exit 0 | 0.3s +$ docker compose --env-file .env exec -T api sh -c df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l) +Filesystem Size Used Avail Use% Mounted on +tmpfs 16M 0 16M 0% /tmp +files_in_tmp=0 + diff --git a/docs/lab4/evidence/sandbox/check.txt b/docs/lab4/evidence/sandbox/check.txt new file mode 100644 index 0000000..395bd0f --- /dev/null +++ b/docs/lab4/evidence/sandbox/check.txt @@ -0,0 +1,6 @@ +# 2026-11-02T13:36:48Z | offline contract +$ python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-02T13:36:48Z | full OpenAPI validation +$ python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/docs/lab4/evidence/sandbox/consistency.txt b/docs/lab4/evidence/sandbox/consistency.txt new file mode 100644 index 0000000..3ca0db1 --- /dev/null +++ b/docs/lab4/evidence/sandbox/consistency.txt @@ -0,0 +1,17 @@ +# 2026-11-02T13:37:02Z | DB<->S3 consistency, aborted upload, injected faults (api-11 s3_error_on_put, api-12 db_error_after_s3_put, api-13 crash_after_s3_put) +$ BASE_URL=... FAULT_*_URL=... python -m unittest discover -s student-tests -p test_attachment_consistency.py -v +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.391s + +OK diff --git a/docs/lab4/evidence/sandbox/failover.txt b/docs/lab4/evidence/sandbox/failover.txt new file mode 100644 index 0000000..d68450f --- /dev/null +++ b/docs/lab4/evidence/sandbox/failover.txt @@ -0,0 +1,12 @@ +# 2026-11-02T13:28:02Z failover run 1: python3 tests/resilience.py --mode failover --action-script +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 4.468} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +2026-11-02T13:27:47Z stop-one api-1 pid 7684 +restored; next run +# 2026-11-02T13:28:23Z failover run 2: python3 tests/resilience.py --mode failover --action-script +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 0, "first_recovery_seconds": 4.475} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +2026-11-02T13:28:08Z stop-one api-1 pid 7937 +restored; next run diff --git a/docs/lab4/evidence/sandbox/failover_series.json b/docs/lab4/evidence/sandbox/failover_series.json new file mode 100644 index 0000000..c69f3b3 --- /dev/null +++ b/docs/lab4/evidence/sandbox/failover_series.json @@ -0,0 +1,268 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "api-1", + "api-2" + ], + "rounds": [ + { + "target_instance": "api-1", + "mode": "stop", + "lb_up_before": [ + "api-1", + "api-2" + ], + "stop_hook_seconds": 5.0, + "lb_detect_down_seconds": 1.3, + "target_last_served_after_stop_seconds": 1.31, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 1.86, + "lb_back_up_seconds": 2.91, + "target_back_in_rotation_seconds": 2.87, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.02, + "requests": 1597, + "ok": 1597, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 318.1, + "write_ok_rps": 187.8, + "latency_ms_ok": { + "p50": 16.7, + "p95": 38.6, + "p99": 50.6 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_first_10s": { + "from_s": 5.02, + "to_s": 15.02, + "requests": 2392, + "ok": 2392, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 239.2, + "write_ok_rps": 140.9, + "latency_ms_ok": { + "p50": 24.4, + "p95": 39.5, + "p99": 48.8 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_rest": { + "from_s": 15.02, + "to_s": 20.02, + "requests": 1057, + "ok": 1057, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 211.4, + "write_ok_rps": 124.2, + "latency_ms_ok": { + "p50": 27.6, + "p95": 44.5, + "p99": 54.8 + }, + "instances": [ + "api-2" + ] + }, + "after_restore": { + "from_s": 20.02, + "to_s": 26.15, + "requests": 1671, + "ok": 1671, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 272.8, + "write_ok_rps": 160.7, + "latency_ms_ok": { + "p50": 21.2, + "p95": 39.0, + "p99": 49.4 + }, + "instances": [ + "api-1", + "api-2" + ] + } + }, + "lb_timeline": [ + [ + 0.02, + [ + "api-1", + "api-2" + ] + ], + [ + 6.32, + [ + "api-2" + ] + ], + [ + 22.93, + [ + "api-1", + "api-2" + ] + ] + ], + "data": { + "confirmed_writes": 3957, + "stored_messages": 3957, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "api-2", + "mode": "stop", + "lb_up_before": [ + "api-1", + "api-2" + ], + "stop_hook_seconds": 5.23, + "lb_detect_down_seconds": 1.53, + "target_last_served_after_stop_seconds": 1.47, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 1.45, + "lb_back_up_seconds": 3.05, + "target_back_in_rotation_seconds": 3.05, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.01, + "requests": 1416, + "ok": 1416, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 282.9, + "write_ok_rps": 164.8, + "latency_ms_ok": { + "p50": 19.4, + "p95": 41.3, + "p99": 53.4 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_first_10s": { + "from_s": 5.01, + "to_s": 15.01, + "requests": 2431, + "ok": 2431, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 243.1, + "write_ok_rps": 142.8, + "latency_ms_ok": { + "p50": 24.8, + "p95": 39.6, + "p99": 46.3 + }, + "instances": [ + "api-1", + "api-2" + ] + }, + "outage_rest": { + "from_s": 15.01, + "to_s": 20.01, + "requests": 1154, + "ok": 1154, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 230.7, + "write_ok_rps": 136.0, + "latency_ms_ok": { + "p50": 25.4, + "p95": 39.9, + "p99": 46.5 + }, + "instances": [ + "api-1" + ] + }, + "after_restore": { + "from_s": 20.01, + "to_s": 26.3, + "requests": 1851, + "ok": 1851, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 293.9, + "write_ok_rps": 172.6, + "latency_ms_ok": { + "p50": 19.8, + "p95": 35.8, + "p99": 42.1 + }, + "instances": [ + "api-1", + "api-2" + ] + } + }, + "lb_timeline": [ + [ + 0.01, + [ + "api-1", + "api-2" + ] + ], + [ + 6.53, + [ + "api-1" + ] + ], + [ + 23.06, + [ + "api-1", + "api-2" + ] + ] + ], + "data": { + "confirmed_writes": 4020, + "stored_messages": 4020, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/sandbox/load.json b/docs/lab4/evidence/sandbox/load.json new file mode 100644 index 0000000..f7d5a05 --- /dev/null +++ b/docs/lab4/evidence/sandbox/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 20.0, + "elapsed_seconds": 20.055, + "concurrency": 4, + "successful_requests": 5544, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 276.435, + "successful_latency_ms": { + "p50": 13.241, + "p95": 23.756, + "p99": 32.552 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab4/evidence/sandbox/persistence.txt b/docs/lab4/evidence/sandbox/persistence.txt new file mode 100644 index 0000000..707b5ef --- /dev/null +++ b/docs/lab4/evidence/sandbox/persistence.txt @@ -0,0 +1,7 @@ +# 2026-11-02T13:24:54Z native sandbox: python3 tests/resilience.py --mode persistence --action-script +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# hook log: PIDs before/after (postgres, seaweedfs, api x2) + S3 warm-up +2026-11-02T13:24:15Z restart begin pg=6347 s3=6937 api=6405,6525 +{"s3_ready_after_s": 16.4, "last_cycle_s": 1.028, "existing_object_read": true} +2026-11-02T13:24:53Z restart end pg=7134 s3=7149 api=7195,7219 diff --git a/docs/lab4/evidence/sandbox/reconcile.txt b/docs/lab4/evidence/sandbox/reconcile.txt new file mode 100644 index 0000000..78029ae --- /dev/null +++ b/docs/lab4/evidence/sandbox/reconcile.txt @@ -0,0 +1,17 @@ +# 2026-11-02T13:37:06Z | reconcile dry-run +$ python -m src.reconcile +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "objects_listed": 238, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.036, + "ready_rows": 238, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.045 +} diff --git a/docs/lab4/evidence/sandbox/s3_bootstrap.txt b/docs/lab4/evidence/sandbox/s3_bootstrap.txt new file mode 100644 index 0000000..2a46dc3 --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_bootstrap.txt @@ -0,0 +1,45 @@ +# 2026-11-02T13:36:49Z | S3 bootstrap, 2nd run on existing bucket (idempotent) +$ python -m src.s3_bootstrap +{ + "endpoint_host": "127.0.0.3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} diff --git a/docs/lab4/evidence/sandbox/s3_outage.json b/docs/lab4/evidence/sandbox/s3_outage.json new file mode 100644 index 0000000..21afe8a --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 16.42, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 1.75s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "1.75s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 2.31s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "2.31s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 0.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "0.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "16.42s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.01s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/sandbox/s3_outage_hang.json b/docs/lab4/evidence/sandbox/s3_outage_hang.json new file mode 100644 index 0000000..4f58582 --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_outage_hang.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 16.09, + "checks": [ + { + "check": "baseline upload 201", + "ok": true, + "detail": "201" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 10.02s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "10.02s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 10.01s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "10.01s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 5.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "5.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "16.09s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 201 after recovery (same token, no API restart)", + "ok": true, + "detail": "201" + } + ], + "pass": true +} diff --git a/docs/lab4/evidence/sandbox/s3_private.txt b/docs/lab4/evidence/sandbox/s3_private.txt new file mode 100644 index 0000000..7398970 --- /dev/null +++ b/docs/lab4/evidence/sandbox/s3_private.txt @@ -0,0 +1,54 @@ +# 2026-11-02T13:37:07Z | private bucket outside API (anonymous S3 requests), key permissions, admin ports +$ python -m src.s3_bootstrap --check-only --sample-from-db +{ + "endpoint_host": "127.0.0.3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} diff --git a/docs/lab4/evidence/sandbox/smoke.txt b/docs/lab4/evidence/sandbox/smoke.txt new file mode 100644 index 0000000..c0127ee --- /dev/null +++ b/docs/lab4/evidence/sandbox/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-02T13:36:50Z | public smoke +$ python3 tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab4/evidence/sandbox/student-tests.txt b/docs/lab4/evidence/sandbox/student-tests.txt new file mode 100644 index 0000000..bd5e966 --- /dev/null +++ b/docs/lab4/evidence/sandbox/student-tests.txt @@ -0,0 +1,120 @@ +# 2026-11-02T13:37:16Z | student tests over HTTP via LB (lab2-4); DB-level tests skip here and run below with DB access +$ python3 student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... + 24 parallel POSTs, one key -> 1 message; replicas: ['api-1', 'api-2'] +ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 74 tests in 6.028s + +OK (skipped=23) + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# DB-level tests (lab2-3) with direct DB access +$ python -m unittest discover -s student-tests -p "test_storage_*.py"; ... -p test_session_cache.py +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.022s + +OK +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.564s + +OK diff --git a/docs/lab4/evidence/sandbox/upload_memory_c10.json b/docs/lab4/evidence/sandbox/upload_memory_c10.json new file mode 100644 index 0000000..15d8fa0 --- /dev/null +++ b/docs/lab4/evidence/sandbox/upload_memory_c10.json @@ -0,0 +1,43 @@ +{ + "label": "final code; peak via VmHWM and 20ms sampling", + "date_utc": "2026-11-02T13:41:45Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 5.06, + "latency_201_s": { + "p50": 1.316, + "p95": 2.069, + "max": 2.088280546000078, + "mean": 1.407 + }, + "memory": { + "pid-9293": { + "baseline_mib": 95.5, + "peak_mib": 163.4, + "growth_mib": 67.9, + "samples": 356 + }, + "pid-9364": { + "baseline_mib": 95.4, + "peak_mib": 156.1, + "growth_mib": 60.6, + "samples": 356 + } + }, + "process_peak_rss_mib": { + "pid-9293": 165.8, + "pid-9364": 165.3 + } +} diff --git a/docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json b/docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json new file mode 100644 index 0000000..dc21474 --- /dev/null +++ b/docs/lab4/evidence/sandbox/upload_memory_c20_limit4.json @@ -0,0 +1,39 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-02T13:11:09Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 40 + }, + "sha256_ok": 40, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 6.14, + "latency_201_s": { + "p50": 2.159, + "p95": 3.077, + "max": 3.089563180000141, + "mean": 2.079 + }, + "memory": { + "pid-3762": { + "baseline_mib": 114.5, + "peak_mib": 161.7, + "growth_mib": 47.2, + "samples": 387 + }, + "pid-3834": { + "baseline_mib": 96.5, + "peak_mib": 159.1, + "growth_mib": 62.6, + "samples": 387 + } + } +} diff --git a/docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json b/docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json new file mode 100644 index 0000000..84924f3 --- /dev/null +++ b/docs/lab4/evidence/sandbox/upload_memory_c20_nolimit.json @@ -0,0 +1,39 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=64 (limiter effectively off)", + "date_utc": "2026-11-02T13:11:34Z", + "host": "Linux x86_64, 2 CPU", + "source": "/proc VmRSS (20 ms)", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 40 + }, + "sha256_ok": 40, + "instances": [ + "api-1", + "api-2" + ], + "duration_s": 6.06, + "latency_201_s": { + "p50": 2.901, + "p95": 3.031, + "max": 3.03438846999984, + "mean": 2.702 + }, + "memory": { + "pid-4062": { + "baseline_mib": 88.2, + "peak_mib": 261.6, + "growth_mib": 173.4, + "samples": 400 + }, + "pid-4133": { + "baseline_mib": 88.7, + "peak_mib": 291.4, + "growth_mib": 202.8, + "samples": 400 + } + } +} diff --git a/docs/lab4/evidence/smoke.txt b/docs/lab4/evidence/smoke.txt new file mode 100644 index 0000000..8d23ad7 --- /dev/null +++ b/docs/lab4/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-04 14:04:01 UTC | Публичный smoke | exit 0 | 0.8s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab4/evidence/stand.txt b/docs/lab4/evidence/stand.txt new file mode 100644 index 0000000..d4cd20d --- /dev/null +++ b/docs/lab4/evidence/stand.txt @@ -0,0 +1,42 @@ +# 2026-11-04 14:03:55 UTC | Стенд: контейнеры | exit 0 | 0.2s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab4-api-1 messenger-api:lab4 "python -m src.server" api 6 minutes ago Up 3 minutes (healthy) 8080/tcp +messenger-lab4-api-2 messenger-api:lab4 "python -m src.server" api 6 minutes ago Up 3 minutes (healthy) 8080/tcp +messenger-lab4-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 6 minutes ago Up 4 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab4-migrate-1 messenger-lab4-migrate "python -m src.migra…" migrate 6 minutes ago Exited (0) 2 minutes ago +messenger-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 6 minutes ago Up 5 minutes (healthy) 5432/tcp +messenger-lab4-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 6 minutes ago Up 5 minutes (healthy) 6379/tcp +messenger-lab4-s3-1 chrislusf/seaweedfs:4.50 "/usr/bin/weed serve…" s3 6 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +messenger-lab4-s3-init-1 messenger-lab4-s3-init "python -m src.s3_bo…" s3-init 6 minutes ago Exited (0) 2 minutes ago +# 2026-11-04 14:03:55 UTC | Стенд: реплики и статус в LB | exit 0 | 0.2s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +c9fab9c9e2c1 messenger-lab4-api-1 running healthy messenger-api:lab4 api1 UP +4fc3ad11e268 messenger-lab4-api-2 running healthy messenger-api:lab4 api2 UP + +# 2026-11-04 14:03:57 UTC | Стенд: лимиты CPU/RAM | exit 0 | 1.4s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab4-lb-1 0.14% 13.6MiB / 128MiB 10.63% +messenger-lab4-api-2 0.26% 90.05MiB / 320MiB 28.14% +messenger-lab4-api-1 0.30% 106MiB / 320MiB 33.13% +messenger-lab4-postgres-1 0.13% 71.34MiB / 512MiB 13.93% +messenger-lab4-redis-1 2.31% 5.629MiB / 96MiB 5.86% +messenger-lab4-s3-1 0.21% 508.3MiB / 512MiB 99.28% + +# 2026-11-04 14:03:57 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.2s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp +s3 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp + +# 2026-11-04 14:03:57 UTC | Образ SeaweedFS (закреплённая версия) | exit 0 | 0.3s +$ docker compose --env-file .env images s3 +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +messenger-lab4-s3-1 chrislusf/seaweedfs 4.50 linux/amd64 c2658820e763 197MB 10 days ago + diff --git a/docs/lab4/evidence/student-tests.txt b/docs/lab4/evidence/student-tests.txt new file mode 100644 index 0000000..edbc587 --- /dev/null +++ b/docs/lab4/evidence/student-tests.txt @@ -0,0 +1,168 @@ +# 2026-11-04 14:04:10 UTC | Собственные тесты (HTTP через LB, включая test_attachments.py) | exit 0 | 9.4s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 74 tests in 7.352s + +OK (skipped=23) + + 24 parallel POSTs, one key -> 1 message; replicas: ['4fc3ad11e268', 'c9fab9c9e2c1'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-11-04 14:04:18 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 7.9s +$ python scripts/ops.py in-network storage + Container messenger-lab4-api-run-7c01226dcaba Creating + Container messenger-lab4-api-run-7c01226dcaba Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.302s + +OK + Container messenger-lab4-api-run-b6f4e53090a5 Creating + Container messenger-lab4-api-run-b6f4e53090a5 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 1.860s + +OK + +# 2026-11-04 14:04:33 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 15.2s +$ python scripts/ops.py in-network expiry + Container messenger-lab4-redis-1 Running + Container messenger-lab4-s3-1 Running + Container messenger-lab4-postgres-1 Running + Container messenger-lab4-api-1 Running + Container messenger-lab4-api-2 Running + Container messenger-lab4-api-short-ttl-1 Creating + Container messenger-lab4-api-short-ttl-1 Created + Container messenger-lab4-s3-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-s3-1 Healthy + Container messenger-lab4-migrate-1 Starting + Container messenger-lab4-s3-init-1 Starting + Container messenger-lab4-migrate-1 Started + Container messenger-lab4-s3-init-1 Started + Container messenger-lab4-s3-init-1 Waiting + Container messenger-lab4-postgres-1 Waiting + Container messenger-lab4-redis-1 Waiting + Container messenger-lab4-migrate-1 Waiting + Container messenger-lab4-redis-1 Healthy + Container messenger-lab4-postgres-1 Healthy + Container messenger-lab4-migrate-1 Exited + Container messenger-lab4-s3-init-1 Exited + Container messenger-lab4-api-2 Waiting + Container messenger-lab4-api-1 Waiting + Container messenger-lab4-api-2 Healthy + Container messenger-lab4-api-1 Healthy + Container messenger-lab4-api-short-ttl-1 Starting + Container messenger-lab4-api-short-ttl-1 Started + Container messenger-lab4-api-run-ae85ff205e89 Creating + Container messenger-lab4-api-run-ae85ff205e89 Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab4-api-run-87803f7efa19 Creating + Container messenger-lab4-api-run-87803f7efa19 Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.644s + +OK + diff --git a/docs/lab4/evidence/upload_memory.txt b/docs/lab4/evidence/upload_memory.txt new file mode 100644 index 0000000..5eb2de4 --- /dev/null +++ b/docs/lab4/evidence/upload_memory.txt @@ -0,0 +1,108 @@ +# 2026-11-04 14:05:09 UTC | Пик памяти: 10 одновременных загрузок по 9.5 MiB | exit 0 | 14.2s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 10 --rounds 3 --docker --label c=10, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c10.json +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:09Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 7.44, + "latency_201_s": { + "p50": 2.016, + "p95": 3.75, + "max": 3.75, + "mean": 2.007 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 119.7, + "peak_mib": 153.5, + "growth_mib": 33.8, + "samples": 6 + }, + "messenger-lab4-api-1": { + "baseline_mib": 117.6, + "peak_mib": 149.7, + "growth_mib": 32.1, + "samples": 6 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 172.1, + "messenger-lab4-api-1": 175.4 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 158.0, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + } +} +# 2026-11-04 14:05:29 UTC | Пик памяти: 20 одновременных загрузок | exit 0 | 20.4s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 20 --rounds 2 --docker --label c=20, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c20.json +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:29Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 39, + "503": 1 + }, + "sha256_ok": 39, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 13.24, + "latency_201_s": { + "p50": 4.609, + "p95": 7.312, + "max": 7.421999999998661, + "mean": 4.437 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 113.6, + "peak_mib": 160.9, + "growth_mib": 47.3, + "samples": 9 + }, + "messenger-lab4-api-1": { + "baseline_mib": 109.6, + "peak_mib": 158.9, + "growth_mib": 49.3, + "samples": 9 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 174.2, + "messenger-lab4-api-1": 176.6 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 176.2, + "messenger-lab4-api-1": 183.2 + } +} + diff --git a/docs/lab4/evidence/upload_memory_c10.json b/docs/lab4/evidence/upload_memory_c10.json new file mode 100644 index 0000000..0505146 --- /dev/null +++ b/docs/lab4/evidence/upload_memory_c10.json @@ -0,0 +1,51 @@ +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:09Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "201": 30 + }, + "sha256_ok": 30, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 7.44, + "latency_201_s": { + "p50": 2.016, + "p95": 3.75, + "max": 3.75, + "mean": 2.007 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 119.7, + "peak_mib": 153.5, + "growth_mib": 33.8, + "samples": 6 + }, + "messenger-lab4-api-1": { + "baseline_mib": 117.6, + "peak_mib": 149.7, + "growth_mib": 32.1, + "samples": 6 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 172.1, + "messenger-lab4-api-1": 175.4 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 158.0, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + } +} diff --git a/docs/lab4/evidence/upload_memory_c20.json b/docs/lab4/evidence/upload_memory_c20.json new file mode 100644 index 0000000..5093d11 --- /dev/null +++ b/docs/lab4/evidence/upload_memory_c20.json @@ -0,0 +1,52 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-04T14:05:29Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "201": 39, + "503": 1 + }, + "sha256_ok": 39, + "instances": [ + "4fc3ad11e268", + "c9fab9c9e2c1" + ], + "duration_s": 13.24, + "latency_201_s": { + "p50": 4.609, + "p95": 7.312, + "max": 7.421999999998661, + "mean": 4.437 + }, + "memory": { + "messenger-lab4-api-2": { + "baseline_mib": 113.6, + "peak_mib": 160.9, + "growth_mib": 47.3, + "samples": 9 + }, + "messenger-lab4-api-1": { + "baseline_mib": 109.6, + "peak_mib": 158.9, + "growth_mib": 49.3, + "samples": 9 + } + }, + "process_peak_rss_mib": { + "messenger-lab4-api-2": 174.2, + "messenger-lab4-api-1": 176.6 + }, + "cgroup_peak_mib_before": { + "messenger-lab4-api-2": 174.8, + "messenger-lab4-api-1": 183.2 + }, + "cgroup_peak_mib_after": { + "messenger-lab4-api-2": 176.2, + "messenger-lab4-api-1": 183.2 + } +} diff --git a/docs/lab5/REPORT.md b/docs/lab5/REPORT.md new file mode 100644 index 0000000..ebbb567 --- /dev/null +++ b/docs/lab5/REPORT.md @@ -0,0 +1,257 @@ +# Отчёт — лабораторная 5 + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, FastAPI 0.115.6, Uvicorn 0.34.0, Pydantic 2.10.4, psycopg 3.2.3 + psycopg-pool 3.2.4, **redis-py 5.2.1 (Redis Streams)**, argon2-cffi 23.1.0, boto3 1.35.99, **Pillow 11.1.0 (JPEG/WebP, ImageOps.exif_transpose)**; PostgreSQL 16.4, Redis 7.4.1 (кэш сессий **и отдельный брокер**), HAProxy 2.8.30, SeaweedFS 4.50. Документация: Redis Streams (XADD, XREADGROUP, XACK, XAUTOCLAIM, XPENDING, XTRIM MINID), Redis persistence (AOF, `appendfsync always`), RabbitMQ reliability/confirms (различие publisher confirm и consumer ack), шаблон transactional outbox, AWS Architecture Blog «Exponential Backoff And Jitter», Pillow (draft, reduce/reducing_gap, сохранение JPEG/WebP), Python `multiprocessing` (forkserver) и `resource` (RLIMIT_AS/RLIMIT_CPU). Помощь ИИ (Claude) — при проектировании outbox/аренд/fencing, написании worker, тестов отказов и отчёта. + +Минимум lab4 перенесён целиком (балансировщик, ≥2 реплики, сессии, идемпотентность, приватный S3, проверки изображений, сборщик БД↔S3). Единственное объявленное изменение API: `POST /chats/{id}/attachments` отвечает **202 queued** вместо 201 ready. + +## Запуск из чистого clone + +**Версии.** API и worker — один образ из `Dockerfile` (`python:3.12.7-slim-bookworm`); `postgres:16.4-alpine`; `redis:7.4.1-alpine` (два сервиса: `redis` — кэш сессий, `broker` — очередь); `haproxy:2.8.30-alpine`; `chrislusf/seaweedfs:4.50`. Нужны Docker Engine/Desktop с Compose v2 и Python 3.9+ на хосте (`scripts/ops.py` и тесты — только стандартная библиотека). Оркестрация — **Compose** (Swarm не используется). + +```sh +$env:PYTHONUTF8 = "1" +# 1. Секреты (один раз; secrets/ в .gitignore, значения не печатаются) и .env +python scripts/init_secrets.py # lab5 добавляет secrets/broker_password и secrets/broker_url +cp .env.example .env # Windows: copy .env.example .env + +# 2. Сборка и запуск: +# postgres, redis, broker -> migrate (001..005); s3 -> s3-init (bucket) -> api x2 + worker -> lb +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build + +# 3. Готовность: 2 реплики healthy и UP в HAProxy, S3 реально пишет/читает, worker healthy +python scripts/ops.py wait +python scripts/ops.py wait-s3 +python scripts/ops.py worker-start # идемпотентно: дождаться healthy worker +python scripts/ops.py ps +``` + +Всё то же одной командой: `make up` (= init_secrets + config + up --build + ожидания). Прогрев после первой сборки ~30–40 с. + +```sh +# Публичные проверки +make check # офлайн, без сервера +make test # smoke lab5: 202 -> ready -> thumbnail 256x160 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh # Windows: scripts\restart.cmd +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh # Windows: scripts\stop-one.cmd +python scripts/ops.py restore + +# Собственные проверки lab5 +make test-pipeline # варианты по фактическим байтам (7 тестов) +make test-worker-outage # сценарий защиты уровня 3 (останавливает/запускает worker, рестарт API и broker) +make test-pipeline-faults # 10 внедрённых отказов уровня 4 (~2–3 мин; worker временно пересоздаётся) +python student-tests/run_all.py --base-url http://localhost:8080 # lab2-4 (+ test_pipeline.py) + +# Всё сразу с записью в evidence/ (~20 мин; только на учебном стенде) +python scripts/collect_evidence.py # или --quick (~10 мин) +``` + +**Операторские команды очереди** (выполняются в one-off контейнере образа worker, секреты не печатаются): + +```sh +python scripts/ops.py jobs stats # задания по статусам, outbox, DLQ, длина/pending/lag потока +python scripts/ops.py jobs failed # реестр dead_letters +python scripts/ops.py jobs show # задание + журнал событий (job_events) +python scripts/ops.py jobs replay # явный replay failed (make replay ID=...) +python scripts/ops.py jobs replay --all-failed --error-code retry_exhausted --dry-run +python scripts/ops.py worker-stop | worker-start | worker-kill | api-restart | broker-restart +python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (staging/, final/, originals/) +``` + +**Остановка.** Безопасная (volumes `pgdata`, `s3data`, `brokerdata` сохраняются): `make down` или `docker compose --env-file .env --profile ttl --profile faults down`. **Намеренное удаление всех учебных данных:** `docker compose --env-file .env --profile ttl --profile faults down -v` (удаляет БД, бакет и очередь безвозвратно). + +## Архитектура и решения + +``` +клиент ──HTTP──> HAProxy ──> API x2 ──[1] INSERT pending ─────────────────────────> PostgreSQL + ├──[2] PUT staging/ (Content-MD5) ───────> S3 (SeaweedFS) + ├──[3] COMMIT: queued + attachment_jobs + job_outbox ──> PostgreSQL + │ └─> 202 queued + └──[4] после ответа: relay XADD (fsync) ─────────> broker (Redis Streams) + │ XREADGROUP (prefetch 1) + worker (WORKER_CONCURRENCY consumer'ов) <───────────────────────────────────────────────┘ + claim: аренда в PostgreSQL (lease_token, lease_until) + читает staging (ключ из БД) -> SHA-256 -> PUT final//original + подпроцесс (RLIMIT_AS, timeout): thumbnail/crop/optimized -> PUT final//p1/ + COMMIT (fenced по lease_token): attachment_variants + state=ready ──> PostgreSQL + XACK ──> broker; DELETE staging (иначе — сборщик) + sweeper outbox + сборщик просроченных аренд: в каждой реплике API и в worker +``` + +**Источник истины** — PostgreSQL: состояние вложения, задание, outbox, результаты, журнал. S3 хранит байты (staging/ и final/), брокер — только транспорт уведомлений (может быть пересоздан из outbox/таблицы заданий). Брокер **не** входит в синхронный путь API: при его недоступности upload всё равно отвечает 202, строка ждёт в `job_outbox`; `/health/ready` от брокера не зависит. + +### Что сохранено в момент 202 + +1. Все байты оригинала в приватном S3 `staging/` (PUT с Content-MD5, SHA-256 в метаданных объекта и в БД). Ключ генерирует сервер, filename в ключ не попадает. +2. Метаданные вложения (`state=queued`, size, sha256, размеры, будущий конечный ключ). +3. Намерение обработать: строка `attachment_jobs` (job_id, attempt=1, max_attempts) и строка `job_outbox` с JSON-сообщением **в той же транзакции**. + +Сообщение в брокере на момент 202 может ещё не существовать — это нормально: его гарантирует outbox, а не HTTP-запрос. Остановка API сразу после 202 работу не теряет (тест A). + +### Состояния и окна отказа + +Внутреннее `pending` (приём идёт) наружу не видно; публичные: `queued → processing → ready | failed`. + +| Окно отказа | Что произойдёт | Где доказано | +| --- | --- | --- | +| после staging PUT до COMMIT задания | строка pending + staging-объект; клиенту ошибка (не 202). Сборщик через grace удаляет и строку, и объект | lab4 `test_attachment_consistency.py` (crash_after_s3_put теперь между staging и COMMIT) | +| после COMMIT до publish | строка outbox осталась; sweeper любого процесса публикует через `RELAY_FAST_PATH_GRACE_SECONDS` | `test_a` (api-fault-publish, `FAULT_INJECTION=crash_after_commit`) | +| после publish до подтверждения в outbox | XADD выполнен, DELETE строки — нет -> повторная публикация, дубль в брокере -> worker no-op | `test_b` (api-fault-dup, `crash_after_publish`) | +| после записи S3 до COMMIT результата | сообщение в PEL; перезапущенный consumer читает свой PEL, перехватывает свою аренду, перезаписывает **те же** байты, COMMIT | `test_c` (`fault-crash-after-effects`) | +| после COMMIT результата до ack | повторная доставка -> claim видит succeeded -> `redelivery_noop already_succeeded`, ack | `test_d` (`fault-crash-before-ack`) | +| worker завис дольше аренды | сборщик аренд -> attempt+1 через outbox -> другой consumer -> ready; проснувшийся старый результат отсечён fencing'ом (`lease_lost`) | `test_h` (`fault-slow`) | + +### Брокер: Redis Streams как durable at-least-once очередь + +Выбран Redis Streams (разрешён README): тот же закреплённый образ, что и для кэша, но **отдельный сервис** с другими гарантиями. Кэш сессий — без persistence и с LRU-вытеснением; объединять их нельзя. + +- **Durability:** `--appendonly yes --appendfsync always` — Redis отвечает на XADD только после fsync AOF. Успешный XADD — наш **publisher confirm**: сообщение переживает рестарт/падение брокера с volume `brokerdata` (проверено в сценарии `worker_outage`: lag=1 до и после рестарта брокера). `--maxmemory-policy noeviction`: поток никогда не вытесняется; при переполнении XADD упадёт, строка останется в outbox. +- **Доставка at-least-once:** consumer group; XREADGROUP кладёт сообщение в PEL до XACK. **Consumer ack** worker делает только после COMMIT результата (или перехода в retry/failed) — это другая граница, чем confirm при публикации. +- **Упавший consumer:** имя consumer = `-<поток>`; после `restart` контейнера hostname тот же, и consumer первым делом читает свой PEL (`XREADGROUP ... 0`). Если контейнер пересоздан, сообщения забирает любой живой consumer через `XAUTOCLAIM` после `JOB_CLAIM_IDLE_SECONDS` (75 с > аренды 60 с). +- **Prefetch/конкуренция:** каждый consumer-поток берёт `COUNT 1`, потоков `WORKER_CONCURRENCY=2`, контейнеров `WORKER_REPLICAS=1` -> одновременно в обработке не больше 2 сообщений. +- **Очистка потока:** `XTRIM MINID` до минимума из (самый старый pending, last-delivered-id+1) — удаляются только доставленные и подтверждённые записи. +- **Содержимое сообщения** — ровно `contracts/jobs.schema.json` (`schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at`): без байтов, credentials, URL и filename. Worker проверяет схему строго; мусор подтверждается и логируется (`test_j`); сообщение, доставленное больше `JOB_MAX_DELIVERIES` раз, переводит задание в `failed poison_message`. + +### Transactional outbox + +`job_outbox` пишется в одной транзакции с `attachment_jobs` и `state=queued`. Публикация: + +- **быстрый путь** — после отправки 202 (FastAPI background task) API берёт свою строку `FOR UPDATE SKIP LOCKED`, делает XADD, удаляет строку, пишет событие `published`, COMMIT; +- **sweeper** — в каждой реплике API и в worker раз в `RELAY_POLL_INTERVAL_SECONDS` публикует строки с `available_at <= now()`. У свежей строки `available_at = now()+2 c`, чтобы не соревноваться с быстрым путём; у retry — время следующей попытки (так реализован отложенный повтор без delayed-очереди брокера). + +`SKIP LOCKED` разносит строки между relay; несколько relay безопасны. Сбой между XADD и COMMIT relay даёт дубль — это и есть at-least-once, дубль гасит идемпотентный worker. + +### Worker, идемпотентность и аренды + +- **Ключ идемпотентности эффекта** — `(attachment_id, pipeline_version, variant)`: PRIMARY KEY `attachment_variants` + **детерминированные ключи S3** `final//original`, `final//p1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Повтор пишет те же байты по тем же ключам (Pillow-вывод детерминирован при одной версии библиотек — проверено в `test_a`/`test_c` сравнением SHA-256 с эталонной загрузкой). `job_id` не меняется при redelivery и retry. +- **Аренда (lease)**: claim атомарно ставит `status=processing, lease_token=, lease_owner=, lease_until=now()+JOB_TIMEOUT_SECONDS` (`SELECT ... FOR UPDATE`). Захват разрешён только для текущей попытки и если задание queued, или аренда истекла, или аренда принадлежит этому же consumer (его прошлая инкарнация мертва: consumer обрабатывает одно сообщение за раз). +- **Fencing**: публикация результата, retry и failed выполняются с условием `lease_token = <мой> AND status='processing'`. Worker, у которого аренду отобрали, ничего не может изменить (`test_h`, событие `lease_lost`). `ready` никогда не откатывается в processing старым дублем; `attachments.pipeline_version` обновляется только если новая версия ≥ текущей. +- **Зависший vs долгий worker:** аренда (60 с) > максимальной длительности попытки (обработка ≤ 30 с + S3-дедлайны), это проверяется при старте конфигурации. Пока аренда не истекла, второй worker задание не возьмёт (`leased_by_other`, ack дубля). Истекла + `JOB_LEASE_GRACE_SECONDS` -> сборщик аренд переводит задание в queued attempt+1 через outbox. Healthcheck worker'а — heartbeat, который пишется, только пока **все** consumer-потоки проходят цикл. +- **Staging** удаляется worker'ом после COMMIT и XACK; если не удалось — сборщик (проход D) удалит staging у ready-вложений. Staging failed-вложений хранится `STAGING_RETENTION_SECONDS` (24 ч) для replay, затем удаляется (replay после этого отклоняется: нужен повторный upload). Текущую работу (queued/processing) сборщик не трогает. + +### Варианты изображений (pipeline_version = 1) + +| Вариант | Формат | Правило | Параметры | +| --- | --- | --- | --- | +| thumbnail | JPEG | вписан в 256×256, пропорции, без увеличения, округление вниз (мин. 1 px): 320×200 → 256×160 | quality 85, optimize, baseline, 4:2:0 | +| crop | JPEG | центрированный квадрат min(w,h) → 128×128 (маленькие увеличиваются) | quality 85 | +| optimized | WebP | вписан в 1280×1280 без увеличения | lossy, quality 80, method 4; уменьшение каждого маленького файла не обещается | + +Общее: EXIF orientation применяется **до** расчёта размеров (400×100 c orientation=6 → thumbnail 64×256); прозрачность компонуется на белый (масштабирование RGBA в Pillow — с premultiplied alpha); **EXIF и XMP не копируются** (Pillow пишет их только при явной передаче), ICC-профиль сохраняется, чтобы не исказить цвет. Целочисленная арифметика размеров исключает ошибки округления float. + +**Ограничения ресурсов worker** (недоверенный файл, даже прошедший проверки API): + +| Ресурс | Ограничение | Поведение | +| --- | --- | --- | +| байты | staging-объект > `MAX_UPLOAD_BYTES` не читается; SHA-256 и размер сверяются с БД | failed `invalid_image` / `processing_failed` | +| пиксели | размеры из заголовка до декодирования, > `MAX_IMAGE_PIXELS` | failed `invalid_image` | +| память | обработка в отдельном процессе (forkserver) с `RLIMIT_AS = WORKER_IMAGE_MEMORY_MB` (1024) | MemoryError/смерть процесса → failed `resource_limit` | +| время | процесс убивается через `WORKER_IMAGE_TIMEOUT_SECONDS` (30), `RLIMIT_CPU` — страховка | временная ошибка `processing_timeout`, повтор ≤ JOB_MAX_ATTEMPTS | + +JPEG декодируется сразу в уменьшенном масштабе (`draft`), поэтому 40 Мп JPEG — пик 46 MiB; худший случай 40 Мп RGBA PNG — **361 MiB** (было 518 MiB, пока альфа компоновалась до масштабирования; порядок операций изменён). Отсюда лимит контейнера worker 1280 MiB при 2 подпроцессах. Замер: `evidence/local/imaging_memory.txt`; проверка самих лимитов: `python -m src.imaging FILE --memory-mb 200` → `resource_limit`, `--timeout 0.3` → `timeout`. + +### Retry, backoff, DLQ, replay + +- **Классификация ошибок.** Временные: S3 недоступен/таймаут, таймаут обработки, внедрённый `injected_transient` → retry. Постоянные: битое/неподдерживаемое изображение, превышение пикселей/байт, лимит памяти, нет источника, несовпадение SHA-256 staging → сразу `failed` без повторов (`test_f`). Если недоступна сама БД, сообщение **не** подтверждается и вернётся из PEL — решение о retry без БД принять нельзя. +- **Backoff с jitter** («equal jitter»): `d = min(JOB_RETRY_MAX_SECONDS, JOB_RETRY_BASE_SECONDS·2^(attempt−1))`, задержка равномерно в `[d/2, d]` (нижняя граница гарантирует паузу, случайная часть разносит повторы многих заданий). Реализован через `job_outbox.available_at`: попытка не начнётся раньше (проверено по журналу в `test_e`). +- **Предел:** `JOB_MAX_ATTEMPTS=5`; затем `failed retry_exhausted`, запись в `dead_letters` (реестр failed/DLQ), событие в `job_events`. +- **Replay** — только явная команда оператора (`ops.py jobs replay`). Разрешён только для `failed`; успешное задание **не исполняется повторно** (`noop`, событие `replay_refused`); без staging-источника — отказ. Новая попытка: attempt+1, свежий бюджет попыток, `replay_count+1`, `dead_letters.replayed_at`, публикация через outbox (`test_g`). + +### Роли и сервисы + +| Роль | Сервис | Версия | Health/readiness | Volume | Сеть/порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | haproxy:2.8.30-alpine | `/lb-health`; проверка `/health/ready` реплик | — | backend; 127.0.0.1:8080 (единственный публичный), 8404 stats | — | +| api ×2 | `api` | сборка из Dockerfile | `/health/ready` (PG + схема 005 + не drain) | нет (read-only rootfs, tmpfs /tmp) | backend, storage, queue; 8080 только внутри | secrets: database_url, cache_url, s3_*, broker_url | +| worker | `worker` | тот же образ | heartbeat-файл < 90 с | нет (read-only, tmpfs) | backend, storage, queue | secrets: database_url, s3_*, broker_url | +| broker | `broker` | redis:7.4.1-alpine, AOF always, noeviction | `redis-cli ping` | brokerdata | queue (internal) | secrets: broker_password | +| postgres | `postgres` | postgres:16.4-alpine | pg_isready | pgdata | backend | secrets: postgres_password | +| кэш сессий | `redis` | redis:7.4.1-alpine, без persistence | ping | — | backend | secrets: redis_password | +| s3 | `s3` | chrislusf/seaweedfs:4.50 | /healthz + прогрев `ops.py wait-s3` | s3data | storage (internal) | secrets: s3_config.json | +| bootstrap | `migrate`, `s3-init` | тот же образ | one-shot, идемпотентны | — | backend / storage | — | + +Топология брокера (поток + группа) создаётся кодом идемпотентно (`XGROUP CREATE ... MKSTREAM`, BUSYGROUP = no-op). + +### Переменные `.env.example` → реальная конфигурация + +| Логическое имя (skeleton) | Реализация | +| --- | --- | +| `BROKER_URL_FILE` | `/run/secrets/broker_url` (создаёт `init_secrets.py`) | +| `WORKER_CONCURRENCY=2` | число consumer-потоков в контейнере worker (prefetch 1 на поток) | +| `JOB_MAX_ATTEMPTS=5` | `attachment_jobs.max_attempts` при создании задания | +| `JOB_TIMEOUT_SECONDS=60` | срок аренды; ≥ `WORKER_IMAGE_TIMEOUT_SECONDS + 20` (проверка при старте) | +| `STAGING_RETENTION_SECONDS=86400` | хранение staging failed-вложений (окно replay) | +| дополнительно | `JOB_LEASE_GRACE_SECONDS`, `JOB_CLAIM_IDLE_SECONDS`, `JOB_MAX_DELIVERIES`, `JOB_RETRY_BASE/MAX_SECONDS`, `WORKER_IMAGE_TIMEOUT_SECONDS`, `WORKER_IMAGE_MEMORY_MB`, `RELAY_*`, `BROKER_STREAM/GROUP`, `PIPELINE_VERSION`, `WORKER_REPLICAS` — описаны в `.env.example` | + +### Что изменено относительно lab4 + +- `migrations/005_async_processing.sql`: состояния `queued/processing/failed`, `staging_key`, `error_code`, `pipeline_version`; таблицы `attachment_variants`, `attachment_jobs`, `job_outbox`, `dead_letters`, `job_events`. +- `src/main.py`: upload → staging + задание + outbox → **202**; content: 409 для queued/processing/failed, варианты; сообщение с failed-вложением → 409, с queued/processing — разрешено. +- Новые модули: `src/broker.py`, `src/outbox.py`, `src/jobs.py` (переходы состояний), `src/worker.py`, `src/imaging.py`, `src/jobs_admin.py`. +- `src/reconcile.py`: префиксы `staging/`, `final/`, `originals/` (lab4), проход D — очистка staging. +- `compose.yaml`: сервисы `broker`, `worker`, `api-fault-publish`, `api-fault-dup`, сеть `queue`; `compose.faults.yaml`. +- Тесты lab4, ожидавшие 201, переведены на 202 + ожидание ready (`test_attachments.py`, `test_attachment_consistency.py`, `s3_outage.py`, `upload_memory.py`). Публичные `tests/` и контракт не изменялись. + +## Доказательства критериев + + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | `PASS: lab5 public API scenario` (202 → ready, оригинал, JPEG thumbnail 256×160) | evidence/local/smoke.txt | +| Минимум lab4 сохранён | `student-tests/run_all.py` через 2 реплики | 91 тест OK (33 skip — требуют сеть compose), race 100/100, persistence и failover PASS | evidence/local/student-tests.txt, persistence.txt, failover.txt | +| 3: 202 только после durable staging + задания | `test_pipeline.py`, `test_i`, `test_a` | 202 приходит до публикации, задача доходит до ready даже при падении API сразу после 202 | pipeline_and_attachments.txt, pipeline_faults.txt | +| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта | compose: `broker` (AOF always) + `worker`; `test_c`, `test_d` | ack только после COMMIT; падение до ack → повторная доставка без дубля | compose.yaml, pipeline_faults_events.txt | +| 3: worker недоступен → API не делает resize | `make test-worker-outage` | worker остановлен: 202, GET queued, content 409, сообщения 201; после старта worker — ready | worker_outage.txt/.json | +| 3: состояния queued/processing/ready/failed, original неизменен, thumbnail по контракту | `test_pipeline.py` | SHA-256 оригинала совпадает; thumbnail 256×160 по SOF-заголовку JPEG | pipeline_and_attachments.txt | +| 3: ограниченный retry, failed с безопасным error_code, повтор не создаёт 2-й набор | `test_e`, `test_g`, `assert_single_result` | ровно 3 строки variants и 1 событие succeeded во всех сценариях; `retry_exhausted`/`invalid_image` | pipeline_faults.txt | +| 3: демонстрация worker stop → upload → рестарт API → рестарт broker → worker → ready | `make test-worker-outage` | lag=1 до и после рестарта брокера, ready через ~0.03 с после старта worker | worker_outage.txt | +| 4: crop 128 и WebP; лимиты bytes/pixels/time/memory | `test_pipeline.py`, `python -m src.imaging` | crop 128×128, WebP по RIFF-заголовку; EXIF удалён; orientation/alpha/малые исходники; `resource_limit`, `timeout` | pipeline_and_attachments.txt, imaging_memory.txt | +| 4: outbox, crash между COMMIT и publish | `test_a` (api-fault-publish) | `fault_injected` от api-fault-publish, `published` другим процессом, attempt=1, ready | pipeline_faults_events.txt | +| 4: backoff+jitter, постоянные/временные, DLQ, безопасный replay | `test_e`, `test_f`, `test_g` | интервалы в [d/2, d]; permanent без повторов; replay → ready; повторный replay → noop | pipeline_faults.txt | +| 4: остановка worker после записи объекта до ack | `test_c`, `test_d` | `lease_takeover` своим consumer после рестарта; те же SHA-256; `redelivery_noop already_succeeded` | worker_fault_log.txt | +| 4: lease для зависшего processing, ограниченная конкуренция, очистка staging | `test_h`, `test_i` | `lease_expired` → attempt 2 у другого consumer → ready; старый `lease_lost`; staging_key=NULL | pipeline_faults_events.txt | + +## Эксперименты и отказы + +Локальная песочница: 2 vCPU, 7 GiB RAM, Linux, Python 3.13 (в образе — 3.12.7), без лимитов cgroup. Набор: синтетические изображения (`tests/fixtures/sample.png` 320×200, генерируемые 40 Мп PNG/JPEG). Повторов: сценарии отказов — 1 полный прогон, все 10 PASS; задержка — 30 последовательных загрузок. Основной compose-стенд: Windows 10, 32GB, Ryzen 5 7500f; доказательства в `/evidence`. + +| Измерение | Результат | +| --- | --- | +| upload → 202 (sample.png, 109 KiB) | p50 16 мс, p95 29 мс | +| upload → ready (опрос каждые 0.2 с) | p50 0.44 с, p95 0.45 с, max 0.45 с | +| пик RSS подпроцесса обработки | 23 MiB (320×200), 46 MiB (40 Мп JPEG), 361 MiB (40 Мп RGBA PNG) | +| время обработки 40 Мп | JPEG 0.2 с, RGBA PNG 1.8 с | + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | +| --- | --- | --- | --- | +| worker остановлен, upload | 202 queued, content 409, остальной API работает | так и есть, сообщения 201 | ready после старта worker; потерь нет | +| рестарт API и broker (volumes целы) при неподтверждённой задаче | задача остаётся в потоке | lag=1 до и после | — / нет | +| API умер после COMMIT до XADD | sweeper другого процесса публикует | published другим actor через ~2 с | ≤ RELAY_FAST_PATH_GRACE + 1 с / нет | +| API умер после XADD до подтверждения outbox | дубль, no-op | `redelivery_noop already_succeeded` | — / нет, 1 набор вариантов | +| worker умер после записи S3 до COMMIT | свой PEL после рестарта, те же байты | `lease_takeover`, SHA-256 = эталон | ~1–2 с (рестарт процесса) / нет | +| worker умер после COMMIT до XACK | повторная доставка no-op | `redelivery_noop already_succeeded` | ~1–2 с / нет | +| worker «завис» дольше аренды | новая попытка, fencing старой | `lease_expired` → ready на attempt 2 → `lease_lost` | аренда + grace + ≤5 с / нет | +| временные ошибки ×2 | backoff+jitter, ready на 3-й | задержки 0.5–1 с и 1–2 с (base 1, cap 4 в стенде отказов) | — | +| временная ошибка всегда | failed retry_exhausted, DLQ, replay | 5 попыток → failed → replay → ready → replay ещё раз = noop | — | +| мусор в потоке | ack, worker жив | pending=0 | — | + + +## Ограничения и следующие шаги + +- **Один Docker-host:** брокер, PostgreSQL и S3 — по одному экземпляру. Выдерживаются рестарты и падения процессов с сохранением volumes, но не потеря диска/хоста. Redis без репликации: при потере volume брокера неопубликованные строки outbox сохранятся, а уже опубликованные, но не обработанные задачи потребуют повторной публикации (`jobs replay` работает только для failed; для queued без сообщения нужен ручной requeue — сборщик аренд покрывает только processing). +- **At-least-once + идемпотентность ≠ exactly-once:** дубли сообщений и повторная запись тех же объектов S3 возможны и ожидаемы; уникален только опубликованный результат. Детерминизм байтов вариантов гарантирован в пределах одной версии Pillow/libjpeg/libwebp; при обновлении библиотек нужна новая `pipeline_version` (версионирование нескольких pipeline — трек 5, не заявлен). +- Fencing защищает БД, но не S3: «проснувшийся» старый worker может перезаписать объект варианта теми же (детерминированными) байтами после публикации результата. +- Классификация ошибок упрощена: таймаут обработки считается временным, поэтому «тяжёлый» файл может занять до 5 попыток по 30 с. +- Маркеры отказов в filename работают **только** при `WORKER_FAULTS=true` (`compose.faults.yaml`); в обычном стенде filename на обработку не влияет. +- Старые ready-вложения lab4 остаются без вариантов (запрос варианта → 404 по контракту); фоновой догенерации нет. +- HTTP без TLS — только локальный учебный стенд (TLS — lab7). + +## Вопросы защиты — кратко + +- **Publisher confirm vs consumer ack.** Confirm — брокер принял и сохранил сообщение (у нас: ответ XADD после fsync AOF), это граница «producer → брокер»; после него relay удаляет строку outbox. Ack (XACK) — consumer сообщает, что обработка устойчиво завершена (COMMIT результата/перевода в retry/failed), это граница «брокер → consumer»; до него сообщение остаётся в PEL и будет доставлено снова. +- **Что сохранено в момент 202?** Байты в S3 staging, метаданные, задание и outbox в одной транзакции PostgreSQL. Сообщения в брокере может ещё не быть — его гарантирует outbox. +- **Почему повторная доставка нормальна и где ключ идемпотентности?** At-least-once неизбежно даёт дубли (падение до ack, повтор publish). Ключ эффекта — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants` + детерминированные ключи S3; задание — `job_id` (UNIQUE `(attachment_id, pipeline_version)`); повтор успешного — no-op с ack. +- **Как отличить зависший worker от долго работающего?** Аренда с заведомо большим сроком, чем максимальная попытка (жёсткий таймаут подпроцесса + дедлайны S3). Пока аренда действует — задание не отдаётся второму worker; после истечения + grace — новая попытка, а результат старой отсекается fencing-токеном. Heartbeat контейнера проверяет, что все consumer-потоки живы. diff --git a/docs/lab5/evidence/README.md b/docs/lab5/evidence/README.md new file mode 100644 index 0000000..8df7001 --- /dev/null +++ b/docs/lab5/evidence/README.md @@ -0,0 +1,5 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. diff --git a/docs/lab5/evidence/SUMMARY.txt b/docs/lab5/evidence/SUMMARY.txt new file mode 100644 index 0000000..12c408b --- /dev/null +++ b/docs/lab5/evidence/SUMMARY.txt @@ -0,0 +1,54 @@ +# 2026-11-10 14:26:03 UTC +OK Контракт (offline) evidence/check.txt +OK Compose config evidence/check.txt +OK Стенд: контейнеры evidence/stand.txt +OK Стенд: реплики и статус в LB evidence/stand.txt +OK Стенд: лимиты CPU/RAM evidence/stand.txt +OK Стенд: опубликованные порты (только lb) evidence/stand.txt +OK Образ SeaweedFS (закреплённая версия) evidence/stand.txt +OK lab5: broker (Redis AOF) и worker evidence/stand.txt +OK lab5: конфигурация durability брокера evidence/stand.txt +OK S3 bootstrap: лог первого запуска s3-init evidence/s3_bootstrap.txt +OK S3 bootstrap: повторный запуск (идемпотентность) evidence/s3_bootstrap.txt +OK Публичный smoke evidence/smoke.txt +OK Собственные тесты (HTTP через LB, включая test_attachments.py) evidence/student-tests.txt +OK Тесты с доступом к БД (внутри compose-сети) evidence/student-tests.txt +OK Истечение сессии (api-short-ttl, TTL 2 с) evidence/student-tests.txt +OK Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) evidence/consistency.txt +OK Сверка БД <-> S3 (dry-run) evidence/reconcile.txt +OK Приватный бакет вне API, права ключей, закрытые админ-порты evidence/s3_private.txt +OK На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes evidence/s3_private.txt +OK API: файлы в /tmp (tmpfs) после загрузок evidence/s3_private.txt +OK Пик памяти: 10 одновременных загрузок по 9.5 MiB evidence/upload_memory.txt +OK Пик памяти: 20 одновременных загрузок evidence/upload_memory.txt +OK Failover #1: реплики до evidence/failover.txt +OK Failover #1: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #1: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #1: restore evidence/failover.txt +OK Failover #2: реплики до evidence/failover.txt +OK Failover #2: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #2: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #2: restore evidence/failover.txt +OK Failover #3: реплики до evidence/failover.txt +OK Failover #3: make test-failover (вкл. скачивание файла через другую реплику) evidence/failover.txt +OK Failover #3: реплики после (остановлена именно цель теста) evidence/failover.txt +OK Failover #3: restore evidence/failover.txt +OK Серия отказов: каждая реплика, graceful stop evidence/failover_series.txt +OK Серия отказов: каждая реплика, SIGKILL evidence/failover_series.txt +OK Недоступность S3 и восстановление evidence/s3_outage.txt +OK Варианты изображений по фактическим байтам (test_pipeline.py) evidence/pipeline.txt +OK Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready evidence/worker_outage.txt +OK Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease) evidence/pipeline_faults.txt +OK Очередь после сценариев: задания, outbox, DLQ, lag evidence/pipeline_faults.txt +OK Реестр failed (DLQ) evidence/pipeline_faults.txt +OK Worker: последние строки лога evidence/worker_log_sample.txt +OK Rolling update lab5 -> lab5.1 под нагрузкой evidence/rolling_update.txt +OK Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой evidence/rolling_update.txt +OK Реплики после обновлений evidence/rolling_update.txt +OK Persistence: контейнеры до evidence/persistence.txt +OK Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB) evidence/persistence.txt +OK Persistence: контейнеры после (Status = Up N seconds) evidence/persistence.txt +OK Persistence: сверка БД <-> S3 после рестарта evidence/persistence.txt +OK Нагрузка 30 с, concurrency 8 evidence/load.txt +OK CPU/RAM после нагрузки evidence/load.txt +OK Лог LB (последние строки, без секретов) evidence/lb_log_sample.txt diff --git a/docs/lab5/evidence/check.txt b/docs/lab5/evidence/check.txt new file mode 100644 index 0000000..a85d620 --- /dev/null +++ b/docs/lab5/evidence/check.txt @@ -0,0 +1,7 @@ +# 2026-11-10 14:13:49 UTC | Контракт (offline) | exit 0 | 0.1s +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +# 2026-11-10 14:13:49 UTC | Compose config | exit 0 | 0.3s +$ docker compose --env-file .env config --quiet + + diff --git a/docs/lab5/evidence/consistency.txt b/docs/lab5/evidence/consistency.txt new file mode 100644 index 0000000..8ecc8f1 --- /dev/null +++ b/docs/lab5/evidence/consistency.txt @@ -0,0 +1,75 @@ +# 2026-11-10 14:15:11 UTC | Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*) | exit 0 | 20.0s +$ python scripts/ops.py in-network consistency + Container messenger-lab5-redis-1 Running + Container messenger-lab5-api-1 Running + Container messenger-lab5-api-2 Running + Container messenger-lab5-postgres-1 Running + Container messenger-lab5-s3-1 Running + Container messenger-lab5-api-fault-s3-1 Creating + Container messenger-lab5-api-fault-db-1 Creating + Container messenger-lab5-api-fault-crash-1 Creating + Container messenger-lab5-api-fault-s3-1 Created + Container messenger-lab5-api-fault-db-1 Created + Container messenger-lab5-api-fault-crash-1 Created + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-s3-init-1 Starting + Container messenger-lab5-migrate-1 Starting + Container messenger-lab5-s3-init-1 Started + Container messenger-lab5-migrate-1 Started + Container messenger-lab5-migrate-1 Waiting + Container messenger-lab5-s3-init-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-redis-1 Waiting + Container messenger-lab5-redis-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Exited + Container messenger-lab5-s3-init-1 Exited + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-db-1 Starting + Container messenger-lab5-api-fault-s3-1 Starting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-crash-1 Starting + Container messenger-lab5-api-fault-db-1 Started + Container messenger-lab5-api-fault-s3-1 Started + Container messenger-lab5-api-fault-crash-1 Started + Container messenger-lab5-api-run-e891dd567ea2 Creating + Container messenger-lab5-api-run-e891dd567ea2 Created +ready: http://api-fault-s3:8080/health/ready + Container messenger-lab5-api-run-a25e14654677 Creating + Container messenger-lab5-api-run-a25e14654677 Created +ready: http://api-fault-db:8080/health/ready + Container messenger-lab5-api-run-53cacd2097d3 Creating + Container messenger-lab5-api-run-53cacd2097d3 Created +ready: http://api-fault-crash:8080/health/ready + Container messenger-lab5-api-run-66f13fe194f4 Creating + Container messenger-lab5-api-run-66f13fe194f4 Created +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... ok +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... ok +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... ok +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... ok +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... ok +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... ok +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... ok +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... ok +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 4.594s + +OK diff --git a/docs/lab5/evidence/failover.txt b/docs/lab5/evidence/failover.txt new file mode 100644 index 0000000..b52781d --- /dev/null +++ b/docs/lab5/evidence/failover.txt @@ -0,0 +1,66 @@ +# 2026-11-10 14:15:40 UTC | Failover #1: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP +# 2026-11-10 14:16:30 UTC | Failover #1: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 49.3s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 4.297} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:16:30 UTC | Failover #1: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 exited unhealthy messenger-api:lab5 - - + +# 2026-11-10 14:16:37 UTC | Failover #1: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-10T14:16:37.028Z restore: started ['messenger-lab5-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-10 14:16:37 UTC | Failover #2: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:16:54 UTC | Failover #2: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 16.8s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 4.532} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:16:54 UTC | Failover #2: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 exited unhealthy messenger-api:lab5 - - + +# 2026-11-10 14:17:01 UTC | Failover #2: restore | exit 0 | 6.6s +$ python scripts/ops.py restore +2026-11-10T14:17:00.997Z restore: started ['messenger-lab5-api-2']; 2 replicas healthy and UP in LB after 6.5s + +# 2026-11-10 14:17:01 UTC | Failover #3: реплики до | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:17:18 UTC | Failover #3: make test-failover (вкл. скачивание файла через другую реплику) | exit 0 | 17.0s +$ python tests/resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.437} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:17:18 UTC | Failover #3: реплики после (остановлена именно цель теста) | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 exited unhealthy messenger-api:lab5 - - +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:17:25 UTC | Failover #3: restore | exit 0 | 6.7s +$ python scripts/ops.py restore +2026-11-10T14:17:25.258Z restore: started ['messenger-lab5-api-1']; 2 replicas healthy and UP in LB after 6.6s + +# остановленные публичным тестом реплики: ['3a0e06f14a63', '3a0e06f14a63', '4b1faad0399c'] diff --git a/docs/lab5/evidence/failover_series.json b/docs/lab5/evidence/failover_series.json new file mode 100644 index 0000000..f6d5800 --- /dev/null +++ b/docs/lab5/evidence/failover_series.json @@ -0,0 +1,294 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "rounds": [ + { + "target_instance": "3a0e06f14a63", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.47, + "lb_detect_down_seconds": 1.88, + "target_last_served_after_stop_seconds": 1.83, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 6.72, + "lb_back_up_seconds": 0.95, + "target_back_in_rotation_seconds": 6.72, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1083, + "ok": 1083, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 216.6, + "write_ok_rps": 125.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1238, + "ok": 1238, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 123.8, + "write_ok_rps": 74.0, + "latency_ms_ok": { + "p50": 31.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 531, + "ok": 531, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 106.2, + "write_ok_rps": 60.8, + "latency_ms_ok": { + "p50": 63.0, + "p95": 109.0, + "p99": 391.0 + }, + "instances": [ + "4b1faad0399c" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 29.98, + "requests": 1831, + "ok": 1831, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 183.4, + "write_ok_rps": 110.1, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 171.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 6.88, + [ + "api1" + ] + ], + [ + 20.95, + [ + "api1", + "api2" + ] + ], + [ + 21.92, + [ + "api1" + ] + ], + [ + 24.47, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2768, + "stored_messages": 2768, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + }, + { + "target_instance": "4b1faad0399c", + "mode": "stop", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 4.55, + "lb_detect_down_seconds": 2.05, + "target_last_served_after_stop_seconds": 1.97, + "target_served_after_hook_returned": false, + "errors_during_outage": 0, + "error_kinds": [], + "error_window_seconds": 0.0, + "restore_hook_seconds": 7.06, + "lb_back_up_seconds": 2.08, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1346, + "ok": 1346, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 269.2, + "write_ok_rps": 160.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 1371, + "ok": 1371, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 137.1, + "write_ok_rps": 82.7, + "latency_ms_ok": { + "p50": 31.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 564, + "ok": 564, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 112.8, + "write_ok_rps": 66.8, + "latency_ms_ok": { + "p50": 62.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.38, + "requests": 1657, + "ok": 1657, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 159.7, + "write_ok_rps": 90.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.05, + [ + "api2" + ] + ], + [ + 22.08, + [ + "api1", + "api2" + ] + ], + [ + 23.05, + [ + "api2" + ] + ], + [ + 25.51, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2905, + "stored_messages": 2905, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab5/evidence/failover_series.txt b/docs/lab5/evidence/failover_series.txt new file mode 100644 index 0000000..003e978 --- /dev/null +++ b/docs/lab5/evidence/failover_series.txt @@ -0,0 +1,17 @@ +# 2026-11-10 14:18:28 UTC | Серия отказов: каждая реплика, graceful stop | exit 0 | 62.8s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --output evidence/failover_series.json +replicas observed through LB: ['3a0e06f14a63', '4b1faad0399c'] +-- stopping 3a0e06f14a63 (mode=stop) under load + hook 4.47s | LB DOWN after 1.88 s | errors 0 (-) | RPS before 216.6, outage 123.8, after restore 183.4 | p95 outage 109.0 ms | back in rotation 6.72 s | data ok=True +-- stopping 4b1faad0399c (mode=stop) under load + hook 4.55s | LB DOWN after 2.05 s | errors 0 (-) | RPS before 269.2, outage 137.1, after restore 159.7 | p95 outage 94.0 ms | back in rotation 7.06 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates +# 2026-11-10 14:19:31 UTC | Серия отказов: каждая реплика, SIGKILL | exit 0 | 63.0s +$ python student-tests/failover_series.py --base-url http://127.0.0.1:8080 --mode kill --output evidence/failover_series_kill.json +replicas observed through LB: ['3a0e06f14a63', '4b1faad0399c'] +-- stopping 3a0e06f14a63 (mode=kill) under load + hook 1.20s | LB DOWN after 2.39 s | errors 3 (502) | RPS before 280.4, outage 89.8, after restore 171.1 | p95 outage 110.0 ms | back in rotation 7.06 s | data ok=True +-- stopping 4b1faad0399c (mode=kill) under load + hook 0.83s | LB DOWN after 2.34 s | errors 6 (502) | RPS before 260.4, outage 89.4, after restore 156.0 | p95 outage 110.0 ms | back in rotation 7.28 s | data ok=True +PASS: every replica stopped and restored; confirmed data intact, no duplicates + diff --git a/docs/lab5/evidence/failover_series_kill.json b/docs/lab5/evidence/failover_series_kill.json new file mode 100644 index 0000000..1291a1e --- /dev/null +++ b/docs/lab5/evidence/failover_series_kill.json @@ -0,0 +1,298 @@ +{ + "scenario": "stop each API replica in turn under load, restore before the next", + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201" + }, + "replicas": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "rounds": [ + { + "target_instance": "3a0e06f14a63", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 1.2, + "lb_detect_down_seconds": 2.39, + "target_last_served_after_stop_seconds": 0.56, + "target_served_after_hook_returned": false, + "errors_during_outage": 3, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.64, + "restore_hook_seconds": 7.12, + "lb_back_up_seconds": 1.42, + "target_back_in_rotation_seconds": 7.06, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1402, + "ok": 1402, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 280.4, + "write_ok_rps": 161.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 94.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 901, + "ok": 898, + "errors": 3, + "error_ratio": 0.0033, + "ok_rps": 89.8, + "write_ok_rps": 52.6, + "latency_ms_ok": { + "p50": 32.0, + "p95": 110.0, + "p99": 1015.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 518, + "ok": 518, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 103.6, + "write_ok_rps": 60.8, + "latency_ms_ok": { + "p50": 78.0, + "p95": 110.0, + "p99": 187.0 + }, + "instances": [ + "4b1faad0399c" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.44, + "requests": 1786, + "ok": 1786, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 171.1, + "write_ok_rps": 101.4, + "latency_ms_ok": { + "p50": 16.0, + "p95": 94.0, + "p99": 187.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.39, + [ + "api1" + ] + ], + [ + 21.42, + [ + "api1", + "api2" + ] + ], + [ + 21.53, + [ + "api1" + ] + ], + [ + 24.02, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2698, + "stored_messages": 2698, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 1, + "ok": true + } + }, + { + "target_instance": "4b1faad0399c", + "mode": "kill", + "lb_up_before": [ + "api1", + "api2" + ], + "stop_hook_seconds": 0.83, + "lb_detect_down_seconds": 2.34, + "target_last_served_after_stop_seconds": 0.09, + "target_served_after_hook_returned": false, + "errors_during_outage": 6, + "error_kinds": [ + "502" + ], + "error_window_seconds": 0.28, + "restore_hook_seconds": 7.3, + "lb_back_up_seconds": 1.33, + "target_back_in_rotation_seconds": 7.28, + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1302, + "ok": 1302, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 260.4, + "write_ok_rps": 153.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 125.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_first_10s": { + "from_s": 5.0, + "to_s": 15.0, + "requests": 900, + "ok": 894, + "errors": 6, + "error_ratio": 0.0067, + "ok_rps": 89.4, + "write_ok_rps": 52.5, + "latency_ms_ok": { + "p50": 32.0, + "p95": 110.0, + "p99": 1016.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "outage_rest": { + "from_s": 15.0, + "to_s": 20.0, + "requests": 565, + "ok": 565, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 113.0, + "write_ok_rps": 66.6, + "latency_ms_ok": { + "p50": 62.0, + "p95": 109.0, + "p99": 187.0 + }, + "instances": [ + "3a0e06f14a63" + ] + }, + "after_restore": { + "from_s": 20.0, + "to_s": 30.5, + "requests": 1638, + "ok": 1638, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 156.0, + "write_ok_rps": 92.5, + "latency_ms_ok": { + "p50": 16.0, + "p95": 109.0, + "p99": 188.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 7.34, + [ + "api2" + ] + ], + [ + 21.33, + [ + "api1", + "api2" + ] + ], + [ + 22.2, + [ + "api2" + ] + ], + [ + 24.74, + [ + "api1", + "api2" + ] + ] + ], + "data": { + "confirmed_writes": 2599, + "stored_messages": 2599, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 4, + "ok": true + } + } + ], + "pass": true +} diff --git a/docs/lab5/evidence/instances.txt b/docs/lab5/evidence/instances.txt new file mode 100644 index 0000000..3a3d302 --- /dev/null +++ b/docs/lab5/evidence/instances.txt @@ -0,0 +1,22 @@ +# 2026-11-10 14:14:03 UTC | одна сессия, 20 запросов GET /users/me через LB + 1 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019A78-1F90-6AF326AB268-0BC1-0008 + 2 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019A88-1F90-6AF326AB27B-0BC2-0008 + 3 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019A8E-1F90-6AF326AB27F-0BC3-0008 + 4 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019A9C-1F90-6AF326AB282-0BC4-0008 + 5 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019AA4-1F90-6AF326AB296-0BC5-0008 + 6 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AA8-1F90-6AF326AB2A5-0BC6-0008 + 7 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019AAA-1F90-6AF326AB2B4-0BC7-0008 + 8 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AB0-1F90-6AF326AB2C4-0BC8-0008 + 9 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019ABA-1F90-6AF326AB2D3-0BC9-0008 +10 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019ABE-1F90-6AF326AB2E2-0BCA-0008 +11 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019ACC-1F90-6AF326AB2E5-0BCB-0008 +12 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019ADC-1F90-6AF326AB2F2-0BCC-0008 +13 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019ADE-1F90-6AF326AB300-0BCD-0008 +14 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AE4-1F90-6AF326AB303-0BCE-0008 +15 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019AEA-1F90-6AF326AB305-0BCF-0008 +16 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019AF6-1F90-6AF326AB320-0BD0-0008 +17 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019B06-1F90-6AF326AB323-0BD1-0008 +18 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019B0C-1F90-6AF326AB33F-0BD2-0008 +19 200 X-Instance-Id=4b1faad0399c X-Request-Id=AC1400019B16-1F90-6AF326AB34E-0BD3-0008 +20 200 X-Instance-Id=3a0e06f14a63 X-Request-Id=AC1400019B24-1F90-6AF326AB351-0BD4-0008 +разных реплик: 2 diff --git a/docs/lab5/evidence/lb_log_sample.txt b/docs/lab5/evidence/lb_log_sample.txt new file mode 100644 index 0000000..4a390a1 --- /dev/null +++ b/docs/lab5/evidence/lb_log_sample.txt @@ -0,0 +1,62 @@ +# 2026-11-10 14:26:03 UTC | Лог LB (последние строки, без секретов) | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 60 lb +lb-1 | 172.20.0.1:36070 [10/Nov/2026:14:26:00.806] public api/api1 0/0/0/24/25 201 421 ---- 7/7/4/3/0 0/0 rid=AC1400018CE6-1F90-6AF32978326-1436-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36082 [10/Nov/2026:14:26:00.818] public api/api1 0/0/0/14/14 201 421 ---- 8/8/4/2/0 0/0 rid=AC1400018CF2-1F90-6AF32978332-1438-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36116 [10/Nov/2026:14:26:00.827] public api/api2 0/0/0/6/6 201 421 ---- 7/7/3/1/0 0/0 rid=AC1400018D14-1F90-6AF3297833B-143B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36130 [10/Nov/2026:14:26:00.831] public api/api2 0/0/0/7/8 201 421 ---- 8/8/4/1/0 0/0 rid=AC1400018D22-1F90-6AF3297833F-143D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36078 [10/Nov/2026:14:26:00.822] public api/api1 0/0/0/18/18 201 421 ---- 8/8/3/2/0 0/0 rid=AC1400018CEE-1F90-6AF3297832C-143A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36104 [10/Nov/2026:14:26:00.838] public api/api2 0/0/0/6/6 201 421 ---- 8/8/4/1/0 0/0 rid=AC1400018D08-1F90-6AF32978338-143F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36182 [10/Nov/2026:14:26:00.842] public api/api2 0/0/0/6/6 201 421 ---- 8/8/4/0/0 0/0 rid=AC1400018D56-1F90-6AF3297834A-1441-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36196 [10/Nov/2026:14:26:00.850] public api/api2 0/0/0/4/4 201 421 ---- 8/8/7/1/0 0/0 rid=AC1400018D64-1F90-6AF32978352-1443-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36122 [10/Nov/2026:14:26:00.854] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/1/0 0/0 rid=AC1400018D1A-1F90-6AF3297833D-1445-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36212 [10/Nov/2026:14:26:00.856] public api/api2 0/0/0/6/6 201 421 ---- 8/8/6/0/0 0/0 rid=AC1400018D74-1F90-6AF32978358-1447-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36224 [10/Nov/2026:14:26:00.884] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018D80-1F90-6AF3297835E-1449-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36118 [10/Nov/2026:14:26:00.828] public api/api1 0/0/0/76/76 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018D16-1F90-6AF3297833C-143C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36156 [10/Nov/2026:14:26:00.834] public api/api1 0/0/0/78/78 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018D3C-1F90-6AF32978342-143E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36276 [10/Nov/2026:14:26:00.914] public api/api2 0/0/0/6/6 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018DB4-1F90-6AF32978392-144B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36190 [10/Nov/2026:14:26:00.846] public api/api1 0/0/0/77/77 201 421 ---- 8/8/7/6/0 0/0 rid=AC1400018D5E-1F90-6AF3297834E-1442-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36168 [10/Nov/2026:14:26:00.841] public api/api1 0/0/0/82/83 201 421 ---- 7/7/6/5/0 0/0 rid=AC1400018D48-1F90-6AF32978349-1440-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36166 [10/Nov/2026:14:26:00.854] public api/api1 0/0/0/72/73 201 421 ---- 8/8/7/5/0 0/0 rid=AC1400018D46-1F90-6AF32978343-1446-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36262 [10/Nov/2026:14:26:00.906] public api/api1 0/0/0/21/21 201 421 ---- 8/8/6/4/0 0/0 rid=AC1400018DA6-1F90-6AF3297838A-144A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36238 [10/Nov/2026:14:26:00.864] public api/api1 0/0/0/64/64 201 421 ---- 7/7/5/3/0 0/0 rid=AC1400018D8E-1F90-6AF32978360-1448-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36146 [10/Nov/2026:14:26:00.853] public api/api1 0/0/1/74/75 201 421 ---- 6/6/5/3/0 0/0 rid=AC1400018D32-1F90-6AF32978340-1444-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36282 [10/Nov/2026:14:26:00.922] public api/api2 0/0/0/6/6 201 421 ---- 5/5/4/1/0 0/0 rid=AC1400018DBA-1F90-6AF3297839A-144D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36250 [10/Nov/2026:14:26:00.915] public api/api1 0/0/0/14/15 201 421 ---- 5/5/3/2/0 0/0 rid=AC1400018D9A-1F90-6AF3297837C-144C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36294 [10/Nov/2026:14:26:00.926] public api/api2 0/0/0/8/8 201 421 ---- 8/8/6/2/0 0/0 rid=AC1400018DC6-1F90-6AF3297839E-144F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36324 [10/Nov/2026:14:26:00.930] public api/api2 0/0/0/10/10 201 421 ---- 8/8/6/2/0 0/0 rid=AC1400018DE4-1F90-6AF329783A2-1451-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36378 [10/Nov/2026:14:26:00.935] public api/api2 0/0/0/7/7 201 421 ---- 8/8/6/1/0 0/0 rid=AC1400018E1A-1F90-6AF329783A7-1455-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36336 [10/Nov/2026:14:26:00.931] public api/api2 0/0/0/12/12 201 421 ---- 7/7/5/0/0 0/0 rid=AC1400018DF0-1F90-6AF329783A3-1453-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36396 [10/Nov/2026:14:26:00.945] public api/api2 0/0/0/5/5 201 421 ---- 8/8/7/1/0 0/0 rid=AC1400018E2C-1F90-6AF329783B1-1457-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36362 [10/Nov/2026:14:26:00.945] public api/api2 0/0/0/7/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018E0A-1F90-6AF329783A4-1459-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36416 [10/Nov/2026:14:26:00.977] public api/api2 0/0/0/7/7 201 421 ---- 8/8/7/0/0 0/0 rid=AC1400018E40-1F90-6AF329783BA-145B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36308 [10/Nov/2026:14:26:00.926] public api/api1 0/0/0/78/78 201 421 ---- 8/8/7/7/0 0/0 rid=AC1400018DD4-1F90-6AF3297839E-144E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36320 [10/Nov/2026:14:26:00.928] public api/api1 0/0/0/77/78 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018DE0-1F90-6AF329783A0-1450-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36428 [10/Nov/2026:14:26:01.006] public api/api2 0/0/0/6/6 201 420 ---- 8/8/6/0/0 0/0 rid=AC1400018E4C-1F90-6AF3297906-145D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36348 [10/Nov/2026:14:26:00.931] public api/api1 0/0/0/86/86 201 421 ---- 8/8/6/6/0 0/0 rid=AC1400018DFC-1F90-6AF329783A3-1452-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36342 [10/Nov/2026:14:26:00.930] public api/api1 0/0/0/88/88 201 421 ---- 8/8/5/5/0 0/0 rid=AC1400018DF6-1F90-6AF329783A2-1454-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36418 [10/Nov/2026:14:26:00.986] public api/api1 0/0/0/35/36 201 421 ---- 8/8/4/4/0 0/0 rid=AC1400018E42-1F90-6AF329783DA-145C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36384 [10/Nov/2026:14:26:00.942] public api/api1 0/0/0/80/81 201 421 ---- 7/7/4/3/0 0/0 rid=AC1400018E20-1F90-6AF329783AE-1456-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36392 [10/Nov/2026:14:26:00.945] public api/api1 0/0/0/77/78 201 421 ---- 6/6/3/2/0 0/0 rid=AC1400018E28-1F90-6AF329783B1-1458-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36402 [10/Nov/2026:14:26:00.952] public api/api1 0/0/0/71/71 201 421 ---- 6/6/2/1/0 0/0 rid=AC1400018E32-1F90-6AF329783B8-145A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36444 [10/Nov/2026:14:26:01.014] public api/api1 0/0/0/14/14 201 420 ---- 8/8/5/2/0 0/0 rid=AC1400018E5C-1F90-6AF329790E-145E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36438 [10/Nov/2026:14:26:01.023] public api/api2 0/0/0/11/11 201 420 ---- 8/8/5/2/0 0/0 rid=AC1400018E56-1F90-6AF3297908-145F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36496 [10/Nov/2026:14:26:01.025] public api/api2 0/0/0/11/11 201 420 ---- 8/8/5/2/0 0/0 rid=AC1400018E90-1F90-6AF3297919-1461-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36482 [10/Nov/2026:14:26:01.025] public api/api1 0/0/0/11/11 201 420 ---- 7/7/4/2/0 0/0 rid=AC1400018E82-1F90-6AF3297919-1462-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36488 [10/Nov/2026:14:26:01.025] public api/api2 0/0/0/12/12 201 420 ---- 6/6/3/1/0 0/0 rid=AC1400018E88-1F90-6AF3297919-1463-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36512 [10/Nov/2026:14:26:01.036] public api/api2 0/0/0/10/10 201 420 ---- 8/8/7/2/0 0/0 rid=AC1400018EA0-1F90-6AF3297924-1465-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36462 [10/Nov/2026:14:26:01.038] public api/api2 0/0/0/11/11 201 420 ---- 8/8/7/2/0 0/0 rid=AC1400018E6E-1F90-6AF3297915-1467-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36524 [10/Nov/2026:14:26:01.039] public api/api2 0/0/0/9/10 201 420 ---- 7/7/6/1/0 0/0 rid=AC1400018EAC-1F90-6AF3297927-1469-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36546 [10/Nov/2026:14:26:01.048] public api/api2 0/0/0/8/8 201 420 ---- 8/8/7/1/0 0/0 rid=AC1400018EC2-1F90-6AF3297930-146B-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36568 [10/Nov/2026:14:26:01.051] public api/api2 0/0/0/7/7 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018ED8-1F90-6AF3297933-146D-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36592 [10/Nov/2026:14:26:01.059] public api/api2 0/0/0/4/4 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018EF0-1F90-6AF329793B-146F-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36508 [10/Nov/2026:14:26:01.030] public api/api1 0/0/0/68/68 201 420 ---- 8/8/7/7/0 0/0 rid=AC1400018E9C-1F90-6AF329791E-1464-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36470 [10/Nov/2026:14:26:01.025] public api/api1 0/0/0/73/73 201 420 ---- 7/7/6/6/0 0/0 rid=AC1400018E76-1F90-6AF3297919-1460-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36612 [10/Nov/2026:14:26:01.100] public api/api2 0/0/0/5/6 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018F04-1F90-6AF3297964-1471-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36620 [10/Nov/2026:14:26:01.108] public api/api2 0/0/0/4/4 201 420 ---- 8/8/7/0/0 0/0 rid=AC1400018F0C-1F90-6AF329796C-1473-0008 inst=cec102b25e41 "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36618 [10/Nov/2026:14:26:01.100] public api/api1 0/0/0/17/17 201 420 ---- 7/7/6/6/0 0/0 rid=AC1400018F0A-1F90-6AF3297964-1472-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36584 [10/Nov/2026:14:26:01.057] public api/api1 0/0/0/61/61 201 420 ---- 6/6/5/5/0 0/0 rid=AC1400018EE8-1F90-6AF3297939-146E-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36536 [10/Nov/2026:14:26:01.040] public api/api1 0/0/0/81/81 201 420 ---- 5/5/4/4/0 0/0 rid=AC1400018EB8-1F90-6AF3297928-146A-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36516 [10/Nov/2026:14:26:01.039] public api/api1 0/0/0/83/83 201 420 ---- 4/4/3/3/0 0/0 rid=AC1400018EA4-1F90-6AF3297927-1468-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36452 [10/Nov/2026:14:26:01.038] public api/api1 0/0/1/84/85 201 420 ---- 3/3/2/2/0 0/0 rid=AC1400018E64-1F90-6AF3297913-1466-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36600 [10/Nov/2026:14:26:01.065] public api/api1 0/0/0/57/58 201 420 ---- 2/2/1/1/0 0/0 rid=AC1400018EF8-1F90-6AF3297941-1470-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" +lb-1 | 172.20.0.1:36562 [10/Nov/2026:14:26:01.051] public api/api1 0/0/0/72/72 201 420 ---- 1/1/0/0/0 0/0 rid=AC1400018ED2-1F90-6AF3297933-146C-0008 inst=9adbb2f64abd "POST /api/v1/chats/2e3b890e-2af9-4acf-8c20-41d8ad16ad39/messages" diff --git a/docs/lab5/evidence/load.json b/docs/lab5/evidence/load.json new file mode 100644 index 0000000..d999394 --- /dev/null +++ b/docs/lab5/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.094, + "concurrency": 8, + "successful_requests": 5150, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 171.13, + "successful_latency_ms": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/docs/lab5/evidence/load.txt b/docs/lab5/evidence/load.txt new file mode 100644 index 0000000..7cc2794 --- /dev/null +++ b/docs/lab5/evidence/load.txt @@ -0,0 +1,31 @@ +# 2026-11-10 14:26:01 UTC | Нагрузка 30 с, concurrency 8 | exit 0 | 30.5s +$ python tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 8 --output evidence/load.json +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.094, + "concurrency": 8, + "successful_requests": 5150, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 171.13, + "successful_latency_ms": { + "p50": 31.0, + "p95": 110.0, + "p99": 188.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +# 2026-11-10 14:26:03 UTC | CPU/RAM после нагрузки | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab5-api-6 20.42% 95.3MiB / 320MiB 29.78% +messenger-lab5-api-5 19.69% 94.15MiB / 320MiB 29.42% +messenger-lab5-worker-1 0.13% 77.54MiB / 1.25GiB 6.06% +messenger-lab5-lb-1 0.34% 13.26MiB / 128MiB 10.36% +messenger-lab5-postgres-1 3.10% 71.41MiB / 512MiB 13.95% +messenger-lab5-redis-1 0.21% 4.559MiB / 96MiB 4.75% +messenger-lab5-s3-1 2.87% 99.73MiB / 512MiB 19.48% +messenger-lab5-broker-1 2.79% 8.617MiB / 320MiB 2.69% + diff --git a/docs/lab5/evidence/local/README.md b/docs/lab5/evidence/local/README.md new file mode 100644 index 0000000..be500a2 --- /dev/null +++ b/docs/lab5/evidence/local/README.md @@ -0,0 +1,33 @@ +# Локальный прогон (без Docker) — что это и чем НЕ является + +Дата: 2026-11-09. Эти файлы получены при разработке в изолированной Linux-песочнице, +где **Docker Hub недоступен** (образы не скачиваются), поэтому compose-стенд там не +поднимался. Те же процессы запускались нативно: + +| Роль | В compose.yaml | В локальном прогоне | +| --- | --- | --- | +| API x2 | `api` (образ из Dockerfile) | `python -m src.server` x2 (INSTANCE_ID=api1/api2) | +| LB | HAProxy 2.8.30 | 40-строчный round-robin TCP-прокси на asyncio (только для теста) | +| PostgreSQL | postgres:16.4-alpine | PostgreSQL 16 (системный) | +| Кэш сессий | redis:7.4.1-alpine | Redis 7.0.15, без persistence | +| Broker | redis:7.4.1-alpine, AOF `appendfsync always`, `noeviction` | Redis 7.0.15 с теми же флагами, отдельный dir | +| S3 | SeaweedFS 4.50 | moto 5.0.28 (S3-эмулятор, in-memory) | +| Worker | `worker` (`python -m src.worker`) | тот же модуль; «restart: unless-stopped» эмулирован циклом в shell | + +Код приложения, миграции и тесты — те же, что в репозитории. Хуки рестарта/остановки +заменены локальными скриптами (kill/pg_ctl/redis SHUTDOWN), поэтому **доказательства +на compose-стенде нужно собрать самостоятельно**: `python scripts/collect_evidence.py` +(запишет evidence/*.txt и evidence/SUMMARY.txt). S3 в локальном persistence-прогоне +не перезапускался (moto хранит данные в памяти). + +| Файл | Что внутри | +| --- | --- | +| smoke.txt, student-tests.txt | `make test` и `student-tests/run_all.py` через 2 реплики: 91 тест + race, PASS | +| pipeline_and_attachments.txt | test_pipeline.py (7) и адаптированный test_attachments.py (17), PASS | +| pipeline_faults.txt | test_pipeline_faults.py: 10 сценариев отказов уровня 4, PASS | +| pipeline_faults_events.txt | журнал job_events этих сценариев (доказательство *как* система восстановилась) | +| worker_fault_log.txt | строки лога worker: внедрённые падения и восстановление своего PEL | +| worker_outage.txt / .json | сценарий защиты уровня 3 (worker stop -> 202 -> рестарт API и broker -> ready) | +| persistence.txt, failover.txt | публичные tests/resilience.py (persistence: API+worker+PG+Redis+broker) | +| latency.json | 30 загрузок: 202 p50 16 мс; до ready p50 0.44 с (с шагом опроса 0.2 с) | +| imaging_memory.txt | пиковый RSS подпроцесса обработки: 40 Мп RGBA PNG — 361 MiB, 40 Мп JPEG — 46 MiB | diff --git a/docs/lab5/evidence/local/failover.txt b/docs/lab5/evidence/local/failover.txt new file mode 100644 index 0000000..364e26c --- /dev/null +++ b/docs/lab5/evidence/local/failover.txt @@ -0,0 +1,4 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 141, "failed_attempts": 0, "first_recovery_seconds": 0.115} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +EXIT=0 diff --git a/docs/lab5/evidence/local/imaging_memory.txt b/docs/lab5/evidence/local/imaging_memory.txt new file mode 100644 index 0000000..5da3946 --- /dev/null +++ b/docs/lab5/evidence/local/imaging_memory.txt @@ -0,0 +1,5 @@ +synthetic/big.jpg 626630 {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (1280, 800)} 0.20s peak RSS 46 MiB +synthetic/big.png 165443 {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (1280, 800)} 1.80s peak RSS 361 MiB +tests/fixtures/sample.png 111817 {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (320, 200)} 0.05s peak RSS 23 MiB +synthetic/alpha.png 86 {'thumbnail': (50, 30), 'crop': (128, 128), 'optimized': (50, 30)} 0.04s peak RSS 23 MiB +synthetic/orient.jpg 1365 {'thumbnail': (64, 256), 'crop': (128, 128), 'optimized': (100, 400)} 0.04s peak RSS 23 MiB diff --git a/docs/lab5/evidence/local/latency.json b/docs/lab5/evidence/local/latency.json new file mode 100644 index 0000000..ec3d3a5 --- /dev/null +++ b/docs/lab5/evidence/local/latency.json @@ -0,0 +1,14 @@ +{ + "n": 30, + "file": "tests/fixtures/sample.png 320x200 109 KiB", + "upload_202_s": { + "p50": 0.016, + "p95": 0.029 + }, + "upload_to_ready_s": { + "p50": 0.435, + "p95": 0.449, + "max": 0.45 + }, + "note": "local sandbox, poll interval 0.2 s, 1 worker x 2 consumers" +} diff --git a/docs/lab5/evidence/local/persistence.txt b/docs/lab5/evidence/local/persistence.txt new file mode 100644 index 0000000..baff3a9 --- /dev/null +++ b/docs/lab5/evidence/local/persistence.txt @@ -0,0 +1,3 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +EXIT=0 diff --git a/docs/lab5/evidence/local/pipeline_and_attachments.txt b/docs/lab5/evidence/local/pipeline_and_attachments.txt new file mode 100644 index 0000000..d695303 --- /dev/null +++ b/docs/lab5/evidence/local/pipeline_and_attachments.txt @@ -0,0 +1,11 @@ +....... +---------------------------------------------------------------------- +Ran 7 tests in 4.403s + +OK +................. +---------------------------------------------------------------------- +Ran 17 tests in 3.869s + +OK +done diff --git a/docs/lab5/evidence/local/pipeline_faults.txt b/docs/lab5/evidence/local/pipeline_faults.txt new file mode 100644 index 0000000..77846c7 --- /dev/null +++ b/docs/lab5/evidence/local/pipeline_faults.txt @@ -0,0 +1,18 @@ +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... ok +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... ok +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... ok +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... ok +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... ok +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... ok +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... ok +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... ok +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... /home/claude/venv/lib/python3.13/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 71.328s + +OK +EXIT=0 diff --git a/docs/lab5/evidence/local/pipeline_faults_events.txt b/docs/lab5/evidence/local/pipeline_faults_events.txt new file mode 100644 index 0000000..c47c7f8 --- /dev/null +++ b/docs/lab5/evidence/local/pipeline_faults_events.txt @@ -0,0 +1,71 @@ + filename | attempt | event | detail | actor +----------------------------------+---------+-----------------+---------------------------------------------------+------------- + a.png | 1 | accepted | | api + a.png | 1 | fault_injected | crash between COMMIT and publish | api:apipub + a.png | 1 | published | stream id 1794253144786-0 | api:apidup + a.png | 1 | claimed | | wf-1 + a.png | 1 | succeeded | thumbnail,crop,optimized | wf-1 + b.png | 1 | accepted | | api + b.png | 1 | fault_injected | crash after XADD before outbox commit | api:apidup + b.png | 1 | claimed | | wf-1 + b.png | 1 | succeeded | thumbnail,crop,optimized | wf-1 + b.png | 1 | published | stream id 1794253148012-0 | api:api1 + b.png | 1 | redelivery_noop | already_succeeded | wf-0 + fault-crash-after-effects.png | 1 | accepted | | api + fault-crash-after-effects.png | 1 | published | stream id 1794253148173-0 | api:api1 + fault-crash-after-effects.png | 1 | claimed | | wf-1 + fault-crash-after-effects.png | 1 | fault_injected | crash after S3 writes before COMMIT | wf-1 + fault-crash-after-effects.png | 1 | lease_takeover | previous owner wf-1 | wf-1 + fault-crash-after-effects.png | 1 | succeeded | thumbnail,crop,optimized | wf-1 + fault-crash-before-ack.png | 1 | accepted | | api + fault-crash-before-ack.png | 1 | published | stream id 1794253150245-0 | api:api1 + fault-crash-before-ack.png | 1 | claimed | | wf-0 + fault-crash-before-ack.png | 1 | succeeded | thumbnail,crop,optimized | wf-0 + fault-crash-before-ack.png | 1 | fault_injected | crash after COMMIT before XACK | wf-0 + fault-crash-before-ack.png | 1 | redelivery_noop | already_succeeded | wf-0 + fault-permanent.png | 1 | accepted | | api + fault-permanent.png | 1 | published | stream id 1794253155274-0 | api:api1 + fault-permanent.png | 1 | claimed | | wf-1 + fault-permanent.png | 1 | failed | invalid_image; injected permanent failure | wf-1 + fault-slow.png | 1 | accepted | | api + fault-slow.png | 1 | published | stream id 1794253167381-0 | api:api1 + fault-slow.png | 1 | claimed | | wf-1 + fault-slow.png | 1 | lease_expired | owner wf-1; next attempt 2 in 0.51s | worker:wf + fault-slow.png | 1 | redelivery_noop | stale_attempt | wf-0 + fault-slow.png | 2 | published | stream id 1794253198039-0 | worker:wf + fault-slow.png | 2 | claimed | | wf-0 + fault-slow.png | 2 | succeeded | thumbnail,crop,optimized | wf-0 + fault-slow.png | 1 | lease_lost | result not published: lease token no longer valid | wf-1 + fault-transient-until-3.png | 1 | accepted | | api + fault-transient-until-3.png | 1 | published | stream id 1794253152182-0 | api:api1 + fault-transient-until-3.png | 1 | claimed | | wf-1 + fault-transient-until-3.png | 1 | retry_scheduled | injected_transient; next attempt 2 in 0.79s | wf-1 + fault-transient-until-3.png | 2 | published | stream id 1794253153023-0 | api:api1 + fault-transient-until-3.png | 2 | claimed | | wf-0 + fault-transient-until-3.png | 2 | retry_scheduled | injected_transient; next attempt 3 in 1.90s | wf-0 + fault-transient-until-3.png | 3 | published | stream id 1794253154959-0 | worker:wf + fault-transient-until-3.png | 3 | claimed | | wf-0 + fault-transient-until-3.png | 3 | succeeded | thumbnail,crop,optimized | wf-0 + fault-transient-until-replay.png | 1 | accepted | | api + fault-transient-until-replay.png | 1 | published | stream id 1794253155506-0 | api:api1 + fault-transient-until-replay.png | 1 | claimed | | wf-0 + fault-transient-until-replay.png | 1 | retry_scheduled | injected_transient; next attempt 2 in 0.54s | wf-0 + fault-transient-until-replay.png | 2 | published | stream id 1794253156965-0 | worker:wf + fault-transient-until-replay.png | 2 | claimed | | wf-1 + fault-transient-until-replay.png | 2 | retry_scheduled | injected_transient; next attempt 3 in 1.97s | wf-1 + fault-transient-until-replay.png | 3 | published | stream id 1794253158970-0 | worker:wf + fault-transient-until-replay.png | 3 | claimed | | wf-1 + fault-transient-until-replay.png | 3 | retry_scheduled | injected_transient; next attempt 4 in 3.54s | wf-1 + fault-transient-until-replay.png | 4 | published | stream id 1794253162979-0 | worker:wf + fault-transient-until-replay.png | 4 | claimed | | wf-1 + fault-transient-until-replay.png | 4 | retry_scheduled | injected_transient; next attempt 5 in 2.54s | wf-1 + fault-transient-until-replay.png | 5 | published | stream id 1794253165987-0 | worker:wf + fault-transient-until-replay.png | 5 | claimed | | wf-1 + fault-transient-until-replay.png | 5 | failed | retry_exhausted; injected_transient | wf-1 + fault-transient-until-replay.png | 6 | replayed | operator replay; new attempt 6 | test:replay + fault-transient-until-replay.png | 6 | published | stream id 1794253166991-0 | worker:wf + fault-transient-until-replay.png | 6 | claimed | | wf-0 + fault-transient-until-replay.png | 6 | succeeded | thumbnail,crop,optimized | wf-0 + fault-transient-until-replay.png | 6 | replay_refused | already succeeded (no duplicate execution) | test:replay +(67 rows) + diff --git a/docs/lab5/evidence/local/smoke.txt b/docs/lab5/evidence/local/smoke.txt new file mode 100644 index 0000000..5c90482 --- /dev/null +++ b/docs/lab5/evidence/local/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab5/evidence/local/student-tests.txt b/docs/lab5/evidence/local/student-tests.txt new file mode 100644 index 0000000..545d8be --- /dev/null +++ b/docs/lab5/evidence/local/student-tests.txt @@ -0,0 +1,111 @@ + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... + 24 parallel POSTs, one key -> 1 message; replicas: ['api1', 'api2'] +ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... skipped 'needs API at http://127.0.0.1:8090 and DB/S3/broker env' +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 91 tests in 11.178s + +OK (skipped=33) + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +EXIT=0 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +EXIT=0 diff --git a/docs/lab5/evidence/local/worker_fault_log.txt b/docs/lab5/evidence/local/worker_fault_log.txt new file mode 100644 index 0000000..0cbbea1 --- /dev/null +++ b/docs/lab5/evidence/local/worker_fault_log.txt @@ -0,0 +1,8 @@ +2026-11-09 22:39:08,453 ERROR messenger.worker FAULT_INJECTION: worker exits after S3 writes before COMMIT (job 689dc4ad-bf5d-4d33-a7f4-d557aef94b2a) +22:39:08 wf exited (os._exit(70)); supervisor restarts it like docker restart: unless-stopped +2026-11-09 22:39:09,883 INFO messenger.worker wf-1: recovering own unacked message 1794253148173-0 +2026-11-09 22:39:10,530 ERROR messenger.worker FAULT_INJECTION: worker exits after COMMIT before XACK (job 6746ae17-f4cc-48b1-9864-c6e80507c2bf) +22:39:10 wf exited (os._exit(70)); supervisor restarts it like docker restart: unless-stopped +2026-11-09 22:39:11,945 INFO messenger.worker wf-0: recovering own unacked message 1794253150245-0 +2026-11-09 22:39:57,386 INFO messenger.worker wf-0: claimed idle message 1794253167381-0 (deliveries=2) +2026-11-09 22:40:12,653 WARNING messenger.worker wf-1: lease lost for job acf17aa8-ef8d-4338-8beb-bd9bab29be8b; result not published (fenced) diff --git a/docs/lab5/evidence/local/worker_outage.json b/docs/lab5/evidence/local/worker_outage.json new file mode 100644 index 0000000..a154d0a --- /dev/null +++ b/docs/lab5/evidence/local/worker_outage.json @@ -0,0 +1,80 @@ +[ + { + "seconds": 0.02, + "step": "worker stopped", + "t": 0.14 + }, + { + "status": 202, + "attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", + "attachment_status": "queued", + "step": "upload accepted", + "t": 0.15 + }, + { + "metadata_status": "queued", + "content_status": 409, + "message_with_attachment": 201, + "plain_message": 201, + "message_id": "d589ad3f-63a3-4258-a981-701a2a7bfb5e", + "step": "while worker is down", + "t": 3.17 + }, + { + "length": 1, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 32, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "broker before restarts", + "t": 3.37 + }, + { + "seconds": 3.06, + "step": "api replicas restarted", + "t": 6.43 + }, + { + "seconds": 2.03, + "step": "broker restarted (volume kept)", + "t": 8.46 + }, + { + "metadata_status": "queued", + "length": 1, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 32, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "after restarts", + "t": 10.66 + }, + { + "seconds": 3.01, + "step": "worker started", + "t": 13.67 + }, + { + "attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "thumbnail_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true, + "step": "ready", + "t": 13.7 + } +] \ No newline at end of file diff --git a/docs/lab5/evidence/local/worker_outage.txt b/docs/lab5/evidence/local/worker_outage.txt new file mode 100644 index 0000000..1db4cd3 --- /dev/null +++ b/docs/lab5/evidence/local/worker_outage.txt @@ -0,0 +1,11 @@ +{"seconds": 0.02, "step": "worker stopped", "t": 0.14} +{"status": 202, "attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", "attachment_status": "queued", "step": "upload accepted", "t": 0.15} +{"metadata_status": "queued", "content_status": 409, "message_with_attachment": 201, "plain_message": 201, "message_id": "d589ad3f-63a3-4258-a981-701a2a7bfb5e", "step": "while worker is down", "t": 3.17} +{"length": 1, "lag": 1, "pending": 0, "jobs": {"succeeded": 32, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "broker before restarts", "t": 3.37} +{"seconds": 3.06, "step": "api replicas restarted", "t": 6.43} +{"seconds": 2.03, "step": "broker restarted (volume kept)", "t": 8.46} +{"metadata_status": "queued", "length": 1, "lag": 1, "pending": 0, "jobs": {"succeeded": 32, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "after restarts", "t": 10.66} +{"seconds": 3.01, "step": "worker started", "t": 13.67} +{"attachment_id": "3985fc73-5e47-411d-8d68-c44551e77cb9", "variants": ["thumbnail", "crop", "optimized"], "thumbnail_jpeg_size": [256, 160], "original_sha256_ok": true, "step": "ready", "t": 13.7} +PASS: accepted job survived worker stop + API restart + broker restart and became ready +EXIT=0 diff --git a/docs/lab5/evidence/persistence.txt b/docs/lab5/evidence/persistence.txt new file mode 100644 index 0000000..40b055c --- /dev/null +++ b/docs/lab5/evidence/persistence.txt @@ -0,0 +1,53 @@ +# 2026-11-10 14:24:42 UTC | Persistence: контейнеры до | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab5-api-5 Up 33 seconds (healthy) 34 seconds ago +messenger-lab5-api-6 Up 22 seconds (healthy) 22 seconds ago +messenger-lab5-broker-1 Up 3 minutes (healthy) 39 minutes ago +messenger-lab5-lb-1 Up 24 minutes (healthy) 39 minutes ago +messenger-lab5-migrate-1 Exited (0) 3 minutes ago 39 minutes ago +messenger-lab5-postgres-1 Up 25 minutes (healthy) 39 minutes ago +messenger-lab5-redis-1 Up 25 minutes (healthy) 39 minutes ago +messenger-lab5-s3-1 Up 4 minutes (healthy) 39 minutes ago +messenger-lab5-s3-init-1 Exited (0) 3 minutes ago 39 minutes ago +messenger-lab5-worker-1 Up About a minute (healthy) About a minute ago +# 2026-11-10 14:25:27 UTC | Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB) | exit 0 | 45.1s +$ python tests/resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# 2026-11-10 14:25:27 UTC | Persistence: контейнеры после (Status = Up N seconds) | exit 0 | 0.3s +$ docker compose --env-file .env ps -a --format table {{.Name}} {{.Status}} {{.RunningFor}} +NAME STATUS CREATED +messenger-lab5-api-5 Up 7 seconds (healthy) About a minute ago +messenger-lab5-api-6 Up 7 seconds (healthy) About a minute ago +messenger-lab5-broker-1 Up 37 seconds (healthy) 40 minutes ago +messenger-lab5-lb-1 Up Less than a second (health: starting) 40 minutes ago +messenger-lab5-migrate-1 Exited (0) 4 minutes ago 40 minutes ago +messenger-lab5-postgres-1 Up 37 seconds (healthy) 40 minutes ago +messenger-lab5-redis-1 Up 37 seconds (healthy) 40 minutes ago +messenger-lab5-s3-1 Up 27 seconds (healthy) 40 minutes ago +messenger-lab5-s3-init-1 Exited (0) 4 minutes ago 40 minutes ago +messenger-lab5-worker-1 Up 7 seconds (healthy) 2 minutes ago + +# 2026-11-10 14:25:30 UTC | Persistence: сверка БД <-> S3 после рестарта | exit 0 | 2.9s +$ python scripts/ops.py reconcile + Container messenger-lab5-api-run-467cf1258171 Creating + Container messenger-lab5-api-run-467cf1258171 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "staging_to_clean": 1, + "objects_listed": 640, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.211, + "ready_rows": 637, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.229 +} + diff --git a/docs/lab5/evidence/pipeline.txt b/docs/lab5/evidence/pipeline.txt new file mode 100644 index 0000000..bbae87b --- /dev/null +++ b/docs/lab5/evidence/pipeline.txt @@ -0,0 +1,14 @@ +# 2026-11-10 14:20:25 UTC | Варианты изображений по фактическим байтам (test_pipeline.py) | exit 0 | 6.6s +$ python -m unittest discover -s student-tests -p test_pipeline.py -v +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 6.481s + +OK diff --git a/docs/lab5/evidence/pipeline_faults.txt b/docs/lab5/evidence/pipeline_faults.txt new file mode 100644 index 0000000..28753d3 --- /dev/null +++ b/docs/lab5/evidence/pipeline_faults.txt @@ -0,0 +1,152 @@ +# 2026-11-10 14:23:04 UTC | Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease) | exit 0 | 126.3s +$ python scripts/ops.py pipeline-faults +2026-11-10T14:20:58.422Z pipeline-faults: recreating worker with compose.faults.yaml + Container messenger-lab5-postgres-1 Running + Container messenger-lab5-s3-1 Running + Container messenger-lab5-api-1 Running + Container messenger-lab5-api-2 Running + Container messenger-lab5-redis-1 Running + Container messenger-lab5-api-fault-publish-1 Creating + Container messenger-lab5-api-fault-dup-1 Creating + Container messenger-lab5-api-fault-publish-1 Created + Container messenger-lab5-api-fault-dup-1 Created + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Starting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-s3-init-1 Starting + Container messenger-lab5-migrate-1 Started + Container messenger-lab5-s3-init-1 Started + Container messenger-lab5-migrate-1 Waiting + Container messenger-lab5-s3-init-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-redis-1 Waiting + Container messenger-lab5-redis-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Exited + Container messenger-lab5-s3-init-1 Exited + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-dup-1 Starting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-fault-publish-1 Starting + Container messenger-lab5-api-fault-dup-1 Started + Container messenger-lab5-api-fault-publish-1 Started + Container messenger-lab5-api-run-f6d1d1271d68 Creating + Container messenger-lab5-api-run-f6d1d1271d68 Created +ready: http://api-fault-publish:8080/health/ready + Container messenger-lab5-api-run-0c827bfb627f Creating + Container messenger-lab5-api-run-0c827bfb627f Created +ready: http://api-fault-dup:8080/health/ready + Container messenger-lab5-api-run-6e712120049d Creating + Container messenger-lab5-api-run-6e712120049d Created +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... ok +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... ok +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... ok +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... ok +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... ok +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... ok +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... ok +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... ok +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... /usr/local/lib/python3.12/site-packages/botocore/auth.py:424: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + datetime_now = datetime.datetime.utcnow() +ok +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... ok + +---------------------------------------------------------------------- +Ran 10 tests in 90.529s + +OK +2026-11-10T14:23:04.629Z pipeline-faults: worker restored to normal configuration +# 2026-11-10 14:23:06 UTC | Очередь после сценариев: задания, outbox, DLQ, lag | exit 0 | 1.7s +$ python scripts/ops.py jobs stats + Container messenger-lab5-worker-run-e232ea69a3ca Creating + Container messenger-lab5-worker-run-e232ea69a3ca Created +{ + "jobs": { + "succeeded": 158, + "failed": 2 + }, + "attachments": { + "ready": 159, + "failed": 2 + }, + "outbox": { + "rows": 0, + "due": 0, + "oldest_age_s": 0.0 + }, + "dead_letters_unreplayed": 2, + "oldest_unfinished_job_age_s": 0.0, + "broker": { + "stream": "attachments.jobs", + "length": 4, + "groups": [ + { + "name": "attachment-workers", + "consumers": 10, + "pending": 0, + "lag": 0, + "last_delivered_id": "1794320565944-0" + } + ] + } +} + +# 2026-11-10 14:23:07 UTC | Реестр failed (DLQ) | exit 0 | 1.6s +$ python scripts/ops.py jobs failed --limit 10 + Container messenger-lab5-worker-run-85c598896740 Creating + Container messenger-lab5-worker-run-85c598896740 Created +[ + { + "attachment_id": "9f72cadb-e186-43cb-8456-ee1d5f3bde45", + "job_id": "05c07c99-eda0-4dbe-b49e-98d1426f3933", + "error_code": "retry_exhausted", + "attempts": 5, + "failed_at": "2026-11-10 14:21:41.960496+00:00", + "replayed_at": "2026-11-10 14:21:42.086003+00:00", + "replay_count": 1, + "job_status": "succeeded", + "replayable": false + }, + { + "attachment_id": "390e3a82-63cd-4a64-a50c-a200f68924c4", + "job_id": "bfe81165-4f51-445a-bf37-07e61e527985", + "error_code": "invalid_image", + "attempts": 1, + "failed_at": "2026-11-10 14:21:32.124484+00:00", + "replayed_at": null, + "replay_count": 0, + "job_status": "failed", + "replayable": true + }, + { + "attachment_id": "5c05fbd5-77b0-4554-87bd-5b315aea3b82", + "job_id": "d9f58adb-d682-4d2f-a94c-16f2ce326ec7", + "error_code": "retry_exhausted", + "attempts": 5, + "failed_at": "2026-11-10 14:09:44.857997+00:00", + "replayed_at": "2026-11-10 14:09:44.981918+00:00", + "replay_count": 1, + "job_status": "succeeded", + "replayable": false + }, + { + "attachment_id": "5a2d84b3-b13f-4c34-83e5-65f42b792752", + "job_id": "41cdf50b-cfd4-4a88-864c-f5a918948014", + "error_code": "invalid_image", + "attempts": 1, + "failed_at": "2026-11-10 14:09:34.398249+00:00", + "replayed_at": null, + "replay_count": 0, + "job_status": "failed", + "replayable": true + } +] + diff --git a/docs/lab5/evidence/reconcile.txt b/docs/lab5/evidence/reconcile.txt new file mode 100644 index 0000000..184a985 --- /dev/null +++ b/docs/lab5/evidence/reconcile.txt @@ -0,0 +1,20 @@ +# 2026-11-10 14:15:14 UTC | Сверка БД <-> S3 (dry-run) | exit 0 | 2.5s +$ python scripts/ops.py reconcile + Container messenger-lab5-api-run-864f3258a8e5 Creating + Container messenger-lab5-api-run-864f3258a8e5 Created +{ + "mode": "dry-run", + "grace_seconds": 900, + "stale_pending": 0, + "stale_pending_ids": [], + "staging_to_clean": 0, + "objects_listed": 266, + "orphan_objects": 0, + "orphan_objects_within_grace": 0, + "orphan_objects_sample": [], + "bucket_scan_seconds": 0.053, + "ready_rows": 265, + "missing_objects": 0, + "missing_object_ids": [], + "total_seconds": 0.066 +} diff --git a/docs/lab5/evidence/rolling_back.json b/docs/lab5/evidence/rolling_back.json new file mode 100644 index 0000000..b6273a7 --- /dev/null +++ b/docs/lab5/evidence/rolling_back.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 29.78, + "replicas_before": [ + "cb14462a28f6", + "cc4d20e47b36" + ], + "replicas_after": [ + "9adbb2f64abd", + "cec102b25e41" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1438, + "ok": 1438, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 287.6, + "write_ok_rps": 163.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 63.0, + "p99": 109.0 + }, + "instances": [ + "cb14462a28f6", + "cc4d20e47b36" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 34.78, + "requests": 8087, + "ok": 8087, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 271.5, + "write_ok_rps": 160.1, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "9adbb2f64abd", + "cb14462a28f6", + "cc4d20e47b36", + "cec102b25e41" + ] + }, + "after": { + "from_s": 34.78, + "to_s": 42.83, + "requests": 1845, + "ok": 1845, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 229.3, + "write_ok_rps": 137.2, + "latency_ms_ok": { + "p50": 16.0, + "p95": 79.0, + "p99": 140.0 + }, + "instances": [ + "9adbb2f64abd", + "cec102b25e41" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api3" + ] + ], + [ + 11.92, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 12.66, + [ + "api1", + "api3" + ] + ], + [ + 15.09, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 19.36, + [ + "api1", + "api2" + ] + ], + [ + 26.97, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 30.62, + [ + "api2", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6690, + "stored_messages": 6690, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab5/evidence/rolling_update.json b/docs/lab5/evidence/rolling_update.json new file mode 100644 index 0000000..eaf4bac --- /dev/null +++ b/docs/lab5/evidence/rolling_update.json @@ -0,0 +1,147 @@ +{ + "scenario": "rolling update of all API replicas under continuous load", + "update_command": "scripts/ops.py rolling-update", + "update_exit_code": 0, + "update_seconds": 31.98, + "replicas_before": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "replicas_after": [ + "cb14462a28f6", + "cc4d20e47b36" + ], + "all_replaced": true, + "errors_during_update": 0, + "error_kinds": [], + "windows": { + "before": { + "from_s": 0, + "to_s": 5.0, + "requests": 1190, + "ok": 1190, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 238.0, + "write_ok_rps": 141.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ] + }, + "during_update": { + "from_s": 5.0, + "to_s": 36.98, + "requests": 7392, + "ok": 7392, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 231.1, + "write_ok_rps": 135.8, + "latency_ms_ok": { + "p50": 16.0, + "p95": 93.0, + "p99": 141.0 + }, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c", + "cb14462a28f6", + "cc4d20e47b36" + ] + }, + "after": { + "from_s": 36.98, + "to_s": 45.02, + "requests": 2038, + "ok": 2038, + "errors": 0, + "error_ratio": 0.0, + "ok_rps": 253.7, + "write_ok_rps": 150.0, + "latency_ms_ok": { + "p50": 16.0, + "p95": 78.0, + "p99": 109.0 + }, + "instances": [ + "cb14462a28f6", + "cc4d20e47b36" + ] + } + }, + "lb_timeline": [ + [ + 0.0, + [ + "api1", + "api2" + ] + ], + [ + 14.41, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 15.08, + [ + "api1", + "api2" + ] + ], + [ + 17.67, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 22.42, + [ + "api2", + "api3" + ] + ], + [ + 30.97, + [ + "api1", + "api2", + "api3" + ] + ], + [ + 33.14, + [ + "api1", + "api3" + ] + ] + ], + "data": { + "confirmed_writes": 6255, + "stored_messages": 6255, + "missing_or_altered": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "same_key_retries": 0, + "ok": true + }, + "load": { + "writers": 4, + "readers": 2, + "client_timeout_s": 2.0 + }, + "pass": true +} diff --git a/docs/lab5/evidence/rolling_update.txt b/docs/lab5/evidence/rolling_update.txt new file mode 100644 index 0000000..8be1631 --- /dev/null +++ b/docs/lab5/evidence/rolling_update.txt @@ -0,0 +1,29 @@ +# 2026-11-10 14:23:55 UTC | Rolling update lab5 -> lab5.1 под нагрузкой | exit 0 | 47.5s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_update.json +replicas before: ['3a0e06f14a63', '4b1faad0399c'] +2026-11-10T14:23:14.819Z rolling-update: start; replicas=2 old=['4b1faad0399c', '3a0e06f14a63'] +2026-11-10T14:23:20.086Z rolling-update: images built in 5.3s (tag lab5.1) +2026-11-10T14:23:22.594Z rolling-update: migrations once in 2.5s: schema is up to date; nothing applied +2026-11-10T14:23:34.419Z rolling-update: step 1/2 new=cb14462a28f6 (messenger-api:lab5.1, ready+UP in 7.5s) replaced old=4b1faad0399c (graceful stop 4.1s) +2026-11-10T14:23:46.022Z rolling-update: step 2/2 new=cc4d20e47b36 (messenger-api:lab5.1, ready+UP in 7.0s) replaced old=3a0e06f14a63 (graceful stop 4.4s) +2026-11-10T14:23:46.260Z rolling-update: done in 31.8s; replicas now ['cb14462a28f6', 'cc4d20e47b36'] +update 32.0s rc=0 | replicas ['3a0e06f14a63', '4b1faad0399c'] -> ['cb14462a28f6', 'cc4d20e47b36'] | errors during update 0 (-) | RPS before 238.0, during 231.1, after 253.7 | p95 during 93.0 ms | data ok=True +PASS: rolling update under load +# 2026-11-10 14:24:41 UTC | Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой | exit 0 | 45.9s +$ python student-tests/rolling_update_load.py --base-url http://127.0.0.1:8080 --output evidence/rolling_back.json +replicas before: ['cb14462a28f6', 'cc4d20e47b36'] +2026-11-10T14:24:02.303Z rolling-update: start; replicas=2 old=['cb14462a28f6', 'cc4d20e47b36'] +2026-11-10T14:24:05.202Z rolling-update: images built in 2.9s (tag lab5) +2026-11-10T14:24:07.607Z rolling-update: migrations once in 2.4s: schema is up to date; nothing applied +2026-11-10T14:24:19.412Z rolling-update: step 1/2 new=cec102b25e41 (messenger-api:lab5, ready+UP in 7.2s) replaced old=cb14462a28f6 (graceful stop 4.4s) +2026-11-10T14:24:31.138Z rolling-update: step 2/2 new=9adbb2f64abd (messenger-api:lab5, ready+UP in 7.1s) replaced old=cc4d20e47b36 (graceful stop 4.4s) +2026-11-10T14:24:31.579Z rolling-update: done in 29.5s; replicas now ['cec102b25e41', '9adbb2f64abd'] +update 29.8s rc=0 | replicas ['cb14462a28f6', 'cc4d20e47b36'] -> ['9adbb2f64abd', 'cec102b25e41'] | errors during update 0 (-) | RPS before 287.6, during 271.5, after 229.3 | p95 during 78.0 ms | data ok=True +PASS: rolling update under load + +# 2026-11-10 14:24:41 UTC | Реплики после обновлений | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +cec102b25e41 messenger-lab5-api-5 running healthy messenger-api:lab5 api2 UP +9adbb2f64abd messenger-lab5-api-6 running healthy messenger-api:lab5 api3 UP + diff --git a/docs/lab5/evidence/s3_bootstrap.txt b/docs/lab5/evidence/s3_bootstrap.txt new file mode 100644 index 0000000..a370a05 --- /dev/null +++ b/docs/lab5/evidence/s3_bootstrap.txt @@ -0,0 +1,225 @@ +# 2026-11-10 14:13:53 UTC | S3 bootstrap: лог первого запуска s3-init | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color s3-init +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "created", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +s3-init-1 | { +s3-init-1 | "endpoint_host": "s3", +s3-init-1 | "bucket": "messenger-attachments", +s3-init-1 | "checks": [ +s3-init-1 | { +s3-init-1 | "check": "api credentials: put/get own bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous GET object is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list bucket is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "anonymous list buckets is denied", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "HTTP 403" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot create buckets", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "api credentials cannot delete the bucket", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "AccessDenied" +s3-init-1 | }, +s3-init-1 | { +s3-init-1 | "check": "SeaweedFS admin/UI ports are not reachable on the shared network", +s3-init-1 | "ok": true, +s3-init-1 | "detail": "checked [9333, 8888, 8080, 19333, 18888]" +s3-init-1 | } +s3-init-1 | ], +s3-init-1 | "bucket_action": "already exists (idempotent no-op)", +s3-init-1 | "ok": true +s3-init-1 | } +# 2026-11-10 14:14:03 UTC | S3 bootstrap: повторный запуск (идемпотентность) | exit 0 | 9.6s +$ docker compose --env-file .env run --rm -T s3-init + Container messenger-lab5-s3-1 Running + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-s3-init-run-ac08328e286f Creating + Container messenger-lab5-s3-init-run-ac08328e286f Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + } + ], + "bucket_action": "already exists (idempotent no-op)", + "ok": true +} + diff --git a/docs/lab5/evidence/s3_outage.json b/docs/lab5/evidence/s3_outage.json new file mode 100644 index 0000000..18fc191 --- /dev/null +++ b/docs/lab5/evidence/s3_outage.json @@ -0,0 +1,98 @@ +{ + "scenario": "S3 outage and recovery", + "max_fail_seconds": 15.0, + "s3_warmup_seconds": 19.66, + "checks": [ + { + "check": "baseline upload 202 (lab5)", + "ok": true, + "detail": "202" + }, + { + "check": "readiness stays 200 while S3 is down (S3 is not a readiness dependency)", + "ok": true, + "detail": "200" + }, + { + "check": "text message 201 during S3 outage", + "ok": true, + "detail": "201" + }, + { + "check": "history 200 during S3 outage", + "ok": true, + "detail": "200" + }, + { + "check": "attachment metadata 200 (PostgreSQL only)", + "ok": true, + "detail": "200" + }, + { + "check": "message referencing existing attachment 201", + "ok": true, + "detail": "201" + }, + { + "check": "upload #1 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 5.06s" + }, + { + "check": "upload #1 failed fast (<= 15s)", + "ok": true, + "detail": "5.06s" + }, + { + "check": "upload #1 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "upload #2 -> 503 unavailable", + "ok": true, + "detail": "503 unavailable in 5.33s" + }, + { + "check": "upload #2 failed fast (<= 15s)", + "ok": true, + "detail": "5.33s" + }, + { + "check": "upload #2 has Retry-After", + "ok": true, + "detail": "" + }, + { + "check": "download -> 503 during outage", + "ok": true, + "detail": "503 in 5.01s" + }, + { + "check": "download failed fast", + "ok": true, + "detail": "5.01s" + }, + { + "check": "session still valid", + "ok": true, + "detail": "200" + }, + { + "check": "S3 read/write warm-up after start", + "ok": true, + "detail": "19.66s" + }, + { + "check": "old file downloads byte-for-byte after recovery", + "ok": true, + "detail": "200 0.02s" + }, + { + "check": "new upload 202 after recovery (same token, no API restart)", + "ok": true, + "detail": "202" + } + ], + "pass": true +} diff --git a/docs/lab5/evidence/s3_outage.txt b/docs/lab5/evidence/s3_outage.txt new file mode 100644 index 0000000..6c80ff1 --- /dev/null +++ b/docs/lab5/evidence/s3_outage.txt @@ -0,0 +1,24 @@ +# 2026-11-10 14:20:18 UTC | Недоступность S3 и восстановление | exit 0 | 47.3s +$ python student-tests/s3_outage.py --base-url http://127.0.0.1:8080 --output evidence/s3_outage.json +PASS baseline upload 202 (lab5) — 202 +$ docker compose --env-file .env stop s3 +PASS readiness stays 200 while S3 is down (S3 is not a readiness dependency) — 200 +PASS text message 201 during S3 outage — 201 +PASS history 200 during S3 outage — 200 +PASS attachment metadata 200 (PostgreSQL only) — 200 +PASS message referencing existing attachment 201 — 201 +PASS upload #1 -> 503 unavailable — 503 unavailable in 5.06s +PASS upload #1 failed fast (<= 15s) — 5.06s +PASS upload #1 has Retry-After +PASS upload #2 -> 503 unavailable — 503 unavailable in 5.33s +PASS upload #2 failed fast (<= 15s) — 5.33s +PASS upload #2 has Retry-After +PASS download -> 503 during outage — 503 in 5.01s +PASS download failed fast — 5.01s +PASS session still valid — 200 +$ docker compose --env-file .env start s3 +$ C:\Users\mrend\AppData\Local\Programs\Python\Python312\python.exe scripts/ops.py wait-s3 +PASS S3 read/write warm-up after start — 19.66s +PASS old file downloads byte-for-byte after recovery — 200 0.02s +PASS new upload 202 after recovery (same token, no API restart) — 202 +PASS diff --git a/docs/lab5/evidence/s3_private.txt b/docs/lab5/evidence/s3_private.txt new file mode 100644 index 0000000..efe8586 --- /dev/null +++ b/docs/lab5/evidence/s3_private.txt @@ -0,0 +1,67 @@ +# 2026-11-10 14:15:16 UTC | Приватный бакет вне API, права ключей, закрытые админ-порты | exit 0 | 2.6s +$ python scripts/ops.py s3-check + Container messenger-lab5-api-run-796deabd49b0 Creating + Container messenger-lab5-api-run-796deabd49b0 Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get own bucket", + "ok": true, + "detail": "" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "111817 bytes" + } + ], + "ok": true +} +# 2026-11-10 14:15:17 UTC | На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes | exit 0 | 0.5s +$ python scripts/ops.py api-fs +messenger-lab5-api-1 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/broker_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/database_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] +messenger-lab5-api-2 ReadonlyRootfs=True Tmpfs={'/tmp': 'size=16m'} Mounts=['bind:/run/secrets/broker_url', 'bind:/run/secrets/cache_url', 'bind:/run/secrets/database_url', 'bind:/run/secrets/s3_access_key_id', 'bind:/run/secrets/s3_secret_access_key'] + +# 2026-11-10 14:15:17 UTC | API: файлы в /tmp (tmpfs) после загрузок | exit 0 | 0.3s +$ docker compose --env-file .env exec -T api sh -c df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l) +Filesystem Size Used Avail Use% Mounted on +tmpfs 16M 0 16M 0% /tmp +files_in_tmp=0 + diff --git a/docs/lab5/evidence/smoke.txt b/docs/lab5/evidence/smoke.txt new file mode 100644 index 0000000..988a689 --- /dev/null +++ b/docs/lab5/evidence/smoke.txt @@ -0,0 +1,3 @@ +# 2026-11-10 14:14:05 UTC | Публичный smoke | exit 0 | 1.8s +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab5/evidence/stand.txt b/docs/lab5/evidence/stand.txt new file mode 100644 index 0000000..81e51f5 --- /dev/null +++ b/docs/lab5/evidence/stand.txt @@ -0,0 +1,61 @@ +# 2026-11-10 14:13:49 UTC | Стенд: контейнеры | exit 0 | 0.3s +$ docker compose --env-file .env ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab5-api-1 messenger-api:lab5 "python -m src.server" api 28 minutes ago Up 10 minutes (healthy) 8080/tcp +messenger-lab5-api-2 messenger-api:lab5 "python -m src.server" api 28 minutes ago Up 10 minutes (healthy) 8080/tcp +messenger-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 28 minutes ago Up 9 minutes (healthy) 6379/tcp +messenger-lab5-lb-1 haproxy:2.8.30-alpine "docker-entrypoint.s…" lb 28 minutes ago Up 13 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +messenger-lab5-migrate-1 messenger-lab5-migrate "python -m src.migra…" migrate 28 minutes ago Exited (0) 4 minutes ago +messenger-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 28 minutes ago Up 14 minutes (healthy) 5432/tcp +messenger-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 28 minutes ago Up 14 minutes (healthy) 6379/tcp +messenger-lab5-s3-1 chrislusf/seaweedfs:4.50 "/usr/bin/weed serve…" s3 28 minutes ago Up 14 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +messenger-lab5-s3-init-1 messenger-lab5-s3-init "python -m src.s3_bo…" s3-init 28 minutes ago Exited (0) 4 minutes ago +messenger-lab5-worker-1 messenger-api:lab5 "python -m src.worker" worker 3 minutes ago Up 2 minutes (healthy) 8080/tcp +# 2026-11-10 14:13:50 UTC | Стенд: реплики и статус в LB | exit 0 | 0.3s +$ python scripts/ops.py ps +X-Instance-Id container state health image lb lb_status +4b1faad0399c messenger-lab5-api-1 running healthy messenger-api:lab5 api1 UP +3a0e06f14a63 messenger-lab5-api-2 running healthy messenger-api:lab5 api2 UP + +# 2026-11-10 14:13:52 UTC | Стенд: лимиты CPU/RAM | exit 0 | 2.2s +$ docker stats --no-stream --format table {{.Name}} {{.CPUPerc}} {{.MemUsage}} {{.MemPerc}} +NAME CPU % MEM USAGE / LIMIT MEM % +messenger-lab5-worker-1 0.09% 76.75MiB / 1.25GiB 6.00% +messenger-lab5-lb-1 0.13% 12.85MiB / 128MiB 10.04% +messenger-lab5-api-1 0.38% 82.19MiB / 320MiB 25.68% +messenger-lab5-api-2 18.24% 77.13MiB / 320MiB 24.10% +messenger-lab5-postgres-1 3.64% 74.93MiB / 512MiB 14.63% +messenger-lab5-redis-1 0.27% 3.957MiB / 96MiB 4.12% +messenger-lab5-s3-1 2.99% 189.7MiB / 512MiB 37.06% +messenger-lab5-broker-1 0.29% 4.012MiB / 320MiB 1.25% + +# 2026-11-10 14:13:52 UTC | Стенд: опубликованные порты (только lb) | exit 0 | 0.4s +$ docker compose --env-file .env ps --format table {{.Service}} {{.Ports}} +SERVICE PORTS +api 8080/tcp +api 8080/tcp +broker 6379/tcp +lb 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +postgres 5432/tcp +redis 6379/tcp +s3 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +worker 8080/tcp + +# 2026-11-10 14:13:53 UTC | Образ SeaweedFS (закреплённая версия) | exit 0 | 0.4s +$ docker compose --env-file .env images s3 +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +messenger-lab5-s3-1 chrislusf/seaweedfs 4.50 linux/amd64 c2658820e763 734MB 2 weeks ago + +# 2026-11-10 14:13:53 UTC | lab5: broker (Redis AOF) и worker | exit 0 | 0.3s +$ docker compose --env-file .env ps broker worker +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 28 minutes ago Up 10 minutes (healthy) 6379/tcp +messenger-lab5-worker-1 messenger-api:lab5 "python -m src.worker" worker 3 minutes ago Up 2 minutes (healthy) 8080/tcp + +# 2026-11-10 14:13:53 UTC | lab5: конфигурация durability брокера | exit 0 | 0.3s +$ docker compose --env-file .env exec -T broker sh -c REDISCLI_AUTH="$(tr -d '\r\n' < /run/secrets/broker_password)" redis-cli CONFIG GET appendfsync; REDISCLI_AUTH="$(tr -d '\r\n' < /run/secrets/broker_password)" redis-cli CONFIG GET maxmemory-policy +appendfsync +always +maxmemory-policy +noeviction + diff --git a/docs/lab5/evidence/student-tests.txt b/docs/lab5/evidence/student-tests.txt new file mode 100644 index 0000000..f333b67 --- /dev/null +++ b/docs/lab5/evidence/student-tests.txt @@ -0,0 +1,185 @@ +# 2026-11-10 14:14:27 UTC | Собственные тесты (HTTP через LB, включая test_attachments.py) | exit 0 | 21.7s +$ python student-tests/run_all.py --base-url http://127.0.0.1:8080 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... skipped 'needs API at http://127.0.0.1:8080 and DB/S3/broker env' +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' + +---------------------------------------------------------------------- +Ran 91 tests in 19.876s + +OK (skipped=33) + + 24 parallel POSTs, one key -> 1 message; replicas: ['3a0e06f14a63', '4b1faad0399c'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED +# 2026-11-10 14:14:36 UTC | Тесты с доступом к БД (внутри compose-сети) | exit 0 | 8.8s +$ python scripts/ops.py in-network storage + Container messenger-lab5-api-run-87d6a97994de Creating + Container messenger-lab5-api-run-87d6a97994de Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 3.408s + +OK + Container messenger-lab5-api-run-7049d7c00dfc Creating + Container messenger-lab5-api-run-7049d7c00dfc Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok + +---------------------------------------------------------------------- +Ran 3 tests in 2.032s + +OK + +# 2026-11-10 14:14:51 UTC | Истечение сессии (api-short-ttl, TTL 2 с) | exit 0 | 15.7s +$ python scripts/ops.py in-network expiry + Container messenger-lab5-postgres-1 Running + Container messenger-lab5-s3-1 Running + Container messenger-lab5-api-1 Running + Container messenger-lab5-api-2 Running + Container messenger-lab5-redis-1 Running + Container messenger-lab5-api-short-ttl-1 Creating + Container messenger-lab5-api-short-ttl-1 Created + Container messenger-lab5-s3-1 Waiting + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-s3-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Starting + Container messenger-lab5-s3-init-1 Starting + Container messenger-lab5-migrate-1 Started + Container messenger-lab5-s3-init-1 Started + Container messenger-lab5-postgres-1 Waiting + Container messenger-lab5-redis-1 Waiting + Container messenger-lab5-migrate-1 Waiting + Container messenger-lab5-s3-init-1 Waiting + Container messenger-lab5-redis-1 Healthy + Container messenger-lab5-postgres-1 Healthy + Container messenger-lab5-migrate-1 Exited + Container messenger-lab5-s3-init-1 Exited + Container messenger-lab5-api-1 Waiting + Container messenger-lab5-api-2 Waiting + Container messenger-lab5-api-1 Healthy + Container messenger-lab5-api-2 Healthy + Container messenger-lab5-api-short-ttl-1 Starting + Container messenger-lab5-api-short-ttl-1 Started + Container messenger-lab5-api-run-f4cd12243eab Creating + Container messenger-lab5-api-run-f4cd12243eab Created +ready: http://api-short-ttl:8080/health/ready + Container messenger-lab5-api-run-249c8116f0a3 Creating + Container messenger-lab5-api-run-249c8116f0a3 Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok + +---------------------------------------------------------------------- +Ran 1 test in 3.743s + +OK + diff --git a/docs/lab5/evidence/upload_memory.txt b/docs/lab5/evidence/upload_memory.txt new file mode 100644 index 0000000..5ddd978 --- /dev/null +++ b/docs/lab5/evidence/upload_memory.txt @@ -0,0 +1,161 @@ +# 2026-11-10 14:15:28 UTC | Пик памяти: 10 одновременных загрузок по 9.5 MiB | exit 0 | 11.3s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 10 --rounds 3 --docker --label c=10, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c10.json +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:28Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "202": 30 + }, + "sha256_ok": 30, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 6.22, + "latency_201_s": { + "p50": 1.719, + "p95": 1.922, + "max": 2.5309999999990396, + "mean": 1.663 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 76.3, + "peak_mib": 151.7, + "growth_mib": 75.4, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 13.0, + "peak_mib": 30.2, + "growth_mib": 17.3, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 134.6, + "peak_mib": 153.6, + "growth_mib": 19.0, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 136.9, + "peak_mib": 152.8, + "growth_mib": 15.9, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 84.9, + "peak_mib": 86.6, + "growth_mib": 1.7, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 227.9, + "peak_mib": 303.6, + "growth_mib": 75.7, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.2, + "peak_mib": 4.5, + "growth_mib": 0.4, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} +# 2026-11-10 14:15:40 UTC | Пик памяти: 20 одновременных загрузок | exit 0 | 11.5s +$ python student-tests/upload_memory.py --base-url http://127.0.0.1:8080 --concurrency 20 --rounds 2 --docker --label c=20, MAX_CONCURRENT_UPLOADS=4 --output evidence/upload_memory_c20.json +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:40Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "202": 40 + }, + "sha256_ok": 40, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 7.16, + "latency_201_s": { + "p50": 2.984, + "p95": 3.562, + "max": 3.577999999997701, + "mean": 2.695 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 152.5, + "peak_mib": 152.5, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 73.6, + "peak_mib": 74.0, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 155.9, + "peak_mib": 171.2, + "growth_mib": 15.3, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 168.7, + "peak_mib": 168.7, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 86.3, + "peak_mib": 86.7, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.2, + "peak_mib": 5.1, + "growth_mib": 0.9, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 348.9, + "peak_mib": 430.3, + "growth_mib": 81.4, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} + diff --git a/docs/lab5/evidence/upload_memory_c10.json b/docs/lab5/evidence/upload_memory_c10.json new file mode 100644 index 0000000..f15c89d --- /dev/null +++ b/docs/lab5/evidence/upload_memory_c10.json @@ -0,0 +1,78 @@ +{ + "label": "c=10, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:28Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 10, + "rounds": 3, + "file_mib": 9.5, + "uploads": 30, + "statuses": { + "202": 30 + }, + "sha256_ok": 30, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 6.22, + "latency_201_s": { + "p50": 1.719, + "p95": 1.922, + "max": 2.5309999999990396, + "mean": 1.663 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 76.3, + "peak_mib": 151.7, + "growth_mib": 75.4, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 13.0, + "peak_mib": 30.2, + "growth_mib": 17.3, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 134.6, + "peak_mib": 153.6, + "growth_mib": 19.0, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 136.9, + "peak_mib": 152.8, + "growth_mib": 15.9, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 84.9, + "peak_mib": 86.6, + "growth_mib": 1.7, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 227.9, + "peak_mib": 303.6, + "growth_mib": 75.7, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.2, + "peak_mib": 4.5, + "growth_mib": 0.4, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} diff --git a/docs/lab5/evidence/upload_memory_c20.json b/docs/lab5/evidence/upload_memory_c20.json new file mode 100644 index 0000000..a790ce9 --- /dev/null +++ b/docs/lab5/evidence/upload_memory_c20.json @@ -0,0 +1,78 @@ +{ + "label": "c=20, MAX_CONCURRENT_UPLOADS=4", + "date_utc": "2026-11-10T14:15:40Z", + "host": "Windows AMD64, 12 CPU", + "source": "docker stats + cgroup peak", + "concurrency": 20, + "rounds": 2, + "file_mib": 9.5, + "uploads": 40, + "statuses": { + "202": 40 + }, + "sha256_ok": 40, + "instances": [ + "3a0e06f14a63", + "4b1faad0399c" + ], + "duration_s": 7.16, + "latency_201_s": { + "p50": 2.984, + "p95": 3.562, + "max": 3.577999999997701, + "mean": 2.695 + }, + "memory": { + "messenger-lab5-worker-1": { + "baseline_mib": 152.5, + "peak_mib": 152.5, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-lb-1": { + "baseline_mib": 73.6, + "peak_mib": 74.0, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-api-1": { + "baseline_mib": 155.9, + "peak_mib": 171.2, + "growth_mib": 15.3, + "samples": 5 + }, + "messenger-lab5-api-2": { + "baseline_mib": 168.7, + "peak_mib": 168.7, + "growth_mib": 0.0, + "samples": 5 + }, + "messenger-lab5-postgres-1": { + "baseline_mib": 86.3, + "peak_mib": 86.7, + "growth_mib": 0.4, + "samples": 5 + }, + "messenger-lab5-redis-1": { + "baseline_mib": 4.2, + "peak_mib": 5.1, + "growth_mib": 0.9, + "samples": 5 + }, + "messenger-lab5-s3-1": { + "baseline_mib": 348.9, + "peak_mib": 430.3, + "growth_mib": 81.4, + "samples": 5 + }, + "messenger-lab5-broker-1": { + "baseline_mib": 4.7, + "peak_mib": 5.3, + "growth_mib": 0.6, + "samples": 5 + } + }, + "process_peak_rss_mib": {}, + "cgroup_peak_mib_before": {}, + "cgroup_peak_mib_after": {} +} diff --git a/docs/lab5/evidence/worker_log_sample.txt b/docs/lab5/evidence/worker_log_sample.txt new file mode 100644 index 0000000..5865ce0 --- /dev/null +++ b/docs/lab5/evidence/worker_log_sample.txt @@ -0,0 +1,3 @@ +# 2026-11-10 14:23:08 UTC | Worker: последние строки лога | exit 0 | 0.3s +$ docker compose --env-file .env logs --no-color --tail 80 worker +worker-1 | 2026-11-10 14:23:00,051 INFO messenger.worker worker 853a915daea3: concurrency=2 max_attempts=5 lease=60s claim_idle=75s image_timeout=30.0s image_memory=1024MiB retry=2.0..60.0s faults=off diff --git a/docs/lab5/evidence/worker_outage.json b/docs/lab5/evidence/worker_outage.json new file mode 100644 index 0000000..7f7b8f4 --- /dev/null +++ b/docs/lab5/evidence/worker_outage.json @@ -0,0 +1,80 @@ +[ + { + "seconds": 2.06, + "step": "worker stopped", + "t": 2.42 + }, + { + "status": 202, + "attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", + "attachment_status": "queued", + "step": "upload accepted", + "t": 2.44 + }, + { + "metadata_status": "queued", + "content_status": 409, + "message_with_attachment": 201, + "plain_message": 201, + "message_id": "310e494b-eccf-49b1-9424-bfec99522d7b", + "step": "while worker is down", + "t": 5.53 + }, + { + "length": 9, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 148, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "broker before restarts", + "t": 7.22 + }, + { + "seconds": 11.0, + "step": "api replicas restarted", + "t": 18.22 + }, + { + "seconds": 4.58, + "step": "broker restarted (volume kept)", + "t": 22.8 + }, + { + "metadata_status": "queued", + "length": 9, + "lag": 1, + "pending": 0, + "jobs": { + "succeeded": 148, + "failed": 1, + "queued": 1 + }, + "outbox_rows": 0, + "step": "after restarts", + "t": 26.67 + }, + { + "seconds": 6.41, + "step": "worker started", + "t": 33.08 + }, + { + "attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", + "variants": [ + "thumbnail", + "crop", + "optimized" + ], + "thumbnail_jpeg_size": [ + 256, + 160 + ], + "original_sha256_ok": true, + "step": "ready", + "t": 33.12 + } +] \ No newline at end of file diff --git a/docs/lab5/evidence/worker_outage.txt b/docs/lab5/evidence/worker_outage.txt new file mode 100644 index 0000000..2b67ab7 --- /dev/null +++ b/docs/lab5/evidence/worker_outage.txt @@ -0,0 +1,12 @@ +# 2026-11-10 14:20:58 UTC | Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready | exit 0 | 33.3s +$ python student-tests/worker_outage.py --base-url http://127.0.0.1:8080 --output evidence/worker_outage.json +{"seconds": 2.06, "step": "worker stopped", "t": 2.42} +{"status": 202, "attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", "attachment_status": "queued", "step": "upload accepted", "t": 2.44} +{"metadata_status": "queued", "content_status": 409, "message_with_attachment": 201, "plain_message": 201, "message_id": "310e494b-eccf-49b1-9424-bfec99522d7b", "step": "while worker is down", "t": 5.53} +{"length": 9, "lag": 1, "pending": 0, "jobs": {"succeeded": 148, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "broker before restarts", "t": 7.22} +{"seconds": 11.0, "step": "api replicas restarted", "t": 18.22} +{"seconds": 4.58, "step": "broker restarted (volume kept)", "t": 22.8} +{"metadata_status": "queued", "length": 9, "lag": 1, "pending": 0, "jobs": {"succeeded": 148, "failed": 1, "queued": 1}, "outbox_rows": 0, "step": "after restarts", "t": 26.67} +{"seconds": 6.41, "step": "worker started", "t": 33.08} +{"attachment_id": "998f7ba6-a5ae-42ab-b965-8c3cb4be2d00", "variants": ["thumbnail", "crop", "optimized"], "thumbnail_jpeg_size": [256, 160], "original_sha256_ok": true, "step": "ready", "t": 33.12} +PASS: accepted job survived worker stop + API restart + broker restart and became ready diff --git a/docs/lab6/REPORT.md b/docs/lab6/REPORT.md new file mode 100644 index 0000000..d0ee4c7 --- /dev/null +++ b/docs/lab6/REPORT.md @@ -0,0 +1,136 @@ +# Отчёт — лабораторная 6 + +- Автор, группа: Попов Константин Дмитриевич, группа БВТ2303 +- Целевая оценка: **4**; трек 5 не заявляется +- Commit/PR: TODO после публикации +- Использованные источники, библиотеки и помощь ИИ: OpenTelemetry Python SDK 1.29.0 (`opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-http`), инструментации 0.50b0 (`fastapi`, `psycopg`, `botocore`); OpenTelemetry Collector contrib 0.115.1, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.6.1, Grafana 11.4.0. Документация: OTel Collector configuration (memory_limiter, batch, sending_queue/retry_on_failure), W3C Trace Context, OTel messaging semantic conventions (producer/consumer, `messaging.*`), HTTP semconv (`http.server.request.duration`), Prometheus histograms / `histogram_quantile`, Loki OTLP ingestion (structured metadata), Grafana provisioning. Остальной стек — как в lab5 (см. `docs/lab5/REPORT.md`). Помощь ИИ (Claude) — при проектировании схемы сигналов, написании `src/telemetry.py`, конфигураций, dashboard, alert rules, сценариев инцидентов и отчёта. + +Минимум lab5 перенесён целиком (HAProxy, 2 реплики API, PostgreSQL, Redis-кэш, SeaweedFS S3, Redis Streams broker, worker, outbox, retry/DLQ). **Бизнес-API не менялся**; новые поля только служебные: `traceparent`/`tracestate` в сообщении задания (уже разрешены `contracts/jobs.schema.json`). + +## Запуск из чистого clone + +**Версии.** API и worker — один образ из `Dockerfile` (`python:3.12.7-slim-bookworm`), `postgres:16.4-alpine`, `redis:7.4.1-alpine` (кэш и broker), `haproxy:2.8.30-alpine`, `chrislusf/seaweedfs:4.50`, **`otel/opentelemetry-collector-contrib:0.115.1`, `prom/prometheus:v2.55.1`, `grafana/loki:3.3.2`, `grafana/tempo:2.6.1`, `grafana/grafana:11.4.0`**. Нужны Docker с Compose v2 и Python 3.9+ на хосте. Оркестрация — Compose. + +```sh +make up # = python scripts/init_secrets.py; cp .env.example .env; + # docker compose --env-file .env up -d --build; ожидание готовности API/S3/worker +make test # публичный smoke через LB http://localhost:8080 +make obs-urls # Grafana http://127.0.0.1:3000 (admin / secrets/grafana_admin_password), Prometheus :9090 +make down # безопасная остановка, все volumes сохраняются +make destroy # НАМЕРЕННОЕ удаление всех данных (down -v): БД, S3, очередь, история телеметрии +``` + +Bootstrap: секреты (`secrets/`, вне Git) создаёт `scripts/init_secrets.py`, в т.ч. новый `grafana_admin_password`. Миграции — one-shot `migrate`, бакет — `s3-init` (как в lab5). Provisioning Grafana (datasources + dashboard) и rules Prometheus подключаются из Git read-only при старте — ручной набор не нужен. Повторный `make up` идемпотентен. + +## Архитектура и решения + +``` +клиент -> HAProxy -> API x2 ──SQL──> PostgreSQL (источник истины, job_outbox) + │ └─S3──> SeaweedFS + └─outbox relay─XADD(traceparent)─> broker (Redis Streams) ─> worker ─S3/SQL─> ... +API/worker ──OTLP/HTTP :4318──> otel-collector ─┬─ traces ─OTLP gRPC─> Tempo (volume tempodata) + ├─ metrics ─/metrics :8889 <─scrape─ Prometheus (promdata, rules) + └─ logs ─OTLP /otlp─> Loki (lokidata) + Grafana (grafanadata) читает все три +``` + +| Роль | Сервис | Образ | Назначение | Volume | Порт наружу | +| --- | --- | --- | --- | --- | --- | +| Collector | `otel-collector` | contrib 0.115.1 | приём OTLP, memory_limiter, batch, ограниченные очереди, экспорт | — (не хранилище) | нет | +| Метрики | `prometheus` | v2.55.1 | TSDB метрик 7 д, вычисление recording/alert rules, состояние firing/resolved | `promdata` | 127.0.0.1:9090 | +| Логи | `loki` | 3.3.2 | хранение логов 7 д; индекс только по `service_name`/`service_instance_id`, остальное — structured metadata | `lokidata` | нет | +| Трассы | `tempo` | 2.6.1 | хранение spans 72 ч, поиск TraceQL | `tempodata` | нет | +| UI | `grafana` | 11.4.0 | запросы и dashboard; сама сигналы не хранит | `grafanadata` | 127.0.0.1:3000 | + +**Почему три хранилища.** Метрика отвечает «сколько и как часто» (дёшево, агрегаты, алерты), но не «какой запрос»; лог отвечает «что именно произошло и почему» в одном процессе; trace отвечает «где ушло время» по цепочке процессов. Prometheus хранит числовые ряды с метками, Loki — строки с минимальным индексом, Tempo — деревья spans по trace_id. + +**Инструментация (`src/telemetry.py`).** Вызывается до создания пула БД/клиента S3: +- Resource: `service.name` = `messenger-api` / `messenger-worker`, `service.version` = `APP_VERSION`, `service.instance.id` = `X-Instance-Id` (hostname контейнера), `deployment.environment.name=lab`. +- Traces: FastAPI (SERVER span, только шаблон маршрута; health исключены; внутренние send/receive spans отключены), psycopg (SQL **без параметров**), botocore (S3). Redis **не** инструментируется: команды `AUTH` и ключи кэша сессий не должны попасть в spans. Sampler `DropOrphanClientSpans`: CLIENT-span без родителя не сохраняется — это фоновые SQL/S3 (sweeper outbox раз в секунду, GC, readiness, callbacks метрик), иначе Tempo заполнялся бы тысячами пустых трасс. Всё внутри SERVER/PRODUCER/CONSUMER сохраняется (ParentBased AlwaysOn). +- Logs: JSON в stdout (для `docker logs`) и OTLP. Поля: `timestamp`, `severity`, `service.name`, `event`, `trace_id`, `span_id` (если есть активный span), плюс технические поля (`request_id`, `http_route`, `outcome`...). Каждый бизнес-запрос даёт строку `event=http.request` с шаблоном маршрута, статусом, длительностью и `request_id`; каждое сообщение worker'а — `event=job.processed`. +- `X-Request-Id` записывается атрибутом `messenger.request_id` в SERVER span: ответ с ошибкой → trace → лог. + +**Перенос контекста через очередь (W3C).** `job_payload()` записывает `traceparent` текущего span в JSON задания, который хранится в `job_outbox` (PostgreSQL) — контекст переживает задержку публикации и рестарты. Relay (быстрый путь API или sweeper любого процесса) создаёт **PRODUCER** span `attachments.jobs publish` — ребёнок span'а из строки outbox — и кладёт в брокер уже его `traceparent`. Worker создаёт **CONSUMER** span `attachments.jobs process` — ребёнок producer. Внутри него — SQL claim/complete и S3 GetObject/PutObject. При временной ошибке `retry_or_fail()` пишет новую строку outbox с `traceparent` consumer span → следующая попытка в **той же** трассе (связь parent-child, а не link: одна логическая операция «upload → ready»). + +**Сигналы и их реальные имена в Prometheus** (Collector `prometheus` exporter: точки → `_`, единица → суффикс, counter → `_total`; `job` = service.name, `instance` = service.instance.id): + +| Логический сигнал (contracts/observability.md) | Имя OTel | Имя в Prometheus | Атрибуты | Ед. | +| --- | --- | --- | --- | --- | +| HTTP requests (counter) | `http.server.request.duration` (count) | `http_server_request_duration_seconds_count` | `http_request_method`, `http_route` (шаблон), `http_response_status_code` → класс через `label_replace` | шт | +| HTTP duration | `http.server.request.duration` | `http_server_request_duration_seconds_bucket` | те же; buckets 5 мс…10 с (View) | с | +| Jobs processed/failed | `messenger.jobs.processed` | `messenger_jobs_processed_total` | `job_type`, `outcome` = succeeded/retry/failed/noop/lease_lost/poison/error | шт | +| Job duration | `messenger.job.duration` | `messenger_job_duration_seconds_bucket` | `job_type`, `outcome`; buckets 50 мс…60 с | с | +| Queue depth | `messenger.queue.depth` | `messenger_queue_depth` | `queue`, `state`=queued/processing | шт | +| Oldest job age | `messenger.queue.oldest_age` | `messenger_queue_oldest_age_seconds` | `queue` | с | +| Outbox backlog | `messenger.outbox.pending` | `messenger_outbox_pending` | `queue` | шт | +| DB pool | `messenger.db.pool.connections` / `.waiting` / `.max` | `messenger_db_pool_connections{state=used\|idle}`, `messenger_db_pool_waiting`, `messenger_db_pool_max` | — | шт | +| Worker saturation | `messenger.worker.busy` / `.capacity` | `messenger_worker_busy`, `messenger_worker_capacity` | — | слоты | +| CPU / RAM процесса | `process.cpu.time`, `process.memory.rss` | `process_cpu_time_seconds_total`, `process_memory_rss_bytes` | — | с, байты | +| End-to-end time to ready | `messenger.attachment.time_to_ready` | `messenger_attachment_time_to_ready_seconds_bucket` | `job_type`; от COMMIT задания (ответ 202) до COMMIT ready | с | +| Публикации relay | `messenger.jobs.published` | `messenger_jobs_published_total` | `job_type` | шт | + +Глубина/возраст очереди считаются из PostgreSQL (источник истины) и публикуются **API-репликами**: при остановленном worker метрика продолжает жить. Реплики дают одинаковые значения — в запросах `max()`. В метках **нет** user_id/chat_id/message_id/attachment_id/request_id/URL/filename. Технические ID (job_id, attachment_id, request_id) есть только в атрибутах spans и логов — они нужны для поиска конкретного upload, данные синтетические. + +**Что не экспортируется:** тела сообщений, байты изображений, filename (в логах/spans нет), заголовки Authorization/Cookie (инструментация их не пишет), параметры SQL (`db.statement` — шаблон с `%s`), подписанные URL (их нет в API), пароли/токены. Проверено: `grep` по всем выгруженным spans/логам локального прогона на `Authorization|Bearer |password` — 0 совпадений. + +**Перцентили** считаются только из histogram: `histogram_quantile(0.95, sum by (le) (rate(..._bucket[5m])))` — buckets всех реплик суммируются, а не усредняются p95 реплик. Важная найденная ошибка: инструментация 0.50b0 при метрике в секундах ставит buckets по умолчанию 0…10000 (миллисекундные) — все запросы попадали бы в первый bucket; заданы явные buckets через View. + +**SLI/SLO** (`infra/prometheus/rules/messenger.yml`, recording rules): +- доступность: `1 − 5xx/все` бизнес-запросы, health исключены, 4xx не ошибка сервиса; **SLO 99% за скользящий 1 ч**; +- задержка: доля запросов быстрее 0.5 с (bucket `le="0.5"`); **SLO 95% за 1 ч**; +- очередь: возраст старейшего незавершённого задания; цель < 60 с. + +**Alert rules** (состояние — Prometheus `/alerts`, Grafana; доставка наружу не нужна), runbook — [docs/runbooks.md](docs/runbooks.md): + +| Alert | Условие | Окно / for | +| --- | --- | --- | +| `ApiHighErrorRate` | доля 5xx > 5% и трафик ≥ 0.2 RPS | rate 1m, for 1m | +| `QueueLagGrowing` | `max(messenger_queue_oldest_age_seconds) > 60` | for 30s | +| `TelemetryExportFailing` | растут `otelcol_exporter_send_failed_*` / `enqueue_failed_*` | rate 1m, for 1m | + +**Collector и ограниченная буферизация** (`infra/otel/collector.yaml`): `memory_limiter` (180 MiB, spike 40) → `batch` (512/1024, 2 с) → exporters с `sending_queue` **20 батчей** и `retry_on_failure` не дольше 60 с; `GOMEMLIMIT=200MiB`, лимит контейнера 384M. Переполнение очереди → отбрасывание с подсчётом (`otelcol_exporter_enqueue_failed_*`). На стороне SDK: `BatchSpanProcessor`/`BatchLogRecordProcessor` с очередью 2048, таймаут экспорта 5 с; при недоступном Collector батч отбрасывается, ошибки SDK — одна JSON-строка в stdout без трейсбека и **не** в OTLP (иначе петля «лог о неудачной отправке логов»). Ни API, ни worker не имеют `depends_on` на телеметрию. + +**Переменные `.env.example` → конфигурация:** `OTEL_SDK_DISABLED` (выключатель), `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318`, `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf`, `OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab`; `OTEL_SERVICE_NAME` задаётся в compose отдельно для API и worker. `GRAFANA_PORT`, `PROMETHEUS_PORT` — порты UI на 127.0.0.1. Остальное — как в lab5. + +**Что изменено относительно lab5:** `src/telemetry.py` (новый); `src/main.py` (setup, FastAPI-инструментация, access-лог, метрики пула/очереди), `src/worker.py` (consumer span, метрики заданий/пула/занятости, time-to-ready), `src/outbox.py` (producer span, повторная инъекция контекста), `src/jobs.py` (`traceparent` в payload, `complete()` возвращает время до ready); `requirements.txt` (+OTel); `compose.yaml` (+5 сервисов, сеть `telemetry`, 4 volume, OTEL_* env); `infra/{otel,prometheus,loki,tempo,grafana}`; `scripts/incident.py`, `scripts/alerts_watch.py`; Makefile (`obs-*`, `incident-*`, `alerts-watch`, `destroy`); `docs/runbooks.md`. Отчёт lab5 и его доказательства перенесены в `docs/lab5/`. + +## Доказательства критериев + +**Важно о среде.** Доказательства в `evidence/local-sandbox/` получены в песочнице **без доступа к Docker Hub**: те же исходники запускались напрямую (Python 3.13, PostgreSQL 16.2, Redis 6.2, S3-эмулятор moto вместо SeaweedFS, 1 реплика API без HAProxy) с **настоящими** бинарниками Collector 0.115, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.6.1 и **конфигурациями из этого репозитория**. Grafana там не запускалась (дистрибутив недоступен) — все PromQL-запросы dashboard выполнены через Prometheus API (0 ошибок), LogQL/TraceQL — через API Loki/Tempo. **Docker-стенд**: Windows 10, Ryzen 5 7500f, 32 GB; доказательства в корне `evidence/`. + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke при включённой телеметрии | `make test` | `PASS: lab6 public API scenario` | `evidence/local-sandbox/smoke.txt` | +| `make check` | `make check` | `PASS: offline skeleton structure` | — | +| API и worker отдают OTLP в Collector; три хранилища | `make obs-probe` → Loki/Tempo API | лог `event=http.request` с `trace_id` в Loki; trace с тем же id в Tempo | `trace_retry_chain.txt` | +| RPS/errors/duration API; jobs success/error и duration worker | dashboard, Prometheus API | все 31 PromQL dashboard возвращают данные | `metric_names.txt` | +| Структурные логи с trace_id/span_id | Loki `{service_name=~"messenger-.+"} \| request_id=""` | API `http.request` и worker `job.processed` одного upload — один `trace_id` | `trace_retry_chain.txt` | +| Импортируемый dashboard | `infra/grafana/dashboards/messenger.json` + provisioning | загружается при старте Grafana, папка Messenger | — | +| Нет секретов/тел сообщений в telemetry | grep выгрузки spans/logs | 0 совпадений `Authorization`, `Bearer `, `password`; SQL без параметров | — | +| Уровень 4: W3C через broker, producer/consumer, retry, S3/DB | upload `fault-transient-until-3.png` при `WORKER_FAULTS=true` | SERVER → PRODUCER → CONSUMER(retry) → PRODUCER → CONSUMER(retry) → PRODUCER → CONSUMER(succeeded) → S3.PutObject×4, SQL — одна трасса из 76 spans | `trace_retry_chain.txt`, `trace_upload_chain.txt` | +| p50/p95/p99, ошибки, queue depth/age, DB pool, saturation worker | dashboard «Messenger lab6» | панели с единицами (s, percentunit, reqps, bytes) | `messenger.json` | +| SLI/SLO и окно; alert error rate | `make incident-db` + `make alerts-watch` | lock 180 с: pending +60 с, **firing +120 с**, **resolved** через 21 с после снятия lock | `alerts_db_lock_180s.txt` | +| Alert queue lag | `make incident-worker` + `make alerts-watch` | queued растёт, `QueueLagGrowing` pending → **firing** → **resolved**; 22 upload'а готовы через 8.7 с после старта worker | `alerts_queue_lag_and_db_short.txt`, `incident_worker_stop.jsonl` | +| Медленная БД отличима от медленного handler | `make incident-db` | 503 только на `/messages`; CLIENT span `INSERT` 5.01 с `QueryCanceled` | `db_lock_span.txt` | +| Collector memory limiter/batch, недоступный backend | `make incident-telemetry` (`SERVICES=otel-collector` и `SERVICES=tempo,loki`) | API 0 ошибок, ~300 RPS; RSS API 135.0→135.4 MiB; Collector пик 234 MiB, очередь 20/20, потери посчитаны | `telemetry_outage.txt` | + +## Эксперименты и отказы + +Локальная песочница: 1 реплика API (без LB), 1 worker (2 потока), нагрузка `tests/load.py` (closed loop, POST сообщений, 2–4 потока), ~300 успешных RPS. **Docker-стенд**: Windows 10, Ryzen 5 7500f, 32 GB; доказательства в корне `evidence/`. + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | +| --- | --- | --- | --- | +| Worker остановлен 110 с, upload каждые 5 с | 202 продолжаются; queue depth и oldest age растут; QueueLagGrowing firing | queued до 22, age > 90 с, firing; API без ошибок | старт worker → все 22 ready за 8.7 с, alert resolved; потерь нет | +| `LOCK TABLE messages` 180 с под нагрузкой | 503 на `/messages` после `statement_timeout` 5 с; ApiHighErrorRate firing; trace показывает SQL | доля 5xx = 100% на маршруте; firing через 120 с; SQL span 5.01 с `QueryCanceled` | resolved через 21 с после снятия lock; запросы с 503 не записаны (клиент повторяет с тем же Idempotency-Key) | +| Collector остановлен 45 с | API работает, RAM не растёт, телеметрия за период теряется | 0 ошибок, 303 RPS; RSS 135.0 → 135.4 MiB | после старта экспорт возобновился; spans/логи/метрики за 45 с потеряны (SDK отбрасывает) | +| Tempo + Loki остановлены 120 с (Collector жив) | очередь Collector заполняется и ограничена; потери считаются | queue 20/20; `enqueue_failed_spans` 131083, `..._log_records` 22016; пик RSS 234 MiB; API 0 ошибок | после старта backend очередь → 0, новые trace/логи видны; потеря — посчитанное число | + +Детекция ошибок не мгновенная: `rate(...[1m])` первую минуту «разбавлен» трафиком до инцидента (при closed-loop нагрузке RPS падает в сотни раз), поэтому pending наступает примерно через 60 с, firing — ещё через `for: 1m`. Инцидент короче ~2 мин до firing не доходит (наблюдалось на lock 100 с — только pending). Это осознанный компромисс против ложных срабатываний; для быстрой детекции нужны burn-rate alerts с короткими окнами (трек 5, не заявлен). + +## Ограничения и следующие шаги + +- Весь стенд на одном Docker-host; Collector, Prometheus, Loki, Tempo — по одному экземпляру без репликации. Потеря host = потеря телеметрии и приложения. +- При недоступном Collector телеметрия за этот период теряется (SDK держит только ограниченную очередь в памяти) — так сделано намеренно: приложение важнее телеметрии. +- Метрики очереди публикуют API-реплики: если упадут все API, метрика пропадёт (alert на отсутствие данных не настроен). +- Sampling 100% (кроме фоновых сиротских SQL) — на учебной нагрузке приемлемо; цену наблюдаемости и tail sampling не исследовал (трек 5 не заявлен). +- Метрики `process_*` — RSS/CPU процесса, а не cgroup контейнера; для контейнеров можно добавить cAdvisor. diff --git a/docs/lab6/evidence/README.md b/docs/lab6/evidence/README.md new file mode 100644 index 0000000..019c34a --- /dev/null +++ b/docs/lab6/evidence/README.md @@ -0,0 +1,11 @@ +# Доказательства + +Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. + +Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Содержимое (lab6) + +- `local-sandbox/` — прогон в песочнице без Docker Hub (те же исходники и конфиги; PostgreSQL 16.2, Redis 6.2, + S3-эмулятор moto, 1 реплика API; настоящие Collector 0.115 / Prometheus 2.55.1 / Loki 3.3.2 / Tempo 2.6.1). Описание условий — REPORT.md, раздел «Доказательства критериев». +- Результаты Docker-стенда - в корне `evidence/`. diff --git a/docs/lab6/evidence/alerts_db.txt b/docs/lab6/evidence/alerts_db.txt new file mode 100644 index 0000000000000000000000000000000000000000..581f622332bb8f48708353017be869f3350b1f65 GIT binary patch literal 662 zcmd6kI}5@<5QL`|{155H$Hkb3U|}Ixi`{@^sTiwNmZLE3%DBqQ|+$&LI1keIKbu*O%Kq2`B4q zwWv1Di$D)BJ8w$QgXZ{-88fDpI^he)BM0$rXXPrCfiA$?$`#yE7w#vp%GGh5J#JI@ z9;oM9qr|>Q9jcjI2)977(={TC$PGOW+EyQEi+|R(g=6`q?EkbSXtTff{oYD`zGAOO Jw~x&I;}bS;S_%LF literal 0 HcmV?d00001 diff --git a/docs/lab6/evidence/alerts_worker.txt b/docs/lab6/evidence/alerts_worker.txt new file mode 100644 index 0000000000000000000000000000000000000000..fbb4d75d3d1e9af443c4841fbe0e1fd2115dfcb4 GIT binary patch literal 690 zcmd6kO%K695Qg8g#DC~rlr1flIJ>w=oLr<)Ux^@9{Chk*B*a0u$z-Opot=+o=X#0M zRHQ_8ImFvYW7MgKjj=7F69qaeQ>mF&_y@d91vcPZB71H&5TD;sNY%UgIw>cwRaJ<7 z>esQZk@XwtsJZEoUP}$=w_}R6ww!`D*>iWR{3UuKYh(F+2trsKy%W9(cWd~L8W}_- zSttHY;qujI`28;?ZK`XOvDz?qt-gbF@+E0Z5C5{ikuI3Ac;O6~{YjcWNS)1;{sC#q UPW{e*CoP!%z|J?eo!nma2JNm~n*aa+ literal 0 HcmV?d00001 diff --git a/docs/lab6/evidence/check.txt b/docs/lab6/evidence/check.txt new file mode 100644 index 0000000000000000000000000000000000000000..80f5d36190aa93798b75f3e0297d5a30df505d6a GIT binary patch literal 180 zcmX|)TMEK35Jk_o;2yF7aRK_(4QN3(Abp@1VzDXM&8x>0WJqr2+{xU}yQbn~?Fc;R zt;mf;XFqaRqbKX7LNZS#8p-6VVO*84NqCt9^;cKZ6?Fw3*!fo<2GEot5>?Z1XJ?GGTN24 z;`L4DcDA&xJ&D#XFpiAXG(t2B+GZ-=t9E^!uQ%j>12qejoG|Qs@!%`=GCq0)3sd>Y#lV z$uBzT*qn OTCVA~dh4n)U)v9te0TZ) literal 0 HcmV?d00001 diff --git a/docs/lab6/evidence/incident_worker.jsonl b/docs/lab6/evidence/incident_worker.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1e1a3fd2e2927b5ecda18d4b770b36310912271f GIT binary patch literal 6762 zcmds+&2Q5{5XI+=#Q%VL>4k>)+u?@b$e|Z*NYOgBDWXk7n+ht#zYhG~I%$HSoOUkA zvc3M=owsk^&Up9dk8gHjx3;yxHk4l4Gme9OWnWQx9Xz~gYT0m}alGg07p`yYQ?&Pt``h^Z#J<~8 zdt!gz)GUV!@>`=d|D(KPuXr z(N0P}0xIvZyJk}~4EUHvUQeA@)Snv10=tEzVv{a1u3C0a9JK=-TaOB+c z#UVcSd3G9|kEo%63q?SAl;WXsE>9dyO3nY-qeH!660ej<3$!I9~1llhWuTLl4>;SZG0@IHI#BAB)IGS{xd$ zWch&Y3v^LamhOFVNFQl&$nKgtm78Rgq;`3kCl2XR5j`yud66>rX{?&Vj-rTfW+Eiw zq)3{Zh=Y#aQC1?Kl46tt`ErucLTV&s9Hpec927N5QKsS>MdZigq)f_*W_)s3nLJEU zQ4!u(@0vFeCux!Nuuw)Or2R?#%29m1D|g^Tex=CX(odQvEZ{_EUnE83B<Q5?wsOP9ZcyA=qNm?X|>#|&Uo%_ESpJ3Nv6YD_5*_;b|6H92%=>FC=JEB3o;Lz(1#5a%BhK&ZH<~Fv zo время ушло в SQL (statement_timeout 5 с), а не в handler. +Длинный прогон (lock 180 с) и состояния ApiHighErrorRate — alerts_db_lock_180s.txt diff --git a/docs/lab6/evidence/local-sandbox/incident_worker_stop.jsonl b/docs/lab6/evidence/local-sandbox/incident_worker_stop.jsonl new file mode 100644 index 0000000..dd82cd0 --- /dev/null +++ b/docs/lab6/evidence/local-sandbox/incident_worker_stop.jsonl @@ -0,0 +1,25 @@ +{"step": "exec", "t": 0.1, "cmd": "/tmp/it/stop_worker.sh"} +{"step": "upload_while_worker_down", "t": 0.1, "status": 202, "request_id": "probe-81b73ce394ae"} +{"step": "upload_while_worker_down", "t": 5.2, "status": 202, "request_id": "probe-099629d749fd"} +{"step": "upload_while_worker_down", "t": 10.2, "status": 202, "request_id": "probe-985b23bb6d03"} +{"step": "upload_while_worker_down", "t": 15.2, "status": 202, "request_id": "probe-2d61f4b6cd93"} +{"step": "upload_while_worker_down", "t": 20.3, "status": 202, "request_id": "probe-30099c838b61"} +{"step": "upload_while_worker_down", "t": 25.3, "status": 202, "request_id": "probe-d789fddd7157"} +{"step": "upload_while_worker_down", "t": 30.3, "status": 202, "request_id": "probe-7fa609b8c10e"} +{"step": "upload_while_worker_down", "t": 35.3, "status": 202, "request_id": "probe-159dfe385e83"} +{"step": "upload_while_worker_down", "t": 40.4, "status": 202, "request_id": "probe-195c69d91227"} +{"step": "upload_while_worker_down", "t": 45.4, "status": 202, "request_id": "probe-c1c1bdea3f27"} +{"step": "upload_while_worker_down", "t": 50.4, "status": 202, "request_id": "probe-1a1a61531996"} +{"step": "upload_while_worker_down", "t": 55.5, "status": 202, "request_id": "probe-8939388f4388"} +{"step": "upload_while_worker_down", "t": 60.5, "status": 202, "request_id": "probe-a23a88d40d77"} +{"step": "upload_while_worker_down", "t": 65.5, "status": 202, "request_id": "probe-bf9e82d4b5b6"} +{"step": "upload_while_worker_down", "t": 70.5, "status": 202, "request_id": "probe-e0e98298a112"} +{"step": "upload_while_worker_down", "t": 75.6, "status": 202, "request_id": "probe-2a5a8eb7a055"} +{"step": "upload_while_worker_down", "t": 80.6, "status": 202, "request_id": "probe-3efa69f3c78e"} +{"step": "upload_while_worker_down", "t": 85.6, "status": 202, "request_id": "probe-281847650ffd"} +{"step": "upload_while_worker_down", "t": 90.7, "status": 202, "request_id": "probe-205546b70ca0"} +{"step": "upload_while_worker_down", "t": 95.7, "status": 202, "request_id": "probe-e352076974d9"} +{"step": "upload_while_worker_down", "t": 100.7, "status": 202, "request_id": "probe-64e5650653c9"} +{"step": "upload_while_worker_down", "t": 105.7, "status": 202, "request_id": "probe-6eee887affe3"} +{"step": "exec", "t": 110.7, "cmd": "/tmp/it/start_worker.sh"} +{"step": "drained", "t": 119.4, "uploads": 22, "not_ready": 0, "drain_seconds": 8.7} diff --git a/docs/lab6/evidence/local-sandbox/metric_names.txt b/docs/lab6/evidence/local-sandbox/metric_names.txt new file mode 100644 index 0000000..242c089 --- /dev/null +++ b/docs/lab6/evidence/local-sandbox/metric_names.txt @@ -0,0 +1,32 @@ +# HELP http_server_active_requests Number of active HTTP server requests. +# TYPE http_server_active_requests gauge +# HELP http_server_request_duration_seconds Duration of HTTP server requests. +# TYPE http_server_request_duration_seconds histogram +# HELP messenger_attachment_time_to_ready_seconds От durable acceptance (COMMIT задания, ответ 202) до публикации ready +# TYPE messenger_attachment_time_to_ready_seconds histogram +# HELP messenger_db_pool_connections +# TYPE messenger_db_pool_connections gauge +# HELP messenger_db_pool_max +# TYPE messenger_db_pool_max gauge +# HELP messenger_db_pool_waiting Запросы, ждущие свободное соединение +# TYPE messenger_db_pool_waiting gauge +# HELP messenger_job_duration_seconds Длительность обработки задания worker'ом +# TYPE messenger_job_duration_seconds histogram +# HELP messenger_jobs_processed_total Обработанные сообщения заданий по исходу (succeeded/retry/failed/noop/...) +# TYPE messenger_jobs_processed_total counter +# HELP messenger_jobs_published_total Сообщения, опубликованные outbox relay в брокер +# TYPE messenger_jobs_published_total counter +# HELP messenger_outbox_pending Строки outbox, ещё не опубликованные в брокер +# TYPE messenger_outbox_pending gauge +# HELP messenger_queue_depth Незавершённые задания +# TYPE messenger_queue_depth gauge +# HELP messenger_queue_oldest_age_seconds Возраст старейшего незавершённого задания +# TYPE messenger_queue_oldest_age_seconds gauge +# HELP messenger_worker_busy Занятые consumer-потоки +# TYPE messenger_worker_busy gauge +# HELP messenger_worker_capacity WORKER_CONCURRENCY +# TYPE messenger_worker_capacity gauge +# HELP process_cpu_time_seconds_total CPU-время процесса и его подпроцессов +# TYPE process_cpu_time_seconds_total counter +# HELP process_memory_rss_bytes RSS процесса +# TYPE process_memory_rss_bytes gauge diff --git a/docs/lab6/evidence/local-sandbox/smoke.txt b/docs/lab6/evidence/local-sandbox/smoke.txt new file mode 100644 index 0000000..7e5c8e2 --- /dev/null +++ b/docs/lab6/evidence/local-sandbox/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. diff --git a/docs/lab6/evidence/local-sandbox/telemetry_outage.txt b/docs/lab6/evidence/local-sandbox/telemetry_outage.txt new file mode 100644 index 0000000..9efc090 --- /dev/null +++ b/docs/lab6/evidence/local-sandbox/telemetry_outage.txt @@ -0,0 +1,21 @@ +Локальный прогон (песочница, без Docker): API x1, worker x1, нагрузка tests/load.py --concurrency 4 (closed loop, POST messages). + +A. Collector остановлен (SDK не может отправить ничего), 45 с: + до: 15 с, 4129 успешных, 0 ошибок, 274.9 RPS; RSS API 126.7 MiB + без коллектора: 45 с, 13663 успешных, 0 ошибок, 303.5 RPS; RSS API 135.0 -> 135.4 MiB (не растёт) + после возврата: 10 с, 2976 успешных, 0 ошибок, 297.1 RPS + SDK: OTLP-экспорт падает с ConnectionError (без повторов), батч отбрасывается; + очереди BatchSpanProcessor/BatchLogRecordProcessor ограничены 2048 записями. + В stdout API — одна JSON-строка на неудачный экспорт (143 строки за 45 с), без трейсбеков, НЕ в OTLP. + +B. Tempo и Loki остановлены, Collector работает, 120 с, ~300 RPS: + API: 36028 успешных, 0 ошибок, 300.2 RPS + Collector (GOMEMLIMIT=200MiB, memory_limiter 180 MiB, sending_queue 20 батчей): пик RSS 234 MiB + otelcol_exporter_queue_size: 20/20 (tempo, loki) — очередь полна, дальше отбрасывание + otelcol_exporter_enqueue_failed_spans 131083, enqueue_failed_log_records 22016 (потери посчитаны) + otelcol_exporter_send_failed_spans 2048, send_failed_log_records 2363 (истёк max_elapsed_time 60 с) + otelcol_receiver_refused_spans 0 + После запуска Tempo/Loki: queue_size -> 0, sent_spans/sent_log_records снова растут, + новый probe виден в Tempo и Loki. + Первый прогон с queue_size 100 дал RSS 290 MiB (> лимита 256M) -> очередь уменьшена до 20, + лимит контейнера Collector поднят до 384M. diff --git a/docs/lab6/evidence/local-sandbox/trace_retry_chain.txt b/docs/lab6/evidence/local-sandbox/trace_retry_chain.txt new file mode 100644 index 0000000..dae3f84 --- /dev/null +++ b/docs/lab6/evidence/local-sandbox/trace_retry_chain.txt @@ -0,0 +1,77 @@ +Tempo search: { span.messenger.request_id = "probe-6f88bf5e5b0d" } -> trace 735b28c5cb7f27799037d11314a8f3c7, spans 76 +messenger-api SERVER POST /api/v1/chats/{chat_id}/attachments 12.5 ms + messenger-api CLIENT messenger 0.1 ms + messenger-api CLIENT SELECT 0.3 ms + messenger-api CLIENT messenger 0.2 ms + messenger-api CLIENT INSERT 0.7 ms + messenger-api CLIENT S3.PutObject 4.7 ms + messenger-api CLIENT messenger 0.2 ms + messenger-api CLIENT UPDATE 0.5 ms + messenger-api CLIENT INSERT 0.3 ms + messenger-api CLIENT INSERT 0.2 ms + messenger-api CLIENT INSERT 0.1 ms + messenger-api CLIENT messenger 1.3 ms + messenger-api CLIENT SELECT 0.7 ms + messenger-api PRODUCER attachments.jobs publish 1.1 ms attempt=1 outcome=None + messenger-worker CONSUMER attachments.jobs process 9.6 ms attempt=1 outcome=retry + messenger-worker CLIENT messenger 0.9 ms + messenger-worker CLIENT SELECT 1.0 ms + messenger-worker CLIENT UPDATE 0.8 ms + messenger-worker CLIENT UPDATE 1.3 ms + messenger-worker CLIENT INSERT 0.5 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.4 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.3 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT UPDATE 0.2 ms + messenger-worker CLIENT INSERT 0.3 ms + messenger-worker CLIENT INSERT 0.1 ms + messenger-api PRODUCER attachments.jobs publish 2.4 ms attempt=2 outcome=None + messenger-worker CONSUMER attachments.jobs process 9.5 ms attempt=2 outcome=retry + messenger-worker CLIENT messenger 0.7 ms + messenger-worker CLIENT SELECT 0.8 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT UPDATE 0.5 ms + messenger-worker CLIENT INSERT 0.2 ms + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT SELECT 0.3 ms + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT SELECT 0.3 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT UPDATE 0.4 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT INSERT 0.3 ms + messenger-worker CLIENT INSERT 0.1 ms + messenger-api PRODUCER attachments.jobs publish 1.1 ms attempt=3 outcome=None + messenger-worker CONSUMER attachments.jobs process 296.9 ms attempt=3 outcome=succeeded + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT SELECT 0.5 ms + messenger-worker CLIENT UPDATE 0.6 ms + messenger-worker CLIENT UPDATE 0.3 ms + messenger-worker CLIENT INSERT 0.1 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT S3.GetObject 5.6 ms + messenger-worker CLIENT S3.HeadObject 6.7 ms + messenger-worker CLIENT S3.PutObject 4.4 ms + messenger-worker CLIENT S3.PutObject 5.3 ms + messenger-worker CLIENT S3.PutObject 13.8 ms + messenger-worker CLIENT S3.PutObject 13.4 ms + messenger-worker CLIENT messenger 0.2 ms + messenger-worker CLIENT UPDATE 0.6 ms + messenger-worker CLIENT INSERT 0.8 ms + messenger-worker CLIENT UPDATE 0.4 ms + messenger-worker CLIENT INSERT 0.2 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT SELECT 0.2 ms + messenger-worker CLIENT S3.DeleteObject 3.3 ms + messenger-worker CLIENT messenger 0.1 ms + messenger-worker CLIENT UPDATE 0.4 ms + messenger-api CLIENT DELETE 1.4 ms + messenger-api CLIENT INSERT 0.6 ms diff --git a/docs/lab6/evidence/local-sandbox/trace_upload_chain.txt b/docs/lab6/evidence/local-sandbox/trace_upload_chain.txt new file mode 100644 index 0000000..26e5979 --- /dev/null +++ b/docs/lab6/evidence/local-sandbox/trace_upload_chain.txt @@ -0,0 +1,43 @@ +TRACE f2e461f369476c9ad5bab71657bd71b9 spans: 40 +messenger-api SERVER POST /api/v1/chats/{chat_id}/attachments 21.9ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT SELECT 1.3ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT INSERT 1.3ms + messenger-api CLIENT S3.PutObject 6.5ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT UPDATE 1.1ms + messenger-api CLIENT INSERT 0.6ms + messenger-api CLIENT INSERT 0.7ms + messenger-api CLIENT INSERT 0.5ms + messenger-api CLIENT messenger 0.2ms + messenger-api CLIENT SELECT 0.3ms + messenger-api PRODUCER attachments.jobs publish 1.4ms + messenger-worker CONSUMER attachments.jobs process 323.1ms + messenger-worker CLIENT messenger 0.6ms + messenger-worker CLIENT SELECT 0.7ms + messenger-worker CLIENT UPDATE 0.4ms + messenger-worker CLIENT UPDATE 0.8ms + messenger-worker CLIENT INSERT 0.6ms + messenger-worker CLIENT messenger 0.3ms + messenger-worker CLIENT SELECT 0.5ms + messenger-worker CLIENT S3.GetObject 10.8ms + messenger-worker CLIENT S3.HeadObject 7.6ms + messenger-worker CLIENT S3.PutObject 15.6ms + messenger-worker CLIENT S3.PutObject 16.5ms + messenger-worker CLIENT S3.PutObject 4.7ms + messenger-worker CLIENT S3.PutObject 5.4ms + messenger-worker CLIENT messenger 0.2ms + messenger-worker CLIENT UPDATE 0.9ms + messenger-worker CLIENT INSERT 1.1ms + messenger-worker CLIENT UPDATE 0.8ms + messenger-worker CLIENT INSERT 0.4ms + messenger-worker CLIENT messenger 0.2ms + messenger-worker CLIENT SELECT 1.9ms + messenger-worker CLIENT S3.DeleteObject 3.8ms + messenger-worker CLIENT messenger 0.4ms + messenger-worker CLIENT UPDATE 0.6ms + messenger-api CLIENT DELETE 1.8ms + messenger-api CLIENT INSERT 2.2ms +total traces 65537 spans 263520 +root CLIENT spans (should be 0): 0 diff --git a/docs/lab6/evidence/probe.txt b/docs/lab6/evidence/probe.txt new file mode 100644 index 0000000000000000000000000000000000000000..fd2c00e5fb8f6a3e4445c311fb6dc2f019c8bbc0 GIT binary patch literal 796 zcma))%}&Bl5QWd$#CK@A(kP}yD%FHHU}NH*Kq*Ruf-RUB)JIpp8EVxC37Pa}e(sz( z_e?)OHVRc~rCdwA8wH$e{FxRSYD{$Qxm8__bfI>gv%VqU8iZ|cTTxRj!F-~YJN`gZ zN83(Vssw*Tt)+yHE@y|Ri^cevCe37i82_6degCvv`fng?=E0J zYtVlyneJ6GM4{X_@X~ZvWHNLsP_jaU75)G&KNCz{+fcV=WUTJ!nzPXbwG*P!waj&^ zzD|AP0n7ssmsAS6P2~zRt?xNYa=W$}sOSI2VcQI9PJNg>!r8d)vyD$L^~^onz`!*$ cUOl~|$A5+`IG*_?0=QBZK4ffJ^6lP@_U`}dXt?<<1V$ov z&Z;+?YGmM{Yv^dLA6;gaTAaMN+8gvweRR57H<{IckqmZclC`fys&ucfgc5S>F$NJ#vIC?}A}hnqG{eLyOuLOoDWL%FAE+D5o>6{kS|Jn-JGH}-DV zj-9l?0a;Gg_RQ?tH?uQi|N40%2l7FVq_M$q3=0SHenTro2oB{V>~n?_c~=SL5J*nt ziyX^g0j5JdKZ3>2_${+JgaoC|WLRltuHh5#oqx|dg$=0Li_vxBE&=vQZ%^AsQdSTj8N&Lj8F=SJvb#&chKU}C>UJYQi&Be}#m!&#I)Y^e@*U|uT(>p{wY zcMZmyz&pqOo+&=B#c=88k9Y0)eR5C9a)2zucr`b`yyY?-^cFe$((P-9jv3; zdqCEEKIA`JRVMZCbf_7o45vMMN>9}9P!nXVGLAL&G2)L|J3&6?uDC>$GCPjsJAV0| zaEa>>JC2cGw@dX^S7TRaVf|ifS63ppnRTz8lIcCn&cnnsGrg}L8hGUfkhQDCzDJIg z#D0U;9hTO%^#klO8`0`~$GUx^-nYsk!0oIuE!`ut%GA$u##i)xW-;GOO;PdMh^wjU zEo|TCHH;21Mx47ep7$JAG zmNpW8^#3k#vDgon_G7a@pp5}WnBd*)idQ0?DnEvJ;d+7-dQ{N6l&gaN_W88-0)5u)rGd<_tz(xi*j3e1#)^oH24A(L+2(s- zD|IT}qrOUgvCT+L`ayjc%E--~&|)fsoCE0dYuz92JW!6`##o4@l*M8XLzT7vtSUoVzRE_ObR onD 0.05` при трафике ≥ 0.2 RPS, `for: 1m`. +SLI = 5xx / все бизнес-запросы (health исключены инструментацией, 4xx не считаются). + +1. Dashboard «Messenger lab6» → «RPS по маршруту и классу статуса»: какие `http_route` дают 5xx? + Один маршрут (например `/api/v1/chats/{chat_id}/messages`) → проблема в его зависимостях; все маршруты → общая зависимость (БД, сеть). +2. «p95 по маршруту» и «DB pool: занято / ожидают». Растёт `waiting` и p95 ≈ 3 с → пул БД исчерпан (`DB_POOL_TIMEOUT_SECONDS=3`). p95 ≈ 5 с → `statement_timeout` (`DB_STATEMENT_TIMEOUT_MS=5000`) — БД медленная или заблокирована. +3. Grafana → Explore → Tempo, TraceQL: `{ resource.service.name="messenger-api" && status=error }`. + Откройте trace: если длинный и красный — CLIENT span `INSERT`/`SELECT` (psycopg) с `QueryCanceled` → виновата БД, handler ни при чём. Если SQL-spans короткие, а SERVER span длинный → медленный код handler'а. +4. Кнопка «Logs for this span» (или Loki `{service_name="messenger-api"} | trace_id=""`) — лог `messenger.api database unavailable: QueryCanceled`. +5. Проверка БД: `docker compose --env-file .env exec postgres psql -U messenger -d messenger -c "select pid, state, wait_event_type, now()-query_start as dur, left(query,60) from pg_stat_activity where datname='messenger' order by dur desc nulls last;"` + Видна блокирующая транзакция (`LOCK TABLE ...`, `idle in transaction`) → завершить: `select pg_terminate_backend();` +6. Resolved: доля 5xx < 5% — через ~20–60 с после устранения (окно 1m). + +## QueueLagGrowing + +**Условие:** `max(messenger_queue_oldest_age_seconds{queue="attachments.jobs"}) > 60`, `for: 30s`. +Метрику публикуют API-реплики из PostgreSQL, поэтому она жива и при остановленном worker. + +1. Dashboard → «Глубина очереди»: растёт `queued`, а `processing` = 0 → никто не берёт задания. + `outbox (не опубликовано)` > 0 → проблема до брокера (брокер недоступен), а не в worker. +2. «Saturation worker»: серии `capacity` нет → worker не работает (`docker compose ps worker`). + busy/capacity = 100% долгое время → worker жив, но не успевает: смотреть «Длительность задания p95» и «Задания по исходу» (много `retry`/`error`?). +3. Tempo: `{ resource.service.name="messenger-worker" && kind=consumer && status=error }` — ошибки обработки; в trace видно, на каком S3/SQL span упала попытка, retry идут в той же трассе. +4. Loki: `{service_name="messenger-worker"} | event="job.processed" | outcome!="succeeded"`. +5. Действие: `python scripts/ops.py worker-start` (или `docker compose --env-file .env up -d worker`); при постоянных ошибках — `make jobs-stats`, `make jobs-failed`. +6. Resolved: после запуска worker очередь разбирается (в замере — за секунды), возраст → 0. + +## TelemetryExportFailing + +**Условие:** Collector не отправляет spans/logs (`otelcol_exporter_send_failed_*` или `enqueue_failed_*` растут), `for: 1m`. + +1. Dashboard → «Collector: принято / отправлено / потеряно» и «очередь экспорта»: `queue_size` упирается в 20 → backend недоступен. +2. `docker compose --env-file .env ps tempo loki` — какой backend лежит; `docker compose logs --tail 50 otel-collector`. +3. API и worker продолжают работать: проверить `make test`. Потери телеметрии = значение `enqueue_failed_*` + `send_failed_*` за период. +4. Действие: `docker compose --env-file .env up -d tempo loki`. После возврата очередь опустошается, `sent_*` растёт. diff --git a/docs/security/SCAN_TRIAGE.md b/docs/security/SCAN_TRIAGE.md new file mode 100644 index 0000000..b00d941 --- /dev/null +++ b/docs/security/SCAN_TRIAGE.md @@ -0,0 +1,38 @@ +# Сканирование зависимостей и образов — разбор применимости (lab7) + +Команды: + +```powershell +python scripts/ops.py scan # pip-audit по requirements.txt + trivy (HIGH,CRITICAL, --ignore-unfixed) по образам стенда + # -> evidence/scan/pip-audit.txt, evidence/scan/trivy_.txt +``` + +Локально (без Docker) pip-audit выполнен до и после обновления: `evidence/sandbox/pip-audit-before.txt`, `evidence/sandbox/pip-audit-after.txt`. + +## 1. Python-зависимости приложения (pip-audit, 2026-11-16) + +**До** (requirements lab6 + `cryptography 44.0.0`): 72 уникальных уязвимости в 4 пакетах. **После** обновления: `No known vulnerabilities found`. После обновления повторно пройдены: публичный smoke и security (HTTPS через LB), 91 собственный HTTP-тест lab2–6 (58 выполнены, 33 требуют in-network окружения), 12 white-box тестов хранилища, 24 проверки `security_http.py`, `leak_check.py` с телеметрией (OTel SDK 1.45) — `evidence/sandbox/`. + +| Пакет (было → стало) | Находки | Применимость к нашему коду | Решение | +| --- | --- | --- | --- | +| **Pillow** 11.1.0 → **13.0.0** | 56: OOB write/read в PSD, TIFF, JPEG2000, FPX, BMP RLE, MSP; декомпрессионные бомбы (FITS, GIF/APNG/MPO seek, EXIF); DoS в PdfParser, XBM-regex; ImageCms, rank filter | **Применимо частично и это самый опасный путь (T6)**: worker открывает *недоверенные* байты из upload. До `Image.open` API проверяет сигнатуру PNG/JPEG и пиксели, а worker декодирует только PNG/JPEG (`src/images.py`, `src/imaging.py`), поэтому плагины PSD/TIFF/FITS/PDF/BMP недостижимы. Достижимы: JPEG-декодер (draft scale, CVE-2026-112403), EXIF-парсинг (CVE-2026-96643, `exif_transpose`), PNG/APNG-разбор. Дополнительно ограничивает ущерб: подпроцесс с RLIMIT_AS 1 GiB и таймаутом 30 с, non-root, read-only, без сети | **Обновлено** до 13.0.0 (устраняет все 56). Оставлены ограничения форматов и песочница подпроцесса как defence-in-depth | +| **starlette** 0.41.3 → **1.10.0** (через **FastAPI** 0.115.6 → **0.163.1**) | 7: multipart spool (CVE-2025-54121), Range в FileResponse (CVE-2025-62727), Host header → `request.url` (CVE-2026-48710), StaticFiles на Windows, `HTTPEndpoint` getattr, лимиты url-encoded форм, путь в `request.url` | В основном **не применимо**: нет `request.form()`/multipart (загрузка — сырые байты), нет `FileResponse`/`StaticFiles`/`HTTPEndpoint`. Условно применимо: `request.url` используется только для `url.path` в логах, абсолютные URL из Host не строятся; Host проходит через HAProxy | **Обновлено** (убирает класс находок без анализа каждой будущей ветки кода). FastAPI-обработчики ошибок и валидации совместимы — тесты зелёные | +| **opentelemetry-api** 1.29.0 → **1.45.2** (+ sdk/exporter 1.45.2, instrumentation 0.66b2) | 2 ReDoS: `TraceState.from_header`, `TraceContextTextMapPropagator.extract` | **Применимо**: FastAPI-инструментация извлекает `traceparent`/`tracestate` из входящего HTTP-запроса, а их присылает внешний клиент → CPU DoS реплики | **Обновлено** + LB удаляет внешние `traceparent`, `tracestate`, `baggage` (клиент вне границы доверия не управляет trace-контекстом) | +| **cryptography** 44.0.0 → **50.0.2** | 7: OpenSSL в wheel (CVE-2024-12797 и др.), проверка точки EC при загрузке ключа, name constraints, PKCS#7 decrypt oracle, рекурсия при построении цепочки | Используется для: разбора клиентского cert и CRL в API (`src/tls.py`, вход — сертификат, уже прошедший проверку OpenSSL, и CRL нашего CA) и `scripts/pki.py` (оператор). PKCS#7, X.509-verifier цепочек и name constraints не используются; статический OpenSSL wheel — `ssl` Python использует системный OpenSSL образа | **Обновлено**; низкая применимость, но пакет находится на mTLS-пути | + +## 2. Образы (trivy, выполняется на стенде) + +`ops.py scan` запускает `aquasec/trivy:0.57.1 image --scanners vuln --severity HIGH,CRITICAL --ignore-unfixed` для образа приложения и образов инфраструктуры (`haproxy:2.8.30-alpine`, `postgres:16.4-alpine`, `redis:7.4.1-alpine`, `openbao/openbao:2.1.0`, `chrislusf/seaweedfs:4.50`). Отчёт trivy с CVE ОС не был получен в среде подготовки (Docker Hub был недоступен), поэтому таблица ниже — **методика разбора**, которую нужно применить к `evidence/scan/trivy_*.txt` после запуска. Заполненные строки внесите сюда; «скриншот количества CVE» не является разбором. + +| Вопрос для каждой находки | Как отвечаем в этом стенде | +| --- | --- | +| Достижим ли уязвимый код? | Пакет ОС/библиотека загружается процессом сервиса? (например, `libexpat` в образе Python, но приложение не парсит XML → не применимо; `openssl`/`libssl` в HAProxy/Python — применимо: внешний TLS) | +| Кто контролирует вход? | Внешний клиент (LB, API) → высокий приоритет; только identity стенда за mTLS (broker, collector, backends) → средний; только оператор (`tools`, `migrate`) → низкий | +| Что ограничивает ущерб? | non-root, `cap_drop: ALL`, read-only rootfs, internal-сети без egress, лимиты (см. `ops.py hardening`) | +| Действие | Обновить тег/digest (закреплённая версия) и повторить `run.ps1`; если исправления нет — зафиксировать остаточный риск и компенсирующий контроль | + +Приоритет обновления при появлении фиксов: образ приложения (`python:3.12.7-slim-bookworm` → актуальный patch 3.12/bookworm), затем `haproxy` (внешний TLS), затем `openbao`. + +## 3. Чего сканирование не покрывает + +Логические ошибки авторизации (покрыты `security_http.py`, `tls_audit.py`), неверную конфигурацию TLS/ACL (покрыта `tls_audit.py`), утечки секретов в телеметрию (`leak_check.py`) и права контейнеров (`hardening_audit.py`). diff --git a/evidence/bao_policy.json b/evidence/bao_policy.json new file mode 100644 index 0000000..992d66b --- /dev/null +++ b/evidence/bao_policy.json @@ -0,0 +1,70 @@ +{ + "ok": true, + "checks": [ + { + "check": "approle login from outside secrets network is denied", + "ok": true, + "detail": "exit 2" + }, + { + "check": "api reads own kv/messenger/api", + "ok": true, + "detail": "" + }, + { + "check": "api cannot read kv/messenger/worker", + "ok": true, + "detail": "" + }, + { + "check": "api cannot write own kv path", + "ok": true, + "detail": "" + }, + { + "check": "api cannot write policies", + "ok": true, + "detail": "" + }, + { + "check": "api cannot mint SecretIDs", + "ok": true, + "detail": "" + }, + { + "check": "api token is short-lived (ttl<=15m) and not root", + "ok": true, + "detail": "ttl=897s policies=['default', 'messenger-api']" + }, + { + "check": "worker reads own kv/messenger/worker", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot read kv/messenger/api", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot write own kv path", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot write policies", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot mint SecretIDs", + "ok": true, + "detail": "" + }, + { + "check": "worker token is short-lived (ttl<=15m) and not root", + "ok": true, + "detail": "ttl=896s policies=['default', 'messenger-worker']" + } + ] +} diff --git a/evidence/hardening.txt b/evidence/hardening.txt new file mode 100644 index 0000000..1c25b37 --- /dev/null +++ b/evidence/hardening.txt @@ -0,0 +1,34 @@ +service container Config.User uid ro drop add mem published result +api messenger-lab7-api-1 10001:10001 10001 True ALL - 320M - ok +api messenger-lab7-api-2 10001:10001 10001 True ALL - 320M - ok +bao-agent-api messenger-lab7-bao-agent-api-1 10001:10001 10001 True ALL - 128M - ok +bao-agent-worker messenger-lab7-bao-agent-worker-1 10001:10001 10001 True ALL - 128M - ok +broker messenger-lab7-broker-1 redis 999 True ALL - 320M - ok +broker-gate messenger-lab7-broker-gate-1 haproxy 99 True ALL - 64M - ok +grafana messenger-lab7-grafana-1 472 472 False ALL - 256M [('127.0.0.1', '3000')] ok + exception: read_only: совместимость Grafana с read-only rootfs не проверена на стенде; caps сброшены, no-new-privileges, uid 472 +lb messenger-lab7-lb-1 haproxy 99 True ALL - 128M [('127.0.0.1', '8443')] ok +loki messenger-lab7-loki-1 10001 10001 True ALL - 512M - ok +migrate messenger-lab7-migrate-1 10001:10001 - True ALL - 256M - ok +openbao messenger-lab7-openbao-1 openbao _rpc True ALL - 256M - ok +otel-collector messenger-lab7-otel-collector-1 10001 10001 True ALL - 384M - ok +otel-gate messenger-lab7-otel-gate-1 haproxy 99 True ALL - 64M - ok +postgres messenger-lab7-postgres-1 70:70 70 True ALL - 512M - ok +prometheus messenger-lab7-prometheus-1 nobody nobody True ALL - 512M - ok +redis messenger-lab7-redis-1 redis 999 True ALL - 96M - ok +s3 messenger-lab7-s3-1 root(image default) root True ALL CAP_CHOWN,CAP_DAC_OVERRIDE,CAP_FOW 512M - ok + exception: root_user: образ SeaweedFS работает от uid 0; capabilities сброшены (CapDrop ALL), rootfs read-only, запись только в volume /data; cap_add: каталог /data в образе принадлежит не root: возвращены только DAC_OVERRIDE, CHOWN, FOWNER +s3-gate messenger-lab7-s3-gate-1 haproxy 99 True ALL - 64M - ok +s3-init messenger-lab7-s3-init-1 10001:10001 - True ALL - 192M - ok +tempo messenger-lab7-tempo-1 10001:10001 10001 True ALL - 512M - ok +tempo-init messenger-lab7-tempo-init-1 root - False ALL CAP_CHOWN 0M - ok + exception: cap_add: one-shot chown volume (CAP_CHOWN), завершается до старта tempo; root_user: one-shot chown volume; limits: one-shot, завершается за доли секунды; read_only: one-shot chown volume, network_mode none +worker messenger-lab7-worker-1 10001:10001 10001 True ALL - 1280M - ok + +network segmentation / egress: + api ok + {"postgres": "reachable", "s3": "reachable", "broker": "reachable", "otel-collector": "reachable", "redis": "reachable", "loki": "no-dns", "tempo": "no-dns", "prometheus": "no-dns", "openbao": "no-dns", "grafana": "no-dns", "api": "reachable", "1.1.1.1": "unreachable"} + worker ok + {"postgres": "reachable", "s3": "reachable", "broker": "reachable", "otel-collector": "reachable", "redis": "reachable", "loki": "no-dns", "tempo": "no-dns", "prometheus": "no-dns", "openbao": "no-dns", "grafana": "no-dns", "api": "reachable", "1.1.1.1": "unreachable"} + +PASS: 0 container(s) with problems diff --git a/evidence/leak_check.json b/evidence/leak_check.json new file mode 100644 index 0000000..5102249 --- /dev/null +++ b/evidence/leak_check.json @@ -0,0 +1,31 @@ +{ + "ok": true, + "controls": { + "loki has messenger-api logs": true, + "tempo has POST /api/v1/auth/sessions spans": true, + "prometheus has http metrics labels": true + }, + "canary_user_created": true, + "searched": { + "loki_lines": 99, + "tempo_traces": 88, + "prometheus_labels": 26 + }, + "needles": [ + "argon2 hash", + "basic header", + "bearer header", + "broker_api_password", + "broker_worker_password", + "cache_api_password", + "db_api_password", + "db_worker_password", + "private key", + "s3_secret_access_key", + "s3_worker_secret_access_key", + "session token", + "user password", + "wrong password" + ], + "findings": [] +} diff --git a/evidence/login_budget.txt b/evidence/login_budget.txt new file mode 100644 index 0000000..989cc27 --- /dev/null +++ b/evidence/login_budget.txt @@ -0,0 +1,28 @@ +Argon2id benchmark: N=15 per row, container CPU quota=0.50, KDF slots/replica=4 +m(KiB),t,p hash p50 verify p50 verify p95 RAM/hash MiB logins/s/replica +19456,2,1 31.3ms 74.3ms 80.0ms 19.0 6.7 <- current +19456,3,1 78.3ms 80.6ms 84.6ms 19.0 6.2 +47104,1,1 90.5ms 95.0ms 167.0ms 46.0 5.3 +65536,2,1 189.0ms 191.7ms 249.2ms 64.0 2.6 +65536,3,1 277.7ms 280.5ms 340.0ms 64.0 1.8 + +{ + "duration_s": 20.1, + "concurrency": 16, + "login_attempts": 1039, + "login_attempts_per_s": 51.7, + "login_status_counts": { + "401": 299, + "429": 740 + }, + "baseline_users_me_ms": { + "p50": 29.7, + "p95": 31.3 + }, + "during_flood_users_me_ms": { + "p50": 100.7, + "p95": 623.5, + "non_200": 0, + "samples": 69 + } +} diff --git a/evidence/rehash.txt b/evidence/rehash.txt new file mode 100644 index 0000000..f02b8e6 --- /dev/null +++ b/evidence/rehash.txt @@ -0,0 +1 @@ +rehash-demo: user=rehash_77e2292d params before=m=8192,t=1,p=1 -> login HTTP 201 -> params after=m=19456,t=2,p=1 diff --git a/evidence/run/SUMMARY.txt b/evidence/run/SUMMARY.txt new file mode 100644 index 0000000..1e147b9 --- /dev/null +++ b/evidence/run/SUMMARY.txt @@ -0,0 +1,34 @@ + +Step Result Seconds Evidence +---- ------ ------- -------- +check-contract PASS 0 evidence\run\check-contract.txt +compose-config PASS 0 evidence\run\compose-config.txt +up PASS 17 evidence\run\up.txt +ps PASS 1 evidence\run\ps.txt +test-smoke PASS 2 evidence\run\test-smoke.txt +test-security PASS 0 evidence\run\test-security.txt +tls-audit PASS 10 evidence\run\tls-audit.txt +security-http PASS 2 evidence\run\security-http.txt +bao-policy PASS 8 evidence\run\bao-policy.txt +hardening PASS 4 evidence\run\hardening.txt +rehash PASS 3 evidence\run\rehash.txt +leak-check PASS 32 evidence\run\leak-check.txt +expiry PASS 18 evidence\run\expiry.txt +test-persistence PASS 54 evidence\run\test-persistence.txt +test-failover-1 PASS 18 evidence\run\test-failover-1.txt +restore-1 PASS 7 evidence\run\restore-1.txt +test-failover-2 PASS 18 evidence\run\test-failover-2.txt +restore-2 PASS 7 evidence\run\restore-2.txt +rotate-db-password PASS 8 evidence\run\rotate-db-password.txt +rotate-lb-cert PASS 22 evidence\run\rotate-lb-cert.txt +rotate-secret-id PASS 5 evidence\run\rotate-secret-id.txt +revoke-identity PASS 26 evidence\run\revoke-identity.txt +bao-sealed PASS 26 evidence\run\bao-sealed.txt +smoke-after-lifecycle PASS 3 evidence\run\smoke-after-lifecycle.txt +student-tests PASS 31 evidence\run\student-tests.txt +storage-tests PASS 10 evidence\run\storage-tests.txt +login-budget PASS 45 evidence\run\login-budget.txt +s3-check PASS 3 evidence\run\s3-check.txt + + + diff --git a/evidence/run/bao-policy.txt b/evidence/run/bao-policy.txt new file mode 100644 index 0000000..74d29dc --- /dev/null +++ b/evidence/run/bao-policy.txt @@ -0,0 +1,74 @@ +# OpenBao: allow/deny политик реальными токенами AppRole, CIDR, TTL +# python scripts/ops.py bao-policy +# 2026-11-17T23:57:40.0157734+03:00 +{ + "ok": true, + "checks": [ + { + "check": "approle login from outside secrets network is denied", + "ok": true, + "detail": "exit 2" + }, + { + "check": "api reads own kv/messenger/api", + "ok": true, + "detail": "" + }, + { + "check": "api cannot read kv/messenger/worker", + "ok": true, + "detail": "" + }, + { + "check": "api cannot write own kv path", + "ok": true, + "detail": "" + }, + { + "check": "api cannot write policies", + "ok": true, + "detail": "" + }, + { + "check": "api cannot mint SecretIDs", + "ok": true, + "detail": "" + }, + { + "check": "api token is short-lived (ttl<=15m) and not root", + "ok": true, + "detail": "ttl=897s policies=['default', 'messenger-api']" + }, + { + "check": "worker reads own kv/messenger/worker", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot read kv/messenger/api", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot write own kv path", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot write policies", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot mint SecretIDs", + "ok": true, + "detail": "" + }, + { + "check": "worker token is short-lived (ttl<=15m) and not root", + "ok": true, + "detail": "ttl=896s policies=['default', 'messenger-worker']" + } + ] +} +2026-11-17T20:57:47.753Z saved evidence/bao_policy.json diff --git a/evidence/run/bao-sealed.txt b/evidence/run/bao-sealed.txt new file mode 100644 index 0000000..d9161d6 --- /dev/null +++ b/evidence/run/bao-sealed.txt @@ -0,0 +1,8 @@ +# OpenBao sealed: API работает и перезапускается на отрисованных секретах, затем unseal +# python scripts/ops.py bao-sealed +# 2026-11-18T00:01:29.3386965+03:00 +OpenBao sealed: agents keep already rendered files; new auth/renew fail until `bao.py unseal` +2026-11-17T21:01:42.296Z bao-sealed: smoke while OpenBao sealed -> PASS +2026-11-17T21:01:53.384Z bao-sealed: api replicas restarted while sealed -> healthy (secrets from agent tmpfs volume) +unsealed=True +2026-11-17T21:01:55.371Z bao-sealed: unsealed after 25s; agents resume renew/render with backoff (<= 1 min) diff --git a/evidence/run/check-contract.txt b/evidence/run/check-contract.txt new file mode 100644 index 0000000..f6560b6 --- /dev/null +++ b/evidence/run/check-contract.txt @@ -0,0 +1,4 @@ +# make check: структура skeleton, контракт, синтаксис тестов +# python scripts/check_contract.py +# 2026-11-17T23:57:07.0451038+03:00 +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/run/compose-config.txt b/evidence/run/compose-config.txt new file mode 100644 index 0000000..9afcfc4 --- /dev/null +++ b/evidence/run/compose-config.txt @@ -0,0 +1,3 @@ +# docker compose config --quiet +# docker compose config --quiet +# 2026-11-17T23:57:07.2224890+03:00 diff --git a/evidence/run/expiry.txt b/evidence/run/expiry.txt new file mode 100644 index 0000000..888a81d --- /dev/null +++ b/evidence/run/expiry.txt @@ -0,0 +1,50 @@ +# Истечение сессии (экземпляр с TTL 2 с, mTLS напрямую к реплике) +# python scripts/ops.py in-network expiry +# 2026-11-17T23:58:26.7184363+03:00 + Container messenger-lab7-s3-1 Running + Container messenger-lab7-postgres-1 Running + Container messenger-lab7-s3-gate-1 Running + Container messenger-lab7-openbao-1 Running + Container messenger-lab7-api-1 Running + Container messenger-lab7-api-2 Running + Container messenger-lab7-bao-agent-api-1 Running + Container messenger-lab7-redis-1 Running + Container messenger-lab7-api-short-ttl-1 Creating + Container messenger-lab7-api-short-ttl-1 Created + Container messenger-lab7-postgres-1 Waiting + Container messenger-lab7-s3-1 Waiting + Container messenger-lab7-postgres-1 Healthy + Container messenger-lab7-s3-1 Healthy + Container messenger-lab7-migrate-1 Starting + Container messenger-lab7-s3-gate-1 Waiting + Container messenger-lab7-migrate-1 Started + Container messenger-lab7-s3-gate-1 Healthy + Container messenger-lab7-s3-init-1 Starting + Container messenger-lab7-s3-init-1 Started + Container messenger-lab7-migrate-1 Waiting + Container messenger-lab7-s3-init-1 Waiting + Container messenger-lab7-bao-agent-api-1 Waiting + Container messenger-lab7-postgres-1 Waiting + Container messenger-lab7-redis-1 Waiting + Container messenger-lab7-migrate-1 Exited + Container messenger-lab7-bao-agent-api-1 Healthy + Container messenger-lab7-redis-1 Healthy + Container messenger-lab7-postgres-1 Healthy + Container messenger-lab7-s3-init-1 Exited + Container messenger-lab7-api-1 Waiting + Container messenger-lab7-api-2 Waiting + Container messenger-lab7-api-1 Healthy + Container messenger-lab7-api-2 Healthy + Container messenger-lab7-api-short-ttl-1 Starting + Container messenger-lab7-api-short-ttl-1 Started + Container messenger-lab7-api-run-a06a9f47848a Creating + Container messenger-lab7-api-run-a06a9f47848a Created +ready: https://api-short-ttl:8443/health/ready + Container messenger-lab7-api-run-2ebe51c30cb2 Creating + Container messenger-lab7-api-run-2ebe51c30cb2 Created +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... ok +System.Management.Automation.RemoteException +---------------------------------------------------------------------- +Ran 1 test in 3.862s +System.Management.Automation.RemoteException +OK diff --git a/evidence/run/hardening.txt b/evidence/run/hardening.txt new file mode 100644 index 0000000..4321bd7 --- /dev/null +++ b/evidence/run/hardening.txt @@ -0,0 +1,38 @@ +# Hardening: non-root, read_only, cap_drop, no-new-privileges, лимиты, порты, сегментация, egress +# python scripts/ops.py hardening +# 2026-11-17T23:57:48.3174507+03:00 +service container Config.User uid ro drop add mem published result +api messenger-lab7-api-1 10001:10001 10001 True ALL - 320M - ok +api messenger-lab7-api-2 10001:10001 10001 True ALL - 320M - ok +bao-agent-api messenger-lab7-bao-agent-api-1 10001:10001 10001 True ALL - 128M - ok +bao-agent-worker messenger-lab7-bao-agent-worker-1 10001:10001 10001 True ALL - 128M - ok +broker messenger-lab7-broker-1 redis 999 True ALL - 320M - ok +broker-gate messenger-lab7-broker-gate-1 haproxy 99 True ALL - 64M - ok +grafana messenger-lab7-grafana-1 472 472 False ALL - 256M [('127.0.0.1', '3000')] ok + exception: read_only: совместимость Grafana с read-only rootfs не проверена на стенде; caps сброшены, no-new-privileges, uid 472 +lb messenger-lab7-lb-1 haproxy 99 True ALL - 128M [('127.0.0.1', '8443')] ok +loki messenger-lab7-loki-1 10001 10001 True ALL - 512M - ok +migrate messenger-lab7-migrate-1 10001:10001 - True ALL - 256M - ok +openbao messenger-lab7-openbao-1 openbao _rpc True ALL - 256M - ok +otel-collector messenger-lab7-otel-collector-1 10001 10001 True ALL - 384M - ok +otel-gate messenger-lab7-otel-gate-1 haproxy 99 True ALL - 64M - ok +postgres messenger-lab7-postgres-1 70:70 70 True ALL - 512M - ok +prometheus messenger-lab7-prometheus-1 nobody nobody True ALL - 512M - ok +redis messenger-lab7-redis-1 redis 999 True ALL - 96M - ok +s3 messenger-lab7-s3-1 root(image default) root True ALL CAP_CHOWN,CAP_DAC_OVERRIDE,CAP_FOW 512M - ok + exception: root_user: образ SeaweedFS работает от uid 0; capabilities сброшены (CapDrop ALL), rootfs read-only, запись только в volume /data; cap_add: каталог /data в образе принадлежит не root: возвращены только DAC_OVERRIDE, CHOWN, FOWNER +s3-gate messenger-lab7-s3-gate-1 haproxy 99 True ALL - 64M - ok +s3-init messenger-lab7-s3-init-1 10001:10001 - True ALL - 192M - ok +tempo messenger-lab7-tempo-1 10001:10001 10001 True ALL - 512M - ok +tempo-init messenger-lab7-tempo-init-1 root - False ALL CAP_CHOWN 0M - ok + exception: cap_add: one-shot chown volume (CAP_CHOWN), завершается до старта tempo; root_user: one-shot chown volume; limits: one-shot, завершается за доли секунды; read_only: one-shot chown volume, network_mode none +worker messenger-lab7-worker-1 10001:10001 10001 True ALL - 1280M - ok + +network segmentation / egress: + api ok + {"postgres": "reachable", "s3": "reachable", "broker": "reachable", "otel-collector": "reachable", "redis": "reachable", "loki": "no-dns", "tempo": "no-dns", "prometheus": "no-dns", "openbao": "no-dns", "grafana": "no-dns", "api": "reachable", "1.1.1.1": "unreachable"} + worker ok + {"postgres": "reachable", "s3": "reachable", "broker": "reachable", "otel-collector": "reachable", "redis": "reachable", "loki": "no-dns", "tempo": "no-dns", "prometheus": "no-dns", "openbao": "no-dns", "grafana": "no-dns", "api": "reachable", "1.1.1.1": "unreachable"} + +PASS: 0 container(s) with problems +2026-11-17T20:57:51.851Z saved evidence/hardening.txt diff --git a/evidence/run/leak-check.txt b/evidence/run/leak-check.txt new file mode 100644 index 0000000..57551f7 --- /dev/null +++ b/evidence/run/leak-check.txt @@ -0,0 +1,35 @@ +# В Loki/Tempo/Prometheus нет паролей, токенов, секретов ролей +# python scripts/ops.py leak-check +# 2026-11-17T23:57:55.1013288+03:00 +{ + "ok": true, + "controls": { + "loki has messenger-api logs": true, + "tempo has POST /api/v1/auth/sessions spans": true, + "prometheus has http metrics labels": true + }, + "canary_user_created": true, + "searched": { + "loki_lines": 99, + "tempo_traces": 88, + "prometheus_labels": 26 + }, + "needles": [ + "argon2 hash", + "basic header", + "bearer header", + "broker_api_password", + "broker_worker_password", + "cache_api_password", + "db_api_password", + "db_worker_password", + "private key", + "s3_secret_access_key", + "s3_worker_secret_access_key", + "session token", + "user password", + "wrong password" + ], + "findings": [] +} +2026-11-17T20:58:26.460Z saved evidence/leak_check.json diff --git a/evidence/run/login-budget.txt b/evidence/run/login-budget.txt new file mode 100644 index 0000000..483e6e8 --- /dev/null +++ b/evidence/run/login-budget.txt @@ -0,0 +1,33 @@ +# Бюджет login DoS: Argon2id на стенде + поток входов +# python scripts/ops.py login-budget +# 2026-11-18T00:02:39.0541070+03:00 +Argon2id benchmark: N=15 per row, container CPU quota=0.50, KDF slots/replica=4 +m(KiB),t,p hash p50 verify p50 verify p95 RAM/hash MiB logins/s/replica +19456,2,1 31.3ms 74.3ms 80.0ms 19.0 6.7 <- current +19456,3,1 78.3ms 80.6ms 84.6ms 19.0 6.2 +47104,1,1 90.5ms 95.0ms 167.0ms 46.0 5.3 +65536,2,1 189.0ms 191.7ms 249.2ms 64.0 2.6 +65536,3,1 277.7ms 280.5ms 340.0ms 64.0 1.8 + +{ + "duration_s": 20.1, + "concurrency": 16, + "login_attempts": 1039, + "login_attempts_per_s": 51.7, + "login_status_counts": { + "401": 299, + "429": 740 + }, + "baseline_users_me_ms": { + "p50": 29.7, + "p95": 31.3 + }, + "during_flood_users_me_ms": { + "p50": 100.7, + "p95": 623.5, + "non_200": 0, + "samples": 69 + } +} + +2026-11-17T21:03:24.128Z saved evidence/login_budget.txt diff --git a/evidence/run/ps.txt b/evidence/run/ps.txt new file mode 100644 index 0000000..38730c5 --- /dev/null +++ b/evidence/run/ps.txt @@ -0,0 +1,6 @@ +# Реплики API: alias, health, статус в LB +# python scripts/ops.py ps +# 2026-11-17T23:57:24.3103206+03:00 +X-Instance-Id container state health lb lb_status +api-1401d400 messenger-lab7-api-1 running healthy api1 UP +api-21bd9440 messenger-lab7-api-2 running healthy api2 UP diff --git a/evidence/run/rehash.txt b/evidence/run/rehash.txt new file mode 100644 index 0000000..6301781 --- /dev/null +++ b/evidence/run/rehash.txt @@ -0,0 +1,5 @@ +# Argon2id: старый hash пересчитывается при успешном входе +# python scripts/ops.py rehash-demo +# 2026-11-17T23:57:52.1167844+03:00 +2026-11-17T20:57:55.086Z rehash-demo: user=rehash_77e2292d params before=m=8192,t=1,p=1 -> login HTTP 201 -> params after=m=19456,t=2,p=1 +2026-11-17T20:57:55.088Z saved evidence/rehash.txt diff --git a/evidence/run/restore-1.txt b/evidence/run/restore-1.txt new file mode 100644 index 0000000..f79b826 --- /dev/null +++ b/evidence/run/restore-1.txt @@ -0,0 +1,4 @@ +# Восстановление реплики +# python scripts/ops.py restore +# 2026-11-17T23:59:55.8260227+03:00 +2026-11-17T21:00:02.727Z restore: started ['messenger-lab7-api-1']; 2 replicas healthy and UP after 6.8s diff --git a/evidence/run/restore-2.txt b/evidence/run/restore-2.txt new file mode 100644 index 0000000..adb55b2 --- /dev/null +++ b/evidence/run/restore-2.txt @@ -0,0 +1,4 @@ +# Восстановление реплики +# python scripts/ops.py restore +# 2026-11-18T00:00:20.7647574+03:00 +2026-11-17T21:00:27.711Z restore: started ['messenger-lab7-api-2']; 2 replicas healthy and UP after 6.9s diff --git a/evidence/run/revoke-identity.txt b/evidence/run/revoke-identity.txt new file mode 100644 index 0000000..3a63752 --- /dev/null +++ b/evidence/run/revoke-identity.txt @@ -0,0 +1,8 @@ +# Отзыв identity worker в OpenBao -> агент без доступа -> восстановление +# python scripts/ops.py revoke-identity worker +# 2026-11-18T00:01:03.4008626+03:00 +revoked identity worker: 1 SecretID, 2 token(s); agent can no longer authenticate +2026-11-17T21:01:24.597Z rotate/revoke: identity worker revoked; agent re-auth denied=True; already rendered files remain -> worker keeps working until restart of the volume +2026-11-17T21:01:24.613Z revoke-identity: LB ready while worker identity is revoked = True +issued new SecretID for worker (ttl 168h, bound to 172.30.250.0/24); previous SecretID destroyed +2026-11-17T21:01:29.326Z revoke-identity: recovered with a new SecretID; agent healthy diff --git a/evidence/run/rotate-db-password.txt b/evidence/run/rotate-db-password.txt new file mode 100644 index 0000000..f1a2280 --- /dev/null +++ b/evidence/run/rotate-db-password.txt @@ -0,0 +1,5 @@ +# Ротация рабочего секрета (пароль роли БД) без пересборки и рестарта API +# python scripts/ops.py rotate-db-password api +# 2026-11-18T00:00:27.7255998+03:00 +2026-11-17T21:00:31.985Z rotate-db-password: role=messenger_api; agent re-rendered after 3.4s; old password accepted=False (['psql: error: connection to server at "127.0.0.1", port 5432 failed: FATAL: password authentication failed for user "messenger_api"']); new password accepted=True; existing sessions terminated=3 +2026-11-17T21:00:36.079Z rotate-db-password: smoke after rotation (new pool connections): ['PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.'] diff --git a/evidence/run/rotate-lb-cert.txt b/evidence/run/rotate-lb-cert.txt new file mode 100644 index 0000000..b8c735e --- /dev/null +++ b/evidence/run/rotate-lb-cert.txt @@ -0,0 +1,19 @@ +# Ротация leaf LB->API + отзыв старого сертификата (CRL), проверка старым и новым +# python scripts/ops.py rotate-lb-cert +# 2026-11-18T00:00:36.0907608+03:00 + Container messenger-lab7-tools-run-b0e6de7a5085 Creating + Container messenger-lab7-tools-run-b0e6de7a5085 Created +issue lb-client serial 44e0a374f70660066d7c277042b7d5a8d9ed8825 until 2026-12-17 +next: перезапустить потребителя нового ключа (без пересборки образа), затем revoke +2026-11-17T21:00:44.120Z rotate-lb-cert: new lb-client in use (lb restarted, no image rebuild) after 7.9s + Container messenger-lab7-tools-run-d38d3e848112 Creating + Container messenger-lab7-tools-run-d38d3e848112 Created +revoke lb-client serial 10b6b35f215923535d3ca55dc76d0c767bebdb0 +crl 3 revoked serial(s), next update 2026-12-17 + Container messenger-lab7-audit-run-fc86d31a5a74 Creating + Container messenger-lab7-audit-run-fc86d31a5a74 Created +System.Management.Automation.RemoteException +{"label": "old-before-revoke", "target": "api:8443", "accepted": true, "observed": ["http", 200]} +{"label": "old-after-revoke", "target": "api:8443", "accepted": false, "observed": ["closed", "connection closed without HTTP response"]} +{"label": "new-after-revoke", "target": "api:8443", "accepted": true, "observed": ["http", 200]} +2026-11-17T21:00:58.270Z rotate-lb-cert: old cert accepted before revoke=True, after revoke=False; new cert accepted=True; LB ready=True diff --git a/evidence/run/rotate-secret-id.txt b/evidence/run/rotate-secret-id.txt new file mode 100644 index 0000000..385bee6 --- /dev/null +++ b/evidence/run/rotate-secret-id.txt @@ -0,0 +1,5 @@ +# Новый SecretID агента API, старый уничтожен +# python scripts/ops.py rotate-secret-id api +# 2026-11-18T00:00:58.3186708+03:00 +issued new SecretID for api (ttl 168h, bound to 172.30.250.0/24); previous SecretID destroyed +2026-11-17T21:01:03.388Z rotate-secret-id: ['api'] agent(s) re-authenticated with the new SecretID; old SecretID destroyed diff --git a/evidence/run/s3-check.txt b/evidence/run/s3-check.txt new file mode 100644 index 0000000..34c6aa4 --- /dev/null +++ b/evidence/run/s3-check.txt @@ -0,0 +1,67 @@ +# Приватность бакета и права ключа API вне API +# python scripts/ops.py s3-check +# 2026-11-18T00:03:24.3699830+03:00 + Container messenger-lab7-api-run-25e8a0a5666b Creating + Container messenger-lab7-api-run-25e8a0a5666b Created +{ + "endpoint_host": "s3", + "bucket": "messenger-attachments", + "checks": [ + { + "check": "api credentials: put/get staging/", + "ok": true, + "detail": "" + }, + { + "check": "api credentials cannot write final/ (worker prefix)", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot list the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "anonymous GET object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list bucket is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "anonymous list buckets is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "api credentials cannot create buckets", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "api credentials cannot delete the bucket", + "ok": true, + "detail": "AccessDenied" + }, + { + "check": "SeaweedFS admin/UI ports are not reachable on the shared network", + "ok": true, + "detail": "checked [9333, 8888, 8080, 19333, 18888, 8334, 18333, 18334]" + }, + { + "check": "anonymous GET of a real attachment object is denied", + "ok": true, + "detail": "HTTP 403" + }, + { + "check": "same object is readable with API credentials and matches sha256", + "ok": true, + "detail": "93 bytes" + } + ], + "ok": true +} diff --git a/evidence/run/security-http.txt b/evidence/run/security-http.txt new file mode 100644 index 0000000..6b1fcbb --- /dev/null +++ b/evidence/run/security-http.txt @@ -0,0 +1,29 @@ +# ACL/вложения, X-User-Id, logout, 429 + Retry-After (общий), 413, ошибки без утечек, TLS снаружи +# python scripts/ops.py security-http +# 2026-11-17T23:57:38.0517038+03:00 +PASS setup: chat + attachment 202 +PASS outsider GET /api/v1/chats/{chat} -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/chats/{chat}/messages -> 404 (no existence leak) 404 +PASS outsider POST /api/v1/chats/{chat}/messages -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/attachments/{att} -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/attachments/{att}/content -> 404 (no existence leak) 404 +PASS outsider attaches foreign attachment to own chat -> 404 404 +PASS X-User-Id cannot override Bearer identity eve_c7dfed486e +PASS X-User-Id alone -> 401 + WWW-Authenticate Bearer Bearer realm="messenger" +PASS Basic on business endpoint -> 401 401 +PASS unknown/forged token -> 401 401 +PASS logout 204 -> token rejected 401 -> repeat logout 401 (204, 401, 401) +PASS wrong password -> 401, body has no username/password/hash {"error":{"code":"unauthorized","message":"Authentication required","request_id" +PASS malformed JSON -> 400 without stack trace/SQL {"error":{"code":"invalid_request","message":"Invalid request","request_id":"AC1 +PASS login brute force -> 429 rate_limited + Retry-After [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429] Retry-After=21 +PASS rate limit is shared: responses from >= 2 replicas ['api-1401d400', 'api-21bd9440'] +PASS correct password while locked out -> 429 (no oracle) 429 +PASS JSON body 70 KB -> 413 payload_too_large from LB (before any replica) instance=None +PASS chunked JSON 70 KB -> 413 payload_too_large (API) instance=api-1401d400 +PASS upload 10 MiB + 1 byte -> 413 413 +PASS wrong server name -> certificate verification error Hostname mismatch, certificate is not valid for 'evil.example'. +PASS empty trust store -> verification error unable to get local issuer certificate +PASS plaintext HTTP on HTTPS port is not served RemoteDisconnected +PASS grafana over HTTPS requires authentication 401 +PASS: 24/24 checks +2026-11-17T20:57:39.809Z saved evidence/security_http.txt diff --git a/evidence/run/smoke-after-lifecycle.txt b/evidence/run/smoke-after-lifecycle.txt new file mode 100644 index 0000000..b2f616d --- /dev/null +++ b/evidence/run/smoke-after-lifecycle.txt @@ -0,0 +1,4 @@ +# Smoke после всех ротаций +# python tests/smoke.py --base-url https://localhost:8443 --ca-file C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\secrets\pki\ca.crt +# 2026-11-18T00:01:55.3847420+03:00 +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/run/storage-tests.txt b/evidence/run/storage-tests.txt new file mode 100644 index 0000000..6e1ff99 --- /dev/null +++ b/evidence/run/storage-tests.txt @@ -0,0 +1,32 @@ +# White-box тесты хранилища внутри сети (identity администратора БД) +# python scripts/ops.py in-network storage +# 2026-11-18T00:02:29.4056637+03:00 + Container messenger-lab7-api-run-0f07d22b265d Creating + Container messenger-lab7-api-run-0f07d22b265d Created +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... ok +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... ok +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... ok +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... ok +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... ok +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... ok +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... ok +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... ok +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... ok +System.Management.Automation.RemoteException +---------------------------------------------------------------------- +Ran 9 tests in 3.350s +System.Management.Automation.RemoteException +OK + Container messenger-lab7-api-run-c198a1fb2736 Creating + Container messenger-lab7-api-run-c198a1fb2736 Created +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... logout: cache invalidation deferred; positive cache disabled +ok +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... ok +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... logout: cache invalidation deferred; positive cache disabled +ok +System.Management.Automation.RemoteException +---------------------------------------------------------------------- +Ran 3 tests in 1.951s +System.Management.Automation.RemoteException +OK diff --git a/evidence/run/student-tests.txt b/evidence/run/student-tests.txt new file mode 100644 index 0000000..dffb83e --- /dev/null +++ b/evidence/run/student-tests.txt @@ -0,0 +1,111 @@ +# Собственные HTTP-тесты lab2-6 через HTTPS +# python student-tests/run_all.py --base-url https://localhost:8443 +# 2026-11-18T00:01:58.0665970+03:00 + +== API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env) +test_chat_list_pagination (test_api.ApiBehaviorTests.test_chat_list_pagination) ... ok +test_cursor_from_other_chat_returns_400 (test_api.ApiBehaviorTests.test_cursor_from_other_chat_returns_400) ... ok +test_duplicate_username_returns_409 (test_api.ApiBehaviorTests.test_duplicate_username_returns_409) ... ok +test_empty_message_returns_400 (test_api.ApiBehaviorTests.test_empty_message_returns_400) ... ok +test_malformed_json_returns_400 (test_api.ApiBehaviorTests.test_malformed_json_returns_400) ... ok +test_naive_timestamp_cursor_returns_400 (test_api.ApiBehaviorTests.test_naive_timestamp_cursor_returns_400) ... ok +test_outsider_cannot_access_chat (test_api.ApiBehaviorTests.test_outsider_cannot_access_chat) ... ok +test_pagination_returns_all_messages_without_duplicates (test_api.ApiBehaviorTests.test_pagination_returns_all_messages_without_duplicates) ... ok +test_password_never_returned (test_api.ApiBehaviorTests.test_password_never_returned) ... ok +test_registration_without_password_returns_400 (test_api.ApiBehaviorTests.test_registration_without_password_returns_400) ... ok +test_sender_id_in_body_rejected (test_api.ApiBehaviorTests.test_sender_id_in_body_rejected) ... ok +test_short_password_returns_400 (test_api.ApiBehaviorTests.test_short_password_returns_400) ... ok +test_crash_between_put_and_commit_dry_run_then_apply_then_noop (test_attachment_consistency.AttachmentConsistencyTests.test_crash_between_put_and_commit_dry_run_then_apply_then_noop) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_fresh_pending_is_kept (test_attachment_consistency.AttachmentConsistencyTests.test_fresh_pending_is_kept) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_missing_object_is_reported_not_deleted (test_attachment_consistency.AttachmentConsistencyTests.test_missing_object_is_reported_not_deleted) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_orphan_object_deleted_only_after_grace (test_attachment_consistency.AttachmentConsistencyTests.test_orphan_object_deleted_only_after_grace) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_pending_cannot_be_attached (test_attachment_consistency.AttachmentConsistencyTests.test_pending_cannot_be_attached) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_ready_and_referenced_are_never_collected (test_attachment_consistency.AttachmentConsistencyTests.test_ready_and_referenced_are_never_collected) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_aborted_upload_leaves_no_row_and_no_object (test_attachment_consistency.UploadFailureTests.test_aborted_upload_leaves_no_row_and_no_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_db_failure_after_put_returns_503_and_deletes_object (test_attachment_consistency.UploadFailureTests.test_db_failure_after_put_returns_503_and_deletes_object) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_process_crash_after_put_is_collected_by_gc (test_attachment_consistency.UploadFailureTests.test_process_crash_after_put_is_collected_by_gc) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_s3_put_failure_returns_503_and_compensates (test_attachment_consistency.UploadFailureTests.test_s3_put_failure_returns_503_and_compensates) ... skipped 'DATABASE_URL and S3 settings are not configured' +test_attachment_from_other_chat_is_404_and_nothing_saved (test_attachments.AttachmentTests.test_attachment_from_other_chat_is_404_and_nothing_saved) ... ok +test_content_type_with_parameters_is_accepted (test_attachments.AttachmentTests.test_content_type_with_parameters_is_accepted) ... ok +test_corrupted_images_are_415 (test_attachments.AttachmentTests.test_corrupted_images_are_415) ... ok +test_dangerous_filenames_are_400 (test_attachments.AttachmentTests.test_dangerous_filenames_are_400) ... ok +test_exactly_max_size_is_accepted (test_attachments.AttachmentTests.test_exactly_max_size_is_accepted) ... ok +test_idempotency_includes_attachment_ids (test_attachments.AttachmentTests.test_idempotency_includes_attachment_ids) ... ok +test_invalid_and_unknown_attachment_ids (test_attachments.AttachmentTests.test_invalid_and_unknown_attachment_ids) ... ok +test_jpeg_baseline_and_progressive (test_attachments.AttachmentTests.test_jpeg_baseline_and_progressive) ... ok +test_message_attachment_validation (test_attachments.AttachmentTests.test_message_attachment_validation) ... ok +test_message_with_attachments_visible_to_members (test_attachments.AttachmentTests.test_message_with_attachments_visible_to_members) ... ok +test_mime_spoofing_is_415 (test_attachments.AttachmentTests.test_mime_spoofing_is_415) ... ok +test_original_variant_and_unknown_variants (test_attachments.AttachmentTests.test_original_variant_and_unknown_variants) ... ok +test_outsider_and_unknown_chat_upload_404 (test_attachments.AttachmentTests.test_outsider_and_unknown_chat_upload_404) ... ok +test_png_roundtrip_member_reads_outsider_404 (test_attachments.AttachmentTests.test_png_roundtrip_member_reads_outsider_404) ... ok +test_too_large_chunked_without_content_length (test_attachments.AttachmentTests.test_too_large_chunked_without_content_length) ... ok +test_too_large_with_content_length (test_attachments.AttachmentTests.test_too_large_with_content_length) ... ok +test_too_many_pixels (test_attachments.AttachmentTests.test_too_many_pixels) ... ok +test_basic_on_business_endpoint_rejected (test_auth.AuthTests.test_basic_on_business_endpoint_rejected) ... ok +test_login_with_body_rejected (test_auth.AuthTests.test_login_with_body_rejected) +Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет. ... ok +test_logout_revokes_only_current_session (test_auth.AuthTests.test_logout_revokes_only_current_session) ... ok +test_malformed_basic_headers (test_auth.AuthTests.test_malformed_basic_headers) ... ok +test_password_with_colon_and_unicode (test_auth.AuthTests.test_password_with_colon_and_unicode) ... ok +test_session_shape_and_token_entropy (test_auth.AuthTests.test_session_shape_and_token_entropy) ... ok +test_unknown_and_garbage_tokens (test_auth.AuthTests.test_unknown_and_garbage_tokens) ... ok +test_wrong_password_and_unknown_user_look_the_same (test_auth.AuthTests.test_wrong_password_and_unknown_user_look_the_same) ... ok +test_x_user_id_gives_no_access (test_auth.AuthTests.test_x_user_id_gives_no_access) ... ok +test_duplicate_and_missing_caller_rejected (test_concurrency.ChatRollbackTests.test_duplicate_and_missing_caller_rejected) ... ok +test_unknown_member_rolls_back_whole_chat (test_concurrency.ChatRollbackTests.test_unknown_member_rolls_back_whole_chat) +Чат вставляется первым, затем участники по порядку (сначала сам владелец). ... ok +test_same_username_parallel_registration (test_concurrency.RegistrationRaceTests.test_same_username_parallel_registration) ... ok +test_session_expires_even_when_cached (test_expiry.ExpiryTests.test_session_expires_even_when_cached) ... skipped 'set EXPIRY_BASE_URL to the short-TTL API instance' +test_auth_checked_before_key (test_idempotency.IdempotencyTests.test_auth_checked_before_key) ... ok +test_concurrent_same_key_across_replicas_creates_one_message (test_idempotency.IdempotencyTests.test_concurrent_same_key_across_replicas_creates_one_message) ... ok +test_concurrent_same_key_different_bodies_one_winner (test_idempotency.IdempotencyTests.test_concurrent_same_key_different_bodies_one_winner) ... ok +test_failed_access_does_not_burn_key (test_idempotency.IdempotencyTests.test_failed_access_does_not_burn_key) ... ok +test_json_formatting_and_key_order_do_not_matter (test_idempotency.IdempotencyTests.test_json_formatting_and_key_order_do_not_matter) ... ok +test_key_survives_new_session_of_same_user (test_idempotency.IdempotencyTests.test_key_survives_new_session_of_same_user) ... ok +test_missing_or_invalid_key_is_400 (test_idempotency.IdempotencyTests.test_missing_or_invalid_key_is_400) ... ok +test_replay_rechecks_session (test_idempotency.IdempotencyTests.test_replay_rechecks_session) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTests.test_scope_is_user_and_chat) ... ok +test_sequential_replay_and_conflict (test_idempotency.IdempotencyTests.test_sequential_replay_and_conflict) ... ok +test_exif_orientation_applied_and_stripped (test_pipeline.PipelineTests.test_exif_orientation_applied_and_stripped) ... ok +test_fixture_320x200 (test_pipeline.PipelineTests.test_fixture_320x200) ... ok +test_progressive_jpeg (test_pipeline.PipelineTests.test_progressive_jpeg) ... ok +test_result_is_stable_and_private (test_pipeline.PipelineTests.test_result_is_stable_and_private) ... ok +test_small_source_not_upscaled_except_crop (test_pipeline.PipelineTests.test_small_source_not_upscaled_except_crop) ... ok +test_transparency_on_white (test_pipeline.PipelineTests.test_transparency_on_white) ... ok +test_unknown_variant_is_400 (test_pipeline.PipelineTests.test_unknown_variant_is_400) ... ok +test_a_crash_between_commit_and_publish (test_pipeline_faults.PipelineFaultTests.test_a_crash_between_commit_and_publish) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_b_crash_after_publish_duplicate_is_noop (test_pipeline_faults.PipelineFaultTests.test_b_crash_after_publish_duplicate_is_noop) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_c_worker_crash_after_side_effects_before_commit (test_pipeline_faults.PipelineFaultTests.test_c_worker_crash_after_side_effects_before_commit) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_d_worker_crash_after_commit_before_ack (test_pipeline_faults.PipelineFaultTests.test_d_worker_crash_after_commit_before_ack) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_e_transient_errors_retry_with_backoff (test_pipeline_faults.PipelineFaultTests.test_e_transient_errors_retry_with_backoff) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_f_permanent_error_fails_without_retry (test_pipeline_faults.PipelineFaultTests.test_f_permanent_error_fails_without_retry) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_g_retry_exhausted_dlq_then_explicit_replay (test_pipeline_faults.PipelineFaultTests.test_g_retry_exhausted_dlq_then_explicit_replay) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_h_hung_worker_lease_expires_and_is_fenced (test_pipeline_faults.PipelineFaultTests.test_h_hung_worker_lease_expires_and_is_fenced) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_i_staging_removed_after_safe_commit (test_pipeline_faults.PipelineFaultTests.test_i_staging_removed_after_safe_commit) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_j_garbage_broker_message_is_acked (test_pipeline_faults.PipelineFaultTests.test_j_garbage_broker_message_is_acked) ... skipped 'needs API at https://localhost:8443 and DB/S3/broker env' +test_debt_survives_api_restart (test_session_cache.SessionRevocationCacheTests.test_debt_survives_api_restart) +Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу. ... skipped 'DATABASE_URL is not configured' +test_normal_logout_leaves_no_debt (test_session_cache.SessionRevocationCacheTests.test_normal_logout_leaves_no_debt) ... skipped 'DATABASE_URL is not configured' +test_revoked_token_rejected_after_partition_heals (test_session_cache.SessionRevocationCacheTests.test_revoked_token_rejected_after_partition_heals) ... skipped 'DATABASE_URL is not configured' +test_cleanup_runs_on_one_replica_at_a_time (test_storage_idempotency.StorageIdempotencyTests.test_cleanup_runs_on_one_replica_at_a_time) ... skipped 'DATABASE_URL is not configured' +test_conflict_on_different_body (test_storage_idempotency.StorageIdempotencyTests.test_conflict_on_different_body) ... skipped 'DATABASE_URL is not configured' +test_expired_key_can_be_reused_and_is_cleaned (test_storage_idempotency.StorageIdempotencyTests.test_expired_key_can_be_reused_and_is_cleaned) ... skipped 'DATABASE_URL is not configured' +test_failure_inside_transaction_stores_nothing (test_storage_idempotency.StorageIdempotencyTests.test_failure_inside_transaction_stores_nothing) ... skipped 'DATABASE_URL is not configured' +test_threads_same_key_single_insert (test_storage_idempotency.StorageIdempotencyTests.test_threads_same_key_single_insert) ... skipped 'DATABASE_URL is not configured' +test_failure_after_inserts_leaves_nothing (test_storage_rollback.StorageRollbackTests.test_failure_after_inserts_leaves_nothing) ... skipped 'DATABASE_URL is not configured' +test_migrations_are_idempotent (test_storage_rollback.StorageRollbackTests.test_migrations_are_idempotent) ... skipped 'DATABASE_URL is not configured' +test_no_orphan_chats_in_database (test_storage_rollback.StorageRollbackTests.test_no_orphan_chats_in_database) ... skipped 'DATABASE_URL is not configured' +test_passwords_stored_as_argon2id (test_storage_rollback.StorageRollbackTests.test_passwords_stored_as_argon2id) ... skipped 'DATABASE_URL is not configured' +System.Management.Automation.RemoteException +---------------------------------------------------------------------- +Ran 91 tests in 28.150s +System.Management.Automation.RemoteException +OK (skipped=33) + + 24 parallel POSTs, one key -> 1 message; replicas: ['api-01803f21', 'api-7e1626c5'] + +== Concurrent write test +PASS: concurrent write test; 10 clients x 10 messages = 100 messages; no lost or duplicate messages + +ALL STUDENT TESTS PASSED diff --git a/evidence/run/test-failover-1.txt b/evidence/run/test-failover-1.txt new file mode 100644 index 0000000..bb09c2d --- /dev/null +++ b/evidence/run/test-failover-1.txt @@ -0,0 +1,6 @@ +# make test-failover: отказ одной API-реплики по HTTPS +# python tests/resilience.py --mode failover --action-script C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\scripts\stop-one.cmd --base-url https://localhost:8443 --ca-file C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\secrets\pki\ca.crt +# 2026-11-17T23:59:38.0003701+03:00 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 0, "first_recovery_seconds": 4.984} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/run/test-failover-2.txt b/evidence/run/test-failover-2.txt new file mode 100644 index 0000000..ab75ea5 --- /dev/null +++ b/evidence/run/test-failover-2.txt @@ -0,0 +1,6 @@ +# Отказ другой реплики (последовательно) +# python tests/resilience.py --mode failover --action-script C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\scripts\stop-one.cmd --base-url https://localhost:8443 --ca-file C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\secrets\pki\ca.crt +# 2026-11-18T00:00:02.7399008+03:00 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 0, "first_recovery_seconds": 5.063} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/run/test-persistence.txt b/evidence/run/test-persistence.txt new file mode 100644 index 0000000..c371c69 --- /dev/null +++ b/evidence/run/test-persistence.txt @@ -0,0 +1,5 @@ +# make test-persistence: рестарт API и всех хранилищ (вкл. OpenBao + unseal), данные и токен живы +# python tests/resilience.py --mode persistence --action-script C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\scripts\restart.cmd --base-url https://localhost:8443 --ca-file C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\secrets\pki\ca.crt +# 2026-11-17T23:58:44.4582689+03:00 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/run/test-security.txt b/evidence/run/test-security.txt new file mode 100644 index 0000000..9f0749c --- /dev/null +++ b/evidence/run/test-security.txt @@ -0,0 +1,4 @@ +# make test-security: доверенный CA работает, недоверенный — ошибка проверки сертификата +# python tests/security.py --base-url https://localhost:8443 --ca-file C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\secrets\pki\ca.crt +# 2026-11-17T23:57:27.7419268+03:00 +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. diff --git a/evidence/run/test-smoke.txt b/evidence/run/test-smoke.txt new file mode 100644 index 0000000..115e9a2 --- /dev/null +++ b/evidence/run/test-smoke.txt @@ -0,0 +1,4 @@ +# make test BASE_URL=https://localhost:8443 CA_FILE=secrets\pki\ca.crt +# python tests/smoke.py --base-url https://localhost:8443 --ca-file C:\Users\mrend\Desktop\VP_lab7\mtusi2026lab7-fork\secrets\pki\ca.crt +# 2026-11-17T23:57:25.2579225+03:00 +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/run/tls-audit.txt b/evidence/run/tls-audit.txt new file mode 100644 index 0000000..892d37a --- /dev/null +++ b/evidence/run/tls-audit.txt @@ -0,0 +1,718 @@ +# Вся trust matrix: TLS/mTLS (нет cert / чужой CA / чужая identity / верная), имена, plaintext, ACL +# python scripts/ops.py tls-audit +# 2026-11-17T23:57:27.8839570+03:00 +{ + "ok": true, + "total": 101, + "failed": 0, + "checks": [ + { + "link": "client -> lb (public TLS)", + "check": "trusted CA + hostname lb/localhost SAN", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "client -> lb (public TLS)", + "check": "empty trust store", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', 'unable to get local issuer certificate')" + }, + { + "link": "client -> lb (public TLS)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "client -> lb (public TLS)", + "check": "plaintext HTTP on :8443", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "client -> lb (public TLS)", + "check": "HAProxy stats not exposed on network", + "expected": "refused", + "ok": true, + "observed": "('refused', 'connection refused')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "identity lb (allowed)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "rogue CA with CN=lb", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity intruder", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity worker", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity api (replica as client)", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "plaintext HTTP on :8443", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "health listener 8081 not reachable from network (loopback only)", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "LB-side check: server name must be api", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api verify-full", + "expected": "TLS session", + "ok": true, + "observed": "(True, 'TLSv1.3')" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "plaintext (sslmode=disable)", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"172.19.0.3\", port 5432 failed: FATAL: pg_hba.conf rejects connection for host \"172.19.0.7\", user \"messenger_api\", database \"messenger\", no encryption" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "rogue CA as trust root", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"172.19.0.3\", port 5432 failed: SSL error: certificate verify failed" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "wrong hostname (IP instead of name)", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"172.19.0.3\", port 5432 failed: server certificate for \"postgres\" (and 1 other name) does not match host name \"172.19.0.3\"" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT count(*) FROM attachments", + "expected": "allowed", + "ok": true, + "observed": "executed" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT password_hash FROM users LIMIT 1", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table users" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT token_hash FROM sessions LIMIT 1", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table sessions" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api: CREATE TABLE audit_ddl (i int)", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for schema public" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api: TRUNCATE users", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table users" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api SET sess:*", + "expected": "ok", + "ok": true, + "observed": "('ok', True)" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api SET other:key 1", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', 'No permissions to access a key')" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'flushall' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api CONFIG GET *", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'config|get' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api KEYS *", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'keys' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "cert without AUTH (default user off)", + "expected": "NOAUTH", + "ok": true, + "observed": "('noauth', 'Authentication required.')" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "no client certificate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:6379 : (1, '[SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (_\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "rogue CA client certificate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:6379 : (1, '[SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:2570)')\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "plaintext", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:6379 : (104, 'Connection reset by peer')\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "no client certificate (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from broker:6380 : (1, '[SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "rogue CA with CN=api (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from broker:6380 : (1, '[SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:2570)')\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity intruder (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity lb (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity prometheus (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "identity api without AUTH", + "expected": "NOAUTH", + "ok": true, + "observed": "('noauth', 'Authentication required.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer PING", + "expected": "ok", + "ok": true, + "observed": "('ok', True)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XREADGROUP", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xreadgroup' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XACK", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xack' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XDEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xdel' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer DEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'del' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'flushall' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XADD other key", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', 'No permissions to access a key')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer XLEN", + "expected": "ok", + "ok": true, + "observed": "('ok', 1)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer SET", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'set' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'flushall' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer CONFIG", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'config|get' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer DEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'del' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "Redis port 6379 not reachable from network (loopback only)", + "expected": "connection refused", + "ok": true, + "observed": "('conn_error', 'Error 111 connecting to broker:6379. Connection refused.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "plaintext to gate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "TLS verify-full GET /healthz", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "plaintext on :8333", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "S3 API behind gate :8334 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "S3 gateway gRPC (unauthenticated) :18334 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "S3 gRPC default port :18333 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "master :9333 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "filer :8888 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "filer gRPC :18888 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api PUT staging/", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api DELETE staging/", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api PUT final/ (worker prefix)", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api LIST bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api CREATE bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker LIST bucket", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker PUT outside its prefixes", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker DELETE bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "identity api POST /v1/logs", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "identity worker POST /v1/logs", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "no client certificate POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "rogue CA with CN=api POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity intruder POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity prometheus POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity lb POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "plaintext on :4318", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity prometheus /metrics :8889", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity prometheus /metrics :8888 (self)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity api /metrics :8889", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 14318 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 18889 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 18888 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 13133 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "client cert grafana (TLS ok, service ready)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "client cert grafana (TLS ok, service ready)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "client cert grafana (TLS ok, service ready)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "collector -> tempo OTLP gRPC (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "TLS verify-full /v1/sys/health", + "expected": "HTTP 200 (unsealed)", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "plaintext on :8200", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "read KV without token", + "expected": "HTTP 403", + "ok": true, + "observed": "('http', 403)" + } + ] +} +2026-11-17T20:57:35.308Z saved evidence/tls_audit.json diff --git a/evidence/run/up.txt b/evidence/run/up.txt new file mode 100644 index 0000000..6fbaad6 --- /dev/null +++ b/evidence/run/up.txt @@ -0,0 +1,162 @@ +# Bootstrap: secrets -> образ -> PKI -> OpenBao init/unseal/configure -> стенд -> ожидание готовности +# python scripts/ops.py up +# 2026-11-17T23:57:07.5355535+03:00 +keep secrets/postgres_password +keep secrets/database_admin_url +keep secrets/db_api_password +keep secrets/db_worker_password +keep secrets/cache_api_password +keep secrets/broker_api_password +keep secrets/broker_worker_password +keep secrets/grafana_admin_password +keep secrets/s3_admin_access_key_id +keep secrets/s3_admin_secret_access_key +keep secrets/s3_access_key_id +keep secrets/s3_secret_access_key +keep secrets/s3_worker_access_key_id +keep secrets/s3_worker_secret_access_key +keep secrets/s3_config.json +2026-11-17T20:57:07.709Z up: building image (api/worker/migrate/s3-init/tools) + Image messenger-api:lab7 Building +#1 [internal] load local bake definitions +#1 reading from stdin 1.50kB 0.0s done +#1 DONE 0.0s + +#2 [internal] load build definition from Dockerfile +#2 transferring dockerfile: 1.53kB done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/python:3.12.7-slim-bookworm +#3 DONE 1.4s + +#4 [internal] load .dockerignore +#4 transferring context: 264B done +#4 DONE 0.0s + +#5 [1/7] FROM docker.io/library/python:3.12.7-slim-bookworm@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d +#5 resolve docker.io/library/python:3.12.7-slim-bookworm@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d 0.0s done +#5 DONE 0.0s + +#6 [internal] load build context +#6 transferring context: 1.37kB done +#6 DONE 0.0s + +#7 [3/7] COPY requirements.txt . +#7 CACHED + +#8 [4/7] RUN pip install --no-cache-dir --disable-pip-version-check -r requirements.txt +#8 CACHED + +#9 [6/7] COPY migrations ./migrations +#9 CACHED + +#10 [2/7] WORKDIR /app +#10 CACHED + +#11 [5/7] COPY src ./src +#11 CACHED + +#12 [7/7] RUN groupadd --gid 10001 appuser && useradd --uid 10001 --gid 10001 --no-create-home --home-dir /nonexistent --shell /usr/sbin/nologin appuser +#12 CACHED + +#13 exporting to image +#13 exporting layers +#13 exporting layers done +#13 exporting manifest sha256:3a3cbef5472c7fe7fb3770fd2a1361ab59ec02b28bda430f29b6c70043b43554 done +#13 exporting config sha256:26f2e3e40f1cba8fe10cba8aa600e656f54656db05c77c043493c6c44acc98a7 done +#13 exporting attestation manifest sha256:2387092eee85d6799f18a319530d1e434643d44ac3ffb5c82531ffd7d12d2d6c 0.0s done +#13 exporting manifest list sha256:4427dfc802d546fd09249101e9efca16c96b46d6bad96682c9d4daae5ab1e21c 0.0s done +#13 naming to docker.io/library/messenger-api:lab7 +#13 naming to docker.io/library/messenger-api:lab7 done +#13 unpacking to docker.io/library/messenger-api:lab7 0.0s done +#13 DONE 0.2s + +#14 resolving provenance for metadata file +#14 DONE 0.0s + Image messenger-api:lab7 Built + Container messenger-lab7-tools-run-d7893944bfb7 Creating + Container messenger-lab7-tools-run-d7893944bfb7 Created +keep ca.crt +keep lb-public +keep lb-client +keep api +keep worker +keep postgres +keep redis +keep broker +keep s3 +keep otel-collector +keep loki +keep tempo +keep prometheus +keep grafana +keep openbao +keep intruder +OpenBao already initialized (keys: present) +OpenBao is unsealed +issued new SecretID for api (ttl 168h, bound to 172.30.250.0/24); previous SecretID destroyed +issued new SecretID for worker (ttl 168h, bound to 172.30.250.0/24); previous SecretID destroyed +configured: kv-v2, approle, policies messenger-api/messenger-worker, secrets written to KV +2026-11-17T20:57:19.647Z up: starting the stand + Container messenger-lab7-grafana-1 Running + Container messenger-lab7-worker-1 Running + Container messenger-lab7-tempo-1 Running + Container messenger-lab7-broker-gate-1 Running + Container messenger-lab7-otel-gate-1 Running + Container messenger-lab7-redis-1 Running + Container messenger-lab7-bao-agent-worker-1 Running + Container messenger-lab7-api-1 Running + Container messenger-lab7-api-2 Running + Container messenger-lab7-openbao-1 Running + Container messenger-lab7-otel-collector-1 Running + Container messenger-lab7-broker-1 Running + Container messenger-lab7-postgres-1 Running + Container messenger-lab7-lb-1 Running + Container messenger-lab7-loki-1 Running + Container messenger-lab7-s3-gate-1 Running + Container messenger-lab7-prometheus-1 Running + Container messenger-lab7-bao-agent-api-1 Running + Container messenger-lab7-s3-1 Running + Container messenger-lab7-tempo-init-1 Starting + Container messenger-lab7-broker-1 Waiting + Container messenger-lab7-prometheus-1 Waiting + Container messenger-lab7-postgres-1 Waiting + Container messenger-lab7-s3-1 Waiting + Container messenger-lab7-tempo-init-1 Started + Container messenger-lab7-tempo-init-1 Waiting + Container messenger-lab7-broker-1 Healthy + Container messenger-lab7-s3-1 Healthy + Container messenger-lab7-s3-gate-1 Waiting + Container messenger-lab7-postgres-1 Healthy + Container messenger-lab7-migrate-1 Starting + Container messenger-lab7-prometheus-1 Healthy + Container messenger-lab7-tempo-init-1 Exited + Container messenger-lab7-migrate-1 Started + Container messenger-lab7-s3-gate-1 Healthy + Container messenger-lab7-s3-init-1 Starting + Container messenger-lab7-s3-init-1 Started + Container messenger-lab7-migrate-1 Waiting + Container messenger-lab7-s3-init-1 Waiting + Container messenger-lab7-bao-agent-worker-1 Waiting + Container messenger-lab7-bao-agent-api-1 Waiting + Container messenger-lab7-postgres-1 Waiting + Container messenger-lab7-postgres-1 Waiting + Container messenger-lab7-broker-gate-1 Waiting + Container messenger-lab7-redis-1 Waiting + Container messenger-lab7-migrate-1 Waiting + Container messenger-lab7-s3-init-1 Waiting + Container messenger-lab7-migrate-1 Exited + Container messenger-lab7-bao-agent-worker-1 Healthy + Container messenger-lab7-broker-gate-1 Healthy + Container messenger-lab7-postgres-1 Healthy + Container messenger-lab7-migrate-1 Exited + Container messenger-lab7-bao-agent-api-1 Healthy + Container messenger-lab7-postgres-1 Healthy + Container messenger-lab7-redis-1 Healthy + Container messenger-lab7-s3-init-1 Exited + Container messenger-lab7-s3-init-1 Exited + Container messenger-lab7-api-1 Waiting + Container messenger-lab7-api-2 Waiting + Container messenger-lab7-api-1 Healthy + Container messenger-lab7-api-2 Healthy +2026-11-17T20:57:24.297Z up: ready in 17s -> https://127.0.0.1:8443 (CA: secrets\pki\ca.crt); Grafana https://127.0.0.1:3000 (admin / secrets/grafana_admin_password) diff --git a/evidence/sandbox/README.md b/evidence/sandbox/README.md new file mode 100644 index 0000000..3e8df9b --- /dev/null +++ b/evidence/sandbox/README.md @@ -0,0 +1,38 @@ +# evidence/sandbox — проверка конфигураций вне Docker (2026-11-16) + +В среде подготовки решения Docker Hub был недоступен (registry отвечал 403), поэтому полный стенд +в Docker там не поднимался. Чтобы не сдавать непроверенные конфиги, **те же файлы** из `infra/`, +`infra/openbao/`, `infra/postgres/`, `infra/redis/`, `infra/haproxy/` и код приложения были запущены +на тех же версиях продуктов нативно на одном Linux-хосте: + +| Компонент | Версия в песочнице | Источник | +| --- | --- | --- | +| OpenBao (server + 2 agent) | 2.1.0 | GitHub release | +| HAProxy (LB, broker-gate, otel-gate) | 2.8.0 (в compose 2.8.30) | сборка из исходников с OpenSSL | +| Redis (кэш, broker) | 7.4.1, BUILD_TLS=yes | сборка из исходников | +| PostgreSQL | 16 | пакет ОС | +| SeaweedFS | 4.50 | GitHub release | +| OTel Collector contrib / Prometheus / Loki / Tempo | 0.115.1 / 2.55.1 / 3.3.2 / 2.6.1 | GitHub release | +| API ×2 + worker | этот репозиторий, Python 3.13, requirements.txt | — | + +Отличия от Docker: пути `/run/secrets/*` — symlink-и, все сервисы на одном хосте (имена резолвятся в +127.0.0.1 через /etc/hosts), порты, конфликтующие на общем хосте, переназначены (LB 9443, кэш Redis +16379, self-метрики Collector 18898/28888), HAProxy с двумя статическими `server` вместо `server-template` +(нет Docker DNS). Grafana не запускалась. + +Файлы: + +* `public-tests.txt` — `tests/smoke.py`, `tests/security.py`, `scripts/check_contract.py` по HTTPS через LB. +* `security_http.txt` — 24/24 проверки ACL, 429, 413, TLS снаружи. +* `student-tests.txt` — 91 тест lab2–6 (58 выполнено, 33 требуют in-network окружения) + 12 white-box тестов хранилища. +* `tls_audit.json` / `tls_audit.txt` — 95 проверок матрицы (прогон до добавления 6 проверок служебных портов S3 — всего теперь 101); **7 FAIL — артефакты общего хоста**: в + песочнице loopback-порты сервисов (`127.0.0.1:8081` health API, `127.0.0.1:6379` Redis broker, + `127.0.0.1:14318/18889/18888/13133` Collector, `127.0.0.1:8404` статистика HAProxy) доступны тесту, + потому что тест работает на том же хосте. В Docker это loopback **внутри контейнера**, и из контейнера + `audit` они недоступны — именно это и проверяют эти 7 пунктов на стенде. +* `bao_policy.json` — 12/12 allow/deny политик OpenBao. +* `leak_check.json` — нет канареек и секретов ролей в Loki/Tempo/Prometheus (с положительным контролем свежих данных). +* `lifecycle.txt` — ротация пароля БД, ротация и отзыв сертификата LB, SecretID, sealed OpenBao, rehash, бюджет login DoS. +* `pip-audit-before.txt` / `pip-audit-after.txt` — 72 уязвимости → 0 после обновления зависимостей. + +На Docker-стенде всё это (и hardening, persistence, failover) повторяет `run.ps1` и пишет в `evidence/run/`. diff --git a/evidence/sandbox/bao_policy.json b/evidence/sandbox/bao_policy.json new file mode 100644 index 0000000..958ecf9 --- /dev/null +++ b/evidence/sandbox/bao_policy.json @@ -0,0 +1,65 @@ +{ + "ok": true, + "checks": [ + { + "check": "api reads own kv/messenger/api", + "ok": true, + "detail": "" + }, + { + "check": "api cannot read kv/messenger/worker", + "ok": true, + "detail": "" + }, + { + "check": "api cannot write own kv path", + "ok": true, + "detail": "" + }, + { + "check": "api cannot write policies", + "ok": true, + "detail": "" + }, + { + "check": "api cannot mint SecretIDs", + "ok": true, + "detail": "" + }, + { + "check": "api token is short-lived (ttl<=15m) and not root", + "ok": true, + "detail": "ttl=899s policies=['default', 'messenger-api']" + }, + { + "check": "worker reads own kv/messenger/worker", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot read kv/messenger/api", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot write own kv path", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot write policies", + "ok": true, + "detail": "" + }, + { + "check": "worker cannot mint SecretIDs", + "ok": true, + "detail": "" + }, + { + "check": "worker token is short-lived (ttl<=15m) and not root", + "ok": true, + "detail": "ttl=899s policies=['default', 'messenger-worker']" + } + ] +} diff --git a/evidence/sandbox/leak_check.json b/evidence/sandbox/leak_check.json new file mode 100644 index 0000000..fa2cd78 --- /dev/null +++ b/evidence/sandbox/leak_check.json @@ -0,0 +1,31 @@ +{ + "ok": true, + "controls": { + "loki has messenger-api logs": true, + "tempo has POST /api/v1/auth/sessions spans": true, + "prometheus has http metrics labels": true + }, + "canary_user_created": true, + "searched": { + "loki_lines": 54, + "tempo_traces": 51, + "prometheus_labels": 26 + }, + "needles": [ + "argon2 hash", + "basic header", + "bearer header", + "broker_api_password", + "broker_worker_password", + "cache_api_password", + "db_api_password", + "db_worker_password", + "private key", + "s3_secret_access_key", + "s3_worker_secret_access_key", + "session token", + "user password", + "wrong password" + ], + "findings": [] +} diff --git a/evidence/sandbox/lifecycle.txt b/evidence/sandbox/lifecycle.txt new file mode 100644 index 0000000..d64497e --- /dev/null +++ b/evidence/sandbox/lifecycle.txt @@ -0,0 +1,58 @@ +# Жизненный цикл секретов/сертификатов — проверка в локальной песочнице (2026-11-16) +# +# ВАЖНО: это НЕ Docker-стенд. Docker Hub из среды подготовки был недоступен, поэтому конфиги +# стенда проверялись на тех же версиях продуктов, запущенных нативно (OpenBao 2.1.0, +# HAProxy 2.8 из исходников, Redis 7.4.1 из исходников, PostgreSQL 16, SeaweedFS 4.50, +# otelcol-contrib 0.115.1, Prometheus 2.55.1, Loki 3.3.2, Tempo 2.6.1) с ровно теми же +# файлами конфигурации из infra/ (подставлены только пути /run/secrets и порты, занятые на +# общем хосте). Повторить на Docker: python scripts/ops.py <команда> (см. RUN.md). + +## 1. Ротация рабочего секрета: пароль роли messenger_api (без рестарта API и пересборки) +ALTER ROLE messenger_api PASSWORD -> bao kv patch kv/messenger/api db_password= +agent re-rendered /run/bao/database_url after 11s (static_secret_render_interval был 15 с; в репозитории 5 с) +old password, NEW connection: password authentication failed for user "messenger_api" +new password, NEW connection: new password works +pg_terminate_backend(все сессии messenger_api) -> 2 сессии разорваны -> пул открыл новые +python3 tests/smoke.py ... -> PASS: lab7 public API scenario +=> ops.py rotate-db-password делает то же самое на Docker-стенде и пишет окно в evidence/ops.log + +## 2. Ротация leaf-сертификата LB->API и отзыв старой identity (CRL), без рестарта API +pki.py rotate lb-client -> issue lb-client serial 7d592e39c7af2af70c237972b0532b84bf75c05 +new lb: 200 old lb (not yet revoked): 200 +pki.py revoke lb-client -> crl 1 revoked serial(s) +(через <= 5 с API перечитал CRL) +new lb: 200 old lb (revoked): 000 (соединение разорвано) +API log: mTLS CRL loaded: 1 revoked serial(s) +API log: mTLS client rejected: identity=lb reason=certificate_revoked +=> ops.py rotate-lb-cert: rotate -> restart lb (без build) -> revoke -> проба старым/новым из контейнера audit + +## 3. Ротация SecretID и отзыв identity AppRole (bao.py) +issued new SecretID for worker (ttl 168h, bound to CIDR); previous SecretID destroyed +old SecretID login: denied +new SecretID login: accepted +revoked identity worker: 1 SecretID, 3 token(s); agent can no longer authenticate +after revoke-role, current SecretID login: denied +recovered with new SecretID: accepted + +## 4. OpenBao sealed / недоступен +bao operator seal -> Success! Vault is sealed. +файлы /run/bao/* агентов остались (-r-------- database_url ...), API продолжает работу +agent log: Vault is sealed (retry attempt 10 after "1m0s") (backoff, процесс агента жив) +bao.py unseal -> unsealed=True; агент сам восстановился и перерисовал секрет через ~60 с: +agent: (runner) rendered "(dynamic)" => ".../database_url" + +## 5. KV -> агент -> файл (доставка изменения) +kv patch -> файл перерисован (ROTATION PROPAGATED) + +## 6. Argon2id rehash при успешном входе +register 201 +before: m=8192,t=1,p=1 (hash подменён на «старые» параметры админским UPDATE) +login 201 +after: m=19456,t=2,p=1 (пересчитан под текущие параметры; лог auth.rehash) + +## 7. Бюджет login DoS (песочница без CPU-лимита; на стенде повторить ops.py login-budget) +Argon2id m=19456,t=2,p=1: hash p50 24.2 ms, verify p50 24.0 ms, RAM 19 MiB на хеш; +оценка при CPU-квоте 0.5 и 4 слотах KDF: ~20.8 входов/с на реплику. +Поток 30 с, 8 потоков, случайные username с одного IP: + login attempts 150.6/s: 401 = 136 (дошли до Argon2id), 429 = 4397 (срезаны rate limit ДО KDF) + GET /users/me p50 9.0 ms -> 39.4 ms во время потока, p95 13.5 -> 112.4 ms, ошибок 0 из 201 diff --git a/evidence/sandbox/pip-audit-after.txt b/evidence/sandbox/pip-audit-after.txt new file mode 100644 index 0000000..029cbc2 --- /dev/null +++ b/evidence/sandbox/pip-audit-after.txt @@ -0,0 +1,2 @@ +# pip-audit ПОСЛЕ обновления (requirements.txt lab7), 2026-11-16T11:26:50Z +No known vulnerabilities found diff --git a/evidence/sandbox/pip-audit-before.txt b/evidence/sandbox/pip-audit-before.txt new file mode 100644 index 0000000..f5254ee --- /dev/null +++ b/evidence/sandbox/pip-audit-before.txt @@ -0,0 +1,104 @@ +# pip-audit ДО обновления (requirements lab6 + cryptography 44.0.0), 2026-11-16T11:26:42Z +Found 100 known vulnerabilities in 4 packages +Name Version ID Fix Versions +----------------- ------- ------------------- ------------ +pillow 11.1.0 PYSEC-2026-2249 12.1.1 +pillow 11.1.0 PYSEC-2026-2250 12.2.0 +pillow 11.1.0 PYSEC-2026-165 12.2.0 +pillow 11.1.0 PYSEC-2026-2874 12.2.0 +pillow 11.1.0 PYSEC-2026-2252 12.2.0 +pillow 11.1.0 PYSEC-2026-165 12.2.0 +pillow 11.1.0 PYSEC-2026-2250 12.2.0 +pillow 11.1.0 PYSEC-2026-2253 12.3.0 +pillow 11.1.0 PYSEC-2026-2255 12.3.0 +pillow 11.1.0 PYSEC-2026-2257 12.3.0 +pillow 11.1.0 PYSEC-2026-2256 12.3.0 +pillow 11.1.0 PYSEC-2026-2254 12.3.0 +pillow 11.1.0 PYSEC-2026-2252 12.2.0 +pillow 11.1.0 PYSEC-2026-2249 12.1.1 +pillow 11.1.0 PYSEC-2026-2874 12.2.0 +pillow 11.1.0 PYSEC-2026-3453 12.3.0 +pillow 11.1.0 PYSEC-2026-3451 12.3.0 +pillow 11.1.0 PYSEC-2026-3493 12.3.0 +pillow 11.1.0 PYSEC-2026-2254 12.3.0 +pillow 11.1.0 PYSEC-2026-2253 12.3.0 +pillow 11.1.0 PYSEC-2026-2256 12.3.0 +pillow 11.1.0 PYSEC-2026-2255 12.3.0 +pillow 11.1.0 PYSEC-2026-3454 12.3.0 +pillow 11.1.0 PYSEC-2026-3451 12.3.0 +pillow 11.1.0 PYSEC-2026-3494 12.3.0 +pillow 11.1.0 PYSEC-2026-3495 12.3.0 +pillow 11.1.0 PYSEC-2026-3496 12.3.0 +pillow 11.1.0 PYSEC-2026-3453 12.3.0 +pillow 11.1.0 PYSEC-2026-3454 12.3.0 +pillow 11.1.0 PYSEC-2026-3495 12.3.0 +pillow 11.1.0 PYSEC-2026-3496 12.3.0 +pillow 11.1.0 PYSEC-2026-3494 12.3.0 +pillow 11.1.0 PYSEC-2026-3493 12.3.0 +pillow 11.1.0 PYSEC-2026-4327 13.0.0 +pillow 11.1.0 PYSEC-2026-4326 13.0.0 +pillow 11.1.0 PYSEC-2026-4339 13.0.0 +pillow 11.1.0 PYSEC-2026-4340 13.0.0 +pillow 11.1.0 PYSEC-2026-4343 13.0.0 +pillow 11.1.0 PYSEC-2026-4334 13.0.0 +pillow 11.1.0 PYSEC-2026-4337 13.0.0 +pillow 11.1.0 PYSEC-2026-4341 13.0.0 +pillow 11.1.0 PYSEC-2026-4330 13.0.0 +pillow 11.1.0 PYSEC-2026-4338 13.0.0 +pillow 11.1.0 PYSEC-2026-4335 13.0.0 +pillow 11.1.0 PYSEC-2026-4331 13.0.0 +pillow 11.1.0 PYSEC-2026-4332 13.0.0 +pillow 11.1.0 PYSEC-2026-4329 13.0.0 +pillow 11.1.0 PYSEC-2026-4342 13.0.0 +pillow 11.1.0 PYSEC-2026-4363 13.0.0 +pillow 11.1.0 CVE-2026-112895 13.0.0 +pillow 11.1.0 CVE-2026-112400 13.0.0 +pillow 11.1.0 CVE-2026-112897 13.0.0 +pillow 11.1.0 CVE-2026-112395 13.0.0 +pillow 11.1.0 CVE-2026-112404 13.0.0 +pillow 11.1.0 CVE-2026-112899 13.0.0 +pillow 11.1.0 CVE-2026-112900 13.0.0 +pillow 11.1.0 CVE-2026-112893 13.0.0 +pillow 11.1.0 CVE-2026-112891 13.0.0 +pillow 11.1.0 CVE-2026-112390 13.0.0 +pillow 11.1.0 CVE-2026-112401 13.0.0 +pillow 11.1.0 CVE-2026-112391 13.0.0 +pillow 11.1.0 CVE-2026-112894 13.0.0 +pillow 11.1.0 CVE-2026-112406 13.0.0 +pillow 11.1.0 CVE-2026-112399 13.0.0 +pillow 11.1.0 CVE-2026-112892 13.0.0 +pillow 11.1.0 CVE-2026-112393 13.0.0 +pillow 11.1.0 CVE-2026-96638 13.0.0 +pillow 11.1.0 CVE-2026-96640 13.0.0 +pillow 11.1.0 CVE-2026-96778 13.0.0 +pillow 11.1.0 GHSA-78g4-xqwv-q3g8 13.0.0 +pillow 11.1.0 CVE-2026-128067 13.0.0 +pillow 11.1.0 CVE-2026-128066 13.0.0 +opentelemetry-api 1.29.0 CVE-2026-114603 1.45.1 +opentelemetry-api 1.29.0 CVE-2026-126360 1.45.2 +cryptography 44.0.0 PYSEC-2026-1284 44.0.1 +cryptography 44.0.0 PYSEC-2026-2141 46.0.5 +cryptography 44.0.0 PYSEC-2026-35 46.0.6 +cryptography 44.0.0 PYSEC-2026-35 46.0.6 +cryptography 44.0.0 PYSEC-2026-1284 44.0.1 +cryptography 44.0.0 PYSEC-2026-2141 46.0.5 +cryptography 44.0.0 PYSEC-2026-3552 50.0.0 +cryptography 44.0.0 PYSEC-2026-3553 49.0.0 +cryptography 44.0.0 PYSEC-2026-3552 50.0.0 +cryptography 44.0.0 PYSEC-2026-3553 49.0.0 +cryptography 44.0.0 PYSEC-2026-3554 49.0.0 +cryptography 44.0.0 GHSA-537c-gmf6-5ccf 48.0.1 +starlette 0.41.3 PYSEC-2026-1941 0.47.2 +starlette 0.41.3 PYSEC-2026-1942 0.49.1 +starlette 0.41.3 PYSEC-2026-161 1.0.1 +starlette 0.41.3 PYSEC-2026-161 1.0.1 +starlette 0.41.3 PYSEC-2026-2281 1.1.0 +starlette 0.41.3 PYSEC-2026-2280 1.1.0 +starlette 0.41.3 PYSEC-2026-249 1.3.1 +starlette 0.41.3 PYSEC-2026-248 1.3.0 +starlette 0.41.3 PYSEC-2026-249 1.3.1 +starlette 0.41.3 PYSEC-2026-248 1.3.0 +starlette 0.41.3 PYSEC-2026-1942 0.49.1 +starlette 0.41.3 PYSEC-2026-1941 0.47.2 +starlette 0.41.3 PYSEC-2026-2281 1.1.0 +starlette 0.41.3 PYSEC-2026-2280 1.1.0 diff --git a/evidence/sandbox/public-tests.txt b/evidence/sandbox/public-tests.txt new file mode 100644 index 0000000..556d2af --- /dev/null +++ b/evidence/sandbox/public-tests.txt @@ -0,0 +1,6 @@ +# python3 tests/smoke.py --base-url https://localhost:9443 --ca-file (2026-11-16T11:39:03Z) +PASS: lab7 public API scenario; infrastructure and grading evidence remain separate. +# python3 tests/security.py ... +PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening. +# python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/sandbox/security_http.txt b/evidence/sandbox/security_http.txt new file mode 100644 index 0000000..c33abed --- /dev/null +++ b/evidence/sandbox/security_http.txt @@ -0,0 +1,26 @@ +# security_http.py via LB, 2 API replicas (2026-11-16T11:39:04Z) +PASS setup: chat + attachment 202 +PASS outsider GET /api/v1/chats/{chat} -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/chats/{chat}/messages -> 404 (no existence leak) 404 +PASS outsider POST /api/v1/chats/{chat}/messages -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/attachments/{att} -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/attachments/{att}/content -> 404 (no existence leak) 404 +PASS outsider attaches foreign attachment to own chat -> 404 404 +PASS X-User-Id cannot override Bearer identity eve_26766f0a44 +PASS X-User-Id alone -> 401 + WWW-Authenticate Bearer Bearer realm="messenger" +PASS Basic on business endpoint -> 401 401 +PASS unknown/forged token -> 401 401 +PASS logout 204 -> token rejected 401 -> repeat logout 401 (204, 401, 401) +PASS wrong password -> 401, body has no username/password/hash {"error":{"code":"unauthorized","message":"Authentication required","request_id" +PASS malformed JSON -> 400 without stack trace/SQL {"error":{"code":"invalid_request","message":"Invalid request","request_id":"7F0 +PASS login brute force -> 429 rate_limited + Retry-After [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429] Retry-After=55 +PASS rate limit is shared: responses from >= 2 replicas ['api-1213c2d7', 'api-21b5d532'] +PASS correct password while locked out -> 429 (no oracle) 429 +PASS JSON body 70 KB -> 413 payload_too_large from LB (before any replica) instance=None +PASS chunked JSON 70 KB -> 413 payload_too_large (API) instance=api-1213c2d7 +PASS upload 10 MiB + 1 byte -> 413 413 +PASS wrong server name -> certificate verification error Hostname mismatch, certificate is not valid for 'evil.example'. +PASS empty trust store -> verification error unable to get local issuer certificate +PASS plaintext HTTP on HTTPS port is not served RemoteDisconnected +PASS grafana over HTTPS requires authentication (skipped: not reachable) ConnectionRefusedError +PASS: 24/24 checks diff --git a/evidence/sandbox/student-tests.txt b/evidence/sandbox/student-tests.txt new file mode 100644 index 0000000..60beaaa --- /dev/null +++ b/evidence/sandbox/student-tests.txt @@ -0,0 +1,12 @@ +# student tests lab2-6 over HTTPS (2026-11-16T11:39:05Z) +............ssssssssss.............................s. + 24 parallel POSTs, one key -> 1 message; replicas: ['api-1213c2d7', 'api-21b5d532'] +................ssssssssssssssssssssss +---------------------------------------------------------------------- +Ran 91 tests in 15.196s + +OK (skipped=33) +# white-box storage tests (admin DB identity) +Ran 12 tests in 4.088s + +OK diff --git a/evidence/sandbox/tls_audit.json b/evidence/sandbox/tls_audit.json new file mode 100644 index 0000000..69e8826 --- /dev/null +++ b/evidence/sandbox/tls_audit.json @@ -0,0 +1,672 @@ +{ + "ok": false, + "total": 95, + "failed": 7, + "checks": [ + { + "link": "client -> lb (public TLS)", + "check": "trusted CA + hostname lb/localhost SAN", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "client -> lb (public TLS)", + "check": "empty trust store", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', 'unable to get local issuer certificate')" + }, + { + "link": "client -> lb (public TLS)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "client -> lb (public TLS)", + "check": "plaintext HTTP on :8443", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "client -> lb (public TLS)", + "check": "HAProxy stats not exposed on network", + "expected": "refused", + "ok": false, + "observed": "('tls_error', 'WRONG_VERSION_NUMBER')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "identity lb (allowed)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "rogue CA with CN=lb", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('reset', 'BrokenPipeError')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity intruder", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('reset', 'BrokenPipeError')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity worker", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('reset', 'BrokenPipeError')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity api (replica as client)", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('reset', 'BrokenPipeError')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "plaintext HTTP on :8443", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "health listener 8081 not reachable from network (loopback only)", + "expected": "refused", + "ok": false, + "observed": "('http', 404)" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "LB-side check: server name must be api", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api verify-full", + "expected": "TLS session", + "ok": true, + "observed": "(True, 'TLSv1.3')" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "plaintext (sslmode=disable)", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"127.0.0.1\", port 5432 failed: FATAL: pg_hba.conf rejects connection for host \"127.0.0.1\", user \"messenger_api\", database \"messenger\", no encryption" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "rogue CA as trust root", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"127.0.0.1\", port 5432 failed: SSL error: certificate verify failed" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "wrong hostname (IP instead of name)", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"127.0.0.1\", port 5432 failed: server certificate for \"postgres\" (and 1 other name) does not match host name \"127.0.0.1\"" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT count(*) FROM attachments", + "expected": "allowed", + "ok": true, + "observed": "executed" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT password_hash FROM users LIMIT 1", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table users" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT token_hash FROM sessions LIMIT 1", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table sessions" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api: CREATE TABLE audit_ddl (i int)", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for schema public" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api: TRUNCATE users", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table users" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api SET sess:*", + "expected": "ok", + "ok": true, + "observed": "('ok', True)" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api SET other:key 1", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', 'No permissions to access a key')" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'flushall' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api CONFIG GET *", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'config|get' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api KEYS *", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'keys' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "cert without AUTH (default user off)", + "expected": "NOAUTH", + "ok": true, + "observed": "('noauth', 'Authentication required.')" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "no client certificate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:16379 : (1, '[SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "rogue CA client certificate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:16379 : (1, '[SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:2663)')\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "plaintext", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:16379 : (104, 'Connection reset by peer')\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "no client certificate (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from broker:6380 : (1, '[SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "rogue CA with CN=api (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from broker:6380 : (1, '[SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:2663)')\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity intruder (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity lb (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity prometheus (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "identity api without AUTH", + "expected": "NOAUTH", + "ok": true, + "observed": "('noauth', 'Authentication required.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer PING", + "expected": "ok", + "ok": true, + "observed": "('ok', True)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XREADGROUP", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xreadgroup' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XACK", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xack' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XDEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xdel' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer DEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'del' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'flushall' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XADD other key", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', 'No permissions to access a key')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer XLEN", + "expected": "ok", + "ok": true, + "observed": "('ok', 149)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer SET", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'set' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'flushall' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer CONFIG", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'config|get' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer DEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'del' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "Redis port 6379 not reachable from network (loopback only)", + "expected": "connection refused", + "ok": false, + "observed": "('ok', True)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "plaintext to gate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "TLS verify-full GET /healthz", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "plaintext on :8333", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api PUT staging/", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api DELETE staging/", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api PUT final/ (worker prefix)", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api LIST bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api CREATE bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker LIST bucket", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker PUT outside its prefixes", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker DELETE bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "identity api POST /v1/logs", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "identity worker POST /v1/logs", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "no client certificate POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "rogue CA with CN=api POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity intruder POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity prometheus POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity lb POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "plaintext on :4318", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity prometheus /metrics :8889", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity prometheus /metrics :8888 (self)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity api /metrics :8889", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 14318 not reachable", + "expected": "refused", + "ok": false, + "observed": "('http', 404)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 18889 not reachable", + "expected": "refused", + "ok": false, + "observed": "('http', 404)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 18888 not reachable", + "expected": "refused", + "ok": false, + "observed": "('closed', \"b'\\\\x00\\\\x00\\\\x18\\\\x04\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x00\\\\x05\\\\x00\\\\x00@\\\\x00\\\\x00\\\\x03\\\\x00\\\\x00\\\\x03'\")" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 13133 not reachable", + "expected": "refused", + "ok": false, + "observed": "('http', 200)" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "client cert grafana", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "client cert grafana", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "client cert grafana", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "collector -> tempo OTLP gRPC (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "TLS verify-full /v1/sys/health", + "expected": "HTTP 200 (unsealed)", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "plaintext on :8200", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "read KV without token", + "expected": "HTTP 403", + "ok": true, + "observed": "('http', 403)" + } + ] +} diff --git a/evidence/sandbox/tls_audit.txt b/evidence/sandbox/tls_audit.txt new file mode 100644 index 0000000..e33ff3c --- /dev/null +++ b/evidence/sandbox/tls_audit.txt @@ -0,0 +1,95 @@ +PASS client -> lb (public TLS) trusted CA + hostname lb/localhost SAN ('http', 200) +PASS client -> lb (public TLS) empty trust store ('verify_error', 'unable to get local issuer certificate') +PASS client -> lb (public TLS) wrong hostname ('verify_error', "Hostname mismatch, certificate is not vali +PASS client -> lb (public TLS) plaintext HTTP on :8443 ('closed', "b''") +FAIL client -> lb (public TLS) HAProxy stats not exposed on network ('tls_error', 'WRONG_VERSION_NUMBER') +PASS lb -> api (mTLS, allowlist lb + CRL) identity lb (allowed) ('http', 200) +PASS lb -> api (mTLS, allowlist lb + CRL) no client certificate ('closed', 'connection closed without HTTP response') +PASS lb -> api (mTLS, allowlist lb + CRL) rogue CA with CN=lb ('reset', 'BrokenPipeError') +PASS lb -> api (mTLS, allowlist lb + CRL) same CA, identity intruder ('reset', 'BrokenPipeError') +PASS lb -> api (mTLS, allowlist lb + CRL) same CA, identity worker ('reset', 'BrokenPipeError') +PASS lb -> api (mTLS, allowlist lb + CRL) same CA, identity api (replica as client) ('reset', 'BrokenPipeError') +PASS lb -> api (mTLS, allowlist lb + CRL) plaintext HTTP on :8443 ('closed', "b''") +FAIL lb -> api (mTLS, allowlist lb + CRL) health listener 8081 not reachable from network (loo ('http', 404) +PASS lb -> api (mTLS, allowlist lb + CRL) LB-side check: server name must be api ('verify_error', "Hostname mismatch, certificate is not vali +PASS api/worker -> postgres (TLS verify-full, roles) messenger_api verify-full (True, 'TLSv1.3') +PASS api/worker -> postgres (TLS verify-full, roles) plaintext (sslmode=disable) connection failed: connection to server at "127.0.0.1", port +PASS api/worker -> postgres (TLS verify-full, roles) rogue CA as trust root connection failed: connection to server at "127.0.0.1", port +PASS api/worker -> postgres (TLS verify-full, roles) wrong hostname (IP instead of name) connection failed: connection to server at "127.0.0.1", port +PASS api/worker -> postgres (TLS verify-full, roles) messenger_worker: SELECT count(*) FROM attachments executed +PASS api/worker -> postgres (TLS verify-full, roles) messenger_worker: SELECT password_hash FROM users LI permission denied for table users +PASS api/worker -> postgres (TLS verify-full, roles) messenger_worker: SELECT token_hash FROM sessions LI permission denied for table sessions +PASS api/worker -> postgres (TLS verify-full, roles) messenger_api: CREATE TABLE audit_ddl (i int) permission denied for schema public +PASS api/worker -> postgres (TLS verify-full, roles) messenger_api: TRUNCATE users permission denied for table users +PASS api -> redis cache (mTLS + ACL) messenger_api SET sess:* ('ok', True) +PASS api -> redis cache (mTLS + ACL) messenger_api SET other:key 1 ('noperm', 'No permissions to access a key') +PASS api -> redis cache (mTLS + ACL) messenger_api FLUSHALL ('noperm', "User messenger_api has no permissions to run the +PASS api -> redis cache (mTLS + ACL) messenger_api CONFIG GET * ('noperm', "User messenger_api has no permissions to run the +PASS api -> redis cache (mTLS + ACL) messenger_api KEYS * ('noperm', "User messenger_api has no permissions to run the +PASS api -> redis cache (mTLS + ACL) cert without AUTH (default user off) ('noauth', 'Authentication required.') +PASS api -> redis cache (mTLS + ACL) no client certificate ('conn_error', "Error while reading from redis:16379 : (1, ' +PASS api -> redis cache (mTLS + ACL) rogue CA client certificate ('conn_error', "Error while reading from redis:16379 : (1, ' +PASS api -> redis cache (mTLS + ACL) plaintext ('conn_error', "Error while reading from redis:16379 : (104, +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke no client certificate (with valid producer password) ('conn_error', "Error while reading from broker:6380 : (1, ' +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke rogue CA with CN=api (with valid producer password) ('conn_error', "Error while reading from broker:6380 : (1, ' +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke same CA, identity intruder (with valid producer pass ('conn_error', 'Connection closed by server.') +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke same CA, identity lb (with valid producer password) ('conn_error', 'Connection closed by server.') +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke same CA, identity prometheus (with valid producer pa ('conn_error', 'Connection closed by server.') +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke identity api without AUTH ('noauth', 'Authentication required.') +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer PING ('ok', True) +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer XREADGROUP ('noperm', "User api_producer has no permissions to run the +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer XACK ('noperm', "User api_producer has no permissions to run the +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer XDEL ('noperm', "User api_producer has no permissions to run the +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer DEL ('noperm', "User api_producer has no permissions to run the +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer FLUSHALL ('noperm', "User api_producer has no permissions to run the +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke api_producer XADD other key ('noperm', 'No permissions to access a key') +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke worker_consumer XLEN ('ok', 149) +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke worker_consumer SET ('noperm', "User worker_consumer has no permissions to run t +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke worker_consumer FLUSHALL ('noperm', "User worker_consumer has no permissions to run t +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke worker_consumer CONFIG ('noperm', "User worker_consumer has no permissions to run t +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke worker_consumer DEL ('noperm', "User worker_consumer has no permissions to run t +FAIL api/worker -> broker-gate (mTLS allowlist api|worker) -> broke Redis port 6379 not reachable from network (loopback ('ok', True) +PASS api/worker -> broker-gate (mTLS allowlist api|worker) -> broke plaintext to gate ('conn_error', 'Connection closed by server.') +PASS api/worker -> s3 (TLS, scoped identities) TLS verify-full GET /healthz ('http', 200) +PASS api/worker -> s3 (TLS, scoped identities) wrong hostname ('verify_error', "Hostname mismatch, certificate is not vali +PASS api/worker -> s3 (TLS, scoped identities) plaintext on :8333 ('http', 400) +PASS api/worker -> s3 (TLS, scoped identities) api PUT staging/ allowed +PASS api/worker -> s3 (TLS, scoped identities) api DELETE staging/ allowed +PASS api/worker -> s3 (TLS, scoped identities) api PUT final/ (worker prefix) AccessDenied +PASS api/worker -> s3 (TLS, scoped identities) api LIST bucket AccessDenied +PASS api/worker -> s3 (TLS, scoped identities) api CREATE bucket AccessDenied +PASS api/worker -> s3 (TLS, scoped identities) worker LIST bucket allowed +PASS api/worker -> s3 (TLS, scoped identities) worker PUT outside its prefixes AccessDenied +PASS api/worker -> s3 (TLS, scoped identities) worker DELETE bucket AccessDenied +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect identity api POST /v1/logs ('http', 200) +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect identity worker POST /v1/logs ('http', 200) +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect no client certificate POST /v1/logs ('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED') +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect rogue CA with CN=api POST /v1/logs ('tls_error', 'TLSV1_ALERT_UNKNOWN_CA') +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect same CA, identity intruder POST /v1/logs ('closed', 'connection closed without HTTP response') +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect same CA, identity prometheus POST /v1/logs ('closed', 'connection closed without HTTP response') +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect same CA, identity lb POST /v1/logs ('closed', 'connection closed without HTTP response') +PASS api/worker -> otel-gate (mTLS allowlist api|worker) -> collect plaintext on :4318 ('closed', "b''") +PASS prometheus -> otel-gate metrics (allowlist prometheus) identity prometheus /metrics :8889 ('http', 200) +PASS prometheus -> otel-gate metrics (allowlist prometheus) identity prometheus /metrics :8888 (self) ('http', 200) +PASS prometheus -> otel-gate metrics (allowlist prometheus) identity api /metrics :8889 ('closed', 'connection closed without HTTP response') +FAIL prometheus -> otel-gate metrics (allowlist prometheus) collector loopback port 14318 not reachable ('http', 404) +FAIL prometheus -> otel-gate metrics (allowlist prometheus) collector loopback port 18889 not reachable ('http', 404) +FAIL prometheus -> otel-gate metrics (allowlist prometheus) collector loopback port 18888 not reachable ('closed', "b'\\x00\\x00\\x18\\x04\\x00\\x00\\x00\\x00\\x00\ +FAIL prometheus -> otel-gate metrics (allowlist prometheus) collector loopback port 13133 not reachable ('http', 200) +PASS collector/grafana -> loki (mTLS) client cert grafana ('http', 200) +PASS collector/grafana -> loki (mTLS) no client certificate ('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED') +PASS collector/grafana -> loki (mTLS) rogue CA client certificate ('tls_error', 'TLSV1_ALERT_UNKNOWN_CA') +PASS collector/grafana -> loki (mTLS) plaintext ('http', 400) +PASS grafana -> tempo (mTLS) client cert grafana ('http', 200) +PASS grafana -> tempo (mTLS) no client certificate ('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED') +PASS grafana -> tempo (mTLS) rogue CA client certificate ('tls_error', 'TLSV1_ALERT_UNKNOWN_CA') +PASS grafana -> tempo (mTLS) plaintext ('http', 400) +PASS grafana -> prometheus (mTLS) client cert grafana ('http', 200) +PASS grafana -> prometheus (mTLS) no client certificate ('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED') +PASS grafana -> prometheus (mTLS) rogue CA client certificate ('tls_error', 'TLSV1_ALERT_UNKNOWN_CA') +PASS grafana -> prometheus (mTLS) plaintext ('http', 400) +PASS collector -> tempo OTLP gRPC (mTLS) no client certificate ('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED') +PASS agents -> openbao (TLS + AppRole) TLS verify-full /v1/sys/health ('http', 200) +PASS agents -> openbao (TLS + AppRole) wrong hostname ('verify_error', "Hostname mismatch, certificate is not vali +PASS agents -> openbao (TLS + AppRole) plaintext on :8200 ('http', 400) +PASS agents -> openbao (TLS + AppRole) read KV without token ('http', 403) diff --git a/evidence/security_http.txt b/evidence/security_http.txt new file mode 100644 index 0000000..1e3996f --- /dev/null +++ b/evidence/security_http.txt @@ -0,0 +1,25 @@ +PASS setup: chat + attachment 202 +PASS outsider GET /api/v1/chats/{chat} -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/chats/{chat}/messages -> 404 (no existence leak) 404 +PASS outsider POST /api/v1/chats/{chat}/messages -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/attachments/{att} -> 404 (no existence leak) 404 +PASS outsider GET /api/v1/attachments/{att}/content -> 404 (no existence leak) 404 +PASS outsider attaches foreign attachment to own chat -> 404 404 +PASS X-User-Id cannot override Bearer identity eve_c7dfed486e +PASS X-User-Id alone -> 401 + WWW-Authenticate Bearer Bearer realm="messenger" +PASS Basic on business endpoint -> 401 401 +PASS unknown/forged token -> 401 401 +PASS logout 204 -> token rejected 401 -> repeat logout 401 (204, 401, 401) +PASS wrong password -> 401, body has no username/password/hash {"error":{"code":"unauthorized","message":"Authentication required","request_id" +PASS malformed JSON -> 400 without stack trace/SQL {"error":{"code":"invalid_request","message":"Invalid request","request_id":"AC1 +PASS login brute force -> 429 rate_limited + Retry-After [401, 401, 401, 401, 401, 401, 401, 401, 401, 429, 429, 429] Retry-After=21 +PASS rate limit is shared: responses from >= 2 replicas ['api-1401d400', 'api-21bd9440'] +PASS correct password while locked out -> 429 (no oracle) 429 +PASS JSON body 70 KB -> 413 payload_too_large from LB (before any replica) instance=None +PASS chunked JSON 70 KB -> 413 payload_too_large (API) instance=api-1401d400 +PASS upload 10 MiB + 1 byte -> 413 413 +PASS wrong server name -> certificate verification error Hostname mismatch, certificate is not valid for 'evil.example'. +PASS empty trust store -> verification error unable to get local issuer certificate +PASS plaintext HTTP on HTTPS port is not served RemoteDisconnected +PASS grafana over HTTPS requires authentication 401 +PASS: 24/24 checks diff --git a/evidence/tls_audit.json b/evidence/tls_audit.json new file mode 100644 index 0000000..6bb137f --- /dev/null +++ b/evidence/tls_audit.json @@ -0,0 +1,714 @@ +{ + "ok": true, + "total": 101, + "failed": 0, + "checks": [ + { + "link": "client -> lb (public TLS)", + "check": "trusted CA + hostname lb/localhost SAN", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "client -> lb (public TLS)", + "check": "empty trust store", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', 'unable to get local issuer certificate')" + }, + { + "link": "client -> lb (public TLS)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "client -> lb (public TLS)", + "check": "plaintext HTTP on :8443", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "client -> lb (public TLS)", + "check": "HAProxy stats not exposed on network", + "expected": "refused", + "ok": true, + "observed": "('refused', 'connection refused')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "identity lb (allowed)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "rogue CA with CN=lb", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity intruder", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity worker", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "same CA, identity api (replica as client)", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "plaintext HTTP on :8443", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "health listener 8081 not reachable from network (loopback only)", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "lb -> api (mTLS, allowlist lb + CRL)", + "check": "LB-side check: server name must be api", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api verify-full", + "expected": "TLS session", + "ok": true, + "observed": "(True, 'TLSv1.3')" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "plaintext (sslmode=disable)", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"172.19.0.3\", port 5432 failed: FATAL: pg_hba.conf rejects connection for host \"172.19.0.7\", user \"messenger_api\", database \"messenger\", no encryption" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "rogue CA as trust root", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"172.19.0.3\", port 5432 failed: SSL error: certificate verify failed" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "wrong hostname (IP instead of name)", + "expected": "rejected", + "ok": true, + "observed": "connection failed: connection to server at \"172.19.0.3\", port 5432 failed: server certificate for \"postgres\" (and 1 other name) does not match host name \"172.19.0.3\"" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT count(*) FROM attachments", + "expected": "allowed", + "ok": true, + "observed": "executed" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT password_hash FROM users LIMIT 1", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table users" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_worker: SELECT token_hash FROM sessions LIMIT 1", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table sessions" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api: CREATE TABLE audit_ddl (i int)", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for schema public" + }, + { + "link": "api/worker -> postgres (TLS verify-full, roles)", + "check": "messenger_api: TRUNCATE users", + "expected": "permission denied", + "ok": true, + "observed": "permission denied for table users" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api SET sess:*", + "expected": "ok", + "ok": true, + "observed": "('ok', True)" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api SET other:key 1", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', 'No permissions to access a key')" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'flushall' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api CONFIG GET *", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'config|get' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "messenger_api KEYS *", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User messenger_api has no permissions to run the 'keys' command\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "cert without AUTH (default user off)", + "expected": "NOAUTH", + "ok": true, + "observed": "('noauth', 'Authentication required.')" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "no client certificate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:6379 : (1, '[SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (_\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "rogue CA client certificate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:6379 : (1, '[SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:2570)')\")" + }, + { + "link": "api -> redis cache (mTLS + ACL)", + "check": "plaintext", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from redis:6379 : (104, 'Connection reset by peer')\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "no client certificate (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from broker:6380 : (1, '[SSL: TLSV13_ALERT_CERTIFICATE_REQUIRED] tlsv13 alert certificate required (\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "rogue CA with CN=api (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', \"Error while reading from broker:6380 : (1, '[SSL: TLSV1_ALERT_UNKNOWN_CA] tlsv1 alert unknown ca (_ssl.c:2570)')\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity intruder (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity lb (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "same CA, identity prometheus (with valid producer password)", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "identity api without AUTH", + "expected": "NOAUTH", + "ok": true, + "observed": "('noauth', 'Authentication required.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer PING", + "expected": "ok", + "ok": true, + "observed": "('ok', True)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XREADGROUP", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xreadgroup' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XACK", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xack' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XDEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'xdel' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer DEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'del' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User api_producer has no permissions to run the 'flushall' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "api_producer XADD other key", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', 'No permissions to access a key')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer XLEN", + "expected": "ok", + "ok": true, + "observed": "('ok', 1)" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer SET", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'set' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer FLUSHALL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'flushall' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer CONFIG", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'config|get' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "worker_consumer DEL", + "expected": "NOPERM", + "ok": true, + "observed": "('noperm', \"User worker_consumer has no permissions to run the 'del' command\")" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "Redis port 6379 not reachable from network (loopback only)", + "expected": "connection refused", + "ok": true, + "observed": "('conn_error', 'Error 111 connecting to broker:6379. Connection refused.')" + }, + { + "link": "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL", + "check": "plaintext to gate", + "expected": "connection rejected", + "ok": true, + "observed": "('conn_error', 'Connection closed by server.')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "TLS verify-full GET /healthz", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "plaintext on :8333", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "S3 API behind gate :8334 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "S3 gateway gRPC (unauthenticated) :18334 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "S3 gRPC default port :18333 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "master :9333 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "filer :8888 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "filer gRPC :18888 not reachable from network", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api PUT staging/", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api DELETE staging/", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api PUT final/ (worker prefix)", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api LIST bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "api CREATE bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker LIST bucket", + "expected": "allowed", + "ok": true, + "observed": "allowed" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker PUT outside its prefixes", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> s3 (TLS, scoped identities)", + "check": "worker DELETE bucket", + "expected": "AccessDenied", + "ok": true, + "observed": "AccessDenied" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "identity api POST /v1/logs", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "identity worker POST /v1/logs", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "no client certificate POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "rogue CA with CN=api POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity intruder POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity prometheus POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "same CA, identity lb POST /v1/logs", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector", + "check": "plaintext on :4318", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('closed', \"b''\")" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity prometheus /metrics :8889", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity prometheus /metrics :8888 (self)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "identity api /metrics :8889", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('closed', 'connection closed without HTTP response')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 14318 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 18889 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 18888 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "prometheus -> otel-gate metrics (allowlist prometheus)", + "check": "collector loopback port 13133 not reachable", + "expected": "refused", + "ok": true, + "observed": "('refused', '[Errno 111] Connection refused')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "client cert grafana (TLS ok, service ready)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "collector/grafana -> loki (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "client cert grafana (TLS ok, service ready)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "grafana -> tempo (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "client cert grafana (TLS ok, service ready)", + "expected": "HTTP 200", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "rogue CA client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV1_ALERT_UNKNOWN_CA')" + }, + { + "link": "grafana -> prometheus (mTLS)", + "check": "plaintext", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "collector -> tempo OTLP gRPC (mTLS)", + "check": "no client certificate", + "expected": "rejected (TLS/handshake/close)", + "ok": true, + "observed": "('tls_error', 'TLSV13_ALERT_CERTIFICATE_REQUIRED')" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "TLS verify-full /v1/sys/health", + "expected": "HTTP 200 (unsealed)", + "ok": true, + "observed": "('http', 200)" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "wrong hostname", + "expected": "verify_error", + "ok": true, + "observed": "('verify_error', \"Hostname mismatch, certificate is not valid for 'evil.example'.\")" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "plaintext on :8200", + "expected": "no successful plaintext response", + "ok": true, + "observed": "('http', 400)" + }, + { + "link": "agents -> openbao (TLS + AppRole)", + "check": "read KV without token", + "expected": "HTTP 403", + "ok": true, + "observed": "('http', 403)" + } + ] +} diff --git a/infra/README.md b/infra/README.md index 5a39e86..e4b69b4 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,18 @@ -# Инфраструктура студента +# Инфраструктура (lab7) -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология — корневой `compose.yaml`; роль → сервис → версия → health → volume → сеть/порт → credentials — таблица в `REPORT.md`. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +| Каталог | Что внутри | +| --- | --- | +| `haproxy/haproxy.cfg` | LB: внешний TLS `:8443`, mTLS к репликам API (identity `lb`), лимиты, JSON-ошибки | +| `haproxy/broker-gate.cfg` | mTLS-gate брокера: CA + CRL + allowlist CN `api`/`worker`, backend — Redis на loopback | +| `haproxy/otel-gate.cfg` | mTLS-gate Collector: OTLP только `api`/`worker`, метрики только `prometheus` | +| `postgres/` | entrypoint (ключ TLS в tmpfs 0600, uid 70), `pg_hba.conf` (только hostssl + SCRAM), роли (initdb) | +| `redis/cache.sh`, `redis/broker.sh` | TLS/ACL кэша сессий и ACL брокера (producer/consumer), ACL-файл в tmpfs с SHA-256 паролей | +| `openbao/` | `server.hcl` (не dev, file storage, TLS), `agent-api.hcl`/`agent-worker.hcl` (AppRole + шаблоны), `policies/` | +| `otel/collector.yaml` | Collector на loopback за gate, mTLS к Loki/Tempo, scrub атрибутов | +| `prometheus/` | scrape Collector по mTLS, `web.yml` (TLS + клиентский cert), `client.yml` (healthcheck), rules lab6 | +| `loki/`, `tempo/` | `http_tls_config` с `RequireAndVerifyClientCert`, внутренний gRPC на 127.0.0.1 | +| `grafana/` | provisioning data sources с mTLS (`$__file{}`), dashboard lab6 | -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Bootstrap (идемпотентно, повторный запуск безопасен): `scripts/init_secrets.py` → `scripts/pki.py init` → `scripts/bao.py init|unseal|configure` → `docker compose up -d` (`migrate`, `s3-init` — one-shot). Всё вместе: `python scripts/ops.py up`. Restart hook перезапускает все хранилища с volumes (и OpenBao с unseal оператором); failover hook останавливает только заданную API-реплику. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..f71ab65 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,944 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger lab6 — API, worker, очередь", + "tags": [ + "lab6", + "messenger" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": false, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "annotations": { + "list": [] + }, + "templating": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "API: RED (бизнес-запросы, health исключены)", + "id": 1, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "type": "timeseries", + "title": "RPS по маршруту и классу статуса", + "id": 2, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "rate(http_server_request_duration_seconds_count[1m]); класс статуса из http_response_status_code", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (http_route, status_class) (label_replace(rate(http_server_request_duration_seconds_count{job=\"messenger-api\"}[1m]), \"status_class\", \"${1}xx\", \"http_response_status_code\", \"(\\\\d)..\"))", + "legendFormat": "{{http_route}} {{status_class}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Доля 5xx (SLI ошибок)", + "id": 3, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "numerator: 5xx; denominator: все бизнес-запросы. Порог алерта 5%", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "custom": { + "thresholdsStyle": { + "mode": "line" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "messenger:http_error_ratio:rate1m", + "legendFormat": "5xx / все, окно 1m", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Латентность p50 / p95 / p99 (из histogram)", + "id": 4, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Квантили из buckets всех реплик (sum by le), не среднее p95 реплик", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "p95 по маршруту", + "id": 5, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\"}[5m])))", + "legendFormat": "{{http_route}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "stat", + "title": "SLO доступности, окно 1 ч (цель 99%)", + "id": 6, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "1 - 5xx/все за 1h", + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "messenger:http_availability:ratio1h", + "legendFormat": "availability 1h", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "stat", + "title": "SLO задержки: доля < 0.5 с, 1 ч (цель 95%)", + "id": 7, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.95 + } + ] + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "messenger:http_fast_ratio:ratio1h", + "legendFormat": "fast 1h", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Worker и очередь attachments.jobs", + "id": 8, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + } + }, + { + "type": "timeseries", + "title": "Задания по исходу", + "id": 9, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "succeeded / retry / failed / noop / lease_lost / poison / error", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{outcome}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Длительность задания p50 / p95 / p99", + "id": 10, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_job_duration_seconds_bucket[5m])))", + "legendFormat": "p99", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Время до ready p50 / p95 (end-to-end)", + "id": 11, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "От COMMIT задания (ответ 202) до COMMIT ready", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Глубина очереди", + "id": 12, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Из PostgreSQL (публикуют API-реплики, max по репликам)", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "max by (state) (messenger_queue_depth{queue=\"attachments.jobs\"})", + "legendFormat": "{{state}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max(messenger_outbox_pending{queue=\"attachments.jobs\"})", + "legendFormat": "outbox (не опубликовано)", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Возраст старейшего задания", + "id": 13, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Порог алерта QueueLagGrowing: 60 с", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "custom": { + "thresholdsStyle": { + "mode": "line" + } + } + }, + "overrides": [] + }, + "targets": [ + { + "expr": "max(messenger_queue_oldest_age_seconds{queue=\"attachments.jobs\"})", + "legendFormat": "oldest age", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Saturation worker (busy / capacity)", + "id": 14, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "capacity (слотов)" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "targets": [ + { + "expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)", + "legendFormat": "busy/capacity", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "capacity (слотов)", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Ресурсы: DB pool, CPU, RAM", + "id": 15, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + } + }, + { + "type": "timeseries", + "title": "DB pool: занято / ожидают / максимум", + "id": 16, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum by (job, instance) (messenger_db_pool_connections{state=\"used\"})", + "legendFormat": "{{job}} {{instance}} used", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum by (job, instance) (messenger_db_pool_waiting)", + "legendFormat": "{{job}} {{instance}} waiting", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max by (job) (messenger_db_pool_max)", + "legendFormat": "{{job}} max", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "CPU процесса (ядра)", + "id": 17, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "rate(process_cpu_time_seconds_total{job=~\"messenger-.+\"}[1m])", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "RSS процесса", + "id": 18, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 35 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "process_memory_rss_bytes{job=~\"messenger-.+\"}", + "legendFormat": "{{job}} {{instance}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Конвейер телеметрии (Collector)", + "id": 19, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + } + }, + { + "type": "timeseries", + "title": "Collector: принято / отправлено / потеряно (spans)", + "id": 20, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 44 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(otelcol_receiver_accepted_spans[1m]))", + "legendFormat": "accepted", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_exporter_sent_spans[1m]))", + "legendFormat": "sent", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "(sum(rate(otelcol_exporter_send_failed_spans[1m])) or vector(0)) + (sum(rate(otelcol_exporter_enqueue_failed_spans[1m])) or vector(0))", + "legendFormat": "lost", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "sum(rate(otelcol_receiver_refused_spans[1m]))", + "legendFormat": "refused (memory_limiter)", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "timeseries", + "title": "Collector: логи и очередь экспорта", + "id": 21, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 44 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(rate(otelcol_exporter_sent_log_records[1m]))", + "legendFormat": "logs sent", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "(sum(rate(otelcol_exporter_send_failed_log_records[1m])) or vector(0)) + (sum(rate(otelcol_exporter_enqueue_failed_log_records[1m])) or vector(0))", + "legendFormat": "logs lost", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "expr": "max by (exporter) (otelcol_exporter_queue_size)", + "legendFormat": "queue {{exporter}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + { + "type": "row", + "title": "Логи и трассы", + "id": 22, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + } + }, + { + "type": "logs", + "title": "Логи WARNING/ERROR API и worker", + "id": 23, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 53 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "expr": "{service_name=~\"messenger-.+\"} | severity_text=~\"WARNING|ERROR|CRITICAL\"", + "queryType": "range", + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + } + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + }, + { + "type": "table", + "title": "Трассы обработки заданий (consumer spans)", + "id": 24, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "description": "", + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 53 + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "targets": [ + { + "query": "{resource.service.name=\"messenger-worker\" && kind=consumer}", + "queryType": "traceql", + "limit": 20, + "tableType": "traces", + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..3d454f6 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..af502c6 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,72 @@ +# Grafana 11.4.0: источники данных (не хранит сигналы сама — только запросы и UI). +# lab7: все источники — HTTPS с проверкой сервера (CA стенда + имя) и клиентским cert grafana. +# Фиксированные uid: на них ссылаются dashboard и переходы trace <-> logs. +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: https://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 10s + tlsAuth: true + tlsAuthWithCACert: true + serverName: prometheus + secureJsonData: + tlsCACert: "$__file{/run/secrets/ca_crt}" + tlsClientCert: "$__file{/run/secrets/grafana_crt}" + tlsClientKey: "$__file{/run/secrets/grafana_key}" + + - name: Loki + uid: loki + type: loki + access: proxy + url: https://loki:3100 + editable: false + secureJsonData: + tlsCACert: "$__file{/run/secrets/ca_crt}" + tlsClientCert: "$__file{/run/secrets/grafana_crt}" + tlsClientKey: "$__file{/run/secrets/grafana_key}" + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + serverName: loki + # trace_id хранится как structured metadata записи -> кнопка перехода в Tempo. + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: "$${__value.raw}" + urlDisplayLabel: "Открыть trace" + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: https://tempo:3200 + editable: false + secureJsonData: + tlsCACert: "$__file{/run/secrets/ca_crt}" + tlsClientCert: "$__file{/run/secrets/grafana_crt}" + tlsClientKey: "$__file{/run/secrets/grafana_key}" + jsonData: + tlsAuth: true + tlsAuthWithCACert: true + serverName: tempo + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: "-5m" + spanEndTimeShift: "5m" + filterByTraceID: true + filterBySpanID: false + customQuery: true + # Все логи API и worker этой трассы (trace_id — structured metadata, не текст строки). + query: '{service_name=~"messenger-.+"} | trace_id="$${__trace.traceId}"' + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true diff --git a/infra/haproxy/broker-gate.cfg b/infra/haproxy/broker-gate.cfg new file mode 100644 index 0000000..bb36691 --- /dev/null +++ b/infra/haproxy/broker-gate.cfg @@ -0,0 +1,42 @@ +# lab7, уровень 4: mTLS-gate брокера (Redis Streams). +# +# Redis 7.4 умеет TLS и проверку CA клиента, но не умеет ограничивать допустимые +# identity (SAN/CN) — любой сертификат нашего CA прошёл бы рукопожатие. Поэтому TLS +# завершается здесь, в sidecar, который делит сетевой namespace с broker +# (network_mode: service:broker), а Redis слушает ТОЛЬКО 127.0.0.1:6379 этого namespace — +# закрытый локальный hop, снаружи контейнера недоступный. +# +# Проверки gate: +# 1. рукопожатие: `verify required` + CA стенда + CRL (нет cert / чужой CA / отозван -> отказ); +# 2. identity: CN клиента должен быть `api` или `worker` (intruder, lb, prometheus... -> reject); +# 3. дальше Redis ACL: у api и worker разные пользователи и разные права +# (api_producer: XADD; worker_consumer: XREADGROUP/XACK/...; default user отключён). +# Ни сертификат без пароля ACL, ни пароль без разрешённого сертификата не дают доступа. + +global + log stdout format raw local0 info + maxconn 512 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + +defaults + mode tcp + log global + option tcplog + timeout connect 2s + timeout client 1h + timeout server 1h + +frontend broker_mtls + bind :6380 ssl crt /run/secrets/broker_pem ca-file /run/secrets/ca_crt crl-file /run/secrets/crl_pem verify required + log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts tls=%sslv client_cn=%{+Q}[ssl_c_s_dn(CN)] verify=%[ssl_c_verify]" + acl allowed_identity ssl_c_s_dn(CN) -m str api worker + tcp-request content reject unless { ssl_c_used } { ssl_c_verify 0 } allowed_identity + default_backend redis_local + +backend redis_local + server redis 127.0.0.1:6379 + +frontend gate_health + mode http + bind 127.0.0.1:8405 + monitor-uri /gate-health diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 0000000..acfe3da --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,108 @@ +# HAProxy 2.8 LTS — единственная публичная точка входа (lab3), lab7: TLS снаружи, mTLS внутрь. +# +# клиент --TLS 1.2+ (сертификат lb-public, SAN localhost/127.0.0.1)--> :8443 frontend public +# LB --mTLS (клиентский cert lb-client, CN/SAN lb; проверка сервера: CA + SAN api)--> api:8443 +# +# Проверка сервера API: `verify required ca-file` + `verifyhost api` (SAN DNS:api) — LB не +# отправит запрос «реплике», чей сертификат не выдан нашим CA на имя api. +# Проверка клиента делает API (src/tls.py): только identity `lb`, CRL. `no-ssl-reuse`: +# каждое новое соединение — полное рукопожатие, поэтому отзыв/ротация сертификата LB +# не маскируется TLS session resumption. +# +# Обнаружение реплик (как в lab3): Docker DNS, server-template до 6 слотов, проверка +# GET /health/ready по mTLS каждую 1 с; fall 2 / rise 2. Повторяются только ошибки +# установления соединения (retry-on conn-failure по умолчанию) — для POST безопасно. +# +# Лимиты (lab7): размер тела не-upload запросов — по Content-Length до реплики (64 KiB); +# загрузка (10 MiB) — в API; заголовки — tune.bufsize; медленный клиент — timeout http-request. +# Ответы, которые формирует сам LB, — в JSON-формате ошибок контракта с X-Request-Id. + +global + log stdout format raw local0 info + maxconn 2000 + # Внешний и внутренний TLS: не ниже 1.2, без TLS tickets (ключ тикета = общий секрет LB). + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + ssl-default-server-options ssl-min-ver TLSv1.2 no-tls-tickets + tune.bufsize 32768 + +resolvers docker + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + option redispatch 1 + retries 2 + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 3s + timeout check 1500ms + default-server inter 1s fastinter 500ms downinter 2s fall 2 rise 2 + +frontend public + bind :8443 ssl crt /run/secrets/lb_public_pem alpn http/1.1 + # Сквозной идентификатор запроса (как в lab3). + unique-id-format %{+X}o\ %ci%cp-%fp-%Ts%ms-%rt-%pid + http-request set-header X-Request-Id %[unique-id] unless { req.hdr(X-Request-Id) -m found } + # X-Forwarded-For от клиента не доверяем: удаляем и ставим адрес TCP-клиента. + # API использует его для rate limit входа только потому, что к API подключается лишь LB (mTLS). + http-request del-header X-Forwarded-For + option forwardfor + # Внешний клиент не управляет trace-контекстом (подмена trace_id, ReDoS-парсеры заголовков). + http-request del-header traceparent + http-request del-header tracestate + http-request del-header baggage + # Транспорт без шифрования не принимается вовсе (порт только TLS), HSTS для браузеров. + http-response set-header Strict-Transport-Security "max-age=31536000" + # Внимание: одноимённые acl в HAProxy объединяются по ИЛИ — поэтому отдельные имена. + acl upload_path path_reg ^/api/v1/chats/[^/]+/attachments$ + # Не-upload запросы: тело > 64 KiB по Content-Length отклоняется до реплики. + # Загрузки (<= 10 MiB) проверяет API: он дочитывает/сбрасывает тело и отвечает 413 + # корректно (ранний ответ LB клиент, ещё пишущий 10 MiB, получил бы broken pipe). + acl body_large req.hdr_val(content-length) gt 65536 + http-request return status 413 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"payload_too_large\",\"message\":\"Request body is too large\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" if body_large !upload_path + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq tls=%sslv rid=%[capture.req.hdr(0)] inst=%[capture.res.hdr(0)] \"%HM %HPO\"" + http-request capture req.hdr(X-Request-Id) len 64 + declare capture response len 32 + http-response capture res.hdr(X-Instance-Id) id 0 + default_backend api + +backend api + balance roundrobin + acl is_upload method POST + acl is_upload_path path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 90s if is_upload is_upload_path + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + http-error status 503 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API instance is available\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + http-error status 504 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API instance timeout\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + http-error status 502 content-type application/json hdr X-Request-Id "%[req.hdr(X-Request-Id)]" lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Bad gateway\",\"request_id\":\"%[req.hdr(X-Request-Id)]\"}}" + # mTLS к каждой реплике; проверки (check) идут тем же mTLS. + server-template api 1-6 api:8443 check resolvers docker init-addr none resolve-prefer ipv4 ssl verify required ca-file /run/secrets/ca_crt crt /run/secrets/lb_client_pem verifyhost api sni str(api) no-ssl-reuse + +# Состояние реплик (UP/DOWN) — только внутри контейнера (127.0.0.1), порт НЕ публикуется. +# scripts/ops.py читает его через `docker compose exec lb wget ...`. +frontend stats + bind 127.0.0.1:8404 + monitor-uri /lb-health + stats enable + stats uri /stats + stats refresh 2s + stats show-legends + http-request set-log-level silent diff --git a/infra/haproxy/otel-gate.cfg b/infra/haproxy/otel-gate.cfg new file mode 100644 index 0000000..5fb5502 --- /dev/null +++ b/infra/haproxy/otel-gate.cfg @@ -0,0 +1,57 @@ +# lab7, уровень 4: mTLS-gate OpenTelemetry Collector. +# +# Collector сам умеет TLS и проверку CA клиента, но не ограничивает identity: любой +# сертификат стенда (lb, intruder...) смог бы писать произвольную телеметрию. Gate делит +# сетевой namespace с otel-collector (network_mode: service:otel-collector); Collector +# слушает OTLP и метрики ТОЛЬКО на 127.0.0.1:14318/18889/18888 этого namespace +# (закрытый локальный hop; порты другие, чтобы не конфликтовать с bind gate). +# +# :4318 OTLP/HTTP ingest — только identity api и worker (разные сертификаты) +# :8889 метрики приложения — только identity prometheus (scrape) +# :8888 self-метрики Collector — только identity prometheus (scrape) +# Везде: verify required + CA стенда + CRL; иначе — отказ на рукопожатии/сессии. + +global + log stdout format raw local0 info + maxconn 1024 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + +defaults + mode tcp + log global + timeout connect 2s + timeout client 60s + timeout server 60s + log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts tls=%sslv client_cn=%{+Q}[ssl_c_s_dn(CN)] verify=%[ssl_c_verify]" + +frontend otlp_ingest + bind :4318 ssl crt /run/secrets/otel_collector_pem ca-file /run/secrets/ca_crt crl-file /run/secrets/crl_pem verify required + acl allowed_identity ssl_c_s_dn(CN) -m str api worker + tcp-request content reject unless { ssl_c_used } { ssl_c_verify 0 } allowed_identity + default_backend otlp_local + +frontend metrics_app + bind :8889 ssl crt /run/secrets/otel_collector_pem ca-file /run/secrets/ca_crt crl-file /run/secrets/crl_pem verify required + acl allowed_identity ssl_c_s_dn(CN) -m str prometheus + tcp-request content reject unless { ssl_c_used } { ssl_c_verify 0 } allowed_identity + default_backend metrics_app_local + +frontend metrics_self + bind :8888 ssl crt /run/secrets/otel_collector_pem ca-file /run/secrets/ca_crt crl-file /run/secrets/crl_pem verify required + acl allowed_identity ssl_c_s_dn(CN) -m str prometheus + tcp-request content reject unless { ssl_c_used } { ssl_c_verify 0 } allowed_identity + default_backend metrics_self_local + +backend otlp_local + server otlp 127.0.0.1:14318 + +backend metrics_app_local + server app 127.0.0.1:18889 + +backend metrics_self_local + server self 127.0.0.1:18888 + +frontend gate_health + mode http + bind 127.0.0.1:8405 + monitor-uri /gate-health diff --git a/infra/haproxy/s3-gate.cfg b/infra/haproxy/s3-gate.cfg new file mode 100644 index 0000000..5f7ee09 --- /dev/null +++ b/infra/haproxy/s3-gate.cfg @@ -0,0 +1,30 @@ +# lab7: TCP-passthrough к S3 API SeaweedFS (TLS завершает сам SeaweedFS — сквозное шифрование). +# +# Зачем: `weed server -s3` открывает рядом с S3 API свой gRPC-порт (s3.port + 10000) на том же +# адресе, что и S3, и выключить его флагом нельзя (-s3.port.grpc=0 = порт по умолчанию). Этот +# gRPC не аутентифицирован (в т.ч. перенастройка identities S3), поэтому SeaweedFS привязан к +# 127.0.0.1 своего сетевого namespace (-s3.ip.bind=127.0.0.1), а в сеть storage наружу выдан +# ТОЛЬКО порт S3 API через этот gate (network_mode: service:s3). Gate не видит содержимого (TLS), +# он лишь пересылает TCP 0.0.0.0:8333 -> 127.0.0.1:8334. +global + log stdout format raw local0 info + maxconn 512 + +defaults + mode tcp + log global + timeout connect 2s + timeout client 120s + timeout server 120s + +frontend s3_api + bind :8333 + default_backend s3_local + +backend s3_local + server s3 127.0.0.1:8334 + +frontend gate_health + mode http + bind 127.0.0.1:8405 + monitor-uri /gate-health diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..89a74a3 --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,57 @@ +# Loki 3.3.2 (single binary) — хранилище ЛОГОВ. Приём — native OTLP (/otlp) от Collector. +# Resource-атрибуты service.name / service.instance.id становятся index labels +# (service_name, service_instance_id), атрибуты записи (event, request_id, trace_id, +# span_id, severity) — structured metadata (не labels: низкая кардинальность индекса). +auth_enabled: false + +# lab7: HTTP API (приём OTLP от Collector, запросы Grafana) — только TLS с обязательным +# клиентским сертификатом нашего CA (mTLS). Внутренний gRPC (querier <-> frontend) +# слушает только loopback контейнера — закрытый локальный hop. +server: + http_listen_port: 3100 + grpc_listen_address: 127.0.0.1 + grpc_listen_port: 9095 + log_level: warn + # Минимальная версия TLS у dskit server по умолчанию — 1.2 (Go crypto/tls). + http_tls_config: + cert_file: /run/secrets/loki_crt + key_file: /run/secrets/loki_key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/ca_crt + +common: + path_prefix: /loki + replication_factor: 1 + instance_addr: 127.0.0.1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + # Защита от взрыва: не больше 2 MiB/s на tenant на учебном стенде. + ingestion_rate_mb: 2 + ingestion_burst_size_mb: 4 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/openbao/agent-api.hcl b/infra/openbao/agent-api.hcl new file mode 100644 index 0000000..66a4a14 --- /dev/null +++ b/infra/openbao/agent-api.hcl @@ -0,0 +1,81 @@ +# lab7: OpenBao Agent для роли `api`. Единственный процесс, у которого есть SecretID. +# Auto-auth AppRole -> короткоживущий токен (TTL 15 мин, max 1 ч, renew агентом) -> +# шаблоны отрисовывают секреты в tmpfs-volume api-secrets (/run/bao), откуда их читает +# приложение (read-only). Токен на диск НЕ пишется (нет sink). Приложение не знает ни +# SecretID, ни токена: «секрета для получения секретов» в образе/приложении нет. +# +# Ротация: KV v2 перечитывается каждые 5 с (static_secret_render_interval); изменённое +# значение перерисовывается в файл, новые соединения БД берут новый пароль (src/db.py). +# OpenBao sealed/недоступен: уже отрисованные файлы остаются, приложение работает со +# старыми значениями; агент повторяет попытки (exit_on_retry_failure = false). +pid_file = "/run/bao/.agent.pid" + +vault { + address = "https://openbao:8200" + ca_cert = "/run/secrets/ca_crt" + retry { + num_retries = 12 + } +} + +auto_auth { + method "approle" { + mount_path = "auth/approle" + config = { + role_id_file_path = "/run/secrets/bao_role_id" + secret_id_file_path = "/run/secrets/bao_secret_id" + remove_secret_id_file_after_reading = false + } + } +} + +template_config { + static_secret_render_interval = "5s" + exit_on_retry_failure = false +} + +template { + destination = "/run/bao/database_url" + perms = "0400" + error_on_missing_key = true + contents = < короткоживущий токен (TTL 15 мин, max 1 ч, renew агентом) -> +# шаблоны отрисовывают секреты в tmpfs-volume worker-secrets (/run/bao), откуда их читает +# приложение (read-only). Токен на диск НЕ пишется (нет sink). Приложение не знает ни +# SecretID, ни токена: «секрета для получения секретов» в образе/приложении нет. +# +# Ротация: KV v2 перечитывается каждые 5 с (static_secret_render_interval); изменённое +# значение перерисовывается в файл, новые соединения БД берут новый пароль (src/db.py). +# OpenBao sealed/недоступен: уже отрисованные файлы остаются, приложение работает со +# старыми значениями; агент повторяет попытки (exit_on_retry_failure = false). +pid_file = "/run/bao/.agent.pid" + +vault { + address = "https://openbao:8200" + ca_cert = "/run/secrets/ca_crt" + retry { + num_retries = 12 + } +} + +auto_auth { + method "approle" { + mount_path = "auth/approle" + config = { + role_id_file_path = "/run/secrets/bao_role_id" + secret_id_file_path = "/run/secrets/bao_secret_id" + remove_secret_id_file_after_reading = false + } + } +} + +template_config { + static_secret_render_interval = "5s" + exit_on_retry_failure = false +} + +template { + destination = "/run/bao/database_url" + perms = "0400" + error_on_missing_key = true + contents = < gate :4318 --loopback--> receivers.otlp 127.0.0.1:14318 +# prometheus --mTLS (CN prometheus)--> gate :8889 --loopback--> exporters.prometheus 127.0.0.1:18889 +# prometheus --mTLS (CN prometheus)--> gate :8888 --loopback--> self-метрики 127.0.0.1:18888 +# Выходы Collector — TLS с проверкой сервера (CA стенда + имя) и клиентским сертификатом +# otel-collector (mTLS) к Tempo (OTLP gRPC) и Loki (OTLP HTTP). +# +# Ограничение памяти и буферов (как в lab6): memory_limiter, sending_queue ограничена, +# retry_on_failure с max_elapsed_time — бесконечного накопления нет. +# +# Утечки: SDK не экспортирует заголовки Authorization/Cookie и тела; processor +# attributes/scrub дополнительно удаляет атрибуты с потенциальными секретами. + +receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:14318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 180 + spike_limit_mib: 40 + batch: + send_batch_size: 512 + send_batch_max_size: 1024 + timeout: 2s + # lab7: страховка от утечки credentials в telemetry (основная защита — в приложении). + attributes/scrub: + actions: + - key: http.request.header.authorization + action: delete + - key: http.request.header.cookie + action: delete + - key: db.user + action: delete + - key: url.query + action: delete + - key: http.url + action: delete + +exporters: + otlp/tempo: + endpoint: tempo:4317 + tls: + ca_file: /run/secrets/ca_crt + cert_file: /run/secrets/otel_collector_crt + key_file: /run/secrets/otel_collector_key + min_version: "1.2" + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 20 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + otlphttp/loki: + endpoint: https://loki:3100/otlp + tls: + ca_file: /run/secrets/ca_crt + cert_file: /run/secrets/otel_collector_crt + key_file: /run/secrets/otel_collector_key + min_version: "1.2" + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 20 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + + prometheus: + endpoint: 127.0.0.1:18889 + resource_to_telemetry_conversion: + enabled: false + metric_expiration: 5m + +extensions: + health_check: + endpoint: 127.0.0.1:13133 + +service: + extensions: [health_check] + telemetry: + logs: + level: info + metrics: + level: normal + address: 127.0.0.1:18888 + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, attributes/scrub, batch] + exporters: [otlphttp/loki] diff --git a/infra/postgres/entrypoint.sh b/infra/postgres/entrypoint.sh new file mode 100644 index 0000000..624f969 --- /dev/null +++ b/infra/postgres/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# lab7: PostgreSQL с TLS. Контейнер работает сразу от uid 70 (postgres), без root и +# без capabilities. PostgreSQL требует, чтобы приватный ключ принадлежал ему и имел +# права 0600; bind-mount secrets (особенно на Windows) этого не гарантирует, поэтому +# ключ копируется в tmpfs /run/pgtls (только этот контейнер, только память) с 0600. +set -eu +umask 077 +install -m 0600 /run/secrets/postgres_key /run/pgtls/server.key +install -m 0644 /run/secrets/postgres_crt /run/pgtls/server.crt +exec docker-entrypoint.sh "$@" diff --git a/infra/postgres/initdb/10-roles.sh b/infra/postgres/initdb/10-roles.sh new file mode 100644 index 0000000..90fd11b --- /dev/null +++ b/infra/postgres/initdb/10-roles.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# lab7: роли приложения создаются ОДИН раз при инициализации нового volume. +# Пароли читаются из secrets оператора внутри psql (\set с `cat`), в argv и логи не попадают. +# Права на таблицы выдаёт миграция 006_security.sql (её выполняет администратор `messenger`). +# Ротация пароля: scripts/ops.py rotate-db-password (ALTER ROLE + OpenBao KV). +set -eu +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<'SQL' +\set api_pw `cat /run/secrets/db_api_password` +\set worker_pw `cat /run/secrets/db_worker_password` +CREATE ROLE messenger_api LOGIN PASSWORD :'api_pw' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT 60; +CREATE ROLE messenger_worker LOGIN PASSWORD :'worker_pw' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT 20; +REVOKE ALL ON DATABASE messenger FROM PUBLIC; +GRANT CONNECT ON DATABASE messenger TO messenger_api, messenger_worker; +SQL diff --git a/infra/postgres/pg_hba.conf b/infra/postgres/pg_hba.conf new file mode 100644 index 0000000..e3c85dd --- /dev/null +++ b/infra/postgres/pg_hba.conf @@ -0,0 +1,10 @@ +# lab7: доступ к PostgreSQL только по TLS и только по SCRAM. +# TYPE DATABASE USER ADDRESS METHOD +# Локальный unix-socket внутри контейнера: initdb-скрипты, pg_isready, операторский psql +# через `docker compose exec` (граница доверия — доступ к Docker на хосте). +local all all scram-sha-256 +# Роли приложения и администратор миграций — только TLS (клиенты: sslmode=verify-full). +hostssl messenger messenger_api,messenger_worker,messenger 0.0.0.0/0 scram-sha-256 +# Любое соединение без TLS отклоняется явно. +hostnossl all all 0.0.0.0/0 reject +host all all ::/0 reject diff --git a/infra/prometheus/client.yml b/infra/prometheus/client.yml new file mode 100644 index 0000000..47076e6 --- /dev/null +++ b/infra/prometheus/client.yml @@ -0,0 +1,6 @@ +# Клиентская TLS-конфигурация для `promtool check healthy` (Docker healthcheck). +tls_config: + ca_file: /run/secrets/ca_crt + cert_file: /run/secrets/prometheus_crt + key_file: /run/secrets/prometheus_key + server_name: prometheus diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..a7ef2f0 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,31 @@ +# Prometheus v2.55.1 — хранилище МЕТРИК (TSDB на volume promdata, retention 7d) +# и вычислитель alert rules (состояние firing/resolved видно в /alerts и Grafana; +# доставка наружу не настроена — Alertmanager не нужен по заданию). +global: + scrape_interval: 10s + evaluation_interval: 10s + +rule_files: + - /etc/prometheus/rules/*.yml + +# lab7: scrape Collector только через otel-gate по mTLS (identity prometheus разрешена +# gate только на портах метрик). Проверка сервера: CA стенда + SAN otel-collector. +scrape_configs: + # Метрики приложения, которые Collector получил по OTLP (job/instance от exporter'а). + - job_name: otel-collector-app + honor_labels: true + scheme: https + tls_config: &collector_tls + ca_file: /run/secrets/ca_crt + cert_file: /run/secrets/prometheus_crt + key_file: /run/secrets/prometheus_key + server_name: otel-collector + min_version: TLS12 + static_configs: + - targets: ["otel-collector:8889"] + # Собственные метрики Collector: принятые/отброшенные/неотправленные spans, logs, points. + - job_name: otel-collector-self + scheme: https + tls_config: *collector_tls + static_configs: + - targets: ["otel-collector:8888"] diff --git a/infra/prometheus/rules/messenger.yml b/infra/prometheus/rules/messenger.yml new file mode 100644 index 0000000..8492445 --- /dev/null +++ b/infra/prometheus/rules/messenger.yml @@ -0,0 +1,66 @@ +# SLI/SLO и alert rules lab6. Единицы: доли (0..1), секунды. +# +# SLI доступности API: доля бизнес-запросов (health исключены инструментацией) +# без ответа 5xx. 4xx (неверный пароль, 404, 409) — ошибка клиента, не сервиса. +# SLO: 99% запросов без 5xx в скользящем окне 1 ч (учебный стенд; бюджет 1%). +# SLI задержки: доля запросов быстрее 0.5 с (корзина le="0.5" гистограммы); SLO 95% за 1 ч. +# SLI очереди: возраст старейшего незавершённого задания; цель < 60 с. +groups: + - name: messenger-sli + interval: 10s + rules: + - record: messenger:http_requests:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api"}[1m])) + - record: messenger:http_errors_5xx:rate1m + expr: sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_response_status_code=~"5.."}[1m])) or vector(0) + - record: messenger:http_error_ratio:rate1m + expr: messenger:http_errors_5xx:rate1m / messenger:http_requests:rate1m + - record: messenger:http_availability:ratio1h + expr: | + 1 - ( + (sum(increase(http_server_request_duration_seconds_count{job="messenger-api",http_response_status_code=~"5.."}[1h])) or vector(0)) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api"}[1h])) + ) + - record: messenger:http_fast_ratio:ratio1h + expr: | + sum(increase(http_server_request_duration_seconds_bucket{job="messenger-api",le="0.5"}[1h])) + / sum(increase(http_server_request_duration_seconds_count{job="messenger-api"}[1h])) + + - name: messenger-alerts + rules: + - alert: ApiHighErrorRate + # >5% ответов 5xx за 1 мин при заметном трафике (>= 0.2 RPS), держится 1 мин. + expr: messenger:http_error_ratio:rate1m > 0.05 and messenger:http_requests:rate1m >= 0.2 + for: 1m + labels: + severity: page + annotations: + summary: "API: доля 5xx {{ $value | humanizePercentage }} (> 5%)" + description: "Бизнес-запросы messenger-api отвечают 5xx. Окно 1m, порог 5%, for 1m." + runbook: "docs/runbooks.md#apihigherrorrate" + + - alert: QueueLagGrowing + # Старейшее незавершённое задание старше 60 с — worker не успевает или стоит. + expr: max(messenger_queue_oldest_age_seconds{queue="attachments.jobs"}) > 60 + for: 30s + labels: + severity: page + annotations: + summary: "Очередь attachments.jobs: старейшее задание {{ $value | humanizeDuration }}" + description: "Возраст старейшего queued/processing задания > 60 с в течение 30 с." + runbook: "docs/runbooks.md#queuelaggrowing" + + - alert: TelemetryExportFailing + # Collector не может отправить данные в backend (Tempo/Loki недоступны). + # Счётчики появляются лениво (enqueue_failed — после первого отказа): "or vector(0)". + expr: | + (sum(rate(otelcol_exporter_send_failed_spans[1m])) or vector(0)) + + (sum(rate(otelcol_exporter_send_failed_log_records[1m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_spans[1m])) or vector(0)) + + (sum(rate(otelcol_exporter_enqueue_failed_log_records[1m])) or vector(0)) > 0 + for: 1m + labels: + severity: ticket + annotations: + summary: "Collector теряет telemetry (backend недоступен)" + runbook: "docs/runbooks.md#telemetryexportfailing" diff --git a/infra/prometheus/web.yml b/infra/prometheus/web.yml new file mode 100644 index 0000000..1cb2b3a --- /dev/null +++ b/infra/prometheus/web.yml @@ -0,0 +1,7 @@ +# lab7: Prometheus HTTP (Grafana, healthcheck) — только TLS + клиентский сертификат CA стенда. +tls_server_config: + cert_file: /run/secrets/prometheus_crt + key_file: /run/secrets/prometheus_key + client_auth_type: RequireAndVerifyClientCert + client_ca_file: /run/secrets/ca_crt + min_version: TLS12 diff --git a/infra/redis/broker.sh b/infra/redis/broker.sh new file mode 100644 index 0000000..4b2cd2b --- /dev/null +++ b/infra/redis/broker.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# lab7: broker (Redis Streams, AOF fsync always — как в lab5). Слушает ТОЛЬКО +# 127.0.0.1:6379 своего сетевого namespace; снаружи доступен лишь через broker-gate +# (mTLS + allowlist identity api/worker, infra/haproxy/broker-gate.cfg). +# ACL — раздельные пользователи для producer (API) и consumer (worker): +# api_producer — XADD и создание группы в потоке attachments.jobs; читать/ack/удалять не может +# worker_consumer — XREADGROUP/XACK/XAUTOCLAIM/XPENDING/XTRIM + XADD (relay outbox в worker) +# ни у кого нет FLUSHALL/CONFIG/DEBUG/KEYS (-@dangerous -@admin), default отключён. +set -eu +umask 077 +api_hash=$(tr -d '\r\n' < /run/secrets/broker_api_password | sha256sum | cut -d' ' -f1) +worker_hash=$(tr -d '\r\n' < /run/secrets/broker_worker_password | sha256sum | cut -d' ' -f1) +cat > /tmp/users.acl < /tmp/users.acl < ($2, $3) +-- ORDER BY created_at, id LIMIT $4 +-- Обоснование и EXPLAIN (ANALYZE, BUFFERS): evidence/explain_history.txt +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +-- Серверные сессии. Источник истины для сессий — PostgreSQL; +-- Redis хранит только кэш проверок токена (см. REPORT.md). +-- Хранится SHA-256 от токена, сам токен в БД не попадает. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_user_id_idx ON sessions (user_id); +CREATE INDEX sessions_expires_at_idx ON sessions (expires_at); diff --git a/migrations/002_session_cache_invalidations.sql b/migrations/002_session_cache_invalidations.sql new file mode 100644 index 0000000..1c26116 --- /dev/null +++ b/migrations/002_session_cache_invalidations.sql @@ -0,0 +1,13 @@ +-- 002: надёжная инвалидация кэша сессий. +-- Строка вставляется В ТОЙ ЖЕ транзакции, что и sessions.revoked_at, и удаляется +-- только после того, как в Redis записано «надгробие» для этого токена. +-- Пока в таблице есть строки, API не доверяет положительным записям кэша +-- и проверяет каждый токен в PostgreSQL (см. src/sessions.py). +CREATE TABLE session_cache_invalidations ( + token_hash bytea PRIMARY KEY, + revoked_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT session_cache_invalidations_hash_len CHECK (octet_length(token_hash) = 32) +); + +CREATE INDEX session_cache_invalidations_revoked_at_idx + ON session_cache_invalidations (revoked_at); diff --git a/migrations/003_idempotency_keys.sql b/migrations/003_idempotency_keys.sql new file mode 100644 index 0000000..af60c76 --- /dev/null +++ b/migrations/003_idempotency_keys.sql @@ -0,0 +1,40 @@ +-- 003 (lab3): общая идемпотентность POST /chats/{chat_id}/messages. +-- +-- Область уникальности по контракту: пользователь + HTTP-метод + путь чата + ключ. +-- Путь однозначно задаётся chat_id, метод сейчас всегда POST (колонка оставлена, +-- чтобы область ключа была записана явно и расширялась без смены PK). +-- +-- Атомарность: строка ключа и сообщение пишутся ОДНОЙ транзакцией. +-- 1. INSERT ключа (ON CONFLICT ...) — конкурентный повтор с тем же ключом +-- ждёт на уникальном индексе, пока первая транзакция не завершится; +-- 2. INSERT сообщения с заранее выбранным id; +-- 3. COMMIT. FK message_id -> messages проверяется в момент COMMIT +-- (DEFERRABLE INITIALLY DEFERRED), поэтому message_id NOT NULL с первого шага. +-- Ошибка/откат на любом шаге не оставляет ни сообщения, ни ключа: +-- транзиентный 5xx не фиксируется как окончательный результат. +-- +-- Совместимость с работающей версией (rolling update): миграция только +-- добавляет новую таблицу, старые таблицы и запросы не меняются. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + method text NOT NULL DEFAULT 'POST', + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL + REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idem_key), + CONSTRAINT idempotency_keys_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_keys_method CHECK (method IN ('POST')), + CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_keys_expiry_after_create CHECK (expires_at > created_at) +); + +-- Периодическая очистка истёкших ключей (src/idempotency.py). +CREATE INDEX idempotency_keys_expires_at_idx ON idempotency_keys (expires_at); +-- ON DELETE CASCADE от messages без полного скана. +CREATE INDEX idempotency_keys_message_id_idx ON idempotency_keys (message_id); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..20561c2 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,68 @@ +-- 004 (lab4): вложения. Метаданные — в PostgreSQL, байты — в приватном S3-бакете. +-- +-- Жизненный цикл строки attachments (внутреннее поле state, в API не видно): +-- pending — намерение записано ДО S3 PUT (одна короткая транзакция); +-- ready — объект записан в S3, затем UPDATE ... WHERE state='pending'. +-- Наружу (GET, ссылки из сообщений) доступны только ready. Строки pending, +-- оставшиеся после сбоя между S3 и COMMIT, удаляет сборщик (src/reconcile.py) +-- после grace period; сначала удаляется строка (атомарный «захват»), потом объект. +-- Миграция только расширяющая (expand): реплики lab3 продолжают работать. + +-- Сообщение с вложениями может иметь пустой текст (text = ""). +-- Запрет «пустой текст без вложений» проверяет приложение. +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); +-- Цель составного внешнего ключа message_attachments (message_id, chat_id). +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE RESTRICT, + owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + -- Только отображаемое имя. Никогда не используется как путь или ключ S3. + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + -- Ключ объекта назначает сервер (случайный UUID), он не равен id вложения + -- и не раскрывается в API. + object_key text NOT NULL, + state text NOT NULL DEFAULT 'pending', + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_state CHECK (state IN ('pending', 'ready')) +); + +-- Сборщик незавершённых загрузок: WHERE state = 'pending' AND created_at < ... +CREATE INDEX attachments_pending_created_idx ON attachments (created_at) WHERE state = 'pending'; +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +-- Связь сообщение -> вложения (порядок attachment_ids сохраняется в position). +-- Составные внешние ключи гарантируют НА УРОВНЕ БД, что вложение и сообщение +-- принадлежат одному чату. ON DELETE RESTRICT: строку вложения, на которую +-- ссылается сообщение, нельзя удалить (сборщик удаляет только pending, а +-- pending прикрепить нельзя — двойная защита). +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_message_fk FOREIGN KEY (message_id, chat_id) + REFERENCES messages (id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/005_async_processing.sql b/migrations/005_async_processing.sql new file mode 100644 index 0000000..49d3df0 --- /dev/null +++ b/migrations/005_async_processing.sql @@ -0,0 +1,116 @@ +-- 005 (lab5): асинхронная обработка вложений через брокер. +-- +-- Жизненный цикл attachments.state (pending — внутренний, в API не виден): +-- pending — строка-намерение записана ДО S3 PUT в staging (как в lab4); +-- queued — оригинал целиком лежит в staging/, в ОДНОЙ транзакции с этим +-- UPDATE записаны attachment_jobs и строка job_outbox -> 202; +-- processing — worker взял аренду (lease) задания; +-- ready — worker записал оригинал в final/ и все варианты, затем одной +-- транзакцией опубликовал результат (variants + state); +-- failed — постоянная ошибка или исчерпаны попытки (error_code). +-- Миграция расширяющая: реплики lab4 продолжают работать (pending/ready). + +ALTER TABLE attachments DROP CONSTRAINT attachments_state; +ALTER TABLE attachments ADD CONSTRAINT attachments_state + CHECK (state IN ('pending', 'queued', 'processing', 'ready', 'failed')); + +-- Ключ staging-объекта (приватный префикс staging/ того же бакета). NULL после +-- безопасной очистки. object_key — конечный ключ оригинала (final//original), +-- назначается сервером при приёме и становится читаемым только в ready. +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key); +ALTER TABLE attachments ADD COLUMN error_code text; +-- Версия pipeline, результат которой опубликован (или ожидается) для вложения. +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code + CHECK ((state = 'failed') = (error_code IS NOT NULL) OR state = 'pending'); + +-- Результат обработки. Ключ эффекта — (attachment_id, pipeline_version, name): +-- повторная доставка не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_object_key_key UNIQUE (object_key), + CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp', 'image/png')), + CONSTRAINT attachment_variants_size CHECK (size_bytes > 0 AND width > 0 AND height > 0), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +-- Одно логическое задание на (вложение, версию pipeline). job_id не меняется +-- при redelivery и retry; attempt растёт только при управляемой повторной попытке. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + status text NOT NULL DEFAULT 'queued', + attempt integer NOT NULL DEFAULT 1, + max_attempts integer NOT NULL, + replay_count integer NOT NULL DEFAULT 0, + -- Аренда: токен-«забор» (fencing), владелец (consumer) и срок. + lease_token uuid, + lease_owner text, + lease_until timestamptz, + next_attempt_at timestamptz, + last_error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + started_at timestamptz, + finished_at timestamptz, + CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'succeeded', 'failed')), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND max_attempts >= 1), + CONSTRAINT attachment_jobs_lease CHECK ((status = 'processing') = (lease_token IS NOT NULL)) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_status_idx ON attachment_jobs (status, updated_at); + +-- Transactional outbox: строка пишется в той же транзакции, что и задание. +-- Relay публикует в брокер (XADD с fsync), затем удаляет строку. Сбой между +-- XADD и DELETE даёт повторную публикацию — worker идемпотентен. +-- available_at: для свежей загрузки = now()+grace (сначала быстрый путь из API), +-- для retry = время следующей попытки (backoff с jitter). +CREATE TABLE job_outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + publish_failures integer NOT NULL DEFAULT 0 +); +CREATE INDEX job_outbox_available_idx ON job_outbox (available_at, id); + +-- Реестр окончательно упавших заданий (DLQ). Replay — только явный, операторский. +CREATE TABLE dead_letters ( + job_id uuid PRIMARY KEY REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + attachment_id uuid NOT NULL, + error_code text NOT NULL, + attempts integer NOT NULL, + failed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + replayed_at timestamptz, + replay_count integer NOT NULL DEFAULT 0 +); + +-- Аудит переходов задания: claim, retry, fail, replay, no-op повторы, публикации. +-- Без секретов и без содержимого файлов. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid, + attempt integer, + event text NOT NULL, + detail text, + actor text, + at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX job_events_job_idx ON job_events (job_id, id); +CREATE INDEX job_events_at_idx ON job_events (at); diff --git a/migrations/006_security.sql b/migrations/006_security.sql new file mode 100644 index 0000000..21726e8 --- /dev/null +++ b/migrations/006_security.sql @@ -0,0 +1,49 @@ +-- 006 (lab7): общий для всех реплик rate limit входа + раздельные роли БД. +-- +-- login_rate_limits: счётчики фиксированного окна. bucket = 'u:'||sha256(username) +-- или 'ip:'||sha256(ip) — открытые username/IP не хранятся. Старые окна удаляет +-- фоновая очистка API (delete_old_rate_limits). +CREATE TABLE IF NOT EXISTS login_rate_limits ( + bucket text NOT NULL, + window_start timestamptz NOT NULL, + hits integer NOT NULL, + PRIMARY KEY (bucket, window_start) +); + +-- Роли приложения создаёт infra/postgres/initdb/10-roles.sh при первой инициализации +-- volume (пароли — из secrets оператора). Если миграция идёт на volume, где ролей +-- ещё нет, создаём их без права входа: GRANT ниже не упадёт, а пароль/LOGIN выдаст +-- `python scripts/ops.py db-roles`. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'messenger_api') THEN + CREATE ROLE messenger_api NOLOGIN; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'messenger_worker') THEN + CREATE ROLE messenger_worker NOLOGIN; + END IF; +END +$$; + +-- Ни у одной роли приложения нет DDL: схемой владеет только администратор (migrate). +REVOKE CREATE ON SCHEMA public FROM PUBLIC; +REVOKE ALL ON ALL TABLES IN SCHEMA public FROM messenger_api, messenger_worker; +GRANT USAGE ON SCHEMA public TO messenger_api, messenger_worker; + +-- API: обязательный синхронный путь (пользователи, сессии, чаты, сообщения, +-- идемпотентность, вложения, задания/outbox для relay) — только DML, без TRUNCATE. +GRANT SELECT, INSERT, UPDATE, DELETE ON + users, chats, chat_members, messages, sessions, session_cache_invalidations, + idempotency_keys, attachments, message_attachments, attachment_variants, + attachment_jobs, job_outbox, dead_letters, job_events, login_rate_limits +TO messenger_api; +GRANT SELECT ON schema_migrations TO messenger_api; + +-- worker: только конвейер вложений. НЕТ доступа к users (password_hash), sessions +-- (token hashes), chats, messages, idempotency_keys, login_rate_limits. +GRANT SELECT, INSERT, UPDATE, DELETE ON + attachments, attachment_variants, attachment_jobs, job_outbox, dead_letters, job_events +TO messenger_worker; +GRANT SELECT ON message_attachments, schema_migrations TO messenger_worker; + +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO messenger_api, messenger_worker; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d099b2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.163.1 +uvicorn==0.34.0 +pydantic==2.14.3 +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +# lab4: S3-клиент и независимый парсер изображений +boto3==1.35.99 +Pillow==13.0.0 +# lab6: OpenTelemetry SDK + OTLP/HTTP exporter + автоинструментация (версии закреплены вместе) +opentelemetry-api==1.45.2 +opentelemetry-sdk==1.45.2 +opentelemetry-exporter-otlp-proto-http==1.45.2 +opentelemetry-instrumentation-fastapi==0.66b2 +opentelemetry-instrumentation-psycopg==0.66b2 +opentelemetry-instrumentation-botocore==0.66b2 +# lab7: X.509 — проверка identity/CRL клиента mTLS (src/tls.py), scripts/pki.py +cryptography==50.0.2 diff --git a/run.ps1 b/run.ps1 new file mode 100644 index 0000000..be78171 --- /dev/null +++ b/run.ps1 @@ -0,0 +1,230 @@ +<# +.SYNOPSIS + Лабораторная 7 (мессенджер): развернуть стенд и проверить всё — одной командой в PowerShell. + +.DESCRIPTION + Запуск из корня репозитория: + powershell -ExecutionPolicy Bypass -File .\run.ps1 # всё: up + все проверки + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action up # только развернуть + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action test # публичные тесты lab7 (HTTPS) + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action security + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action resilience + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action lifecycle + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action extra + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action scan + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action down # остановить, данные сохраняются + powershell -ExecutionPolicy Bypass -File .\run.ps1 -Action destroy # НАМЕРЕННО удалить данные стенда + + Нужно: Docker Desktop (Compose v2), Python 3.9+ (только стандартная библиотека; PKI создаётся + в контейнере, если на хосте нет пакета cryptography), свободные порты 127.0.0.1:8443 и :3000. + Вывод каждого шага сохраняется в evidence\run\<шаг>.txt, итоговая таблица — в конце. +#> +[CmdletBinding()] +param( + [ValidateSet('all', 'check', 'up', 'test', 'security', 'resilience', 'lifecycle', 'extra', 'scan', 'down', 'destroy')] + [string]$Action = 'all', + # Не разворачивать стенд в режиме all (если он уже запущен) + [switch]$SkipUp, + # Остановиться на первом проваленном шаге + [switch]$StopOnFailure +) + +$ErrorActionPreference = 'Continue' +Set-StrictMode -Version 2.0 +$Root = Split-Path -Parent $MyInvocation.MyCommand.Path +Set-Location $Root +try { [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 } catch { } +$env:PYTHONIOENCODING = 'utf-8' +$env:PYTHONUTF8 = '1' +$env:COMPOSE_PROJECT_NAME = 'messenger-lab7' + +$CaFile = Join-Path $Root 'secrets\pki\ca.crt' +$HttpsPort = 8443 +if (Test-Path (Join-Path $Root '.env')) { + foreach ($line in Get-Content (Join-Path $Root '.env')) { + if ($line -match '^\s*HTTPS_PORT\s*=\s*(\d+)') { $HttpsPort = [int]$Matches[1] } + } +} +$BaseUrl = "https://localhost:$HttpsPort" +$EvidenceDir = Join-Path $Root 'evidence\run' +New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null +$Results = New-Object System.Collections.Generic.List[object] + +# ------------------------------------------------------------------ Python +$Py = $null +$PyArgs = @() +foreach ($candidate in @(@('python'), @('py', '-3'), @('python3'))) { + $exe = Get-Command $candidate[0] -ErrorAction SilentlyContinue + if (-not $exe) { continue } + $extra = @() + if ($candidate.Count -gt 1) { $extra = @($candidate[1..($candidate.Count - 1)]) } + $ver = & $candidate[0] @extra -c "import sys; print('%d.%d' % sys.version_info[:2])" 2>$null + if ($LASTEXITCODE -eq 0 -and $ver -and ([version]"$ver" -ge [version]'3.9')) { + $Py = $candidate[0]; $PyArgs = $extra; break + } +} + +function Write-Title([string]$Text) { + Write-Host '' + Write-Host ('=' * 78) -ForegroundColor DarkCyan + Write-Host " $Text" -ForegroundColor Cyan + Write-Host ('=' * 78) -ForegroundColor DarkCyan +} + +function Invoke-Step { + <# Выполнить внешнюю команду, показать вывод построчно и сохранить его в evidence\run\.txt #> + param([string]$Name, [string]$Title, [string]$Exe, [string[]]$Arguments, [hashtable]$EnvVars = @{}) + Write-Title "$Name — $Title" + $file = Join-Path $EvidenceDir ($Name + '.txt') + "# $Title`r`n# $Exe $($Arguments -join ' ')`r`n# $(Get-Date -Format o)" | Out-File -FilePath $file -Encoding utf8 + $saved = @{} + foreach ($k in $EnvVars.Keys) { $saved[$k] = [Environment]::GetEnvironmentVariable($k); [Environment]::SetEnvironmentVariable($k, $EnvVars[$k]) } + $sw = [Diagnostics.Stopwatch]::StartNew() + & $Exe @Arguments 2>&1 | ForEach-Object { + $line = "$_" + Write-Host $line + Add-Content -Path $file -Value $line -Encoding UTF8 + } + $code = $LASTEXITCODE + $sw.Stop() + foreach ($k in $saved.Keys) { [Environment]::SetEnvironmentVariable($k, $saved[$k]) } + $ok = ($code -eq 0) + $Results.Add([pscustomobject]@{ Step = $Name; Result = $(if ($ok) { 'PASS' } else { "FAIL ($code)" }); + Seconds = [int]$sw.Elapsed.TotalSeconds; Evidence = "evidence\run\$Name.txt" }) | Out-Null + if ($ok) { Write-Host "[PASS] $Name" -ForegroundColor Green } else { Write-Host "[FAIL] $Name (exit $code)" -ForegroundColor Red } + if (-not $ok -and $StopOnFailure) { Show-Summary; exit 1 } + return $ok +} + +function Invoke-Py { + param([string]$Name, [string]$Title, [string[]]$Arguments, [hashtable]$EnvVars = @{}) + return Invoke-Step -Name $Name -Title $Title -Exe $Py -Arguments (@($PyArgs) + @($Arguments)) -EnvVars $EnvVars +} + +function Show-Summary { + Write-Title 'Итог' + $Results | Format-Table -AutoSize | Out-String -Width 200 | Write-Host + $failed = @($Results | Where-Object { $_.Result -ne 'PASS' }).Count + $summary = Join-Path $EvidenceDir 'SUMMARY.txt' + $Results | Format-Table -AutoSize | Out-String -Width 200 | Out-File -FilePath $summary -Encoding utf8 + if ($failed -eq 0) { Write-Host "Все шаги пройдены. Сводка: evidence\run\SUMMARY.txt" -ForegroundColor Green } + else { Write-Host "Провалено шагов: $failed. Подробности — в файлах evidence\run\*.txt" -ForegroundColor Red } +} + +# ------------------------------------------------------------------ шаги +function Step-Check { + Write-Title 'Проверка окружения' + $ok = $true + if (-not $Py) { Write-Host 'Нужен Python 3.9+ (python или py -3 в PATH)' -ForegroundColor Red; $ok = $false } + else { Write-Host "Python: $Py $($PyArgs -join ' ') ($(& $Py @PyArgs --version 2>&1))" } + if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { Write-Host 'Нужен Docker Desktop' -ForegroundColor Red; $ok = $false } + else { + docker info --format '{{.ServerVersion}}' 1>$null 2>$null + if ($LASTEXITCODE -ne 0) { Write-Host 'Docker daemon не запущен (запустите Docker Desktop)' -ForegroundColor Red; $ok = $false } + $cv = docker compose version --short 2>$null + if ($LASTEXITCODE -ne 0) { Write-Host 'Нужен Docker Compose v2 (docker compose)' -ForegroundColor Red; $ok = $false } + else { Write-Host "Docker Compose: $cv" } + } + foreach ($port in @($HttpsPort, 3000)) { + $busy = $false + try { $busy = @(Get-NetTCPConnection -LocalPort $port -State Listen -ErrorAction Stop).Count -gt 0 } catch { } + if ($busy) { + $own = docker ps --filter "label=com.docker.compose.project=messenger-lab7" -q 2>$null + if (-not $own) { Write-Host "Порт $port занят другим процессом" -ForegroundColor Yellow } + } + } + if (-not $ok) { Write-Host 'Окружение не готово.' -ForegroundColor Red; exit 2 } + Invoke-Py 'check-contract' 'make check: структура skeleton, контракт, синтаксис тестов' @('scripts/check_contract.py') | Out-Null + Invoke-Step 'compose-config' 'docker compose config --quiet' 'docker' @('compose', 'config', '--quiet') | Out-Null +} + +function Step-Up { + Invoke-Py 'up' 'Bootstrap: secrets -> образ -> PKI -> OpenBao init/unseal/configure -> стенд -> ожидание готовности' ` + @('scripts/ops.py', 'up') | Out-Null + Invoke-Py 'ps' 'Реплики API: alias, health, статус в LB' @('scripts/ops.py', 'ps') | Out-Null +} + +function Step-Test { + Invoke-Py 'test-smoke' "make test BASE_URL=$BaseUrl CA_FILE=secrets\pki\ca.crt" ` + @('tests/smoke.py', '--base-url', $BaseUrl, '--ca-file', $CaFile) | Out-Null + Invoke-Py 'test-security' 'make test-security: доверенный CA работает, недоверенный — ошибка проверки сертификата' ` + @('tests/security.py', '--base-url', $BaseUrl, '--ca-file', $CaFile) | Out-Null +} + +function Step-Security { + Invoke-Py 'tls-audit' 'Вся trust matrix: TLS/mTLS (нет cert / чужой CA / чужая identity / верная), имена, plaintext, ACL' ` + @('scripts/ops.py', 'tls-audit') | Out-Null + Invoke-Py 'security-http' 'ACL/вложения, X-User-Id, logout, 429 + Retry-After (общий), 413, ошибки без утечек, TLS снаружи' ` + @('scripts/ops.py', 'security-http') | Out-Null + Invoke-Py 'bao-policy' 'OpenBao: allow/deny политик реальными токенами AppRole, CIDR, TTL' ` + @('scripts/ops.py', 'bao-policy') | Out-Null + Invoke-Py 'hardening' 'Hardening: non-root, read_only, cap_drop, no-new-privileges, лимиты, порты, сегментация, egress' ` + @('scripts/ops.py', 'hardening') | Out-Null + Invoke-Py 'rehash' 'Argon2id: старый hash пересчитывается при успешном входе' @('scripts/ops.py', 'rehash-demo') | Out-Null + Invoke-Py 'leak-check' 'В Loki/Tempo/Prometheus нет паролей, токенов, секретов ролей' @('scripts/ops.py', 'leak-check') | Out-Null + Invoke-Py 'expiry' 'Истечение сессии (экземпляр с TTL 2 с, mTLS напрямую к реплике)' @('scripts/ops.py', 'in-network', 'expiry') | Out-Null +} + +function Step-Resilience { + $restart = Join-Path $Root 'scripts\restart.cmd' + $stopOne = Join-Path $Root 'scripts\stop-one.cmd' + Invoke-Py 'test-persistence' 'make test-persistence: рестарт API и всех хранилищ (вкл. OpenBao + unseal), данные и токен живы' ` + @('tests/resilience.py', '--mode', 'persistence', '--action-script', $restart, '--base-url', $BaseUrl, '--ca-file', $CaFile) | Out-Null + Invoke-Py 'test-failover-1' 'make test-failover: отказ одной API-реплики по HTTPS' ` + @('tests/resilience.py', '--mode', 'failover', '--action-script', $stopOne, '--base-url', $BaseUrl, '--ca-file', $CaFile) | Out-Null + Invoke-Py 'restore-1' 'Восстановление реплики' @('scripts/ops.py', 'restore') | Out-Null + Invoke-Py 'test-failover-2' 'Отказ другой реплики (последовательно)' ` + @('tests/resilience.py', '--mode', 'failover', '--action-script', $stopOne, '--base-url', $BaseUrl, '--ca-file', $CaFile) | Out-Null + Invoke-Py 'restore-2' 'Восстановление реплики' @('scripts/ops.py', 'restore') | Out-Null +} + +function Step-Lifecycle { + Invoke-Py 'rotate-db-password' 'Ротация рабочего секрета (пароль роли БД) без пересборки и рестарта API' ` + @('scripts/ops.py', 'rotate-db-password', 'api') | Out-Null + Invoke-Py 'rotate-lb-cert' 'Ротация leaf LB->API + отзыв старого сертификата (CRL), проверка старым и новым' ` + @('scripts/ops.py', 'rotate-lb-cert') | Out-Null + Invoke-Py 'rotate-secret-id' 'Новый SecretID агента API, старый уничтожен' @('scripts/ops.py', 'rotate-secret-id', 'api') | Out-Null + Invoke-Py 'revoke-identity' 'Отзыв identity worker в OpenBao -> агент без доступа -> восстановление' ` + @('scripts/ops.py', 'revoke-identity', 'worker') | Out-Null + Invoke-Py 'bao-sealed' 'OpenBao sealed: API работает и перезапускается на отрисованных секретах, затем unseal' ` + @('scripts/ops.py', 'bao-sealed') | Out-Null + Invoke-Py 'smoke-after-lifecycle' 'Smoke после всех ротаций' @('tests/smoke.py', '--base-url', $BaseUrl, '--ca-file', $CaFile) | Out-Null +} + +function Step-Extra { + Invoke-Py 'student-tests' 'Собственные HTTP-тесты lab2-6 через HTTPS' ` + @('student-tests/run_all.py', '--base-url', $BaseUrl) -EnvVars @{ CA_FILE = $CaFile } | Out-Null + Invoke-Py 'storage-tests' 'White-box тесты хранилища внутри сети (identity администратора БД)' ` + @('scripts/ops.py', 'in-network', 'storage') | Out-Null + Invoke-Py 'login-budget' 'Бюджет login DoS: Argon2id на стенде + поток входов' @('scripts/ops.py', 'login-budget') | Out-Null + Invoke-Py 's3-check' 'Приватность бакета и права ключа API вне API' @('scripts/ops.py', 's3-check') | Out-Null +} + +# ------------------------------------------------------------------ main +switch ($Action) { + 'check' { Step-Check } + 'up' { Step-Check; Step-Up } + 'test' { Step-Test } + 'security' { Step-Security } + 'resilience' { Step-Resilience } + 'lifecycle' { Step-Lifecycle } + 'extra' { Step-Extra } + 'scan' { Invoke-Py 'scan' 'pip-audit + trivy образов (разбор: docs/security/SCAN_TRIAGE.md)' @('scripts/ops.py', 'scan') | Out-Null } + 'down' { Invoke-Py 'down' 'Остановка стенда (volumes сохраняются)' @('scripts/ops.py', 'down') | Out-Null } + 'destroy' { + $answer = Read-Host 'Удалить ВСЕ данные стенда (БД, S3, broker, OpenBao, телеметрия)? Введите yes' + if ($answer -eq 'yes') { Invoke-Py 'destroy' 'Удаление volumes стенда' @('scripts/ops.py', 'destroy') | Out-Null } + } + 'all' { + Step-Check + if (-not $SkipUp) { Step-Up } + Step-Test + Step-Security + Step-Resilience + Step-Lifecycle + Step-Extra + } +} +if ($Results.Count -gt 0) { Show-Summary } +if (@($Results | Where-Object { $_.Result -ne 'PASS' }).Count -gt 0) { exit 1 } +exit 0 diff --git a/scripts/bao.py b/scripts/bao.py new file mode 100644 index 0000000..83d100b --- /dev/null +++ b/scripts/bao.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python3 +"""lab7: операции оператора над OpenBao. Python 3.9+, только стандартная библиотека. + + python scripts/bao.py init # operator init (3 shares, порог 2) -> operator-only/openbao-init.json, затем unseal + python scripts/bao.py unseal # unseal после КАЖДОГО рестарта контейнера openbao (ручное действие оператора) + python scripts/bao.py configure # kv-v2, AppRole, политики, роли; секреты приложения из secrets/ -> KV + python scripts/bao.py issue-secret-ids [api|worker ...] # новый SecretID в secrets/bao/, старый отзывается + python scripts/bao.py revoke-role api # отозвать identity: все SecretID и токены роли (агент теряет доступ) + python scripts/bao.py policy-test # allow/deny политик реальными токенами AppRole (отрицательные проверки) + python scripts/bao.py seal # запечатать (демонстрация sealed-поведения) + python scripts/bao.py status + +Модель доверия (bootstrap): + * root token и unseal shares создаются один раз командой init и хранятся только в + operator-only/openbao-init.json (0600, .gitignore, .dockerignore). В учебном стенде все + shares у одного оператора; в реальной эксплуатации — у разных людей, root token + отзывается после настройки и создаётся заново `bao operator generate-root`. + * Приложение не получает ни root token, ни unseal keys, ни SecretID: SecretID читает + только OpenBao Agent своей роли (secrets/bao/_secret_id), SecretID живёт 7 дней и + принимается только из сети secrets (secret_id_bound_cidrs), токен агента — 15 мин (max 1 ч). + * Значения секретов не печатаются. Токен/ключ передаются в `docker exec` через + переменную окружения процесса (-e NAME без значения), а не через argv хоста. + +Переменные: COMPOSE_PROJECT_NAME (messenger-lab7), BAO_LOCAL=1 — выполнять `bao` локально +(для проверки скрипта без Docker; BAO_ADDR/BAO_CACERT берутся из окружения). +""" +import json +import os +import secrets as pysecrets +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _dotenv(name, default): + """Значение из окружения, затем из .env (как у docker compose), затем по умолчанию.""" + if os.environ.get(name): + return os.environ[name] + env = Path(os.environ.get("ENV_FILE", str(ROOT / ".env"))) + if env.is_file(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith(name + "="): + return line.split("=", 1)[1].strip().strip('"').strip("'") + return default + + +OPERATOR = Path(os.environ.get("OPERATOR_DIR", str(ROOT / "operator-only"))) +SECRETS = Path(os.environ.get("SECRETS_DIR", str(ROOT / "secrets"))) +INIT_FILE = OPERATOR / "openbao-init.json" +PROJECT = _dotenv("COMPOSE_PROJECT_NAME", "messenger-lab7") +LOCAL = os.environ.get("BAO_LOCAL") == "1" +SECRETS_CIDR = _dotenv("BAO_SECRETS_CIDR", _dotenv("BAO_SECRETS_SUBNET", "172.30.250.0/24")) +ROLES = { + "api": {"policy": "messenger-api", "kv": "messenger/api"}, + "worker": {"policy": "messenger-worker", "kv": "messenger/worker"}, +} +POLICY_DIR = ROOT / "infra" / "openbao" / "policies" + + +def say(msg): + print(msg, flush=True) + + +def container_id(service="openbao"): + out = subprocess.run(["docker", "compose", "-p", PROJECT, "ps", "-q", service], + cwd=ROOT, capture_output=True, text=True, check=False).stdout.split() + if not out: + raise SystemExit("service %s is not running (docker compose up -d %s)" % (service, service)) + return out[0] + + +def bao(*args, token=None, stdin=None, env_extra=None, check=True, shell=None, service="openbao"): + """Выполнить `bao ...` в контейнере `service` (по умолчанию openbao) или локально. + + Секреты — через env процесса docker exec, не argv. Токены AppRole привязаны к подсети + сети secrets (token_bound_cidrs), поэтому команды С ТОКЕНОМ РОЛИ выполняются из контейнера + агента этой роли (service="bao-agent-"), а не из openbao (там источник 127.0.0.1). + """ + env = dict(os.environ) + names = [] + if token: + env["BAO_TOKEN"] = token + names.append("BAO_TOKEN") + for k, v in (env_extra or {}).items(): + env[k] = v + names.append(k) + if LOCAL: + cmd = ["sh", "-c", shell] if shell else ["/opt/bin/bao" if Path("/opt/bin/bao").exists() else "bao"] + list(args) + else: + cmd = ["docker", "exec", "-i"] + for n in names: + cmd += ["-e", n] # значение берётся из окружения этого процесса, в argv его нет + addr = "https://127.0.0.1:8200" if service == "openbao" else "https://openbao:8200" + cmd += ["-e", "BAO_ADDR=" + addr, "-e", "BAO_CACERT=/run/secrets/ca_crt", + "-e", "BAO_TLS_SERVER_NAME=openbao", container_id(service)] + cmd += ["sh", "-c", shell] if shell else ["bao"] + list(args) + res = subprocess.run(cmd, input=stdin, capture_output=True, text=True, env=env, cwd=ROOT) + if check and res.returncode != 0: + # stderr OpenBao не содержит значений секретов; обрезаем на всякий случай. + raise SystemExit("bao %s failed (%d): %s" % (args[:3] if args else "sh", res.returncode, res.stderr.strip()[-300:])) + return res + + +def status(): + res = bao("status", "-format=json", check=False) + if res.returncode not in (0, 2): + raise SystemExit("OpenBao is not reachable: %s" % res.stderr.strip()[-200:]) + return json.loads(res.stdout) + + +def root_token(): + if not INIT_FILE.exists(): + raise SystemExit("no %s: run `python scripts/bao.py init` first" % INIT_FILE) + return json.loads(INIT_FILE.read_text(encoding="utf-8"))["root_token"] + + +def write_private(path: Path, value: str): + path.parent.mkdir(parents=True, exist_ok=True) + # Запись «на месте» (тот же inode): bind-mount файла в работающем контейнере видит новое значение. + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write(value + "\n") + try: + os.chmod(path, 0o600 if path.is_relative_to(OPERATOR) else 0o644) + except (OSError, AttributeError): + pass + + +def read_secret(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + + +# ---------------------------------------------------------------- commands + +def cmd_wait(_args, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + return status() + except SystemExit: + time.sleep(1) + raise SystemExit("OpenBao did not start within %ss" % timeout) + + +def cmd_init(args): + st = cmd_wait(args) + if st.get("initialized"): + say("OpenBao already initialized (keys: %s)" % ("present" if INIT_FILE.exists() else "MISSING")) + else: + res = bao("operator", "init", "-key-shares=3", "-key-threshold=2", "-format=json") + OPERATOR.mkdir(parents=True, exist_ok=True) + try: + os.chmod(OPERATOR, 0o700) + except OSError: + pass + data = json.loads(res.stdout) + write_private(INIT_FILE, json.dumps({"root_token": data["root_token"], + "unseal_keys_b64": data["unseal_keys_b64"], + "threshold": 2}, indent=2)) + shown = INIT_FILE.relative_to(ROOT) if INIT_FILE.is_relative_to(ROOT) else INIT_FILE + say("initialized: 3 unseal shares (threshold 2) and root token saved to %s (operator only)" % shown) + cmd_unseal(args) + + +def cmd_unseal(_args): + st = cmd_wait(_args) + if not st.get("sealed"): + say("OpenBao is unsealed") + return + keys = json.loads(INIT_FILE.read_text(encoding="utf-8"))["unseal_keys_b64"] + for key in keys[:2]: + bao(shell='bao operator unseal "$BAO_UNSEAL_KEY" >/dev/null', env_extra={"BAO_UNSEAL_KEY": key}) + st = status() + say("unsealed=%s" % (not st["sealed"])) + if st["sealed"]: + raise SystemExit(1) + + +def cmd_seal(_args): + bao("operator", "seal", token=root_token()) + say("OpenBao sealed: agents keep already rendered files; new auth/renew fail until `bao.py unseal`") + + +def _enabled(kind, token): + data = json.loads(bao(kind, "list", "-format=json", token=token).stdout) + return set(data) + + +def cmd_configure(_args): + token = root_token() + if "kv/" not in _enabled("secrets", token): + bao("secrets", "enable", "-path=kv", "kv-v2", token=token) + if "approle/" not in _enabled("auth", token): + bao("auth", "enable", "approle", token=token) + for role, spec in ROLES.items(): + policy = (POLICY_DIR / (spec["policy"] + ".hcl")).read_text(encoding="utf-8") + bao("policy", "write", spec["policy"], "-", token=token, stdin=policy) + bao("write", "auth/approle/role/" + spec["policy"], + "token_policies=" + spec["policy"], "token_ttl=15m", "token_max_ttl=1h", "token_type=service", + "secret_id_ttl=168h", "secret_id_num_uses=0", + "secret_id_bound_cidrs=" + SECRETS_CIDR, "token_bound_cidrs=" + SECRETS_CIDR, + "token_no_default_policy=false", token=token) + role_id = json.loads(bao("read", "-format=json", "auth/approle/role/%s/role-id" % spec["policy"], + token=token).stdout)["data"]["role_id"] + write_private(SECRETS / "bao" / (role + "_role_id"), role_id) + put_app_secrets(token) + # SecretID выпускаются заново при каждом configure: старые файлы могли остаться от + # предыдущего (удалённого) хранилища OpenBao и там недействительны. После configure + # перезапустите агентов: docker compose restart bao-agent-api bao-agent-worker + cmd_issue_secret_ids(type("A", (), {"roles": list(ROLES)})()) + say("configured: kv-v2, approle, policies messenger-api/messenger-worker, secrets written to KV") + + +def put_app_secrets(token): + api = { + "db_password": read_secret("db_api_password"), + "cache_password": read_secret("cache_api_password"), + "broker_password": read_secret("broker_api_password"), + "s3_access_key_id": read_secret("s3_access_key_id"), + "s3_secret_access_key": read_secret("s3_secret_access_key"), + } + worker = { + "db_password": read_secret("db_worker_password"), + "broker_password": read_secret("broker_worker_password"), + "s3_access_key_id": read_secret("s3_worker_access_key_id"), + "s3_secret_access_key": read_secret("s3_worker_secret_access_key"), + } + bao("kv", "put", "-mount=kv", "messenger/api", "-", token=token, stdin=json.dumps(api)) + bao("kv", "put", "-mount=kv", "messenger/worker", "-", token=token, stdin=json.dumps(worker)) + + +def _accessors_file(): + return OPERATOR / "approle-accessors.json" + + +def cmd_issue_secret_ids(args): + token = root_token() + accessors = json.loads(_accessors_file().read_text()) if _accessors_file().exists() else {} + for role in args.roles: + spec = ROLES[role] + res = json.loads(bao("write", "-f", "-format=json", "auth/approle/role/%s/secret-id" % spec["policy"], + token=token).stdout)["data"] + write_private(SECRETS / "bao" / (role + "_secret_id"), res["secret_id"]) + old = accessors.get(role) + accessors[role] = res["secret_id_accessor"] + if old: + # Старый SecretID больше не нужен: отзываем, чтобы утёкшая копия была бесполезна. + bao("write", "auth/approle/role/%s/secret-id-accessor/destroy" % spec["policy"], + "secret_id_accessor=" + old, token=token, check=False) + say("issued new SecretID for %s (ttl 168h, bound to %s)%s" + % (role, SECRETS_CIDR, "; previous SecretID destroyed" if old else "")) + OPERATOR.mkdir(parents=True, exist_ok=True) + write_private(_accessors_file(), json.dumps(accessors, indent=2)) + + +def cmd_revoke_role(args): + """Отзыв identity роли: все её SecretID и все выданные ей токены.""" + token = root_token() + for role in args.roles: + spec = ROLES[role] + listed = bao("list", "-format=json", "auth/approle/role/%s/secret-id" % spec["policy"], + token=token, check=False) + n_ids = 0 + if listed.returncode == 0 and listed.stdout.strip(): + for acc in json.loads(listed.stdout): + bao("write", "auth/approle/role/%s/secret-id-accessor/destroy" % spec["policy"], + "secret_id_accessor=" + acc, token=token) + n_ids += 1 + n_tokens = 0 + accs = bao("list", "-format=json", "auth/token/accessors", token=token, check=False) + for acc in (json.loads(accs.stdout) if accs.returncode == 0 and accs.stdout.strip() else []): + info = bao("token", "lookup", "-format=json", "-accessor", acc, token=token, check=False) + if info.returncode != 0: + continue + meta = (json.loads(info.stdout)["data"].get("meta") or {}) + if meta.get("role_name") == spec["policy"]: + bao("token", "revoke", "-accessor", acc, token=token) + n_tokens += 1 + say("revoked identity %s: %d SecretID, %d token(s); agent can no longer authenticate" + % (role, n_ids, n_tokens)) + + +def _login(role): + role_id = (SECRETS / "bao" / (role + "_role_id")).read_text().strip() + secret_id = (SECRETS / "bao" / (role + "_secret_id")).read_text().strip() + res = bao(shell='bao write -format=json auth/approle/login role_id="$R" secret_id="$S"', + env_extra={"R": role_id, "S": secret_id}, check=False) + return res + + +def cmd_policy_test(_args): + """Отрицательные проверки политик. Логин AppRole выполняется в контейнере АГЕНТА + (сеть secrets, адрес из secret_id_bound_cidrs); из контейнера openbao (127.0.0.1) + тот же SecretID отклоняется — это тоже проверка.""" + results = [] + + def record(name, ok, detail=""): + results.append({"check": name, "ok": bool(ok), "detail": detail}) + + # 1. SecretID вне разрешённой сети не работает (openbao сам себе 127.0.0.1). + if not LOCAL: + res = _login("api") + record("approle login from outside secrets network is denied", res.returncode != 0, + "exit %d" % res.returncode) + for role, other in (("api", "worker"), ("worker", "api")): + tok = agent_token(role) + svc = "bao-agent-" + role # источник запросов — подсеть secrets (token_bound_cidrs) + if not tok: + record("%s: agent login" % role, False, LAST_LOGIN_ERROR.get(role, "no token")) + continue + own = bao("kv", "get", "-mount=kv", "-format=json", ROLES[role]["kv"], token=tok, check=False, service=svc) + record("%s reads own kv/%s" % (role, ROLES[role]["kv"]), own.returncode == 0) + foreign = bao("kv", "get", "-mount=kv", ROLES[other]["kv"], token=tok, check=False, service=svc) + record("%s cannot read kv/%s" % (role, ROLES[other]["kv"]), foreign.returncode != 0 + and "permission denied" in foreign.stderr) + write = bao("kv", "put", "-mount=kv", ROLES[role]["kv"], "x=y", token=tok, check=False, service=svc) + record("%s cannot write own kv path" % role, write.returncode != 0 and "permission denied" in write.stderr) + pol = bao("policy", "write", "pwn", "-", token=tok, stdin='path "*" { capabilities = ["sudo"] }', check=False, + service=svc) + record("%s cannot write policies" % role, pol.returncode != 0 and "permission denied" in pol.stderr) + sid = bao("write", "-f", "auth/approle/role/%s/secret-id" % ROLES[role]["policy"], token=tok, check=False, + service=svc) + record("%s cannot mint SecretIDs" % role, sid.returncode != 0 and "permission denied" in sid.stderr) + look = json.loads(bao("token", "lookup", "-format=json", token=tok, service=svc).stdout)["data"] + record("%s token is short-lived (ttl<=15m) and not root" % role, + 0 < look["ttl"] <= 900 and "root" not in look["policies"], + "ttl=%ss policies=%s" % (look["ttl"], look["policies"])) + ok = all(r["ok"] for r in results) + print(json.dumps({"ok": ok, "checks": results}, ensure_ascii=False, indent=2)) + return 0 if ok else 1 + + +def agent_token(role): + """Получить токен роли ТАК ЖЕ, как агент: логин из контейнера агента (разрешённая сеть).""" + role_id = (SECRETS / "bao" / (role + "_role_id")).read_text().strip() + secret_id = (SECRETS / "bao" / (role + "_secret_id")).read_text().strip() + env = dict(os.environ, R=role_id, S=secret_id) + script = 'bao write -field=token auth/approle/login role_id="$R" secret_id="$S"' + if LOCAL: + cmd = ["sh", "-c", script.replace("bao ", "/opt/bin/bao ", 1)] + else: + cmd = ["docker", "exec", "-i", "-e", "R", "-e", "S", "-e", "BAO_ADDR=https://openbao:8200", + "-e", "BAO_CACERT=/run/secrets/ca_crt", container_id("bao-agent-" + role), "sh", "-c", script] + res = subprocess.run(cmd, capture_output=True, text=True, env=env, cwd=ROOT) + if res.returncode != 0: + LAST_LOGIN_ERROR[role] = " ".join((res.stderr or res.stdout).split())[-200:] + return None + return res.stdout.strip() + + +LAST_LOGIN_ERROR = {} + + +def cmd_status(_args): + st = status() + print(json.dumps({k: st.get(k) for k in ("initialized", "sealed", "t", "n", "progress", "version", + "storage_type")}, indent=2)) + + +def main(): + import argparse + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + for name, fn in (("init", cmd_init), ("unseal", cmd_unseal), ("seal", cmd_seal), ("configure", cmd_configure), + ("status", cmd_status), ("policy-test", cmd_policy_test), ("wait", cmd_wait)): + sub.add_parser(name).set_defaults(fn=fn) + for name, fn in (("issue-secret-ids", cmd_issue_secret_ids), ("revoke-role", cmd_revoke_role)): + p = sub.add_parser(name) + p.add_argument("roles", nargs="*", default=list(ROLES)) + p.set_defaults(fn=fn) + args = parser.parse_args() + if getattr(args, "roles", None): + bad = [r for r in args.roles if r not in ROLES] + if bad: + raise SystemExit("unknown role(s): %s" % bad) + return args.fn(args) or 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..29f667f --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Создаёт локальные учебные секреты в ./secrets (каталог в .gitignore и .dockerignore). + +lab7: у каждой роли свои учётные данные (least privilege), и приложение получает их +не из этих файлов, а из OpenBao (scripts/bao.py configure кладёт их в KV, агент +отрисовывает в tmpfs). Файлы здесь — материал ОПЕРАТОРА для bootstrap: + + postgres_password суперпользователь messenger (только postgres и migrate) + database_admin_url URL миграций (TLS verify-full) — только one-shot migrate + db_api_password роль messenger_api -> OpenBao kv/messenger/api + db_worker_password роль messenger_worker -> OpenBao kv/messenger/worker + cache_api_password Redis-кэш, ACL-пользователь messenger_api -> kv/messenger/api + broker_api_password broker, ACL api_producer -> kv/messenger/api + broker_worker_password broker, ACL worker_consumer -> kv/messenger/worker + s3_admin_* bootstrap-admin SeaweedFS (только one-shot s3-init) + s3_access_key_id/secret S3 identity API (staging/ запись, final/ чтение) + s3_worker_access_key_id/... S3 identity worker (final/ запись, staging/ чтение+удаление, List) + s3_config.json identities SeaweedFS (собирается из ключей выше) + grafana_admin_password вход оператора в Grafana (admin) + +Повторный запуск ничего не перезаписывает. Значения не печатаются. +TLS-материал создаёт scripts/pki.py (secrets/pki/), SecretID/RoleID — scripts/bao.py (secrets/bao/). +""" +import json +import os +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DIR = Path(os.environ.get("SECRETS_DIR", str(ROOT / "secrets"))) + + +def write(name: str, value: str) -> None: + path = DIR / name + if path.exists(): + print("keep secrets/%s" % name) + return + # newline="\n": на Windows иначе CRLF, и `tr -d '\\r\\n'`/чтение в контейнере дадут другой пароль. + with path.open("w", encoding="utf-8", newline="\n") as handle: + handle.write(value + "\n") + # Каталог 0700, файлы 0644: внутри контейнеров их читают непривилегированные + # пользователи с другими uid (redis, postgres, appuser), а на хосте каталог закрыт. + os.chmod(path, 0o644) + print("create secrets/%s" % name) + + +def read(name: str) -> str: + return (DIR / name).read_text(encoding="utf-8").strip() + + +def main() -> None: + DIR.mkdir(exist_ok=True) + (DIR / "bao").mkdir(exist_ok=True) + try: + os.chmod(DIR, 0o700) + except OSError: + pass + write("postgres_password", secrets.token_hex(24)) + write("database_admin_url", + "postgresql://messenger:%s@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/run/secrets/ca_crt" + % read("postgres_password")) + write("db_api_password", secrets.token_hex(24)) + write("db_worker_password", secrets.token_hex(24)) + write("cache_api_password", secrets.token_hex(24)) + write("broker_api_password", secrets.token_hex(24)) + write("broker_worker_password", secrets.token_hex(24)) + write("grafana_admin_password", secrets.token_urlsafe(18)) + # S3 (SeaweedFS). Access key — ASCII 20 символов, secret — 40 (как у AWS). + write("s3_admin_access_key_id", "ADM" + secrets.token_hex(8).upper() + "X") + write("s3_admin_secret_access_key", secrets.token_urlsafe(30)[:40]) + write("s3_access_key_id", "API" + secrets.token_hex(8).upper() + "X") + write("s3_secret_access_key", secrets.token_urlsafe(30)[:40]) + write("s3_worker_access_key_id", "WRK" + secrets.token_hex(8).upper() + "X") + write("s3_worker_secret_access_key", secrets.token_urlsafe(30)[:40]) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + b = bucket + config = { + "identities": [ + { + "name": "bootstrap_admin", + "credentials": [{"accessKey": read("s3_admin_access_key_id"), + "secretKey": read("s3_admin_secret_access_key")}], + "actions": ["Admin", "Read", "List", "Tagging", "Write"], + }, + { + # API: пишет только staging/ (приём загрузки, компенсация, очистка staging), + # читает final/ и originals/ (скачивание). Не может писать final/, листать + # бакет, создавать/удалять бакеты. + "name": "messenger_api", + "credentials": [{"accessKey": read("s3_access_key_id"), + "secretKey": read("s3_secret_access_key")}], + "actions": ["Read:%s/staging/*" % b, "Read:%s/final/*" % b, "Read:%s/originals/*" % b, + "Write:%s/staging/*" % b], + }, + { + # worker: читает staging/, публикует final/ (оригинал + варианты), удаляет + # staging/ после публикации, листает бакет для сверки сирот (reconcile). + "name": "messenger_worker", + "credentials": [{"accessKey": read("s3_worker_access_key_id"), + "secretKey": read("s3_worker_secret_access_key")}], + "actions": ["Read:%s/staging/*" % b, "Read:%s/final/*" % b, "Read:%s/originals/*" % b, + "Write:%s/final/*" % b, "Write:%s/staging/*" % b, "Write:%s/originals/*" % b, + "List:%s" % b], + }, + ] + # Identity "anonymous" не объявлена -> анонимные запросы получают 403. + } + path = DIR / "s3_config.json" + if not path.exists(): + write("s3_config.json", json.dumps(config, indent=2)) + else: + print("keep secrets/s3_config.json") + + +if __name__ == "__main__": + main() diff --git a/scripts/ops.py b/scripts/ops.py new file mode 100644 index 0000000..8ca7908 --- /dev/null +++ b/scripts/ops.py @@ -0,0 +1,871 @@ +#!/usr/bin/env python3 +"""Операции над compose-стендом lab7. Python 3.9+, только стандартная библиотека. + +Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает `docker compose` +и `docker` без shell-интерполяции. Значения секретов не печатает. + +Стенд: + python scripts/ops.py up # bootstrap с нуля: secrets -> образ -> PKI -> OpenBao init/unseal/configure -> стенд -> ожидание + python scripts/ops.py wait # дождаться N здоровых реплик, UP в LB и HTTPS /health/ready + python scripts/ops.py ps # реплики: alias (X-Instance-Id), контейнер, health, статус в LB + python scripts/ops.py down | destroy # остановить (volumes целы) | НАМЕРЕННО удалить данные стенда +Hooks публичных тестов: + python scripts/ops.py stop-one # TARGET_INSTANCE_ID=; реплика остаётся остановленной + python scripts/ops.py restore # вернуть остановленные реплики + python scripts/ops.py restart # persistence: API, worker, PostgreSQL, Redis, broker(+gate), S3(+gate), OpenBao(+unseal), LB +lab7 — безопасность (результаты дописываются в evidence/): + python scripts/ops.py tls-audit # вся trust-matrix: TLS/mTLS/identity/ACL отрицательные и положительные проверки + python scripts/ops.py security-http # ACL/вложения, X-User-Id, logout, 429, 413, утечки в ошибках (через LB) + python scripts/ops.py leak-check # отсутствие паролей/токенов/секретов в Loki, Tempo, Prometheus + python scripts/ops.py hardening # docker inspect: user, read_only, caps, no-new-privileges, limits, порты, docker.sock + python scripts/ops.py bao-policy # OpenBao: allow/deny политик реальными токенами AppRole + python scripts/ops.py rehash-demo # Argon2id: старый hash -> успешный вход -> пересчёт под текущие параметры + python scripts/ops.py rotate-db-password [api|worker] # ротация рабочего секрета без пересборки/рестарта + python scripts/ops.py rotate-lb-cert # ротация leaf LB->API + отзыв старого (CRL), проверка старым/новым + python scripts/ops.py rotate-secret-id [api|worker] # новый SecretID агента, старый уничтожен + python scripts/ops.py revoke-identity [api|worker] # отзыв AppRole identity -> агент без доступа -> восстановление + python scripts/ops.py bao-sealed # OpenBao sealed: API продолжает работать на отрисованных секретах, unseal + python scripts/ops.py login-budget # бюджет login DoS: Argon2id на стенде + поток входов + python scripts/ops.py scan # pip-audit + trivy образов -> evidence/scan/ +lab3–6 (как раньше): rolling-update, in-network storage|expiry|consistency, wait-s3, s3-check, reconcile [--apply], + api-fs, worker-stop|worker-kill|worker-start, api-restart, broker-restart, jobs ..., pipeline-faults + +Переменные: COMPOSE_PROJECT_NAME (messenger-lab7), ENV_FILE (.env), API_REPLICAS (2), HTTPS_PORT (8443), +STOP_MODE (stop|kill), CA_FILE (secrets/pki/ca.crt). +""" +import csv +import io +import json +import os +import secrets as pysecrets +import ssl +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +OPS_LOG = EVIDENCE / "ops.log" +SECRETS = ROOT / "secrets" + +try: # консоль Windows может быть не в UTF-8 + sys.stdout.reconfigure(errors="replace") + sys.stderr.reconfigure(errors="replace") +except AttributeError: + pass + + +# ---------------------------------------------------------------- settings + +def _dotenv(): + path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env"))) + values = {} + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + values[k.strip()] = v.strip().strip('"').strip("'") + return path if path.is_file() else None, values + + +ENV_PATH, DOTENV = _dotenv() + + +def setting(name, default): + return os.environ.get(name) or DOTENV.get(name) or default + + +PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab7") +os.environ.setdefault("COMPOSE_PROJECT_NAME", PROJECT) +REPLICAS = int(setting("API_REPLICAS", "2")) +HTTPS_PORT = int(setting("HTTPS_PORT", "8443")) +LB_URL = "https://127.0.0.1:%d" % HTTPS_PORT +CA_FILE = setting("CA_FILE", str(SECRETS / "pki" / "ca.crt")) + + +def tls_context(): + return ssl.create_default_context(cafile=CA_FILE if Path(CA_FILE).exists() else None) + + +def now_iso(): + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def log(msg): + line = "%s %s" % (now_iso(), msg) + print(line, flush=True) + try: + EVIDENCE.mkdir(exist_ok=True) + with OPS_LOG.open("a", encoding="utf-8") as f: + f.write(line + "\n") + except OSError: + pass + + +# ---------------------------------------------------------------- docker + +def compose_cmd(*args): + cmd = ["docker", "compose", "-p", PROJECT] + if ENV_PATH: + cmd += ["--env-file", str(ENV_PATH)] + return cmd + list(args) + + +def run(cmd, check=True, capture=False, timeout=300, show=False, env=None, stdin=None): + """capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо.""" + out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL) + result = subprocess.run(cmd, cwd=ROOT, check=False, timeout=timeout, stdout=out, stderr=out, text=True, + env=env, input=stdin) + if check and result.returncode != 0: + detail = (result.stderr or "").strip()[-400:] if capture else "" + raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:5]), detail)) + return result + + +def compose(*args, timeout=300): + return run(compose_cmd(*args), timeout=timeout, capture=True) + + +def py(): + return sys.executable or "python" + + +def service_ids(service, include_stopped=False): + cmd = ["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service, + "--filter", "label=com.docker.compose.oneoff=False"] + if include_stopped: + cmd.insert(2, "-a") + return run(cmd, capture=True).stdout.split() + + +def instance_alias(container_id): + """X-Instance-Id реплики: случайный alias, который процесс записал в /tmp/instance-id.""" + res = run(["docker", "exec", container_id, "cat", "/tmp/instance-id"], check=False, capture=True, timeout=10) + return res.stdout.strip() if res.returncode == 0 else "" + + +def api_containers(include_stopped=True, with_alias=True): + cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}", + "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=api", + "--filter", "label=com.docker.compose.oneoff=False"] + if include_stopped: + cmd.insert(2, "-a") + ids = [x for x in run(cmd, capture=True).stdout.split() if x] + if not ids: + return [] + result = [] + for item in json.loads(run(["docker", "inspect"] + ids, capture=True).stdout): + state = item["State"] + nets = item.get("NetworkSettings", {}).get("Networks") or {} + ingress = [n.get("IPAddress") for k, n in nets.items() if k.endswith("_ingress") and n.get("IPAddress")] + running_ = state.get("Status") == "running" + result.append({ + "id": item["Id"], "short": item["Id"][:12], "name": item["Name"].lstrip("/"), + "alias": instance_alias(item["Id"]) if (with_alias and running_) else "", + "state": state.get("Status"), "health": (state.get("Health") or {}).get("Status", "-"), + "image": item["Config"].get("Image"), "ip": ingress[0] if ingress else "", + "created": item.get("Created", "")[:19], + }) + result.sort(key=lambda c: c["name"]) + return result + + +def running(containers=None): + return [c for c in (containers if containers is not None else api_containers(with_alias=False)) + if c["state"] == "running"] + + +# ---------------------------------------------------------------- load balancer + +def lb_servers(): + """Статистика HAProxy (порт 8404 только внутри контейнера lb) через docker exec.""" + out = run(compose_cmd("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv"), + capture=True, check=False, timeout=15).stdout or "" + servers = [] + for r in csv.DictReader(io.StringIO(out.lstrip("# "))): + if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"): + continue + addr = r.get("addr") or "" + status = r.get("status", "") + if status.startswith("MAINT") and not addr: + continue + servers.append({"name": r["svname"], "status": status, "addr": addr}) + return servers + + +def lb_up_count(): + return sum(1 for s in lb_servers() if s["status"].startswith("UP")) + + +def lb_ready(): + try: + with urllib.request.urlopen(LB_URL + "/health/ready", timeout=3, context=tls_context()) as r: + return r.status == 200 + except (urllib.error.URLError, OSError): + return False + + +def wait_until(predicate, timeout, what, interval=0.5): + deadline = time.monotonic() + timeout + last_exc = None + while time.monotonic() < deadline: + try: + if predicate(): + return True + except (urllib.error.URLError, OSError, ValueError, KeyError) as exc: + last_exc = exc + time.sleep(interval) + raise SystemExit("timeout after %ss waiting for %s%s" % ( + timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else "")) + + +def wait_healthy(ids, timeout=90): + def ok(): + states = {c["id"]: c for c in api_containers(with_alias=False)} + return all(states.get(i, {}).get("health") == "healthy" for i in ids) + wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids)) + + +def wait_stack(replicas, timeout=180): + wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas, + timeout, "%d healthy api replicas" % replicas) + wait_until(lambda: lb_up_count() >= replicas, 60, "%d replicas UP in HAProxy (mTLS checks)" % replicas) + wait_until(lb_ready, 60, "LB https /health/ready = 200 (CA %s)" % CA_FILE) + + +def wait_service_healthy(service, timeout): + def ok(): + ids = service_ids(service) + if not ids: + return False + out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip() + return out == "healthy" + wait_until(ok, timeout, service + " healthy") + + +def container_started(service): + ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split() + if not ids: + return "-" + return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23] + + +# ---------------------------------------------------------------- bootstrap + +def pki(*args): + """scripts/pki.py: на хосте, если есть cryptography, иначе в контейнере tools (сеть none).""" + try: + import cryptography # noqa: F401 + run([py(), "scripts/pki.py"] + list(args), show=True, timeout=120) + except ImportError: + run(compose_cmd("--profile", "tools", "run", "--rm", "-T", "tools", "scripts/pki.py") + list(args), + show=True, timeout=300) + + +def bao(*args, check=True): + return run([py(), "scripts/bao.py"] + list(args), show=True, timeout=300, check=check) + + +def cmd_up(_args): + t0 = time.monotonic() + if not ENV_PATH and (ROOT / ".env.example").exists(): + (ROOT / ".env").write_text((ROOT / ".env.example").read_text(encoding="utf-8"), encoding="utf-8") + log("up: .env created from .env.example") + run([py(), "scripts/init_secrets.py"], show=True) + log("up: building image (api/worker/migrate/s3-init/tools)") + run(compose_cmd("build", "api"), show=True, timeout=1800) + pki("init") + compose("up", "-d", "openbao", timeout=180) + bao("init") # init (однократно) + unseal; ключи -> operator-only/ + bao("configure") # kv, approle, политики, секреты -> KV, RoleID/SecretID -> secrets/bao/ + log("up: starting the stand") + run(compose_cmd("up", "-d"), show=True, timeout=1200) + wait_stack(REPLICAS) + wait_service_healthy("worker", 120) + log("up: ready in %.0fs -> %s (CA: %s); Grafana https://127.0.0.1:%s (admin / secrets/grafana_admin_password)" + % (time.monotonic() - t0, LB_URL, Path(CA_FILE).relative_to(ROOT) if Path(CA_FILE).is_relative_to(ROOT) + else CA_FILE, setting("GRAFANA_PORT", "3000"))) + + +def cmd_down(_args): + run(compose_cmd("--profile", "ttl", "--profile", "faults", "--profile", "audit", "down"), show=True, timeout=600) + + +def cmd_destroy(_args): + """Намеренное удаление учебных данных: volumes (БД, S3, broker, OpenBao, telemetry).""" + run(compose_cmd("--profile", "ttl", "--profile", "faults", "--profile", "audit", "down", "-v"), + show=True, timeout=600) + log("destroy: volumes removed; secrets/ and operator-only/ kept (OpenBao keys now refer to a deleted storage:" + " remove operator-only/openbao-init.json and secrets/bao/* before the next `up`)") + + +# ---------------------------------------------------------------- stand state + +def cmd_wait(_args): + t0 = time.monotonic() + wait_stack(REPLICAS) + print("ready: %d api replicas healthy and UP in LB, HTTPS ready (%.1fs)" % (REPLICAS, time.monotonic() - t0)) + + +def cmd_ps(_args): + containers = api_containers() + try: + by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()} + except OSError: + by_addr = {} + print("%-14s %-28s %-9s %-10s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "lb", "lb_status")) + for c in containers: + s = by_addr.get(c["ip"], {}) + print("%-14s %-28s %-9s %-10s %-6s %s" % (c["alias"] or "-", c["name"], c["state"], c["health"], + s.get("name", "-"), s.get("status", "-"))) + + +def find_target(target): + for c in api_containers(): + if target and (c["alias"] == target or c["id"].startswith(target)): + return c + return None + + +def cmd_stop_one(_args): + target = os.environ.get("TARGET_INSTANCE_ID", "").strip() + if not target: + raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)") + mode = os.environ.get("STOP_MODE", "stop").strip().lower() + c = find_target(target) + if c is None: + raise SystemExit("no api container of project %s has X-Instance-Id %s" % (PROJECT, target)) + if c["state"] != "running": + raise SystemExit("container %s is already %s" % (c["name"], c["state"])) + t0 = time.monotonic() + log("stop-one: mode=%s alias=%s container=%s" % (mode, target, c["name"])) + if mode == "kill": + run(["docker", "update", "--restart", "no", c["id"]]) + run(["docker", "kill", "--signal", "KILL", c["id"]]) + else: + run(["docker", "stop", "--time", "20", c["id"]], timeout=40) + log("stop-one: container=%s stopped after %.2fs; left stopped (restore: python scripts/ops.py restore)" + % (c["name"], time.monotonic() - t0)) + + +def cmd_restore(_args): + t0 = time.monotonic() + stopped = [c for c in api_containers(with_alias=False) if c["state"] != "running"] + for c in stopped: + run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False) + compose("up", "-d", "--no-deps", "--no-recreate", "api") + wait_stack(REPLICAS) + log("restore: started %s; %d replicas healthy and UP after %.1fs" + % ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0)) + + +def cmd_restart(_args): + """Hook make test-persistence: перезапуск API, worker, всех хранилищ (PostgreSQL, Redis, + broker, S3, OpenBao) и LB. Volumes не удаляются, данные не пересоздаются. + + OpenBao после рестарта запечатан: оператор распечатывает его (`bao.py unseal`, ключи из + operator-only/). Пока он sealed, агенты держат уже отрисованные секреты (tmpfs volume), + поэтому API/worker стартуют и без OpenBao. Gates делят сетевой namespace с broker / + collector и пересоздаются после рестарта своего сервиса. + """ + t0 = time.monotonic() + stores = ("postgres", "redis", "broker", "s3", "openbao") + started_before = {svc: container_started(svc) for svc in stores} + before = sorted(c["short"] for c in running()) + compose("stop", "lb", "api", "worker", timeout=180) + compose("restart", *stores, timeout=180) + compose("up", "-d", "--no-deps", "--force-recreate", "broker-gate", "s3-gate", timeout=120) + wait_service_healthy("broker", 60) + wait_service_healthy("broker-gate", 60) + bao("unseal") + log("restart: openbao restarted -> sealed -> unsealed by operator (keys: operator-only/)") + wait_service_healthy("s3", 60) + wait_service_healthy("s3-gate", 60) + t_warm = time.monotonic() + warm = s3_warmup(70) + log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm)) + compose("start", "api", "worker", timeout=120) + wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS, + 90, "api healthy after restart") + compose("start", "lb", timeout=60) + log("restart: started_at %s; api %s restarted; worker restarted; lb restarted; %.1fs" + % ("; ".join("%s %s -> %s" % (s, started_before[s], container_started(s)) for s in stores), + before, time.monotonic() - t0)) + + +def cmd_worker_stop(_args): + t0 = time.monotonic() + ids = service_ids("worker") + compose("stop", "worker", timeout=120) + log("worker-stop: stopped %s in %.1fs" % ([i[:12] for i in ids], time.monotonic() - t0)) + + +def cmd_worker_kill(_args): + ids = service_ids("worker") + for i in ids: + run(["docker", "update", "--restart", "no", i]) + run(["docker", "kill", "--signal", "KILL", i]) + log("worker-kill: SIGKILL %s (restart policy disabled until worker-start)" % [i[:12] for i in ids]) + + +def cmd_worker_start(_args): + t0 = time.monotonic() + for i in service_ids("worker", include_stopped=True): + run(["docker", "update", "--restart", "unless-stopped", i], check=False) + compose("up", "-d", "--no-deps", "--no-recreate", "worker", timeout=120) + wait_service_healthy("worker", 90) + log("worker-start: worker healthy after %.1fs" % (time.monotonic() - t0)) + + +def cmd_api_restart(_args): + t0 = time.monotonic() + before = [c["short"] for c in running()] + compose("restart", "api", timeout=120) + wait_stack(REPLICAS) + log("api-restart: %s restarted, %d replicas UP after %.1fs" % (before, REPLICAS, time.monotonic() - t0)) + + +def cmd_broker_restart(_args): + t0 = time.monotonic() + before = container_started("broker") + compose("restart", "broker", timeout=60) + compose("up", "-d", "--no-deps", "--force-recreate", "broker-gate", timeout=120) + wait_service_healthy("broker", 60) + wait_service_healthy("broker-gate", 60) + log("broker-restart: started_at %s -> %s, healthy after %.1fs (volume brokerdata kept, gate recreated)" + % (before, container_started("broker"), time.monotonic() - t0)) + + +def cmd_jobs(args): + run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.jobs_admin") + list(args), + show=True, timeout=300) + + +def s3_warmup(timeout=70): + out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", + "--wait-ready", str(timeout)), capture=True, timeout=timeout + 60, check=False) + lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")] + return lines[-1] if lines else "" + + +def cmd_wait_s3(_args): + wait_service_healthy("s3", 60) + wait_service_healthy("s3-gate", 60) + print(s3_warmup(90)) + + +def cmd_s3_check(_args): + """Приватный бакет вне API: анонимные запросы, права ключа API по префиксам, закрытые порты.""" + run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap", + "--check-only", "--sample-from-db"), show=True, timeout=180) + + +def cmd_api_fs(_args): + for c in api_containers(with_alias=False): + info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0] + mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])] + print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % ( + c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none")) + + +def cmd_reconcile(args): + # lab7: обход бакета требует права List — оно есть только у identity worker. + run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.reconcile") + list(args), + show=True, timeout=600) + + +def cmd_rolling_update(args): + """Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.""" + t_all = time.monotonic() + if [c for c in api_containers(with_alias=False) if c["state"] != "running"]: + raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first") + old = running() + n = len(old) + if n < 1: + raise SystemExit("no running api replicas") + log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old])) + if "--no-build" not in args: + compose("build", "api", timeout=900) + out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines() + log("rolling-update: migrations once: %s" % (out[-1] if out else "")) + for i, victim in enumerate(old, 1): + before = {c["id"] for c in api_containers(with_alias=False)} + t = time.monotonic() + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180) + new = [c for c in api_containers(with_alias=False) if c["id"] not in before] + if len(new) != 1: + raise SystemExit("expected exactly one new container, got %d" % len(new)) + fresh = new[0] + wait_healthy([fresh["id"]], timeout=90) + wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy") + t_ready = time.monotonic() - t + t = time.monotonic() + run(["docker", "stop", "--time", "20", victim["id"]], timeout=40) + run(["docker", "rm", victim["id"]]) + log("rolling-update: step %d/%d new=%s ready+UP in %.1fs; old=%s stopped in %.1fs" + % (i, n, fresh["short"], t_ready, victim["short"], time.monotonic() - t)) + wait_stack(n) + log("rolling-update: done in %.1fs" % (time.monotonic() - t_all)) + + +# ---------------------------------------------------------------- in-network тесты (lab2-5) + +def _in_network_base(): + """One-off контейнер образа API (сети api) + тесты + CA и identity lb для прямых вызовов реплик.""" + return compose_cmd("run", "--rm", "-T", "--no-deps", + "-v", "%s:/app/student-tests:ro" % (ROOT / "student-tests"), + "-v", "%s:/app/tests/fixtures:ro" % (ROOT / "tests" / "fixtures"), + "-v", "%s:/opt/test/lb.pem:ro" % (SECRETS / "pki" / "lb-client.pem"), + "-e", "CA_FILE=/run/secrets/ca_crt", "-e", "CLIENT_CERT_FILE=/opt/test/lb.pem", + "-e", "BASE_URL=https://api:8443") + + +def cmd_in_network(args): + which = args[0] if args else "" + base = _in_network_base() + if which == "storage": + # White-box тесты хранилища применяют миграции (DDL) -> нужна identity администратора + # (как у one-shot migrate); у роли messenger_api прав на DDL нет намеренно. + admin = ["-v", "%s:/opt/test/admin_url:ro" % (SECRETS / "database_admin_url"), + "-e", "DATABASE_URL_FILE=/opt/test/admin_url"] + for pattern in ("test_storage_*.py", "test_session_cache.py"): + run(base + admin + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, + "-v"], show=True, timeout=600) + elif which == "expiry": + prof = compose_cmd("--profile", "ttl") + run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300) + env = ["-e", "EXPIRY_BASE_URL=https://api-short-ttl:8443"] + try: + run(base + env + ["api", "python", "student-tests/wait_url.py", "https://api-short-ttl:8443/health/ready"], + show=True, timeout=120) + run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", + "-p", "test_expiry.py", "-v"], show=True, timeout=300) + finally: + run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120) + else: + raise SystemExit("usage: python scripts/ops.py in-network storage|expiry") + + +def faults_compose(*args): + return run(["docker", "compose", "-p", PROJECT] + (["--env-file", str(ENV_PATH)] if ENV_PATH else []) + + ["-f", "compose.yaml", "-f", "compose.faults.yaml"] + list(args), capture=True, timeout=300) + + +def cmd_pipeline_faults(_args): + """Отказы worker'а (маркеры в filename, compose.faults.yaml) — сценарии c..j lab5.""" + log("pipeline-faults: recreating worker with compose.faults.yaml") + faults_compose("up", "-d", "--no-deps", "--force-recreate", "worker") + wait_service_healthy("worker", 90) + try: + run(_in_network_base() + ["-e", "BASE_URL=https://api:8443", "api", "python", "-m", "unittest", "discover", + "-s", "student-tests", "-p", "test_pipeline_faults.py", "-v"], + show=True, timeout=900, check=False) + finally: + compose("up", "-d", "--no-deps", "--force-recreate", "worker", timeout=180) + wait_service_healthy("worker", 90) + log("pipeline-faults: worker restored to normal configuration") + + +# ---------------------------------------------------------------- lab7: безопасность + +def save(name, text): + if not text.strip(): + log("evidence/%s not saved: empty output (see the error above)" % name) + return + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / name).write_text(text, encoding="utf-8") + log("saved evidence/%s" % name) + + +def audit_run(script, *args, extra=None, timeout=600): + cmd = compose_cmd("--profile", "audit", "run", "--rm", "-T", "--no-deps") + list(extra or []) \ + + ["audit", "student-tests/security/" + script] + list(args) + res = run(cmd, capture=True, check=False, timeout=timeout) + sys.stdout.write(res.stdout or "") + if res.returncode != 0: + sys.stderr.write((res.stderr or "")[-2000:]) + return res + + +def cmd_tls_audit(_args): + res = audit_run("tls_audit.py") + save("tls_audit.json", res.stdout or "") + if res.returncode != 0: + raise SystemExit("tls-audit: FAILED checks above") + + +def cmd_leak_check(_args): + res = audit_run("leak_check.py", timeout=300) + save("leak_check.json", res.stdout or "") + if res.returncode != 0: + raise SystemExit("leak-check: FAILED") + + +def cmd_security_http(_args): + res = run([py(), "student-tests/security/security_http.py", "--base-url", LB_URL, "--ca-file", CA_FILE], + capture=True, check=False, timeout=600) + sys.stdout.write(res.stdout or "") + sys.stderr.write((res.stderr or "")[-2000:]) + save("security_http.txt", (res.stdout or "") + (res.stderr or "")) + if res.returncode != 0: + raise SystemExit("security-http: FAILED") + + +def cmd_hardening(_args): + res = run([py(), "student-tests/security/hardening_audit.py", "--project", PROJECT], capture=True, check=False) + sys.stdout.write(res.stdout or "") + sys.stderr.write(res.stderr or "") + save("hardening.txt", res.stdout or "") + if res.returncode != 0: + raise SystemExit("hardening: FAILED") + + +def cmd_bao_policy(_args): + res = run([py(), "scripts/bao.py", "policy-test"], capture=True, check=False) + sys.stdout.write(res.stdout or "") + save("bao_policy.json", res.stdout or "") + if res.returncode != 0: + raise SystemExit("bao-policy: FAILED") + + +def psql_admin(sql, timeout=60): + """SQL от имени администратора внутри контейнера postgres (unix socket, пароль из secret).""" + return run(compose_cmd("exec", "-T", "postgres", "sh", "-c", + 'PGPASSWORD="$(cat /run/secrets/postgres_password)" psql -v ON_ERROR_STOP=1 -qAt ' + '-U messenger -d messenger -h /var/run/postgresql -f -'), + capture=True, timeout=timeout, stdin=sql) + + +def pg_login_ok(role, password): + """Новое TLS-соединение роли с паролем (из контейнера postgres, hostssl 127.0.0.1).""" + env = dict(os.environ, PGCHECK_PW=password) + res = run(["docker", "exec", "-i", "-e", "PGCHECK_PW", service_ids("postgres")[0], "sh", "-c", + 'PGPASSWORD="$PGCHECK_PW" psql "host=127.0.0.1 dbname=messenger user=%s sslmode=require" -qAtc "select 1"' + % role], capture=True, check=False, env=env, timeout=30) + return res.returncode == 0, (res.stderr or "").strip().splitlines()[-1:] if res.returncode else "" + + +def rendered_hash(role, name="database_url"): + ids = service_ids("bao-agent-" + role) + res = run(["docker", "exec", ids[0], "sha256sum", "/run/bao/" + name], capture=True, check=False, timeout=15) + return res.stdout.split()[0] if res.returncode == 0 and res.stdout else "" + + +def cmd_rotate_db_password(args): + """Ротация рабочего секрета (пароль роли БД) без пересборки образа и рестарта API. + + 1) ALTER ROLE ... PASSWORD (админ, внутри контейнера postgres); + 2) OpenBao KV patch (root token оператора); 3) файл оператора secrets/db__password; + 4) агент перерисовывает /run/bao/database_url (<= 5 с + backoff); + 5) старый пароль отклоняется на НОВОМ соединении, новый принимается; + 6) существующие соединения роли разрываются (pg_terminate_backend) — пул открывает + новые уже с новым паролем; затем публичный smoke проходит. + Окно: между шагами 1 и 4 НОВЫЕ соединения со старым паролем получают отказ (уже открытые + соединения пула работают); измеренное окно пишется в лог. + """ + sys.path.insert(0, str(ROOT / "scripts")) + import bao as baomod + role = (args[0] if args else "api").strip() + if role not in ("api", "worker"): + raise SystemExit("usage: rotate-db-password [api|worker]") + secret_file = SECRETS / ("db_%s_password" % role) + old = secret_file.read_text(encoding="utf-8").strip() + new = pysecrets.token_hex(24) + before = rendered_hash(role) + t0 = time.monotonic() + psql_admin("ALTER ROLE messenger_%s PASSWORD '%s';\n" % (role, new)) + baomod.bao("kv", "patch", "-mount=kv", "messenger/" + role, "-", token=baomod.root_token(), + stdin=json.dumps({"db_password": new})) + with open(secret_file, "w", encoding="utf-8", newline="\n") as fh: + fh.write(new + "\n") + wait_until(lambda: rendered_hash(role) not in ("", before), 120, "agent re-rendered database_url", interval=1) + window = time.monotonic() - t0 + old_ok, old_err = pg_login_ok("messenger_" + role, old) + new_ok, _ = pg_login_ok("messenger_" + role, new) + killed = psql_admin("SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE usename = 'messenger_%s';\n" + % role).stdout.strip() + log("rotate-db-password: role=messenger_%s; agent re-rendered after %.1fs; old password accepted=%s (%s); " + "new password accepted=%s; existing sessions terminated=%s" % (role, window, old_ok, old_err, new_ok, killed)) + if old_ok or not new_ok: + raise SystemExit("rotation check failed") + time.sleep(2) + if role == "api": + res = run([py(), "tests/smoke.py", "--base-url", LB_URL, "--ca-file", CA_FILE], capture=True, check=False, + timeout=300) + log("rotate-db-password: smoke after rotation (new pool connections): %s" + % (res.stdout.strip().splitlines()[-1:] or res.stderr.strip()[-200:])) + if res.returncode != 0: + raise SystemExit(1) + else: + wait_service_healthy("worker", 60) + log("rotate-db-password: worker healthy after its sessions were terminated") + + +def probe_api_cert(pem_host_path, label): + """mTLS-проба реплики API заданным сертификатом из контейнера audit (сеть ingress).""" + res = audit_run("mtls_probe.py", "--target", "api:8443", "--server-name", "api", "--cert", "/opt/probe/cert.pem", + "--label", label, extra=["-v", "%s:/opt/probe/cert.pem:ro" % pem_host_path], timeout=120) + return res.returncode == 0 + + +def cmd_rotate_lb_cert(_args): + """Ротация leaf LB->API без пересборки образов и отзыв старой identity через CRL.""" + t0 = time.monotonic() + pki("rotate", "lb-client") + compose("restart", "lb", timeout=60) + wait_service_healthy("lb", 60) + wait_until(lambda: lb_up_count() >= REPLICAS, 60, "replicas UP with the new LB certificate") + wait_until(lb_ready, 30, "LB ready") + log("rotate-lb-cert: new lb-client in use (lb restarted, no image rebuild) after %.1fs" % (time.monotonic() - t0)) + old_before = probe_api_cert(SECRETS / "pki" / "lb-client.prev.pem", "old-before-revoke") + pki("revoke", "lb-client") + time.sleep(7) # API перечитывает CRL не реже раза в 5 с + old_after = probe_api_cert(SECRETS / "pki" / "lb-client.prev.pem", "old-after-revoke") + new_after = probe_api_cert(SECRETS / "pki" / "lb-client.pem", "new-after-revoke") + ok = lb_ready() + log("rotate-lb-cert: old cert accepted before revoke=%s, after revoke=%s; new cert accepted=%s; LB ready=%s" + % (old_before, old_after, new_after, ok)) + if old_after or not new_after or not ok: + raise SystemExit("rotation/revocation check failed") + + +def cmd_rotate_secret_id(args): + roles = args or ["api"] + bao("issue-secret-ids", *roles) + for role in roles: + compose("restart", "bao-agent-" + role, timeout=60) + wait_service_healthy("bao-agent-" + role, 120) + log("rotate-secret-id: %s agent(s) re-authenticated with the new SecretID; old SecretID destroyed" % roles) + + +def cmd_revoke_identity(args): + role = (args[0] if args else "worker") + bao("revoke-role", role) + compose("restart", "bao-agent-" + role, timeout=60) + time.sleep(15) + logs = run(compose_cmd("logs", "--since", "30s", "bao-agent-" + role), capture=True, check=False).stdout or "" + denied = "invalid" in logs.lower() or "permission denied" in logs.lower() or "error" in logs.lower() + log("rotate/revoke: identity %s revoked; agent re-auth denied=%s; already rendered files remain -> %s keeps" + " working until restart of the volume" % (role, denied, role)) + ok = lb_ready() + log("revoke-identity: LB ready while %s identity is revoked = %s" % (role, ok)) + bao("issue-secret-ids", role) + compose("restart", "bao-agent-" + role, timeout=60) + wait_service_healthy("bao-agent-" + role, 120) + log("revoke-identity: recovered with a new SecretID; agent healthy") + + +def cmd_bao_sealed(_args): + bao("seal") + t0 = time.monotonic() + time.sleep(10) + res = run([py(), "tests/smoke.py", "--base-url", LB_URL, "--ca-file", CA_FILE], capture=True, check=False) + log("bao-sealed: smoke while OpenBao sealed -> %s" % ("PASS" if res.returncode == 0 else "FAIL")) + compose("restart", "api", timeout=120) + wait_stack(REPLICAS) + log("bao-sealed: api replicas restarted while sealed -> healthy (secrets from agent tmpfs volume)") + bao("unseal") + log("bao-sealed: unsealed after %.0fs; agents resume renew/render with backoff (<= 1 min)" % (time.monotonic() - t0)) + if res.returncode != 0: + raise SystemExit(1) + + +def cmd_rehash_demo(_args): + """Argon2id rehash: hash пользователя заменяется на старый (m=8 MiB, t=1), вход его пересчитывает.""" + import base64 + username = "rehash_" + pysecrets.token_hex(4) + password = "rehash-demo-" + pysecrets.token_urlsafe(12) + ctx = tls_context() + + def req(method, path, body=None, headers=None): + r = urllib.request.Request(LB_URL + path, data=json.dumps(body).encode() if body else None, + headers=dict({"Content-Type": "application/json"}, **(headers or {})), + method=method) + with urllib.request.urlopen(r, context=ctx, timeout=10) as resp: + return resp.status + req("POST", "/api/v1/users", {"username": username, "display_name": "Rehash", "password": password}) + old_hash = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-c", + "import sys; from argon2 import PasswordHasher as P; " + "print(P(time_cost=1, memory_cost=8192, parallelism=1).hash(sys.stdin.read()))"), + capture=True, stdin=password).stdout.strip().splitlines()[-1] + psql_admin("UPDATE users SET password_hash = '%s' WHERE username = '%s';\n" % (old_hash, username)) + q = "SELECT split_part(password_hash, '$', 4) FROM users WHERE username = '%s';\n" % username + before = psql_admin(q).stdout.strip() + token = base64.b64encode(("%s:%s" % (username, password)).encode()).decode() + status = req("POST", "/api/v1/auth/sessions", None, {"Authorization": "Basic " + token}) + after = psql_admin(q).stdout.strip() + text = "rehash-demo: user=%s params before=%s -> login HTTP %d -> params after=%s" % (username, before, status, after) + log(text) + save("rehash.txt", text + "\n") + if before == after: + raise SystemExit("hash was not upgraded") + + +def cmd_login_budget(_args): + res = run(compose_cmd("run", "--rm", "-T", "--no-deps", "-v", "%s:/app/student-tests:ro" % (ROOT / "student-tests"), + "api", "python", "student-tests/security/argon2_bench.py"), capture=True, check=False, + timeout=600) + bench = res.stdout or "" + print(bench) + res2 = run([py(), "student-tests/security/login_flood.py", "--base-url", LB_URL, "--ca-file", CA_FILE], + capture=True, check=False, timeout=600) + print(res2.stdout or "", res2.stderr[-1000:] if res2.stderr else "") + save("login_budget.txt", bench + "\n" + (res2.stdout or "")) + + +def cmd_scan(_args): + """Сканирование зависимостей (pip-audit) и образов (trivy). Разбор — docs/security/SCAN_TRIAGE.md.""" + out_dir = EVIDENCE / "scan" + out_dir.mkdir(parents=True, exist_ok=True) + image = "messenger-api:" + setting("APP_VERSION", "lab7") + res = run(["docker", "run", "--rm", "-v", "%s:/req.txt:ro" % (ROOT / "requirements.txt"), + "python:3.12.7-slim-bookworm", "sh", "-c", + "pip install -q pip-audit==2.7.3 >/dev/null 2>&1 && pip-audit -r /req.txt --format columns"], + capture=True, check=False, timeout=900) + (out_dir / "pip-audit.txt").write_text((res.stdout or "") + (res.stderr or ""), encoding="utf-8") + images = [image, "haproxy:2.8.30-alpine", "postgres:16.4-alpine", "redis:7.4.1-alpine", + "openbao/openbao:" + setting("OPENBAO_VERSION", "2.1.0"), + "chrislusf/seaweedfs:" + setting("SEAWEEDFS_VERSION", "4.50")] + for img in images: + res = run(["docker", "run", "--rm", "-v", "/var/run/docker.sock:/var/run/docker.sock:ro", + "aquasec/trivy:0.57.1", "image", "--quiet", "--scanners", "vuln", "--severity", "HIGH,CRITICAL", + "--ignore-unfixed", img], capture=True, check=False, timeout=1800) + name = img.replace("/", "_").replace(":", "_") + (out_dir / ("trivy_%s.txt" % name)).write_text((res.stdout or "") + (res.stderr or ""), encoding="utf-8") + log("scan: %s -> evidence/scan/trivy_%s.txt (exit %d)" % (img, name, res.returncode)) + log("scan: done; triage in docs/security/SCAN_TRIAGE.md") + + +COMMANDS = { + "up": cmd_up, "down": cmd_down, "destroy": cmd_destroy, "wait": cmd_wait, "ps": cmd_ps, + "stop-one": cmd_stop_one, "restore": cmd_restore, "restart": cmd_restart, + "rolling-update": cmd_rolling_update, "in-network": cmd_in_network, "wait-s3": cmd_wait_s3, + "s3-check": cmd_s3_check, "reconcile": cmd_reconcile, "api-fs": cmd_api_fs, + "worker-stop": cmd_worker_stop, "worker-kill": cmd_worker_kill, "worker-start": cmd_worker_start, + "api-restart": cmd_api_restart, "broker-restart": cmd_broker_restart, "jobs": cmd_jobs, + "pipeline-faults": cmd_pipeline_faults, + "tls-audit": cmd_tls_audit, "security-http": cmd_security_http, "leak-check": cmd_leak_check, + "hardening": cmd_hardening, "bao-policy": cmd_bao_policy, "rehash-demo": cmd_rehash_demo, + "rotate-db-password": cmd_rotate_db_password, "rotate-lb-cert": cmd_rotate_lb_cert, + "rotate-secret-id": cmd_rotate_secret_id, "revoke-identity": cmd_revoke_identity, + "bao-sealed": cmd_bao_sealed, "login-budget": cmd_login_budget, "scan": cmd_scan, +} + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS: + print(__doc__) + sys.exit(2) + COMMANDS[sys.argv[1]](sys.argv[2:]) + + +if __name__ == "__main__": + main() diff --git a/scripts/pki.py b/scripts/pki.py new file mode 100644 index 0000000..9f4dd0f --- /dev/null +++ b/scripts/pki.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +"""lab7: локальная PKI учебного стенда (CA, leaf-сертификаты, CRL, ротация, отзыв). + +Запуск (нужен пакет cryptography; он есть в образе приложения): + python scripts/pki.py init # CA + все leaf + пустой CRL (существующие не трогает) + python scripts/pki.py status # serial, срок, SAN всех сертификатов + python scripts/pki.py rotate lb-client # новый leaf (старый -> lb-client.prev.*), ключ тоже новый + python scripts/pki.py revoke lb-client # отозвать ПРЕДЫДУЩИЙ lb-client (.prev) -> crl.pem + python scripts/pki.py renew [--days 7] # перевыпустить всё, что истекает раньше чем через N дней + python scripts/pki.py crl # перевыпустить CRL (nextUpdate +30 дней) + +Без Python-зависимостей на хосте (Windows): + docker compose --profile tools run --rm tools python scripts/pki.py init + +Что где лежит (каталог secrets/ — в .gitignore, в образ не попадает: .dockerignore): + secrets/pki/ca.crt публичный сертификат CA — его можно раздавать (CA_FILE тестов) + secrets/pki/ca.key приватный ключ CA — ТОЛЬКО у оператора, ни в один контейнер не монтируется + secrets/pki/crl.pem список отзыва (читают API и mTLS-gates) + secrets/pki/.crt/.key leaf; .pem = cert + key (формат HAProxy) + secrets/pki/state.json журнал выпусков: serial, срок, кем заменён (без ключей) + +Модель identities: одна identity = один сервис и одна роль. Сервер проверяет не только +цепочку CA, но и SAN/CN клиента по allowlist (API: только lb; broker/collector gate: +только api и worker). Сертификат intruder подписан тем же CA, но не входит ни в один +allowlist — им доказывается, что «доверенный CA» ещё не даёт прав. +""" +import argparse +import datetime as dt +import ipaddress +import json +import os +import shutil +import sys +from pathlib import Path + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + +ROOT = Path(__file__).resolve().parents[1] +DIR = Path(os.environ.get("PKI_DIR", str(ROOT / "secrets" / "pki"))) +CA_DAYS = 3650 +LEAF_DAYS = int(os.environ.get("PKI_LEAF_DAYS", "30")) +CRL_DAYS = 30 +SERVER = "server" +CLIENT = "client" + +# name -> (CN, DNS SAN, IP SAN, роли). CN совпадает с identity, которую проверяют allowlist. +LEAVES = { + # Внешний ingress: hostname/SAN, по которым клиент обращается к стенду. + "lb-public": ("localhost", ["localhost", "lb"], ["127.0.0.1", "::1"], [SERVER]), + # LB как клиент к API-репликам (единственная identity в allowlist API). + "lb-client": ("lb", ["lb"], [], [CLIENT]), + # API: сервер для LB (SAN api) и клиент к broker/collector/redis/postgres. + "api": ("api", ["api", "api-short-ttl"], [], [SERVER, CLIENT]), + # worker: только клиент (broker, collector, postgres). + "worker": ("worker", ["worker"], [], [CLIENT]), + "postgres": ("postgres", ["postgres"], [], [SERVER]), + # redis: сервер + клиент (Docker healthcheck внутри контейнера идёт по mTLS своим cert). + "redis": ("redis", ["redis"], [], [SERVER, CLIENT]), + # broker-gate слушает в сетевом namespace broker -> имя broker. + "broker": ("broker", ["broker"], [], [SERVER]), + "s3": ("s3", ["s3"], [], [SERVER]), + # Collector: сервер (gate, prometheus-экспортёр) и клиент к Loki/Tempo. + "otel-collector": ("otel-collector", ["otel-collector"], [], [SERVER, CLIENT]), + "loki": ("loki", ["loki"], [], [SERVER]), + "tempo": ("tempo", ["tempo"], [], [SERVER]), + # Prometheus: сервер для Grafana и клиент при scrape Collector. + "prometheus": ("prometheus", ["prometheus"], [], [SERVER, CLIENT]), + # Grafana: HTTPS для оператора (localhost) и клиент к data sources. + "grafana": ("grafana", ["grafana", "localhost"], ["127.0.0.1"], [SERVER, CLIENT]), + "openbao": ("openbao", ["openbao"], [], [SERVER]), + # Подписан тем же CA, НЕ входит ни в один allowlist (отрицательные тесты). + "intruder": ("intruder", ["intruder"], [], [SERVER, CLIENT]), +} + + +def now(): + return dt.datetime.now(dt.timezone.utc) + + +def write(path: Path, data: bytes, private: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + # Запись НА МЕСТЕ (тот же inode), а не rename: Docker монтирует secrets как bind-mount + # отдельного файла, и работающий контейнер видит новое содержимое (CRL, ротация leaf) + # без пересоздания; atomic rename оставил бы в контейнере старый inode. + with open(path, "wb") as handle: + handle.write(data) + # Каталог secrets/ — 0700 у оператора. Файлы 0644: внутри контейнеров их читают + # непривилегированные пользователи с другими uid (haproxy, redis, appuser...). + # Исключение: PostgreSQL требует 0600 — entrypoint копирует ключ в tmpfs (infra/postgres). + os.chmod(path, 0o600 if private and os.environ.get("PKI_STRICT_MODE") else 0o644) + + +def load_state() -> dict: + path = DIR / "state.json" + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {"issued": [], "revoked": []} + + +def save_state(state: dict) -> None: + write(DIR / "state.json", json.dumps(state, indent=2, ensure_ascii=False).encode("utf-8")) + + +def new_key(): + return ec.generate_private_key(ec.SECP256R1()) + + +def key_pem(key) -> bytes: + return key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption()) + + +def cert_pem(cert) -> bytes: + return cert.public_bytes(serialization.Encoding.PEM) + + +def load_ca(): + cert = x509.load_pem_x509_certificate((DIR / "ca.crt").read_bytes()) + key = serialization.load_pem_private_key((DIR / "ca.key").read_bytes(), password=None) + return cert, key + + +def init_ca() -> None: + if (DIR / "ca.crt").exists(): + print("keep ca.crt") + return + key = new_key() + name = x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI lab7"), + x509.NameAttribute(NameOID.COMMON_NAME, "Messenger Lab7 Root CA")]) + ski = x509.SubjectKeyIdentifier.from_public_key(key.public_key()) + cert = (x509.CertificateBuilder().subject_name(name).issuer_name(name) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now() - dt.timedelta(minutes=5)) + .not_valid_after(now() + dt.timedelta(days=CA_DAYS)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .add_extension(x509.KeyUsage(digital_signature=True, key_cert_sign=True, crl_sign=True, + content_commitment=False, key_encipherment=False, + data_encipherment=False, key_agreement=False, + encipher_only=False, decipher_only=False), critical=True) + .add_extension(ski, critical=False) + .sign(key, hashes.SHA256())) + write(DIR / "ca.key", key_pem(key), private=True) + write(DIR / "ca.crt", cert_pem(cert)) + print("create ca.crt (serial %x)" % cert.serial_number) + + +def issue(name: str) -> x509.Certificate: + cn, dns, ips, roles = LEAVES[name] + ca_cert, ca_key = load_ca() + key = new_key() + sans = [x509.DNSName(d) for d in dns] + [x509.IPAddress(ipaddress.ip_address(i)) for i in ips] + eku = [] + if SERVER in roles: + eku.append(ExtendedKeyUsageOID.SERVER_AUTH) + if CLIENT in roles: + eku.append(ExtendedKeyUsageOID.CLIENT_AUTH) + cert = (x509.CertificateBuilder() + .subject_name(x509.Name([x509.NameAttribute(NameOID.ORGANIZATION_NAME, "MTUSI lab7"), + x509.NameAttribute(NameOID.COMMON_NAME, cn)])) + .issuer_name(ca_cert.subject).public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now() - dt.timedelta(minutes=5)) + .not_valid_after(now() + dt.timedelta(days=LEAF_DAYS)) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension(x509.KeyUsage(digital_signature=True, key_encipherment=False, key_agreement=True, + content_commitment=False, data_encipherment=False, + key_cert_sign=False, crl_sign=False, encipher_only=False, + decipher_only=False), critical=True) + .add_extension(x509.ExtendedKeyUsage(eku), critical=False) + .add_extension(x509.SubjectAlternativeName(sans), critical=False) + .add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), critical=False) + .sign(ca_key, hashes.SHA256())) + base = DIR / name + # Предыдущая версия сохраняется для отзыва (rotate -> revoke). + if (DIR / (name + ".crt")).exists(): + for ext in (".crt", ".key", ".pem"): + src = DIR / (name + ext) + if src.exists(): + shutil.copyfile(src, DIR / (name + ".prev" + ext)) + write(base.with_suffix(".key"), key_pem(key), private=True) + write(base.with_suffix(".crt"), cert_pem(cert)) + # HAProxy: один файл cert+key. + write(base.with_suffix(".pem"), cert_pem(cert) + key_pem(key), private=True) + state = load_state() + state["issued"].append({"name": name, "serial": "%x" % cert.serial_number, "cn": cn, + "not_after": cert.not_valid_after_utc.isoformat(), "issued_at": now().isoformat()}) + save_state(state) + print("issue %-15s serial %x until %s" % (name, cert.serial_number, cert.not_valid_after_utc.date())) + return cert + + +def write_crl() -> None: + ca_cert, ca_key = load_ca() + state = load_state() + builder = (x509.CertificateRevocationListBuilder().issuer_name(ca_cert.subject) + .last_update(now() - dt.timedelta(minutes=1)) + .next_update(now() + dt.timedelta(days=CRL_DAYS))) + for item in state["revoked"]: + builder = builder.add_revoked_certificate( + x509.RevokedCertificateBuilder().serial_number(int(item["serial"], 16)) + .revocation_date(dt.datetime.fromisoformat(item["revoked_at"])).build()) + crl = builder.sign(ca_key, hashes.SHA256()) + write(DIR / "crl.pem", crl.public_bytes(serialization.Encoding.PEM)) + # HAProxy ожидает CRL рядом с CA в одном файле ca-file не обязательно; отдельный crl-file. + print("crl %d revoked serial(s), next update %s" % (len(state["revoked"]), crl.next_update_utc.date())) + + +def cmd_init(_args) -> None: + DIR.mkdir(parents=True, exist_ok=True) + try: + os.chmod(DIR.parent, 0o700) + except OSError: + pass + init_ca() + for name in LEAVES: + if (DIR / (name + ".crt")).exists(): + print("keep %s" % name) + else: + issue(name) + if not (DIR / "crl.pem").exists(): + write_crl() + + +def cmd_rotate(args) -> None: + for name in args.names: + if name not in LEAVES: + raise SystemExit("unknown identity %s; known: %s" % (name, ", ".join(LEAVES))) + issue(name) + print("next: перезапустить потребителя нового ключа (без пересборки образа), затем revoke") + + +def cmd_revoke(args) -> None: + state = load_state() + for name in args.names: + path = DIR / ((name + ".prev.crt") if not args.current else (name + ".crt")) + if not path.exists(): + raise SystemExit("%s not found (rotate first, or use --current)" % path.name) + cert = x509.load_pem_x509_certificate(path.read_bytes()) + serial = "%x" % cert.serial_number + if any(r["serial"] == serial for r in state["revoked"]): + print("already revoked %s %s" % (name, serial)) + continue + state["revoked"].append({"name": name, "serial": serial, "revoked_at": now().isoformat()}) + print("revoke %s serial %s" % (name, serial)) + save_state(state) + write_crl() + + +def cmd_renew(args) -> None: + limit = now() + dt.timedelta(days=args.days) + for name in LEAVES: + path = DIR / (name + ".crt") + cert = x509.load_pem_x509_certificate(path.read_bytes()) + if cert.not_valid_after_utc < limit: + issue(name) + write_crl() + + +def cmd_status(_args) -> None: + ca = x509.load_pem_x509_certificate((DIR / "ca.crt").read_bytes()) + print("CA %s serial %x until %s" % (ca.subject.rfc4514_string(), ca.serial_number, ca.not_valid_after_utc.date())) + revoked = {r["serial"] for r in load_state()["revoked"]} + for name in LEAVES: + for suffix in ("", ".prev"): + path = DIR / (name + suffix + ".crt") + if not path.exists(): + continue + cert = x509.load_pem_x509_certificate(path.read_bytes()) + san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + serial = "%x" % cert.serial_number + print("%-20s serial %-40s until %s SAN=%s%s" % ( + name + suffix, serial, cert.not_valid_after_utc.date(), + ",".join(str(v.value) for v in san), " REVOKED" if serial in revoked else "")) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("init").set_defaults(fn=cmd_init) + sub.add_parser("status").set_defaults(fn=cmd_status) + sub.add_parser("crl").set_defaults(fn=lambda a: write_crl()) + p = sub.add_parser("rotate") + p.add_argument("names", nargs="+") + p.set_defaults(fn=cmd_rotate) + p = sub.add_parser("revoke") + p.add_argument("names", nargs="+") + p.add_argument("--current", action="store_true", help="отозвать текущий, а не предыдущий сертификат") + p.set_defaults(fn=cmd_revoke) + p = sub.add_parser("renew") + p.add_argument("--days", type=int, default=7) + p.set_defaults(fn=cmd_renew) + args = parser.parse_args() + args.fn(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..030e8b3 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows-вариант restart.sh (lab7): API, worker, PostgreSQL, Redis, broker(+gate), S3, OpenBao(+unseal), LB; volumes сохраняются. +rem python tests\resilience.py --mode persistence --action-script scripts\restart.cmd +setlocal +cd /d "%~dp0.." +set PY=python +where python >nul 2>nul || set PY=py -3 +%PY% scripts\ops.py restart +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..3e8e78a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Hook для make test-persistence (lab7): перезапускает API (все реплики), worker, PostgreSQL, +# Redis, broker (+ пересоздаёт broker-gate в его сетевом namespace), S3 и OpenBao, затем +# распечатывает OpenBao ключами оператора (operator-only/) и поднимает API, worker и LB. +# Volumes сохраняются: никаких down -v / flush / повторного seed / пересоздания бакета. +set -eu +cd "$(dirname "$0")/.." +PY=$(command -v python3 || command -v python) +exec "$PY" scripts/ops.py restart diff --git a/scripts/sql/explain_history.sql b/scripts/sql/explain_history.sql new file mode 100644 index 0000000..d2ee413 --- /dev/null +++ b/scripts/sql/explain_history.sql @@ -0,0 +1,93 @@ +-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на >= 10 000 сообщений. +-- Всё выполняется в ОДНОЙ транзакции и откатывается (ROLLBACK): синтетические +-- данные и временное удаление индекса не остаются в БД. +-- Запуск (compose): +-- docker compose exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 \ +-- < scripts/sql/explain_history.sql > evidence/explain_history.txt +-- Внимание: DROP INDEX внутри транзакции берёт эксклюзивную блокировку messages +-- до ROLLBACK (несколько секунд) — запускать на учебном стенде. + +\set ON_ERROR_STOP 1 +\timing off +SET TimeZone = 'UTC'; +-- Убираем мёртвые версии строк от прошлых запусков (VACUUM нельзя внутри транзакции). +VACUUM (ANALYZE) messages, chat_members, chats, users; +BEGIN; + +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('00000000-0000-4000-8000-00000000e001', 'explain_alice', 'Explain', 'x'), + ('00000000-0000-4000-8000-00000000e002', 'explain_bob', 'Explain', 'x'); + +-- 1 большой чат (12 000 сообщений) + 40 чатов по 1 000 = 52 000 сообщений. +INSERT INTO chats (id, title, created_at) +SELECT CASE WHEN g = 0 THEN '00000000-0000-4000-8000-0000000c0000'::uuid ELSE gen_random_uuid() END, + 'explain-' || g, timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM generate_series(0, 40) AS g; + +INSERT INTO chat_members (chat_id, user_id, position) +SELECT c.id, u.id, u.pos +FROM chats c +CROSS JOIN (VALUES ('00000000-0000-4000-8000-00000000e001'::uuid, 0), + ('00000000-0000-4000-8000-00000000e002'::uuid, 1)) AS u(id, pos) +WHERE c.title LIKE 'explain-%'; + +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), c.id, + CASE WHEN g % 2 = 0 THEN '00000000-0000-4000-8000-00000000e001'::uuid + ELSE '00000000-0000-4000-8000-00000000e002'::uuid END, + 'synthetic message ' || g, + timestamptz '2026-01-02 00:00:00+00' + g * interval '1 millisecond' + (c.created_at - timestamptz '2026-01-01 00:00:00+00') +FROM chats c +CROSS JOIN LATERAL generate_series(1, CASE WHEN c.title = 'explain-0' THEN 12000 ELSE 1000 END) AS g +WHERE c.title LIKE 'explain-%'; + +ANALYZE users, chats, chat_members, messages; + +\echo '=== dataset' +SELECT count(*) AS total_messages, + count(*) FILTER (WHERE chat_id = '00000000-0000-4000-8000-0000000c0000') AS big_chat_messages +FROM messages; + +-- Позиция курсора после 10 000-го сообщения большого чата (как в next_cursor). +SELECT created_at AS cur_ts, id AS cur_id +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== 1. first page (no cursor), LIMIT 51 = limit+1' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id +LIMIT 51; + +\echo '=== 2. deep page: keyset cursor after message #10000 (query used by API)' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' + AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid) +ORDER BY created_at, id +LIMIT 51; + +\echo '=== 3. for comparison: OFFSET pagination to the same page (NOT used by API)' +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' +ORDER BY created_at, id +OFFSET 10000 LIMIT 51; + +\echo '=== 4. same keyset query WITHOUT messages_chat_created_id_idx (index dropped inside the transaction)' +DROP INDEX messages_chat_created_id_idx; +EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '00000000-0000-4000-8000-0000000c0000' + AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid) +ORDER BY created_at, id +LIMIT 51; + +ROLLBACK; +\echo '=== rolled back: synthetic data removed, index restored' diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..4d01493 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,9 @@ +@echo off +rem Windows-вариант stop-one.sh (lab7): TARGET_INSTANCE_ID (alias реплики) -> один контейнер api, остаётся остановленным. +rem python tests\resilience.py --mode failover --action-script scripts\stop-one.cmd +setlocal +cd /d "%~dp0.." +set PY=python +where python >nul 2>nul || set PY=py -3 +%PY% scripts\ops.py stop-one +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..16c935a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Hook для make test-failover (lab7): останавливает ОДНУ API-реплику, чей X-Instance-Id +# (случайный alias, без hostname/IP) тест передал в TARGET_INSTANCE_ID. Alias сопоставляется +# контейнеру через /tmp/instance-id внутри реплики (docker exec). Реплика остаётся +# остановленной; восстановление: python3 scripts/ops.py restore. STOP_MODE=kill — SIGKILL. +set -eu +cd "$(dirname "$0")/.." +PY=$(command -v python3 || command -v python) +exec "$PY" scripts/ops.py stop-one diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..afb92cc 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,63 @@ -# Модель угроз — заполнить +# Модель угроз — мессенджер, lab7 + +Связанные документы: [матрица доверия](../contracts/trust-matrix.md) (протокол и identity каждой связи), [разбор сканирования](../docs/security/SCAN_TRIAGE.md), [REPORT.md](../REPORT.md). Все тесты ниже — воспроизводимые команды `python scripts/ops.py …` (или `run.ps1`), результаты пишутся в `evidence/`. ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +**Активы.** + +| Актив | Где живёт | Чем защищён | +| --- | --- | --- | +| Пароли пользователей | только hash Argon2id (PHC: алгоритм, версия, m/t/p, salt) в `users.password_hash` | Argon2id ≥ 19 MiB/t=2/p=1, rehash при входе, доступ к таблице только у роли API | +| Сессии | токен 256 бит у клиента; в БД — SHA-256 токена + expiry; кэш Redis по ключу `sess:` | TLS снаружи, TTL, отзыв в PostgreSQL, worker таблицу `sessions` не видит | +| Сообщения, чаты, вложения | PostgreSQL, приватный бакет S3 (`staging/`, `final/`) | Авторизация участника чата в API (404 постороннему), раздельные роли БД/S3 | +| Service identities | leaf-сертификаты в `secrets/pki/` (по одному на сервис) | Allowlist CN/SAN на каждом mTLS-входе, CRL, 30 дней | +| CA | `secrets/pki/ca.key` | Только у оператора; не монтируется ни в один контейнер | +| Секреты приложения | OpenBao KV `messenger/api`, `messenger/worker` → tmpfs `/run/bao` | AppRole, политика на роль, токен 15 мин, SecretID только агенту | +| Root/unseal OpenBao | `operator-only/openbao-init.json` (0600) | Вне Git/образов; в контейнеры не передаётся | +| Telemetry | Loki, Tempo, Prometheus (сеть `telemetry`) | Без credentials/тел/токенов (проверено `leak-check`), mTLS, сегментация | + +**Акторы.** (1) Участник чата — легальный пользователь. (2) Посторонний пользователь — имеет свой аккаунт и токен, хочет чужие данные. (3) Внешний клиент без аккаунта / в той же сети, что и хост (перехват, подмена сертификата, перебор паролей, DoS входа). (4) Скомпрометированный сервис внутри стенда (RCE в worker через изображение, утечка ключа/сертификата одной роли). (5) Оператор — доверенный: доступ к Docker хоста = доступ ко всему стенду. + +**Границы учебного стенда.** Один Docker-host; атакующий не имеет доступа к Docker API хоста (иначе он оператор). Сети Docker считаются наблюдаемыми другими контейнерами в той же сети. Физическая безопасность хоста, ядро/рантайм Docker, supply chain базовых образов (кроме сканирования) — вне модели. ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +``` + Интернет/хост ──TLS──► [lb :8443]──mTLS(lb)──►[api ×N]──TLS verify-full──►[postgres] + (127.0.0.1) │ edge+ingress │ ├─mTLS+ACL────────────►[redis] + │ │ ├─TLS, ключ API────────►[s3]◄──TLS, ключ worker──┐ + │ │ ├─mTLS(api)─►[broker-gate]─lo─►[broker]◄─mTLS(worker)─[worker] + │ │ └─mTLS(api)─►[otel-gate]─lo─►[collector]─mTLS─►[loki/tempo] + оператор ──HTTPS+пароль──► [grafana :3000] ─mTLS─► prometheus/loki/tempo ▲mTLS(prometheus) + оператор ──docker exec──► [openbao] ◄──TLS+AppRole── [bao-agent-api/worker] ──tmpfs──► /run/bao (ro) в api/worker +``` -| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | -| --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | +Граница 1 — хост ↔ стенд (только `127.0.0.1:8443` и `:3000`). Граница 2 — LB ↔ реплики (mTLS identity `lb`). Граница 3 — приложение ↔ каждое хранилище (своя identity и права на роль). Граница 4 — секреты (OpenBao, только агенты). Граница 5 — telemetry (отдельные сети ingest/store). -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +## Угрозы, контроли и отрицательные тесты + +| # | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | +| --- | --- | --- | --- | --- | --- | +| T1 | **Перехват/подмена на внешнем канале**: клиент в той же сети получает чужой сертификат (MITM) или шлёт Basic-пароль по HTTP | Конфиденциальность credentials; клиент говорит с настоящим LB | Только TLS 1.2+ на `:8443`, HTTP-порта нет; сертификат LB от CA стенда с SAN `localhost/127.0.0.1`; HSTS; тесты без `--insecure` | `make test-security` (пустой trust store → `SSLCertVerificationError`), `security-http` (чужое имя → ошибка проверки, plaintext не обслуживается), `tls-audit` client→lb | Доверие к CA стенда клиент устанавливает вручную (`CA_FILE`); нет OCSP/CT | +| T2 | **Обход LB напрямую к реплике**: процесс в сети `ingress`/`backend` (или скомпрометированный worker) шлёт запросы прямо на `api:8443` или на health-порт, минуя rate limit и логи LB | Все внешние запросы проходят через LB; `X-Forwarded-For` подделать нельзя | API принимает только mTLS с identity `lb` (allowlist после рукопожатия, `src/tls.py`); health — `127.0.0.1:8081` внутри контейнера; worker не в сети `ingress`; XFF доверяется только из-за mTLS | `tls-audit` lb→api: нет cert, чужой CA с CN=lb, `intruder`, `worker`, `api` → отказ; `api:8081` → refused; `hardening`: из worker `api` → `no-dns` | Компрометация самого LB (ключ `lb-client`) даёт обход до отзыва: ротация + CRL (`rotate-lb-cert`) | +| T3 | **Доверенный CA ≠ право**: ключ одного сервиса (например, `prometheus` или `lb`) используется, чтобы писать в очередь заданий или заливать фальшивую телеметрию | Каждая identity делает только своё | Gates broker и collector: CRL + allowlist CN (`api`,`worker` / `prometheus`); Redis ACL с отдельными пользователями; `default off` | `tls-audit`: `intruder`, `lb`, `prometheus` с ВЕРНЫМ паролем producer → отказ на broker-gate; `prometheus`/`lb` на OTLP → отказ; `api` на `:8889` → отказ | Redis не связывает cert с ACL-пользователем: ключ `api` + пароль `worker_consumer` дали бы права consumer. Пароли ролей разведены политиками OpenBao (api не читает `messenger/worker`) | +| T4 | **Перебор паролей и login DoS**: поток `POST /auth/sessions` по разным репликам (лимит на одной реплике обходится round-robin), каждый запрос стоит 19 MiB и ~24 мс CPU Argon2id | Пароли не подбираются; вход не роняет реплики (OOM) и не вытесняет остальные запросы | Rate limit в PostgreSQL (общий для реплик) ДО KDF: 10/мин на username, 300/мин на IP → 429 + Retry-After; семафор KDF (4/реплику) + таймаут слота 2 с → 503 Retry-After; хеш-«пустышка» для несуществующих username | `security-http`: 12 неверных входов → 429, ответы от ≥2 реплик, верный пароль во время блокировки тоже 429; `login-budget`: поток 150 попыток/с → 136 до KDF, 4397 × 429, `/users/me` без ошибок | Блокировка по username позволяет атакующему на ≤60 с запереть чужой аккаунт; распределённый перебор с многих IP ограничен только лимитом на username | +| T5 | **Утечка credentials через ошибки и telemetry**: пароль из Basic, токен, пароль БД попадают в 401-ответ, stack trace, лог или span | Credentials не покидают процесс | Ответы ошибок — общий JSON без деталей; логи без заголовков/тел; OTel без заголовков и параметров SQL; `attributes/scrub` в Collector; Redis не инструментирован | `leak-check`: канарейки (пароль, неверный пароль, токен) и все рабочие секреты ролей ищутся в Loki, Tempo (100 трасс), Prometheus (все label) — 0 совпадений при положительном контроле; `security-http`: 401/400 без username/пароля/трейсбека | Логи процессов в stdout Docker (`docker logs`) — доступны оператору; проверка по образцам, не полный DLP | +| T6 | **Компрометация worker** (RCE через вредное изображение в Pillow) → чтение `users.password_hash`/`sessions`, подделка `final/` других вложений, чтение секретов API | Worker видит только конвейер вложений | Роль БД `messenger_worker` (нет `users`, `sessions`, `messages`); S3-ключ worker (нет доступа вне своих префиксов, нет admin); ACL broker consumer; политика OpenBao только `messenger/worker`; non-root, read-only, без caps, без egress; обработка изображения в подпроцессе с RLIMIT_AS 1 GiB, timeout 30 с, CPU 1.0/RAM 1280M контейнера; Pillow 13.0 (без известных CVE) | `tls-audit`: worker `SELECT users/sessions` → permission denied, worker PUT вне префиксов → AccessDenied; `bao-policy`: worker не читает `messenger/api`; `hardening`: worker non-root, `1.1.1.1` unreachable, `openbao/loki` → no-dns; `revoke-identity worker` | Worker по роли пишет `final/` всех вложений и `staging/` — может подменить вариант любого вложения; таблица `attachments` общая | +| T7 | **Кража секрета для получения секретов**: SecretID/токен OpenBao утёк из образа, env или логов и используется извне | Секреты получает только легитимный процесс роли и недолго | В образах/приложении нет SecretID и токенов; SecretID только у агента, TTL 7 дней, `secret_id_bound_cidrs` = сеть `secrets`; токен 15 мин/max 1 ч, только в памяти агента; root/unseal только у оператора | `bao-policy`: логин тем же SecretID из контейнера openbao (вне CIDR) отклонён; `rotate-secret-id`: старый SecretID после ротации не работает; `revoke-identity`: после отзыва агент не логинится, восстановление новым SecretID | Внутри сети `secrets` SecretID работает до истечения/ротации; root token хранится рядом со стендом (учебный компромисс) | +| T8 | **Отозванная identity продолжает работать** из-за кэшей/соединений: старый сертификат LB, старый пароль БД, отозванная сессия | Отзыв действует в ограниченное время | CRL перечитывается API ≤ 5 с и проверяется на каждом новом соединении, `no-ssl-reuse` у LB; пароль БД проверяется PostgreSQL на новом соединении, существующие закрываются `pg_terminate_backend`/`max_lifetime`; сессия — `revoked_at` в PG + «надгробие» в Redis | `rotate-lb-cert`: старый cert до отзыва принят, после — отклонён, новый принят; `rotate-db-password`: старый пароль на новом соединении → `password authentication failed`, smoke после разрыва сессий PASS; `security-http`: logout → 401 | Уже установленные соединения LB→API живут до keep-alive (30 с) — проверка отзыва делается новым соединением | +| T9 | **Повышение привилегий/побег из контейнера**: эксплойт в любом сервисе использует root, capabilities, docker.sock или запись в rootfs | Компрометация сервиса не даёт хоста и соседей | Все сервисы `cap_drop: ALL`, `no-new-privileges`, без privileged/host-namespace/docker.sock; почти все non-root и `read_only` с tmpfs; лимиты CPU/RAM/PIDs; сети internal без egress; опубликованы только `127.0.0.1:8443/3000` | `hardening` (docker inspect + docker top: uid, ro, caps, limits, порты, сегментация) | Исключения: SeaweedFS uid 0 (без caps), Grafana с записываемым rootfs, `tempo-init` с CAP_CHOWN (one-shot) — REPORT.md | +| T10 | **Изменение/чтение данных посторонним пользователем**: Eve с собственным токеном перебирает UUID чатов/вложений, подставляет `X-User-Id`, прикрепляет чужой файл к своему чату | Только участник видит чат и вложения; identity = Bearer | Авторизация в API по членству, 404 вместо 403; `X-User-Id` игнорируется; вложение проверяется на принадлежность чату | `security-http`: 5 запросов Eve → 404, прикрепление чужого вложения → 404, `X-User-Id` не меняет `/users/me`, Basic на бизнес-endpoint → 401 | UUID v4 не секрет, но 404 не раскрывает существование | +| T12 | **Обход прав S3 через служебный порт SeaweedFS**: скомпрометированный api/worker в сети `storage` обращается к неаутентифицированному gRPC S3-шлюза (`18333`, в lab6 слушал `0.0.0.0`) и переписывает identities (себе — Admin), либо к master/filer | Права S3 по префиксам нельзя расширить изнутри сети | SeaweedFS привязан к `127.0.0.1` (S3 на 8334, gRPC 18334, master/filer/volume), в сеть выдан только S3 API через `s3-gate` (TCP passthrough, TLS сквозной); admin-ключ только у one-shot `s3-init` | `tls-audit`: `s3:8334`, `18334`, `18333`, `9333`, `8888`, `18888` → refused; `s3-init`: admin-порты закрыты | Найдено при ревизии lab7; другие служебные порты будущих версий SeaweedFS нужно перепроверять тем же аудитом | +| T11 | **Ресурсное истощение телом запроса**: огромный JSON/chunked-тело, 10+ MiB «изображение», декомпрессионная бомба | Реплика не падает по памяти | LB режет тело > 64 KiB по Content-Length (не-upload); API — буфер ≤ 64 KiB и 413 для chunked; upload ≤ 10 MiB, ≤ 40 Мп до распаковки; worker — подпроцесс с RLIMIT_AS | `security-http`: 70 KB JSON → 413 от LB (без X-Instance-Id), chunked → 413 от API, 10 MiB+1 → 413 | Медленные клиенты ограничены таймаутами HAProxy, а не скоростью | ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +* **Bootstrap.** `ops.py up`: `init_secrets.py` (пароли ролей, ключи S3) → `pki.py init` (CA + leaf + CRL; в контейнере `tools` без сети, если на хосте нет `cryptography`) → `bao.py init` (3 shares, порог 2 → `operator-only/`) → `unseal` → `configure` (kv-v2, AppRole, политики, секреты в KV, RoleID/SecretID в `secrets/bao/`) → стенд. Приложение получает только `/run/bao/*` от агента. +* **Expiry/renewal.** Токен агента продлевается агентом; по `token_max_ttl` — новый логин SecretID; SecretID — ротация не реже 7 дней (`rotate-secret-id`). Leaf/CRL — `pki.py renew --days 7` + рестарт потребителей; просроченный CRL = отказ всем клиентам API и gates (fail closed, это видно в логе `mtls.crl_error`). +* **Revoke/rotation.** Сертификат: `pki.py rotate ` → рестарт потребителя → `pki.py revoke ` (CRL). Пароль роли БД: `ops.py rotate-db-password `. AppRole: `bao.py issue-secret-ids` / `revoke-role`. +* **Компрометация identity (runbook).** (1) `pki.py rotate ` и `revoke` (или `bao.py revoke-role `); (2) сменить секреты роли (`rotate-db-password`, новые ключи S3/broker в `secrets/` + `bao.py configure` + рестарт сервиса хранения); (3) проверить `tls-audit` и `leak-check`; (4) для worker — `ops.py jobs failed` и `replay`, задачи сохраняются в PostgreSQL/outbox. +* **Sealed OpenBao.** Чтение секретов приложением не нужно в синхронном пути: api/worker работают и перезапускаются на отрисованных файлах; не доставляются только ротации. Восстановление — `bao.py unseal` оператором (2 из 3 shares); агенты возобновляют работу сами (backoff ≤ 1 мин). Потеря `operator-only/openbao-init.json` = потеря доступа к KV: секреты пересоздаются из `secrets/` новым `init` после `destroy` volume `baodata`. +* **Владелец CA/unseal** — оператор стенда (в отчёте — автор работы). Защита bootstrap-credentials: файлы вне Git/образов (`.gitignore`, `.dockerignore`), каталог `secrets/` 0700, `operator-only/` 0700 + файлы 0600 (на Windows — ACL профиля пользователя). +* **Scan findings** сопоставлены с применимостью в [SCAN_TRIAGE.md](../docs/security/SCAN_TRIAGE.md): применимые уязвимости Python-зависимостей устранены обновлением (pip-audit после — 0), OS-пакеты образов — по отчёту trivy со стенда. diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..5404ef3 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,225 @@ +"""Брокер заданий: Redis Streams с consumer group (lab5). + +Почему это durable at-least-once очередь (подробно — REPORT.md): + +* сервис `broker` — отдельный Redis с AOF и `appendfsync always`: ответ на XADD + приходит только после fsync записи в журнал. Успешный XADD — наш аналог + publisher confirm: сообщение переживёт рестарт/падение брокера с volume; +* `maxmemory-policy noeviction`: поток никогда не вытесняется (при нехватке + памяти XADD получит ошибку, строка outbox останется и будет опубликована позже); +* consumer group: XREADGROUP выдаёт сообщение одному consumer и кладёт его в + PEL (pending entries list) до XACK. Consumer ack — отдельная граница: worker + делает XACK только ПОСЛЕ устойчивого эффекта (COMMIT результата в PostgreSQL); +* упавший consumer: его сообщения остаются в PEL. Тот же consumer после + рестарта контейнера (то же имя) читает свой PEL с id 0; иначе любой живой + consumer забирает их XAUTOCLAIM после JOB_CLAIM_IDLE_SECONDS простоя. + +В сообщении — только JSON по contracts/jobs.schema.json (ID и служебные поля): +ни байтов файла, ни credentials, ни URL, ни filename. +""" +import json +import logging +from typing import Dict, List, Optional, Tuple + +import redis + +from .tls import redis_tls_kwargs + +log = logging.getLogger("messenger.broker") + +FIELD = "job" + + +class BrokerUnavailable(Exception): + """Брокер недоступен/таймаут/ошибка записи.""" + + +class Delivery: + __slots__ = ("message_id", "raw", "deliveries") + + def __init__(self, message_id: str, raw: Optional[str], deliveries: int = 1) -> None: + self.message_id = message_id + self.raw = raw + self.deliveries = deliveries + + def payload(self) -> Optional[dict]: + if self.raw is None: + return None + try: + data = json.loads(self.raw) + except (TypeError, ValueError): + return None + return data if isinstance(data, dict) else None + + +def _s(value) -> str: + return value.decode("utf-8", "replace") if isinstance(value, (bytes, bytearray)) else str(value) + + +class Broker: + def __init__(self, url: str, stream: str, group: str, timeout: float = 2.0, + block_ms: int = 2000) -> None: + self.stream = stream + self.group = group + self.timeout = timeout + self.block_ms = block_ms + common = dict(socket_connect_timeout=timeout, health_check_interval=15, + retry_on_timeout=False, decode_responses=False, + # lab7: rediss:// через broker-gate — mTLS (identity api/worker). + **redis_tls_kwargs("BROKER")) + # Обычные команды: короткий таймаут сокета -> быстрый BrokerUnavailable. + self.client = redis.Redis.from_url(url, socket_timeout=timeout, **common) + # Блокирующий XREADGROUP: таймаут сокета больше времени блокировки. + self.blocking = redis.Redis.from_url(url, socket_timeout=timeout + block_ms / 1000.0, **common) + self._group_ready = False + + def close(self) -> None: + for c in (self.client, self.blocking): + try: + c.close() + except Exception: + pass + + # -------------------------------------------------------------- producer + + def ensure_group(self) -> None: + """Идемпотентно: поток + группа с позиции 0 (сообщения, опубликованные до + создания группы, тоже будут доставлены).""" + if self._group_ready: + return + try: + self.client.xgroup_create(self.stream, self.group, id="0", mkstream=True) + log.info("broker: created stream %s group %s", self.stream, self.group) + except redis.ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise BrokerUnavailable() from exc + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + self._group_ready = True + + def publish(self, payload: Dict) -> str: + """XADD; возврат = подтверждение приёма брокером (AOF fsync always).""" + body = json.dumps(payload, separators=(",", ":"), sort_keys=True) + try: + self.ensure_group() + return _s(self.client.xadd(self.stream, {FIELD: body})) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + # -------------------------------------------------------------- consumer + + @staticmethod + def _parse(entries) -> List[Delivery]: + out = [] + for message_id, fields in entries or []: + if fields is None: # запись удалена из потока, но осталась в PEL + out.append(Delivery(_s(message_id), None)) + continue + raw = fields.get(FIELD.encode()) if isinstance(fields, dict) else None + out.append(Delivery(_s(message_id), _s(raw) if raw is not None else None)) + return out + + def read(self, consumer: str, own_pending: bool = False) -> List[Delivery]: + """Одно сообщение (prefetch = 1 на consumer). own_pending: свой PEL с id 0 — + сообщения, полученные прошлой инкарнацией этого consumer и не подтверждённые.""" + try: + self.ensure_group() + if own_pending: + resp = self.client.xreadgroup(self.group, consumer, {self.stream: "0"}, count=1) + else: + resp = self.blocking.xreadgroup(self.group, consumer, {self.stream: ">"}, + count=1, block=self.block_ms) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + result = [] + for _stream, entries in resp or []: + result.extend(self._parse(entries)) + return result + + def autoclaim(self, consumer: str, min_idle_ms: int) -> List[Delivery]: + """Забрать одно сообщение, которое другой consumer держит дольше min_idle_ms.""" + try: + self.ensure_group() + resp = self.client.xautoclaim(self.stream, self.group, consumer, min_idle_ms, + start_id="0-0", count=1) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + entries = resp[1] if resp and len(resp) > 1 else [] + deliveries = self._parse(entries) + for d in deliveries: + d.deliveries = self.delivery_count(d.message_id) + return deliveries + + def delivery_count(self, message_id: str) -> int: + try: + info = self.client.xpending_range(self.stream, self.group, message_id, message_id, 1) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + return int(info[0]["times_delivered"]) if info else 1 + + def ack(self, message_id: str) -> None: + try: + self.client.xack(self.stream, self.group, message_id) + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + # -------------------------------------------------------- housekeeping + + def trim_acknowledged(self) -> int: + """XTRIM MINID: удалить только записи, которые уже доставлены И подтверждены + (id меньше самого старого pending и не больше last-delivered-id группы). + Недоставленные и неподтверждённые записи никогда не удаляются.""" + try: + groups = self.client.xinfo_groups(self.stream) + if not groups: + return 0 + bounds = [] + for g in groups: + last = _s(g.get("last-delivered-id") or g.get(b"last-delivered-id") or "0-0") + bounds.append(_increment(last)) + pending = self.client.xpending(self.stream, _s(g["name"])) + if pending and pending.get("pending"): + bounds.append(_s(pending["min"])) + minid = min(bounds, key=_id_key) + if minid == "0-0" or minid == "0-1": + return 0 + return int(self.client.xtrim(self.stream, minid=minid, approximate=False) or 0) + except redis.ResponseError: + return 0 + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + def stats(self) -> Dict: + """Длина потока, pending (в обработке/не подтверждено), lag (ещё не выдано).""" + try: + length = int(self.client.xlen(self.stream)) + out = {"stream": self.stream, "length": length, "groups": []} + for g in self.client.xinfo_groups(self.stream): + out["groups"].append({ + "name": _s(g["name"]), + "consumers": int(g["consumers"]), + "pending": int(g["pending"]), + "lag": g.get("lag"), + "last_delivered_id": _s(g["last-delivered-id"]), + }) + return out + except redis.ResponseError: + return {"stream": self.stream, "length": 0, "groups": []} + except redis.RedisError as exc: + raise BrokerUnavailable() from exc + + def ping(self) -> bool: + try: + return bool(self.client.ping()) + except redis.RedisError: + return False + + +def _id_key(message_id: str) -> Tuple[int, int]: + ms, _, seq = message_id.partition("-") + return int(ms), int(seq or 0) + + +def _increment(message_id: str) -> str: + ms, seq = _id_key(message_id) + return "%d-%d" % (ms, seq + 1) diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..48ce8c7 --- /dev/null +++ b/src/config.py @@ -0,0 +1,255 @@ +"""Конфигурация из переменных окружения. + +Секреты (строки подключения с паролями) читаются из файлов *_FILE, +которые Compose монтирует как secrets в /run/secrets/. Для локального +запуска без Docker допускаются и прямые переменные DATABASE_URL / CACHE_URL. +""" +import os +from dataclasses import dataclass +from typing import Optional + + +def _read_secret(name: str) -> Optional[str]: + path = os.environ.get(name + "_FILE") + if path: + with open(path, "r", encoding="utf-8") as handle: + value = handle.read().strip() + return value or None + value = os.environ.get(name, "").strip() + return value or None + + +def _int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return int(raw) + + +def _float(name: str, default: float) -> float: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return float(raw) + + +def _bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +@dataclass(frozen=True) +class Settings: + app_env: str + app_version: str + instance_id: str + database_url: str + cache_url: Optional[str] + session_store: str + session_ttl_seconds: int + session_cache_ttl_seconds: int + db_pool_min: int + db_pool_max: int + db_pool_timeout: float + db_connect_timeout: int + db_statement_timeout_ms: int + cache_timeout_seconds: float + cache_retry_after_seconds: float + migrate_wait_seconds: int + run_migrations_on_start: bool + idempotency_ttl_seconds: int + idempotency_cleanup_interval_seconds: int + ready_db_timeout_seconds: float + drain_seconds: float + graceful_timeout_seconds: int + keep_alive_seconds: int + password_hash_concurrency: int + # lab4: S3 и вложения + s3_endpoint: Optional[str] + s3_region: str + s3_bucket: str + s3_access_key_id: Optional[str] + s3_secret_access_key: Optional[str] + s3_connect_timeout_seconds: float + s3_read_timeout_seconds: float + s3_max_attempts: int + s3_download_deadline_seconds: float + s3_upload_deadline_seconds: float + s3_circuit_open_seconds: float + max_upload_bytes: int + max_image_pixels: int + max_concurrent_uploads: int + upload_queue_timeout_seconds: float + upload_read_timeout_seconds: float + image_decode_concurrency: int + attachment_gc_interval_seconds: int + attachment_gc_grace_seconds: int + orphan_sweep_interval_seconds: int + fault_injection: str + # lab5: брокер, outbox и worker + broker_url: Optional[str] + broker_stream: str + broker_group: str + broker_timeout_seconds: float + pipeline_version: int + worker_concurrency: int + job_max_attempts: int + job_timeout_seconds: int + job_lease_grace_seconds: int + job_claim_idle_seconds: int + job_max_deliveries: int + job_retry_base_seconds: float + job_retry_max_seconds: float + worker_image_timeout_seconds: float + worker_image_memory_mb: int + worker_faults: bool + staging_retention_seconds: int + relay_enabled: bool + relay_poll_interval_seconds: float + relay_fast_path_grace_seconds: float + relay_batch: int + # lab7: безопасность + health_port: int + kdf_queue_timeout_seconds: float + login_rate_window_seconds: int + login_rate_per_user: int + login_rate_per_ip: int + + +# Жёсткие пределы контракта lab4/lab5; конфигурация может их только ужесточить. +CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 +# Grace period сборщика должен быть заметно больше самой долгой загрузки +# (чтение тела + S3 PUT с повтором), см. REPORT.md. +MIN_GC_GRACE_SECONDS = 300 + + +def load_settings() -> Settings: + database_url = _read_secret("DATABASE_URL") + if not database_url: + raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE must be set") + + ttl = _int("SESSION_TTL_SECONDS", 3600) + if ttl < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") + + session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() + # Источник истины для сессий в этой реализации всегда PostgreSQL; + # Redis (CACHE_URL) — только кэш проверок токена. + if session_store != "postgres": + raise RuntimeError("only SESSION_STORE=postgres is supported") + + idem_ttl = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + if idem_ttl < 86400: + # Контракт: повтор в течение 24 часов возвращает тот же результат. + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be >= 86400") + + # X-Instance-Id: по умолчанию hostname контейнера (= короткий ID контейнера + # Docker), стабилен в пределах жизни процесса. INSTANCE_ID — для запуска без Docker. + from .runtime import INSTANCE_ID as instance_id + + max_upload = _int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES) + max_pixels = _int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS) + if not 1 <= max_upload <= CONTRACT_MAX_UPLOAD_BYTES: + raise RuntimeError("MAX_UPLOAD_BYTES must be within 1..10485760") + if not 1 <= max_pixels <= CONTRACT_MAX_IMAGE_PIXELS: + raise RuntimeError("MAX_IMAGE_PIXELS must be within 1..40000000") + gc_grace = _int("ATTACHMENT_GC_GRACE_SECONDS", 900) + # Меньше минимума — только для явных тестов сверки (ALLOW_SHORT_GC_GRACE=true). + if gc_grace < MIN_GC_GRACE_SECONDS and not _bool("ALLOW_SHORT_GC_GRACE", False): + raise RuntimeError("ATTACHMENT_GC_GRACE_SECONDS must be >= %d" % MIN_GC_GRACE_SECONDS) + + job_timeout = _int("JOB_TIMEOUT_SECONDS", 60) + image_timeout = _float("WORKER_IMAGE_TIMEOUT_SECONDS", 30.0) + # Аренда обязана покрывать самую долгую попытку: обработка изображения + + # чтение staging и запись 4 объектов (дедлайны S3) + COMMIT. + if job_timeout < image_timeout + 20: + raise RuntimeError("JOB_TIMEOUT_SECONDS must be >= WORKER_IMAGE_TIMEOUT_SECONDS + 20") + max_attempts = _int("JOB_MAX_ATTEMPTS", 5) + if not 1 <= max_attempts <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS must be within 1..20") + claim_idle = _int("JOB_CLAIM_IDLE_SECONDS", job_timeout + 15) + if claim_idle <= job_timeout: + raise RuntimeError("JOB_CLAIM_IDLE_SECONDS must be > JOB_TIMEOUT_SECONDS") + + return Settings( + app_env=os.environ.get("APP_ENV", "lab"), + app_version=os.environ.get("APP_VERSION", "lab7"), + instance_id=instance_id, + database_url=database_url, + cache_url=_read_secret("CACHE_URL"), + session_store=session_store, + session_ttl_seconds=ttl, + session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60), + db_pool_min=_int("DB_POOL_MIN", 1), + db_pool_max=_int("DB_POOL_MAX", 10), + db_pool_timeout=_float("DB_POOL_TIMEOUT_SECONDS", 3.0), + db_connect_timeout=_int("DB_CONNECT_TIMEOUT_SECONDS", 3), + db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000), + cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.25), + cache_retry_after_seconds=_float("CACHE_RETRY_AFTER_SECONDS", 5.0), + migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60), + # lab3: миграции выполняет отдельный one-shot сервис `migrate`, + # реплики API их не запускают (нет гонки N реплик за схему). + run_migrations_on_start=_bool("RUN_MIGRATIONS_ON_START", False), + idempotency_ttl_seconds=idem_ttl, + idempotency_cleanup_interval_seconds=_int("IDEMPOTENCY_CLEANUP_INTERVAL_SECONDS", 300), + ready_db_timeout_seconds=_float("READY_DB_TIMEOUT_SECONDS", 1.0), + drain_seconds=_float("DRAIN_SECONDS", 3.0), + graceful_timeout_seconds=_int("GRACEFUL_TIMEOUT_SECONDS", 10), + keep_alive_seconds=_int("KEEP_ALIVE_SECONDS", 30), + password_hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 4), + s3_endpoint=os.environ.get("S3_ENDPOINT", "").strip() or None, + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=os.environ.get("S3_BUCKET", "messenger-attachments").strip(), + s3_access_key_id=_read_secret("S3_ACCESS_KEY_ID"), + s3_secret_access_key=_read_secret("S3_SECRET_ACCESS_KEY"), + s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 2.0), + s3_read_timeout_seconds=_float("S3_READ_TIMEOUT_SECONDS", 10.0), + s3_max_attempts=_int("S3_MAX_ATTEMPTS", 2), + # Полный бюджет вызова S3 из запроса (DNS + connect + повторы). Скачивание + # обязано уложиться в timeout server HAProxy (10 с), загрузка — 10 с (PUT 10 MiB локально ~0.1 с). + s3_download_deadline_seconds=_float("S3_DOWNLOAD_DEADLINE_SECONDS", 5.0), + s3_upload_deadline_seconds=_float("S3_UPLOAD_DEADLINE_SECONDS", 10.0), + s3_circuit_open_seconds=_float("S3_CIRCUIT_OPEN_SECONDS", 5.0), + max_upload_bytes=max_upload, + max_image_pixels=max_pixels, + max_concurrent_uploads=max(1, _int("MAX_CONCURRENT_UPLOADS", 4)), + upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0), + upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 60.0), + image_decode_concurrency=max(1, _int("IMAGE_DECODE_CONCURRENCY", 2)), + attachment_gc_interval_seconds=_int("ATTACHMENT_GC_INTERVAL_SECONDS", 300), + attachment_gc_grace_seconds=gc_grace, + orphan_sweep_interval_seconds=_int("ORPHAN_SWEEP_INTERVAL_SECONDS", 3600), + # Только для тестового экземпляра api-faults (профиль faults), см. REPORT.md. + fault_injection=os.environ.get("FAULT_INJECTION", "").strip().lower(), + broker_url=_read_secret("BROKER_URL"), + broker_stream=os.environ.get("BROKER_STREAM", "attachments.jobs").strip(), + broker_group=os.environ.get("BROKER_GROUP", "attachment-workers").strip(), + broker_timeout_seconds=_float("BROKER_TIMEOUT_SECONDS", 2.0), + pipeline_version=max(1, _int("PIPELINE_VERSION", 1)), + worker_concurrency=max(1, _int("WORKER_CONCURRENCY", 2)), + job_max_attempts=max_attempts, + job_timeout_seconds=job_timeout, + job_lease_grace_seconds=_int("JOB_LEASE_GRACE_SECONDS", 10), + job_claim_idle_seconds=claim_idle, + job_max_deliveries=max(2, _int("JOB_MAX_DELIVERIES", 10)), + job_retry_base_seconds=_float("JOB_RETRY_BASE_SECONDS", 2.0), + job_retry_max_seconds=_float("JOB_RETRY_MAX_SECONDS", 60.0), + worker_image_timeout_seconds=image_timeout, + worker_image_memory_mb=_int("WORKER_IMAGE_MEMORY_MB", 1024), + # Только для стенда отказов (compose.faults.yaml): маркеры в filename. + worker_faults=_bool("WORKER_FAULTS", False), + staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400), + relay_enabled=_bool("RELAY_ENABLED", True), + relay_poll_interval_seconds=_float("RELAY_POLL_INTERVAL_SECONDS", 1.0), + relay_fast_path_grace_seconds=_float("RELAY_FAST_PATH_GRACE_SECONDS", 2.0), + relay_batch=max(1, _int("RELAY_BATCH", 100)), + health_port=_int("HEALTH_PORT", 8081), + kdf_queue_timeout_seconds=_float("KDF_QUEUE_TIMEOUT_SECONDS", 2.0), + login_rate_window_seconds=max(1, _int("LOGIN_RATE_WINDOW_SECONDS", 60)), + login_rate_per_user=max(1, _int("LOGIN_RATE_PER_USER", 10)), + login_rate_per_ip=max(1, _int("LOGIN_RATE_PER_IP", 300)), + ) diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..8b62460 --- /dev/null +++ b/src/db.py @@ -0,0 +1,158 @@ +"""Пул соединений PostgreSQL и версионируемые миграции.""" +import logging +import os +import time +from pathlib import Path +from typing import Optional + +import psycopg +from psycopg_pool import ConnectionPool + +from .config import Settings + +log = logging.getLogger("messenger.db") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + +# Произвольная константа: один процесс за раз применяет миграции, +# даже если одновременно стартуют несколько реплик API (lab3). +MIGRATION_LOCK_ID = 2026_0002 + + +def conninfo_kwargs(settings: Settings) -> dict: + return { + "connect_timeout": settings.db_connect_timeout, + # Все timestamptz приходят в UTC; любой запрос ограничен по времени. + "options": "-c TimeZone=UTC -c statement_timeout=%d" + % settings.db_statement_timeout_ms, + "application_name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), + } + + +def _dsn_file() -> Optional[str]: + path = os.environ.get("DATABASE_URL_FILE", "").strip() + return path or None + + +class RotatingConnection(psycopg.Connection): + """lab7: каждое НОВОЕ соединение пула перечитывает DATABASE_URL_FILE. + + Файл отрисовывает OpenBao Agent из KV; после ротации пароля роли (ALTER ROLE + + kv patch, scripts/ops.py rotate-db-password) новые соединения идут с новым паролем + без рестарта процесса и пересборки образа. Открытые соединения PostgreSQL не рвёт; + они закрываются по max_lifetime пула (DB_POOL_MAX_LIFETIME_SECONDS). + """ + + @classmethod + def connect(cls, conninfo: str = "", **kwargs): # type: ignore[override] + path = _dsn_file() + if path: + with open(path, "r", encoding="utf-8") as handle: + current = handle.read().strip() + if current: + conninfo = current + return super().connect(conninfo, **kwargs) + + +def create_pool(settings: Settings) -> ConnectionPool: + pool = ConnectionPool( + conninfo=settings.database_url, + connection_class=RotatingConnection, + max_lifetime=float(os.environ.get("DB_POOL_MAX_LIFETIME_SECONDS", "600")), + kwargs=conninfo_kwargs(settings), + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + # Сколько запрос ждёт свободное соединение, прежде чем получить 503. + timeout=settings.db_pool_timeout, + # Битые после рестарта БД соединения отбрасываются при выдаче из пула. + check=ConnectionPool.check_connection, + max_idle=300, + reconnect_timeout=3600, + open=False, + name="messenger", + ) + # wait=False: процесс поднимается, даже если БД ещё/уже недоступна; + # readiness это покажет, а запросы получат 503. + pool.open(wait=False) + return pool + + +def _migration_files(): + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + if not files: + raise RuntimeError("no migrations found in %s" % MIGRATIONS_DIR) + return files + + +def required_schema_version() -> str: + """Последняя миграция, которую знает этот образ. + + Реплика готова (readiness) только если она уже применена: миграции + запускает отдельный сервис `migrate`, а не сама реплика. + """ + return _migration_files()[-1].stem + + +def schema_has_version(pool: ConnectionPool, version: str, timeout: float) -> bool: + with pool.connection(timeout=timeout) as conn: + row = conn.execute( + "SELECT to_regclass('schema_migrations') IS NOT NULL" + ).fetchone() + if not row[0]: + return False + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = %s)", + (version,), + ).fetchone()[0] + + +def apply_migrations(settings: Settings) -> list: + """Применяет недостающие миграции; возвращает список применённых версий.""" + applied_now = [] + with psycopg.connect( + settings.database_url, autocommit=True, **conninfo_kwargs(settings) + ) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + done = { + row[0] + for row in conn.execute("SELECT version FROM schema_migrations") + } + for path in _migration_files(): + version = path.stem + if version in done: + continue + sql = path.read_text(encoding="utf-8") + # DDL и отметка о версии — одна транзакция: либо всё, либо ничего. + with conn.transaction(): + conn.execute(sql) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + applied_now.append(version) + log.info("applied migration %s", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,)) + return applied_now + + +def wait_and_migrate(settings: Settings) -> list: + deadline = time.monotonic() + settings.migrate_wait_seconds + last_error: Optional[Exception] = None + while time.monotonic() < deadline: + try: + return apply_migrations(settings) + except psycopg.OperationalError as exc: + last_error = exc + log.warning("database not ready for migrations, retrying") + time.sleep(1) + raise RuntimeError("database unavailable for migrations") from last_error diff --git a/src/health.py b/src/health.py new file mode 100644 index 0000000..34624d5 --- /dev/null +++ b/src/health.py @@ -0,0 +1,51 @@ +"""lab7: минимальный health listener только для Docker healthcheck. + +Слушает 127.0.0.1:HEALTH_PORT ВНУТРИ контейнера (loopback, наружу недоступен даже +из сети compose), обслуживает только GET /health/live и /health/ready и больше ничего: +бизнес-API через него недоступен, заголовки/тела не читаются. LB проверяет реплики +по основному порту через mTLS, этот listener нужен лишь Docker (без клиентского cert). +""" +import json +import logging +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable, Optional + +log = logging.getLogger("messenger.health") + + +def start_health_listener(check_ready: Callable[[], None], port: int) -> Optional[ThreadingHTTPServer]: + if port <= 0: + return None + + class Handler(BaseHTTPRequestHandler): + server_version = "health" + sys_version = "" + + def _send(self, status: int, payload: dict) -> None: + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/health/live": + return self._send(200, {"status": "ok"}) + if self.path == "/health/ready": + try: + check_ready() + return self._send(200, {"status": "ok"}) + except Exception: + return self._send(503, {"status": "unavailable"}) + return self._send(404, {"status": "not_found"}) + + def log_message(self, *args) -> None: # тишина: пробы каждые 3 с + return + + server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, name="health-listener", daemon=True).start() + log.info("health listener on 127.0.0.1:%d (loopback only, /health/* only)", port) + return server diff --git a/src/images.py b/src/images.py new file mode 100644 index 0000000..6503e97 --- /dev/null +++ b/src/images.py @@ -0,0 +1,285 @@ +"""Проверка загружаемых изображений по содержимому, с ограниченной памятью. + +Порядок для каждого файла (лимит байт уже применён при чтении тела): +1. Сигнатура: PNG (\\x89PNG\\r\\n\\x1a\\n) или JPEG (FF D8 FF). Иное -> 415. +2. Сигнатура должна совпасть с заявленным Content-Type (подмена MIME -> 415). +3. Размеры читаются из заголовка (PNG IHDR / JPEG SOFn) ДО распаковки пикселей; + width*height > MAX_IMAGE_PIXELS -> 413. Распаковка не начинается. +4. Целостность без полного декодирования в память: + * PNG: обход всех чанков с проверкой CRC, порядка IHDR..IDAT..IEND и + потоковая распаковка zlib с выводом кусками по 64 KiB «в никуда»: + объём распакованных данных должен в точности равняться ожидаемому + (height*(1+rowbytes), с учётом Adam7). Память — O(64 KiB) при любом + размере картинки; «zip-бомба» обрывается, как только вывод превысил ожидание. + * JPEG: обход маркеров (длины сегментов, SOF до SOS, обязательный EOI). + Baseline дополнительно декодируется Pillow в режиме draft (масштаб 1/8, + ~5 MiB даже для 40 Мп). Progressive Pillow не декодируется: libjpeg держит + буфер коэффициентов всего кадра (~120 MiB на 40 Мп, замер в REPORT.md), + для него — только структурная проверка. +5. Pillow открывает файл независимым парсером: формат и размеры должны совпасть. +""" +import io +import struct +import zlib +from typing import Tuple + +from PIL import Image, ImageFile + +# Свою проверку пикселей делаем сами (шаг 3) — до вызова Pillow. +Image.MAX_IMAGE_PIXELS = None +ImageFile.LOAD_TRUNCATED_IMAGES = False + +PNG_SIG = b"\x89PNG\r\n\x1a\n" +JPEG_SIG = b"\xff\xd8\xff" +INFLATE_CHUNK = 64 * 1024 + +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +# SOF0..SOF15 без DHT(C4), JPG(C8), DAC(CC) +SOF_MARKERS = set(range(0xC0, 0xD0)) - {0xC4, 0xC8, 0xCC} +PROGRESSIVE_SOF = {0xC2, 0xC6, 0xCA, 0xCE} + + +class ImageRejected(Exception): + def __init__(self, status: int, code: str, message: str) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def unsupported(message: str = "Unsupported or corrupted image") -> ImageRejected: + return ImageRejected(415, "unsupported_media_type", message) + + +def too_many_pixels() -> ImageRejected: + return ImageRejected(413, "payload_too_large", "Image has too many pixels") + + +def sniff(data) -> str: + head = bytes(data[:8]) + if head.startswith(PNG_SIG): + return "image/png" + if head.startswith(JPEG_SIG): + return "image/jpeg" + return "" + + +# ------------------------------------------------------------------- PNG + +def _png_expected_raw(width: int, height: int, depth: int, color: int, interlace: int) -> int: + bpp = depth * PNG_CHANNELS[color] + + def rows(w: int, h: int) -> int: + if w == 0 or h == 0: + return 0 + return h * (1 + (w * bpp + 7) // 8) + + if interlace == 0: + return rows(width, height) + total = 0 + for x0, y0, dx, dy in ADAM7: + pw = (width - x0 + dx - 1) // dx if width > x0 else 0 + ph = (height - y0 + dy - 1) // dy if height > y0 else 0 + total += rows(pw, ph) + return total + + +def inspect_png(data, max_pixels: int) -> Tuple[int, int, bool]: + view = memoryview(data) + size = len(view) + pos = 8 + width = height = 0 + expected = 0 + inflater = None + produced = 0 + seen_ihdr = seen_plte = idat_done = False + idat_started = False + color = 0 + + while True: + if pos + 12 > size: + raise unsupported("Truncated PNG") + length, ctype = struct.unpack(">I4s", view[pos:pos + 8]) + if length > 0x7FFFFFFF or pos + 12 + length > size: + raise unsupported("Truncated PNG") + if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype): + raise unsupported("Invalid PNG chunk") + body = view[pos + 8:pos + 8 + length] + (crc,) = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length]) + if zlib.crc32(body, zlib.crc32(ctype)) & 0xFFFFFFFF != crc: + raise unsupported("PNG CRC mismatch") + pos += 12 + length + + if not seen_ihdr: + if ctype != b"IHDR" or length != 13: + raise unsupported("PNG must start with IHDR") + width, height, depth, color, comp, flt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF): + raise unsupported("Invalid PNG dimensions") + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or comp or flt or interlace > 1: + raise unsupported("Invalid PNG header") + # Лимит пикселей — до любой распаковки. + if width * height > max_pixels: + raise too_many_pixels() + expected = _png_expected_raw(width, height, depth, color, interlace) + seen_ihdr = True + continue + + if ctype == b"IHDR": + raise unsupported("Duplicate IHDR") + if ctype == b"PLTE": + if idat_started: + raise unsupported("PLTE after IDAT") + seen_plte = True + elif ctype == b"IDAT": + if idat_done: + raise unsupported("Non-contiguous IDAT") + if color == 3 and not seen_plte: + raise unsupported("Missing PLTE") + if not idat_started: + idat_started = True + inflater = zlib.decompressobj() + produced += _inflate(inflater, body, expected - produced) + elif ctype == b"IEND": + if length != 0 or not idat_started: + raise unsupported("Invalid PNG end") + produced += _inflate(inflater, b"", expected - produced, final=True) + if not inflater.eof or produced != expected: + raise unsupported("Corrupted PNG image data") + # Данные после IEND не используются декодерами; допускаем. + return width, height, bool(interlace) + else: + if idat_started: + idat_done = True + if ctype[0] & 0x20 == 0: + # Неизвестный КРИТИЧЕСКИЙ чанк (заглавная первая буква): декодер обязан + # отказаться от такого файла. Вспомогательные (строчная буква) пропускаем. + raise unsupported("Unknown critical PNG chunk") + + +def _inflate(inflater, data, budget: int, final: bool = False) -> int: + """Распаковать data, не держа в памяти больше ~2 x INFLATE_CHUNK; вернуть объём вывода. + + Вход подаётся срезами memoryview по INFLATE_CHUNK: unconsumed_tail — копия + остатка входа, поэтому подача всего IDAT целиком давала бы копию размером с + файл на каждом шаге (замер: пик 2x размера файла и квадратичное копирование). + """ + produced = 0 + view = memoryview(data) + try: + for start in range(0, len(view), INFLATE_CHUNK): + buf = view[start:start + INFLATE_CHUNK] + while True: + out = inflater.decompress(buf, INFLATE_CHUNK) + produced += len(out) + if produced > budget: + raise unsupported("Corrupted PNG image data") + buf = inflater.unconsumed_tail + if not buf: + break + if final: + while True: + out = inflater.flush(INFLATE_CHUNK) if not inflater.eof else b"" + produced += len(out) + if produced > budget: + raise unsupported("Corrupted PNG image data") + if not out: + break + except zlib.error: + raise unsupported("Corrupted PNG image data") + return produced + + +# ------------------------------------------------------------------ JPEG + +def inspect_jpeg(data, max_pixels: int) -> Tuple[int, int, bool]: + raw = data if isinstance(data, (bytes, bytearray)) else bytes(data) + size = len(raw) + pos = 2 + width = height = 0 + progressive = False + seen_sof = False + while True: + # Маркер: один или несколько 0xFF, затем код. + if pos >= size or raw[pos] != 0xFF: + raise unsupported("Corrupted JPEG structure") + while pos < size and raw[pos] == 0xFF: + pos += 1 + if pos >= size: + raise unsupported("Truncated JPEG") + marker = raw[pos] + pos += 1 + if marker == 0xD9: # EOI + if not seen_sof: + raise unsupported("JPEG without frame") + return width, height, progressive + if marker == 0x01 or 0xD0 <= marker <= 0xD7: + continue + if marker == 0xD8 or marker == 0x00: + raise unsupported("Corrupted JPEG structure") + if pos + 2 > size: + raise unsupported("Truncated JPEG") + (length,) = struct.unpack(">H", raw[pos:pos + 2]) + if length < 2 or pos + length > size: + raise unsupported("Truncated JPEG") + segment = raw[pos + 2:pos + length] + pos += length + if marker in SOF_MARKERS: + if seen_sof or len(segment) < 6: + raise unsupported("Invalid JPEG frame") + _precision, height, width, ncomp = struct.unpack(">BHHB", segment[:6]) + if width == 0 or height == 0 or ncomp not in (1, 3, 4) or len(segment) < 6 + 3 * ncomp: + raise unsupported("Invalid JPEG frame") + if width * height > max_pixels: + raise too_many_pixels() + progressive = marker in PROGRESSIVE_SOF + seen_sof = True + elif marker == 0xDA: # SOS: дальше энтропийные данные до следующего маркера + if not seen_sof: + raise unsupported("JPEG scan before frame") + while True: + nxt = raw.find(b"\xff", pos) + if nxt < 0 or nxt + 1 >= size: + raise unsupported("Truncated JPEG") + code = raw[nxt + 1] + if code == 0x00 or 0xD0 <= code <= 0xD7 or code == 0xFF: + pos = nxt + 1 if code == 0xFF else nxt + 2 + continue + pos = nxt + break + + +# -------------------------------------------------------------- common + +def inspect(data, declared_type: str, max_pixels: int) -> Tuple[str, int, int]: + """Вернуть (content_type, width, height) или бросить ImageRejected.""" + detected = sniff(data) + if not detected: + raise unsupported("Only PNG and JPEG images are accepted") + if detected != declared_type: + raise unsupported("Content-Type does not match image content") + if detected == "image/png": + width, height, _ = inspect_png(data, max_pixels) + pil_format = "PNG" + decode = False # целостность IDAT уже проверена потоковой распаковкой + else: + width, height, progressive = inspect_jpeg(data, max_pixels) + pil_format = "JPEG" + decode = not progressive + # Независимый парсер Pillow: только заголовок (open ленивый), затем при + # необходимости декодирование уменьшенной копии. + try: + # lab7: только плагины PNG/JPEG — уязвимые декодеры других форматов недостижимы. + with Image.open(io.BytesIO(data), formats=("PNG", "JPEG")) as img: + if img.format != pil_format or img.size != (width, height): + raise unsupported("Inconsistent image header") + if decode: + img.draft(None, (max(1, width // 8), max(1, height // 8))) + img.load() + except ImageRejected: + raise + except (OSError, SyntaxError, ValueError, struct.error, Image.DecompressionBombError, EOFError): + raise unsupported("Corrupted image") + return detected, width, height diff --git a/src/imaging.py b/src/imaging.py new file mode 100644 index 0000000..b84be7a --- /dev/null +++ b/src/imaging.py @@ -0,0 +1,274 @@ +"""Варианты изображения (lab5) и их изолированное вычисление. + +Варианты pipeline_version=1 (параметры фиксированы, вывод детерминирован для +одной версии Pillow/libjpeg/libwebp, поэтому повтор той же job пишет те же байты): + + thumbnail JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, + размеры округляются вниз (минимум 1 px): 320x200 -> 256x160. + quality=85, optimize, baseline, 4:2:0. + crop JPEG 128x128: центрированный квадрат min(w,h) оригинала, масштаб + до 128 (маленькие исходники увеличиваются). quality=85. + optimized WebP, вписан в 1280x1280 без увеличения, quality=80, method=4. + Уменьшение размера каждого маленького файла НЕ обещается. + +Для всех вариантов: EXIF orientation применяется до вычисления размеров; +прозрачность (RGBA/LA/P с tRNS) компонуется на белый фон; EXIF и XMP НЕ +копируются (Pillow сохраняет их только при явной передаче exif=/xmp=), ICC-профиль +сохраняется, чтобы не исказить цвет широкого охвата. + +Ограничения ресурсов worker (недоверенный файл, даже прошедший проверки API): + * bytes: staging-объект больше MAX_UPLOAD_BYTES не читается (worker.py); + * pixels: размеры читаются из заголовка до декодирования; > MAX_IMAGE_PIXELS -> отказ; + * память: обработка идёт в отдельном процессе с RLIMIT_AS = WORKER_IMAGE_MEMORY_MB; + MemoryError/убийство процесса -> постоянная ошибка resource_limit; + * время: процесс убивается через WORKER_IMAGE_TIMEOUT_SECONDS (временная ошибка, + повтор ограничен JOB_MAX_ATTEMPTS); RLIMIT_CPU — страховка. + Процессы создаёт forkserver: чистый однопоточный родитель, без унаследованных + соединений PostgreSQL/Redis и потоков worker. +""" +import hashlib +import io +import multiprocessing +import os +import resource +import signal +from typing import Dict, Tuple + +from PIL import Image, ImageFile, ImageOps + +ImageFile.LOAD_TRUNCATED_IMAGES = False + +THUMB_BOX = 256 +CROP_SIDE = 128 +OPTIMIZED_BOX = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +VARIANT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} +VARIANT_EXT = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + +# EXIF orientation 5..8 поворачивает изображение на 90°: ширина и высота меняются. +SWAPPING_ORIENTATIONS = {5, 6, 7, 8} + + +class ImageFailure(Exception): + """Постоянная ошибка: повтор того же файла даст тот же результат.""" + + def __init__(self, code: str, message: str = "") -> None: + super().__init__(message or code) + self.code = code + + +class ImageTimeout(Exception): + """Обработка не уложилась в WORKER_IMAGE_TIMEOUT_SECONDS (временная ошибка).""" + + +def fit_within(width: int, height: int, box: int) -> Tuple[int, int]: + """Вписать в box x box без увеличения; целочисленно, округление вниз, минимум 1.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def _orientation(img: Image.Image) -> int: + try: + return int(img.getexif().get(0x0112, 1) or 1) + except Exception: + return 1 + + +def _has_alpha(img: Image.Image) -> bool: + return img.mode in ("RGBA", "LA", "PA") or "transparency" in img.info + + +def _working_mode(img: Image.Image) -> Image.Image: + """RGB или RGBA без лишних копий: уже подходящий режим не конвертируется + (convert в тот же режим всё равно копирует кадр — на 40 Мп это +160 MiB).""" + if img.mode in ("RGB", "RGBA") and not (img.mode == "RGB" and "transparency" in img.info): + return img + return img.convert("RGBA" if _has_alpha(img) else "RGB") + + +def _flatten(img: Image.Image) -> Image.Image: + """Уже УМЕНЬШЕННЫЙ вариант -> RGB: альфа компонуется на белый фон. + Масштабирование RGBA в Pillow идёт с premultiplied alpha, ореолов нет.""" + if img.mode != "RGBA": + return img + background = Image.new("RGB", img.size, (255, 255, 255)) + background.paste(img, mask=img.getchannel("A")) + return background + + +def _encode(img: Image.Image, fmt: str, icc) -> bytes: + out = io.BytesIO() + extra = {"icc_profile": icc} if icc else {} + if fmt == "JPEG": + img.save(out, "JPEG", quality=JPEG_QUALITY, optimize=True, progressive=False, + subsampling="4:2:0", **extra) + else: + img.save(out, "WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, lossless=False, **extra) + return out.getvalue() + + +def render(data: bytes, max_pixels: int) -> Dict[str, Tuple[bytes, int, int, str]]: + """{name: (bytes, width, height, content_type)} для thumbnail, crop, optimized.""" + Image.MAX_IMAGE_PIXELS = None # свой лимит ниже, до декодирования + try: + img = Image.open(io.BytesIO(data), formats=("PNG", "JPEG")) # lab7: только PNG/JPEG-плагины + if img.format not in ("PNG", "JPEG"): + raise ImageFailure("invalid_image", "unsupported format") + raw_w, raw_h = img.size + if raw_w < 1 or raw_h < 1 or raw_w * raw_h > max_pixels: + raise ImageFailure("invalid_image", "too many pixels") + orientation = _orientation(img) + icc = img.info.get("icc_profile") + # Целевые размеры считаются по ОРИЕНТИРОВАННОМУ оригиналу. + w, h = (raw_h, raw_w) if orientation in SWAPPING_ORIENTATIONS else (raw_w, raw_h) + thumb_size = fit_within(w, h, THUMB_BOX) + opt_size = fit_within(w, h, OPTIMIZED_BOX) + if img.format == "JPEG": + # Декодирование сразу в уменьшенном масштабе (1/2..1/8): память и время + # пропорциональны нужному размеру. draft выбирает масштаб >= запрошенного. + want = (opt_size[1], opt_size[0]) if orientation in SWAPPING_ORIENTATIONS else opt_size + img.draft("RGB", want) + img.load() + img = ImageOps.exif_transpose(img) # копия только если orientation != 1 + base = _working_mode(img) + del img + bw, bh = base.size + + result: Dict[str, Tuple[bytes, int, int, str]] = {} + # Сначала масштабирование (reducing_gap: быстрый целочисленный reduce, затем + # LANCZOS), потом компоновка альфы на белый — уже на маленьком кадре. + thumb = _flatten(base.resize(thumb_size, Image.LANCZOS, reducing_gap=3.0) + if base.size != thumb_size else base) + result["thumbnail"] = (_encode(thumb, "JPEG", icc),) + thumb.size + ("image/jpeg",) + del thumb + + side = min(bw, bh) + left, top = (bw - side) // 2, (bh - side) // 2 + crop = _flatten(base.resize((CROP_SIDE, CROP_SIDE), Image.LANCZOS, box=(left, top, left + side, top + side), + reducing_gap=3.0 if side > CROP_SIDE else None)) + result["crop"] = (_encode(crop, "JPEG", icc),) + crop.size + ("image/jpeg",) + del crop + + opt = _flatten(base.resize(opt_size, Image.LANCZOS, reducing_gap=3.0) if base.size != opt_size else base) + result["optimized"] = (_encode(opt, "WEBP", icc),) + opt.size + ("image/webp",) + return result + except ImageFailure: + raise + except MemoryError: + raise ImageFailure("resource_limit", "memory limit exceeded") + except (OSError, SyntaxError, ValueError, EOFError, Image.DecompressionBombError) as exc: + raise ImageFailure("invalid_image", type(exc).__name__) + + +def sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +# ---------------------------------------------------------------- sandbox + +def _child(conn, memory_mb: int, cpu_seconds: int, max_pixels: int) -> None: + """Тело дочернего процесса: лимиты -> получить байты -> вычислить -> отправить.""" + try: + limit = memory_mb * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1)) + except (ValueError, OSError): + pass + signal.signal(signal.SIGTERM, signal.SIG_DFL) + try: + data = conn.recv_bytes() + result = render(data, max_pixels) + del data + peak_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024.0 + conn.send(("ok", (result, peak_mb))) + except ImageFailure as exc: + conn.send(("fail", exc.code)) + except MemoryError: + try: + conn.send(("fail", "resource_limit")) + except Exception: + pass + except Exception as exc: # неожиданное — тоже постоянная ошибка обработки + try: + conn.send(("fail", "processing_failed:" + type(exc).__name__)) + except Exception: + pass + finally: + conn.close() + + +class Sandbox: + def __init__(self, memory_mb: int, timeout: float, max_pixels: int) -> None: + self.memory_mb = memory_mb + self.timeout = timeout + self.max_pixels = max_pixels + self.last_peak_mb = 0.0 + method = "forkserver" if "forkserver" in multiprocessing.get_all_start_methods() else "spawn" + self.ctx = multiprocessing.get_context(method) + if method == "forkserver": + self.ctx.set_forkserver_preload(["src.imaging"]) + + def render(self, data: bytes) -> Dict[str, Tuple[bytes, int, int, str]]: + parent, child = self.ctx.Pipe(duplex=True) + proc = self.ctx.Process( + target=_child, + args=(child, self.memory_mb, int(self.timeout) + 5, self.max_pixels), + daemon=True, + ) + proc.start() + child.close() + try: + parent.send_bytes(data) + if not parent.poll(self.timeout): + raise ImageTimeout() + try: + status, value = parent.recv() + except (EOFError, OSError): + # Процесс умер без ответа: OOM/RLIMIT (SIGKILL/SIGSEGV при нехватке памяти). + proc.join(1) + raise ImageFailure("resource_limit", "worker subprocess died (exit %s)" % proc.exitcode) + if status == "ok": + result, self.last_peak_mb = value + return result + code = str(value) + if code.startswith("processing_failed"): + raise ImageFailure("processing_failed", code) + raise ImageFailure(code) + finally: + parent.close() + if proc.is_alive(): + proc.kill() + proc.join(5) + + +def main() -> None: + """Самопроверка: python -m src.imaging FILE [--memory-mb N] [--timeout S].""" + import argparse + import json + parser = argparse.ArgumentParser() + parser.add_argument("file") + parser.add_argument("--memory-mb", type=int, default=1024) + parser.add_argument("--timeout", type=float, default=30) + parser.add_argument("--max-pixels", type=int, default=40_000_000) + args = parser.parse_args() + with open(args.file, "rb") as f: + data = f.read() + try: + box = Sandbox(args.memory_mb, args.timeout, args.max_pixels) + out = box.render(data) + print(json.dumps({**{k: {"bytes": len(v[0]), "width": v[1], "height": v[2], "type": v[3]} + for k, v in out.items()}, "peak_rss_mb": round(box.last_peak_mb, 1)})) + except ImageFailure as exc: + print(json.dumps({"failed": exc.code})) + except ImageTimeout: + print(json.dumps({"failed": "timeout"})) + + +if __name__ == "__main__": + main() diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..9ced989 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,473 @@ +"""Переходы состояний задания обработки вложения (lab5). Только PostgreSQL. + +Таблица переходов (contracts/attachment-states.md) и где она реализована: + + нет -> queued Storage.accept_attachment (API, одна транзакция с outbox) + queued -> processing claim(): аренда = lease_token + lease_owner + lease_until + processing -> ready complete(): fenced по lease_token; варианты + state одной транзакцией + processing -> queued retry(): временная ошибка, attempt+1, backoff+jitter через outbox + processing -> failed fail(): постоянная ошибка / попытки исчерпаны; запись в dead_letters + processing -> queued reap_expired(): аренда истекла (worker умер/завис) — через outbox + failed -> queued replay(): явная операторская команда, аудит в job_events + +Fencing: любая запись результата worker'а выполняется условием +`lease_token = <мой токен> AND status = 'processing'`. Worker, у которого аренду +уже отобрали (истекла и задание перезапущено), не может ни опубликовать результат, +ни перевести задание в retry/failed — его транзакция ничего не изменит. + +Ключ идемпотентности эффекта — (attachment_id, pipeline_version, variant): +PRIMARY KEY attachment_variants + детерминированные ключи объектов S3. +""" +import json +import random +import uuid +from datetime import datetime, timezone +from typing import Dict, List, Optional, Tuple +from uuid import UUID + +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from . import telemetry + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 + + +def iso_now() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def job_payload(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int) -> Dict: + """Сообщение брокера по contracts/jobs.schema.json: только ID и служебные поля. + + lab6: + W3C traceparent/tracestate ТЕКУЩЕГО span (HTTP upload в API, consumer + span при retry в worker). Строка outbox хранит контекст в PostgreSQL, поэтому + связь upload -> publish -> process сохраняется, даже если публикует sweeper + другого процесса через минуту. Без активного span (reaper) полей нет. + """ + return telemetry.inject({ + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": iso_now(), + }) + + +def validate_payload(data: Optional[Dict]) -> Optional[Dict]: + """Строгая проверка входящего сообщения; None — сообщение отравлено (poison).""" + if not isinstance(data, dict): + return None + allowed = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", + "attempt", "created_at", "traceparent", "tracestate"} + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if not required <= set(data) or not set(data) <= allowed: + return None + try: + if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE: + return None + if type(data["attempt"]) is not int or data["attempt"] < 1: + return None + if type(data["pipeline_version"]) is not int or data["pipeline_version"] < 1: + return None + return { + "job_id": UUID(str(data["job_id"])), + "attachment_id": UUID(str(data["attachment_id"])), + "pipeline_version": data["pipeline_version"], + "attempt": data["attempt"], + } + except (ValueError, TypeError, AttributeError): + return None + + +def backoff_seconds(attempt: int, base: float, cap: float, rng=random) -> float: + """Экспоненциальный backoff с «equal jitter»: d = min(cap, base*2^(n-1)), + задержка равномерно в [d/2, d]. Нижняя граница d/2 гарантирует паузу, + случайная часть разносит повторы многих заданий во времени.""" + d = min(cap, base * (2 ** max(0, attempt - 1))) + return d / 2 + rng.uniform(0, d / 2) + + +class Claim: + __slots__ = ("job_id", "attachment_id", "attempt", "max_attempts", "replay_count", + "pipeline_version", "lease_token", "takeover") + + def __init__(self, row: Dict, takeover: bool) -> None: + self.job_id = row["job_id"] + self.attachment_id = row["attachment_id"] + self.attempt = row["attempt"] + self.max_attempts = row["max_attempts"] + self.replay_count = row["replay_count"] + self.pipeline_version = row["pipeline_version"] + self.lease_token = row["lease_token"] + self.takeover = takeover + + +class JobStore: + def __init__(self, pool: ConnectionPool, settings) -> None: + self.pool = pool + self.settings = settings + + # ------------------------------------------------------------ helpers + + @staticmethod + def _event(conn, job_id, attachment_id, attempt, event, detail=None, actor=None) -> None: + conn.execute( + "INSERT INTO job_events (job_id, attachment_id, attempt, event, detail, actor) " + "VALUES (%s, %s, %s, %s, %s, %s)", + (job_id, attachment_id, attempt, event, detail, actor), + ) + + def event(self, job_id, attachment_id, attempt, event, detail=None, actor=None) -> None: + with self.pool.connection() as conn: + self._event(conn, job_id, attachment_id, attempt, event, detail, actor) + + def has_event(self, job_id, event: str) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM job_events WHERE job_id = %s AND event = %s)", + (job_id, event), + ).fetchone()[0] + + # ------------------------------------------------------------ claim + + def claim(self, msg: Dict, owner: str) -> Tuple[Optional[Claim], str]: + """queued -> processing. Возвращает (Claim, "") или (None, причина no-op). + + Захват разрешён, если сообщение относится к ТЕКУЩЕЙ попытке и задание + queued, либо processing с истёкшей арендой, либо processing с арендой + этого же consumer (его прошлая инкарнация умерла: consumer обрабатывает + не больше одного сообщения за раз, поэтому живой аренды с его именем быть не может). + """ + token = uuid.uuid4() + with self.pool.connection() as conn: + with conn.transaction(): + cur = conn.cursor(row_factory=dict_row).execute( + """ + SELECT job_id, attachment_id, status, attempt, max_attempts, replay_count, + pipeline_version, lease_owner, + (lease_until IS NOT NULL AND lease_until < clock_timestamp()) AS lease_expired + FROM attachment_jobs WHERE job_id = %s + FOR UPDATE + """, + (msg["job_id"],), + ).fetchone() + if cur is None or cur["attachment_id"] != msg["attachment_id"]: + return None, "unknown_job" + same_attempt = cur["attempt"] == msg["attempt"] and cur["pipeline_version"] == msg["pipeline_version"] + takeover = cur["status"] == "processing" and (cur["lease_expired"] or cur["lease_owner"] == owner) + if same_attempt and (cur["status"] == "queued" or takeover): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', lease_token = %s, lease_owner = %s, + lease_until = clock_timestamp() + make_interval(secs => %s), + started_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = %s + """, + (token, owner, self.settings.job_timeout_seconds, cur["job_id"]), + ) + conn.execute( + "UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() " + "WHERE id = %s AND state IN ('queued', 'processing')", + (cur["attachment_id"],), + ) + self._event(conn, cur["job_id"], cur["attachment_id"], cur["attempt"], + "lease_takeover" if takeover else "claimed", + "previous owner %s" % cur["lease_owner"] if takeover else None, owner) + cur["lease_token"] = token + return Claim(cur, takeover), "" + if cur["status"] == "succeeded": + reason = "already_succeeded" + elif cur["status"] == "failed": + reason = "already_failed" + elif cur["attempt"] > msg["attempt"]: + reason = "stale_attempt" + elif cur["status"] == "processing": + reason = "leased_by_other" + else: + reason = "attempt_mismatch" + # Повторная доставка — нормальна; фиксируем no-op в аудите. + self._event(conn, cur["job_id"], cur["attachment_id"], msg["attempt"], + "redelivery_noop", reason, owner) + return None, reason + + def attachment_source(self, attachment_id: UUID) -> Optional[Dict]: + """Доверенные метаданные для worker'а: только из БД, не из сообщения.""" + with self.pool.connection() as conn: + return conn.cursor(row_factory=dict_row).execute( + "SELECT id, content_type, size_bytes, sha256, width, height, object_key, " + "staging_key, filename, state FROM attachments WHERE id = %s", + (attachment_id,), + ).fetchone() + + # ------------------------------------------------------------ outcomes + + def complete(self, claim: Claim, variants: List[Dict], original_key: str, actor: str) -> Optional[float]: + """processing -> ready. None — аренда потеряна (fencing), ничего не изменено. + + lab6: возвращает секунды от создания задания (COMMIT при приёме, ответ 202) + до COMMIT ready — end-to-end time to ready.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'succeeded', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + finished_at = clock_timestamp(), updated_at = clock_timestamp(), last_error_code = NULL + WHERE job_id = %s AND lease_token = %s AND status = 'processing' + RETURNING attempt, EXTRACT(EPOCH FROM clock_timestamp() - created_at) + """, + (claim.job_id, claim.lease_token), + ).fetchone() + if row is None: + return None + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + [(claim.attachment_id, claim.pipeline_version, v["name"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"], v["object_key"]) + for v in variants], + ) + # Никогда не откатываем более новую версию: pipeline_version <= текущей задачи. + conn.execute( + """ + UPDATE attachments + SET state = 'ready', error_code = NULL, object_key = %s, + pipeline_version = %s, updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version <= %s AND state <> 'pending' + """, + (original_key, claim.pipeline_version, claim.attachment_id, claim.pipeline_version), + ) + self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "succeeded", + ",".join(v["name"] for v in variants), actor) + return float(row[1]) + + def retry_or_fail(self, claim: Claim, code: str, actor: str) -> Tuple[str, float]: + """Временная ошибка: следующая попытка через outbox (backoff+jitter) или + failed retry_exhausted. Возвращает ("retry", delay) | ("failed", 0) | ("lost", 0).""" + if claim.attempt >= claim.max_attempts: + return ("failed", 0.0) if self.fail(claim, "retry_exhausted", actor, detail=code) else ("lost", 0.0) + delay = backoff_seconds(claim.attempt, self.settings.job_retry_base_seconds, + self.settings.job_retry_max_seconds) + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, last_error_code = %s, updated_at = clock_timestamp(), + next_attempt_at = clock_timestamp() + make_interval(secs => %s) + WHERE job_id = %s AND lease_token = %s AND status = 'processing' + RETURNING attempt + """, + (code, delay, claim.job_id, claim.lease_token), + ).fetchone() + if row is None: + return "lost", 0.0 + self._requeue_side_effects(conn, claim.job_id, claim.attachment_id, claim.pipeline_version, + row[0], delay) + self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "retry_scheduled", + "%s; next attempt %d in %.2fs" % (code, row[0], delay), actor) + return "retry", delay + + def _requeue_side_effects(self, conn, job_id, attachment_id, pipeline_version, attempt, delay) -> None: + conn.execute( + "UPDATE attachments SET state = 'queued', updated_at = clock_timestamp() " + "WHERE id = %s AND state IN ('queued', 'processing', 'failed')", + (attachment_id,), + ) + conn.execute( + "INSERT INTO job_outbox (job_id, payload, available_at) " + "VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))", + (job_id, Jsonb(job_payload(job_id, attachment_id, pipeline_version, attempt)), delay), + ) + + def fail(self, claim: Claim, code: str, actor: str, detail: Optional[str] = None) -> bool: + """processing -> failed (fenced) + запись в реестр dead_letters.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + last_error_code = %s, finished_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = %s AND lease_token = %s AND status = 'processing' + RETURNING attempt + """, + (code, claim.job_id, claim.lease_token), + ).fetchone() + if row is None: + return False + self._mark_failed(conn, claim.job_id, claim.attachment_id, code, row[0]) + self._event(conn, claim.job_id, claim.attachment_id, claim.attempt, "failed", + code + ("; " + detail if detail else ""), actor) + return True + + @staticmethod + def _mark_failed(conn, job_id, attachment_id, code, attempts) -> None: + conn.execute( + "UPDATE attachments SET state = 'failed', error_code = %s, updated_at = clock_timestamp() " + "WHERE id = %s AND state IN ('queued', 'processing')", + (code, attachment_id), + ) + conn.execute( + """ + INSERT INTO dead_letters (job_id, attachment_id, error_code, attempts) + VALUES (%s, %s, %s, %s) + ON CONFLICT (job_id) DO UPDATE + SET error_code = EXCLUDED.error_code, attempts = EXCLUDED.attempts, + failed_at = clock_timestamp() + """, + (job_id, attachment_id, code, attempts), + ) + + def fail_poison(self, job_id: UUID, code: str, actor: str) -> bool: + """Сообщение доставлялось слишком много раз (worker падает на нём) — + перевести задание в failed без аренды, если оно ещё не завершено.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + last_error_code = %s, finished_at = clock_timestamp(), updated_at = clock_timestamp() + WHERE job_id = %s AND status IN ('queued', 'processing') + RETURNING attachment_id, attempt + """, + (code, job_id), + ).fetchone() + if row is None: + return False + self._mark_failed(conn, job_id, row[0], code, row[1]) + self._event(conn, job_id, row[0], row[1], "failed", code, actor) + return True + + # ------------------------------------------------------------ recovery + + def reap_expired(self, actor: str, limit: int = 100) -> List[Tuple[str, str]]: + """processing с истёкшей арендой (+grace) -> queued attempt+1 через outbox, + или failed processing_timeout, если попытки исчерпаны. SKIP LOCKED: несколько + relay не мешают друг другу.""" + done = [] + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT job_id, attachment_id, attempt, max_attempts, pipeline_version, lease_owner + FROM attachment_jobs + WHERE status = 'processing' + AND lease_until < clock_timestamp() - make_interval(secs => %s) + ORDER BY lease_until LIMIT %s + FOR UPDATE SKIP LOCKED + """, + (self.settings.job_lease_grace_seconds, limit), + ).fetchall() + for r in rows: + if r["attempt"] >= r["max_attempts"]: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + last_error_code = 'processing_timeout', finished_at = clock_timestamp(), + updated_at = clock_timestamp() + WHERE job_id = %s + """, + (r["job_id"],), + ) + self._mark_failed(conn, r["job_id"], r["attachment_id"], "processing_timeout", r["attempt"]) + self._event(conn, r["job_id"], r["attachment_id"], r["attempt"], "failed", + "processing_timeout (lease expired, owner %s)" % r["lease_owner"], actor) + done.append((str(r["job_id"]), "failed")) + continue + delay = backoff_seconds(r["attempt"], self.settings.job_retry_base_seconds, + self.settings.job_retry_max_seconds) + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = attempt + 1, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, last_error_code = 'lease_expired', updated_at = clock_timestamp(), + next_attempt_at = clock_timestamp() + make_interval(secs => %s) + WHERE job_id = %s + """, + (delay, r["job_id"]), + ) + self._requeue_side_effects(conn, r["job_id"], r["attachment_id"], r["pipeline_version"], + r["attempt"] + 1, delay) + self._event(conn, r["job_id"], r["attachment_id"], r["attempt"], "lease_expired", + "owner %s; next attempt %d in %.2fs" % (r["lease_owner"], r["attempt"] + 1, delay), + actor) + done.append((str(r["job_id"]), "requeued")) + return done + + def replay(self, attachment_id: UUID, actor: str, pipeline_version: Optional[int] = None) -> Tuple[str, str]: + """failed -> queued: явный replay. Успешное задание НЕ исполняется повторно. + + Новая попытка получает attempt+1 и свежий бюджет JOB_MAX_ATTEMPTS. Логический + эффект защищён тем же ключом (attachment_id, pipeline_version, variant). + """ + with self.pool.connection() as conn: + with conn.transaction(): + job = conn.cursor(row_factory=dict_row).execute( + """ + SELECT j.job_id, j.status, j.attempt, j.pipeline_version, a.staging_key, a.state + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.attachment_id = %s AND (%s::int IS NULL OR j.pipeline_version = %s) + ORDER BY j.pipeline_version DESC LIMIT 1 + FOR UPDATE OF j, a + """, + (attachment_id, pipeline_version, pipeline_version), + ).fetchone() + if job is None: + return "refused", "no job for this attachment" + if job["status"] == "succeeded" or job["state"] == "ready": + self._event(conn, job["job_id"], attachment_id, job["attempt"], "replay_refused", + "already succeeded (no duplicate execution)", actor) + return "noop", "already succeeded; nothing to replay" + if job["status"] != "failed": + return "refused", "job is %s, only failed jobs can be replayed" % job["status"] + if job["staging_key"] is None: + self._event(conn, job["job_id"], attachment_id, job["attempt"], "replay_refused", + "staging source already removed (retention)", actor) + return "refused", "staging source was removed by retention; re-upload required" + attempt = job["attempt"] + 1 + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, max_attempts = %s, replay_count = replay_count + 1, + last_error_code = NULL, next_attempt_at = NULL, finished_at = NULL, + updated_at = clock_timestamp() + WHERE job_id = %s AND status = 'failed' + """, + (attempt, attempt - 1 + self.settings.job_max_attempts, job["job_id"]), + ) + conn.execute( + "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() " + "WHERE id = %s AND state = 'failed'", + (attachment_id,), + ) + conn.execute( + "UPDATE dead_letters SET replayed_at = clock_timestamp(), replay_count = replay_count + 1 " + "WHERE job_id = %s", + (job["job_id"],), + ) + conn.execute( + "INSERT INTO job_outbox (job_id, payload) VALUES (%s, %s)", + (job["job_id"], Jsonb(job_payload(job["job_id"], attachment_id, job["pipeline_version"], attempt))), + ) + self._event(conn, job["job_id"], attachment_id, attempt, "replayed", + "operator replay; new attempt %d" % attempt, actor) + return "replayed", "attempt %d queued" % attempt + + +def dumps(obj) -> str: + return json.dumps(obj, ensure_ascii=False, default=str) diff --git a/src/jobs_admin.py b/src/jobs_admin.py new file mode 100644 index 0000000..c09084f --- /dev/null +++ b/src/jobs_admin.py @@ -0,0 +1,156 @@ +"""Операторские команды очереди (lab5). Секреты и токены не печатаются. + + python -m src.jobs_admin stats # состояния заданий, outbox, брокер (длина/pending/lag) + python -m src.jobs_admin failed [--limit 50] # реестр dead_letters (DLQ) + python -m src.jobs_admin show ATTACHMENT_ID # задание + журнал событий + python -m src.jobs_admin replay ATTACHMENT_ID [...] # явный replay failed-задания + python -m src.jobs_admin replay --all-failed [--error-code retry_exhausted] [--dry-run] + +Replay безопасен: успешное задание не исполняется повторно (no-op с записью в +аудит), replay без staging-источника отклоняется, новая попытка получает attempt+1. +""" +import argparse +import json +import sys +from uuid import UUID + +from psycopg.rows import dict_row + +from .broker import Broker, BrokerUnavailable +from .config import load_settings +from .db import create_pool +from .jobs import JobStore + + +def _print(obj) -> None: + print(json.dumps(obj, ensure_ascii=False, indent=2, default=str)) + + +def cmd_stats(pool, settings, _args) -> int: + with pool.connection() as conn: + jobs = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall()) + atts = dict(conn.execute("SELECT state, count(*) FROM attachments GROUP BY state").fetchall()) + outbox = conn.execute( + "SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()), " + "COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0) FROM job_outbox" + ).fetchone() + dlq = conn.execute("SELECT count(*) FROM dead_letters WHERE replayed_at IS NULL").fetchone()[0] + oldest_queued = conn.execute( + "SELECT COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0) " + "FROM attachment_jobs WHERE status IN ('queued', 'processing')" + ).fetchone()[0] + out = { + "jobs": jobs, "attachments": atts, + "outbox": {"rows": outbox[0], "due": outbox[1], "oldest_age_s": round(float(outbox[2]), 1)}, + "dead_letters_unreplayed": dlq, + "oldest_unfinished_job_age_s": round(float(oldest_queued), 1), + } + if settings.broker_url: + broker = Broker(settings.broker_url, settings.broker_stream, settings.broker_group, + settings.broker_timeout_seconds) + try: + out["broker"] = broker.stats() + except BrokerUnavailable: + out["broker"] = "unavailable" + broker.close() + _print(out) + return 0 + + +def cmd_failed(pool, _settings, args) -> int: + with pool.connection() as conn: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT d.attachment_id, d.job_id, d.error_code, d.attempts, d.failed_at, d.replayed_at, + d.replay_count, j.status AS job_status, (a.staging_key IS NOT NULL) AS replayable + FROM dead_letters d + JOIN attachment_jobs j ON j.job_id = d.job_id + JOIN attachments a ON a.id = d.attachment_id + ORDER BY d.failed_at DESC LIMIT %s + """, + (args.limit,), + ).fetchall() + _print(rows) + return 0 + + +def cmd_show(pool, _settings, args) -> int: + with pool.connection() as conn: + att = conn.cursor(row_factory=dict_row).execute( + "SELECT id, state, error_code, pipeline_version, size_bytes, sha256, " + "(staging_key IS NOT NULL) AS has_staging, created_at, updated_at FROM attachments WHERE id = %s", + (args.attachment_id,), + ).fetchone() + job = conn.cursor(row_factory=dict_row).execute( + "SELECT job_id, status, attempt, max_attempts, replay_count, lease_owner, lease_until, " + "last_error_code, next_attempt_at, created_at, finished_at FROM attachment_jobs " + "WHERE attachment_id = %s ORDER BY pipeline_version", + (args.attachment_id,), + ).fetchall() + variants = conn.cursor(row_factory=dict_row).execute( + "SELECT pipeline_version, name, content_type, width, height, size_bytes, sha256 " + "FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name", + (args.attachment_id,), + ).fetchall() + events = conn.cursor(row_factory=dict_row).execute( + "SELECT at, attempt, event, detail, actor FROM job_events WHERE attachment_id = %s " + "OR job_id IN (SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id", + (args.attachment_id, args.attachment_id), + ).fetchall() + _print({"attachment": att, "jobs": job, "variants": variants, "events": events}) + return 0 if att else 1 + + +def cmd_replay(pool, settings, args) -> int: + jobs = JobStore(pool, settings) + ids = [UUID(x) for x in args.attachment_ids] + if args.all_failed: + with pool.connection() as conn: + q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'" + params = [] + if args.error_code: + q += " AND last_error_code = %s" + params.append(args.error_code) + ids += [r[0] for r in conn.execute(q + " ORDER BY updated_at", params).fetchall()] + if not ids: + print("nothing to replay") + return 0 + results = [] + for att in ids: + if args.dry_run: + results.append({"attachment_id": att, "result": "dry-run"}) + continue + outcome, detail = jobs.replay(att, actor="operator:jobs_admin") + results.append({"attachment_id": att, "result": outcome, "detail": detail}) + _print(results) + return 0 if all(r["result"] in ("replayed", "noop", "dry-run") for r in results) else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Attachment job queue operations") + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("stats") + p = sub.add_parser("failed") + p.add_argument("--limit", type=int, default=50) + p = sub.add_parser("show") + p.add_argument("attachment_id", type=UUID) + p = sub.add_parser("replay") + p.add_argument("attachment_ids", nargs="*") + p.add_argument("--all-failed", action="store_true") + p.add_argument("--error-code") + p.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + if args.cmd == "replay" and not (args.attachment_ids or args.all_failed): + parser.error("replay: give ATTACHMENT_ID(s) or --all-failed") + settings = load_settings() + pool = create_pool(settings) + pool.wait(timeout=15) + try: + return {"stats": cmd_stats, "failed": cmd_failed, "show": cmd_show, "replay": cmd_replay}[args.cmd]( + pool, settings, args) + finally: + pool.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..1558f17 --- /dev/null +++ b/src/main.py @@ -0,0 +1,1020 @@ +import asyncio +import base64 +import hashlib +import json +import logging +import os +import re +import threading +import time +import unicodedata +import uuid as uuid_mod +from contextlib import asynccontextmanager +from datetime import datetime +from typing import Callable, Dict, Optional, Tuple +from uuid import UUID, uuid4 + +import psycopg +from opentelemetry import trace +from fastapi import BackgroundTasks, Depends, FastAPI, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.concurrency import run_in_threadpool +from fastapi.responses import JSONResponse, StreamingResponse +from psycopg_pool import PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect +from starlette.middleware.base import BaseHTTPMiddleware + +from . import telemetry +from .config import load_settings +from .db import create_pool, required_schema_version, schema_has_version, wait_and_migrate +from .images import ImageRejected, inspect as inspect_image +from .models import ( + Attachment, + Chat, + ChatCreate, + ChatPage, + Message, + MessageCreate, + MessagePage, + Session, + User, + UserCreate, +) +from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable +from .broker import Broker +from .jobs import job_payload +from .outbox import Relay +from .reconcile import STAGING_PREFIX, gc_loop +from .runtime import DRAIN, INSTANCE_ID +from .security import ( + KdfBusy, + configure_kdf_concurrency, + hash_password, + needs_rehash, + parse_basic, + parse_bearer, + verify_password, +) +from .health import start_health_listener +from .sessions import SessionCache, SessionService +from .storage import ( + AttachmentFailed, + AttachmentNotFound, + IdempotencyConflict, + IdempotencyRace, + NotChatMember, + Storage, + UnknownMember, + UsernameTaken, +) + +# lab6: OTLP traces/metrics/logs + JSON-логи. До create_pool (инструментация psycopg). +telemetry.setup("messenger-api") +log = logging.getLogger("messenger.api") +access_log = logging.getLogger("messenger.http") + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +# Контракт lab3: Idempotency-Key — ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def _idempotency_cleanup_loop(storage: Storage, interval: float, stop: threading.Event) -> None: + """Фоновая очистка истёкших ключей и старых окон rate limit. Advisory lock — одна реплика за раз.""" + while not stop.wait(interval): + try: + deleted = storage.delete_expired_idempotency_keys() + if deleted: + log.info("deleted %d expired idempotency keys", deleted) + storage.delete_old_rate_limits() + except Exception as exc: # БД недоступна — попробуем в следующий раз + log.warning("idempotency cleanup skipped: %s", type(exc).__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = load_settings() + if settings.run_migrations_on_start: + # Только для локальной разработки. В compose миграции выполняет + # отдельный one-shot сервис `migrate` до старта реплик. + applied = wait_and_migrate(settings) + log.info("migrations applied at startup: %s", applied or "none") + configure_kdf_concurrency(settings.password_hash_concurrency, settings.kdf_queue_timeout_seconds) + pool = create_pool(settings) + storage = Storage(pool) + cache = SessionCache(settings) + app.state.settings = settings + app.state.storage = storage + app.state.cache = cache + app.state.sessions = SessionService(storage, cache, settings) + app.state.required_schema = required_schema_version() + # lab6: DB pool и очередь (из PostgreSQL) — метрики живут, даже когда worker остановлен. + telemetry.register_pool_metrics(pool) + telemetry.register_queue_metrics(pool, settings.broker_stream) + app.state.schema_ok = False + # lab7: health только на 127.0.0.1 внутри контейнера (Docker healthcheck без mTLS). + health_server = start_health_listener(lambda: check_ready(app.state), settings.health_port) + stop_cleanup = threading.Event() + cleanup = threading.Thread( + target=_idempotency_cleanup_loop, + args=(storage, settings.idempotency_cleanup_interval_seconds, stop_cleanup), + name="idempotency-cleanup", + daemon=True, + ) + cleanup.start() + # lab4: S3 и ограничители ресурсов загрузок (на реплику). + app.state.objects = ObjectStore(settings) if ObjectStore.configured(settings) else None + app.state.upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads) + app.state.decode_slots = threading.BoundedSemaphore(settings.image_decode_concurrency) + if app.state.objects is not None: + threading.Thread( + target=gc_loop, + args=(pool, app.state.objects, settings, stop_cleanup), + name="attachment-gc", + daemon=True, + ).start() + else: + log.warning("S3 is not configured: attachment endpoints return 503") + # lab5: outbox relay в каждой реплике (быстрый путь после 202 + sweeper). + # Брокер НЕ входит в синхронный путь: при его недоступности upload всё равно + # отвечает 202, задание ждёт в job_outbox (PostgreSQL). + app.state.relay = None + if settings.broker_url and settings.relay_enabled: + broker = Broker(settings.broker_url, settings.broker_stream, settings.broker_group, + settings.broker_timeout_seconds) + app.state.relay = Relay(pool, broker, settings, "api:" + INSTANCE_ID) + threading.Thread(target=app.state.relay.loop, args=(stop_cleanup,), name="outbox-relay", + daemon=True).start() + elif not settings.broker_url: + log.warning("BROKER_URL is not configured: jobs stay in job_outbox until a relay publishes them") + log.info( + "attachments: s3=%s bucket=%s max_bytes=%d max_pixels=%d uploads<=%d decode<=%d gc_grace=%ss fault=%s", + "on" if app.state.objects is not None else "off", + settings.s3_bucket, + settings.max_upload_bytes, + settings.max_image_pixels, + settings.max_concurrent_uploads, + settings.image_decode_concurrency, + settings.attachment_gc_grace_seconds, + settings.fault_injection or "none", + ) + log.info( + "started: instance=%s version=%s session_ttl=%ss cache=%s pool=%s..%s schema>=%s", + INSTANCE_ID, + settings.app_version, + settings.session_ttl_seconds, + "redis" if cache.client is not None else "disabled", + settings.db_pool_min, + settings.db_pool_max, + app.state.required_schema, + ) + try: + yield + finally: + log.info("shutdown: instance=%s closing pool", INSTANCE_ID) + stop_cleanup.set() + if health_server is not None: + health_server.shutdown() + pool.close() + if cache.client is not None: + cache.client.close() + telemetry.shutdown() + + +app = FastAPI( + title="MTUSI Messenger (lab7: TLS/mTLS, OpenBao)", + version="7.0.0", + lifespan=lifespan, +) + + +# Request ID + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next: Callable): + incoming = request.headers.get("X-Request-Id", "") + request_id = incoming if REQUEST_ID_RE.match(incoming) else str(uuid4()) + request.state.request_id = request_id + # lab6: X-Request-Id в server span — связь «ответ с ошибкой -> trace -> лог». + span = trace.get_current_span() + if span.is_recording(): + span.set_attribute("messenger.request_id", request_id) + started = time.perf_counter() + response = await call_next(request) + path = request.url.path + if not path.startswith("/health/"): + route = request.scope.get("route") + # Только шаблон маршрута, без реальных UUID и query string. + access_log.info( + "%s %s -> %d", request.method, getattr(route, "path", "unmatched"), response.status_code, + extra={"event": "http.request", "http_method": request.method, + "http_route": getattr(route, "path", "unmatched"), + "http_status_code": response.status_code, "request_id": request_id, + "duration_ms": round((time.perf_counter() - started) * 1000, 1)}, + ) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + if DRAIN.active: + # Во время осушения не держим keep-alive: следующий запрос клиент/LB + # откроет новым соединением — уже к другой реплике. + response.headers["Connection"] = "close" + return response + + +class BodyLimitMiddleware: + """lab7: лимит тела не-upload запросов (MAX_JSON_BODY_BYTES), ASGI-уровень. + + Content-Length больше лимита -> 413 сразу, тело не читается. Без Content-Length + (chunked) байты считаются по мере чтения; превышение -> 413. Загрузка вложения + (POST .../attachments) ограничена отдельно MAX_UPLOAD_BYTES в обработчике. + HAProxy дублирует лимит по Content-Length до того, как запрос дойдёт до реплики. + """ + + def __init__(self, app, limit: int) -> None: + self.app = app + self.limit = limit + + async def __call__(self, scope, receive, send): + if scope["type"] != "http" or scope["path"].endswith("/attachments"): + return await self.app(scope, receive, send) + limit = self.limit + for name, value in scope.get("headers", []): + if name == b"content-length": + try: + if int(value) > limit: + return await self._reject(scope, send) + except ValueError: + return await self._reject(scope, send, 400) + if scope.get("method") not in ("POST", "PUT", "PATCH", "DELETE"): + return await self.app(scope, receive, send) + # Тело (не больше limit) читается заранее и отдаётся приложению одним сообщением. + # Так превышение без Content-Length (chunked) даёт честный 413, а не 400 «ошибка + # разбора» FastAPI, и лишние байты не попадают в память обработчика. + chunks = [] + seen = 0 + while True: + message = await receive() + if message["type"] != "http.request": + return # клиент отключился + body = message.get("body", b"") + seen += len(body) + if seen > limit: + return await self._reject(scope, send) + chunks.append(body) + if not message.get("more_body", False): + break + pending = [{"type": "http.request", "body": b"".join(chunks), "more_body": False}] + + async def replay(): + if pending: + return pending.pop() + return await receive() + + await self.app(scope, replay, send) + + @staticmethod + async def _reject(scope, send, status: int = 413): + request_id = None + for name, value in scope.get("headers", []): + if name == b"x-request-id" and REQUEST_ID_RE.match(value.decode("latin-1")): + request_id = value.decode("latin-1") + request_id = request_id or str(uuid4()) + code = "payload_too_large" if status == 413 else "invalid_request" + body = json.dumps({"error": {"code": code, "message": "Request body is too large" if status == 413 + else "Invalid request", "request_id": request_id}}).encode() + await send({"type": "http.response.start", "status": status, "headers": [ + (b"content-type", b"application/json"), (b"content-length", str(len(body)).encode()), + (b"x-request-id", request_id.encode()), (b"x-instance-id", INSTANCE_ID.encode()), + (b"connection", b"close")]}) + await send({"type": "http.response.body", "body": body}) + + +app.add_middleware(RequestIdMiddleware) +app.add_middleware(BodyLimitMiddleware, limit=int(os.environ.get("MAX_JSON_BODY_BYTES", "65536"))) +# Добавляется последним -> самый внешний middleware: RequestIdMiddleware и +# обработчики работают внутри server span (логи получают trace_id/span_id). +telemetry.instrument_fastapi(app) + + +# errors + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", "Authentication required", {"WWW-Authenticate": challenge}) + + +def bad_request(message: str = "Invalid request") -> ApiError: + return ApiError(400, "invalid_request", message) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Not found") + + +def error_response( + request: Request, + status_code: int, + code: str, + message: str, + headers: Optional[Dict[str, str]] = None, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", None) or str(uuid4()) + all_headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID} + all_headers.update(headers or {}) + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=all_headers, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError) -> JSONResponse: + return error_response(request, exc.status_code, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Invalid request") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + status_to_code = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + } + status_code = exc.status_code if exc.status_code in status_to_code else 500 + code = status_to_code.get(status_code, "internal_error") + message = str(exc.detail) if status_code != 500 else "Internal server error" + return error_response(request, status_code, code, message) + + +async def dependency_unavailable(request: Request, exc: Exception) -> JSONResponse: + # Наружу — только общий 503, без SQL, адресов и текста драйвера. + log.warning("database unavailable: %s", type(exc).__name__) + return error_response(request, 503, "unavailable", "Service temporarily unavailable") + + +app.add_exception_handler(psycopg.OperationalError, dependency_unavailable) +app.add_exception_handler(psycopg.InterfaceError, dependency_unavailable) +app.add_exception_handler(PoolTimeout, dependency_unavailable) + + +@app.exception_handler(Exception) +async def unexpected_exception_handler(request: Request, exc: Exception) -> JSONResponse: + log.exception("unexpected error") + return error_response(request, 500, "internal_error", "Internal server error") + + +# helpers + +def require_json_content_type(request: Request) -> None: + content_type = request.headers.get("content-type", "") + if not content_type.lower().startswith("application/json"): + raise ApiError(415, "unsupported_media_type", "Unsupported media type") + + +def ensure_not_blank(value: str) -> None: + if not value.strip(): + raise bad_request() + + +def storage_of(request: Request) -> Storage: + return request.app.state.storage + + +def current_user(request: Request) -> User: + """Только Authorization: Bearer. Basic и X-User-Id здесь не дают доступа.""" + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise unauthorized() + user = request.app.state.sessions.resolve(token) + if user is None: + raise unauthorized() + request.state.token = token + return user + + +def encode_cursor(resource_id: UUID, created_at: datetime, item_id: UUID) -> str: + payload = { + "resource_id": str(resource_id), + "created_at": created_at.isoformat(), + "item_id": str(item_id), + } + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: Optional[str], expected_resource_id: UUID) -> Optional[Tuple[datetime, UUID]]: + if cursor is None: + return None + try: + padding = "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(cursor + padding) + payload = json.loads(raw.decode("utf-8")) + resource_id = UUID(payload["resource_id"]) + created_at = datetime.fromisoformat(payload["created_at"]) + item_id = UUID(payload["item_id"]) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("invalid_cursor") + except (ValueError, KeyError, TypeError, AttributeError, json.JSONDecodeError): + raise bad_request("Invalid cursor") + if resource_id != expected_resource_id: + raise bad_request("Invalid cursor") + return created_at, item_id + + +# health + +@app.get("/health/live", response_model=Dict[str, str]) +def liveness() -> Dict[str, str]: + return {"status": "ok"} + + +def check_ready(state) -> None: + """Общая проверка для /health/ready (через LB по mTLS) и loopback health listener.""" + if DRAIN.active: + raise ApiError(503, "unavailable", "Instance is shutting down") + timeout = state.settings.ready_db_timeout_seconds + try: + if state.schema_ok: + with state.storage.pool.connection(timeout=timeout) as conn: + conn.execute("SELECT 1") + elif schema_has_version(state.storage.pool, state.required_schema, timeout): + state.schema_ok = True + else: + raise ApiError(503, "unavailable", "Database schema is not migrated") + except (psycopg.Error, PoolTimeout): + raise ApiError(503, "unavailable", "Database is not available") + + +@app.get("/health/ready", response_model=Dict[str, str]) +def readiness(request: Request) -> Dict[str, str]: + """Готовность к обязательному синхронному пути. + + * SIGTERM получен (фаза осушения) -> 503: LB выводит реплику из ротации; + * PostgreSQL недоступна за READY_DB_TIMEOUT_SECONDS -> 503; + * схема ещё не мигрирована до версии, которую знает образ -> 503 + (миграции выполняет отдельный сервис `migrate`). + Redis не критичен: это кэш с обходом (см. REPORT.md). OpenBao тоже не в + синхронном пути: секреты уже отрисованы agent'ом в файлы (REPORT.md, lab7). + """ + check_ready(request.app.state) + return {"status": "ok"} + + +# users + +@app.post("/api/v1/users", response_model=User, status_code=201) +def register_user(request: Request, data: UserCreate) -> User: + require_json_content_type(request) + ensure_not_blank(data.display_name) + ensure_not_blank(data.password) + try: + password_hash = hash_password(data.password) + except KdfBusy: + raise kdf_busy() + try: + return storage_of(request).create_user(data.username, data.display_name, password_hash) + except UsernameTaken: + raise ApiError(409, "conflict", "Username already exists") + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(user: User = Depends(current_user)) -> User: + return user + + +# sessions + +async def require_empty_body(request: Request) -> None: + """Вход — только заголовок Authorization: Basic, без тела (контракт). + + Любые байты в теле (JSON или нет) — 400. Проверка выполняется до + дорогой проверки пароля и до создания сессии. + """ + if await request.body(): + raise bad_request("Request body is not allowed") + + +def kdf_busy() -> ApiError: + return ApiError(503, "unavailable", "Service temporarily unavailable", {"Retry-After": "1"}) + + +def client_ip(request: Request) -> str: + """IP клиента для rate limit. X-Forwarded-For доверяем ТОЛЬКО потому, что + соединение к API может открыть лишь LB (mTLS identity lb); HAProxy дописывает + реальный адрес последним элементом (option forwardfor).""" + forwarded = request.headers.get("X-Forwarded-For", "") + if forwarded: + return forwarded.split(",")[-1].strip()[:64] + return request.client.host if request.client else "-" + + +def _bucket(prefix: str, value: str) -> str: + return prefix + hashlib.sha256(value.encode("utf-8")).hexdigest()[:32] + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201, + dependencies=[Depends(require_empty_body)]) +def create_session(request: Request) -> Session: + credentials = parse_basic(request.headers.get("Authorization")) + settings = request.app.state.settings + storage = storage_of(request) + # lab7: общий для реплик лимит попыток входа (PostgreSQL) ДО дорогого Argon2id: + # перебор пароля одного пользователя и поток входов с одного IP получают 429. + buckets = [(_bucket("ip:", client_ip(request)), settings.login_rate_per_ip)] + if credentials is not None: + buckets.append((_bucket("u:", credentials[0]), settings.login_rate_per_user)) + retry_after = storage.rate_limit_hit(buckets, settings.login_rate_window_seconds) + if retry_after is not None: + raise ApiError(429, "rate_limited", "Too many login attempts", {"Retry-After": str(retry_after)}) + if credentials is None: + raise unauthorized(BASIC_CHALLENGE) + username, password = credentials + found = storage.get_credentials(username) + user, password_hash = found if found else (None, None) + # verify_password выполняется и для несуществующего username (с хешем-пустышкой). + try: + valid = verify_password(password_hash, password) + except KdfBusy: + raise kdf_busy() + if not valid or user is None: + raise unauthorized(BASIC_CHALLENGE) + if needs_rehash(password_hash): + # Старый hash (другие параметры Argon2id) пересчитывается под текущие при + # успешном входе: только здесь у сервера есть открытый пароль. + try: + if storage.update_password_hash(user.id, password_hash, hash_password(password)): + log.info("password hash upgraded to current Argon2id parameters", + extra={"event": "auth.rehash"}) + except KdfBusy: + pass # не критично: пересчитаем при следующем входе + token, expires_at = request.app.state.sessions.create(user) + return Session(token=token, token_type="Bearer", expires_at=expires_at, user=user) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_current_session(request: Request, user: User = Depends(current_user)) -> Response: + if not request.app.state.sessions.revoke(request.state.token): + raise unauthorized() + return Response(status_code=204) + + +# chats + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, data: ChatCreate, user: User = Depends(current_user)) -> Chat: + require_json_content_type(request) + ensure_not_blank(data.title) + if len(data.member_ids) != len(set(data.member_ids)): + raise bad_request("Duplicate member ids") + if user.id not in data.member_ids: + raise bad_request("Caller must be a chat member") + try: + return storage_of(request).create_chat(data.title, data.member_ids) + except UnknownMember: + raise bad_request("Unknown member") + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + user: User = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> ChatPage: + after = decode_cursor(cursor, user.id) + items, has_more = storage_of(request).list_user_chats(user.id, limit, after) + next_cursor = None + if has_more and items: + next_cursor = encode_cursor(user.id, items[-1].created_at, items[-1].id) + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID, user: User = Depends(current_user)) -> Chat: + chat = storage_of(request).get_chat_for_member(chat_id, user.id) + if chat is None: + raise not_found() + return chat + + +# messages + +def request_fingerprint(data: MessageCreate) -> bytes: + """SHA-256 канонического JSON тела. + + Канонизация: проверенная модель -> JSON с сортировкой ключей и без пробелов. + Поэтому порядок ключей и форматирование исходного JSON не влияют на hash. + lab4: attachment_ids имеет значение по умолчанию [], поэтому отсутствующее + поле и явный [] дают одинаковый hash (контракт). Порядок id значим. + """ + canonical = json.dumps( + data.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +def create_message( + request: Request, + response: Response, + chat_id: UUID, + data: MessageCreate, + user: User = Depends(current_user), +) -> Message: + # Порядок: аутентификация (dependency) -> ключ -> тело -> доступ к чату. + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise bad_request("Missing or invalid Idempotency-Key") + require_json_content_type(request) + # lab4: пустой/пробельный текст допустим только вместе с вложениями. + if not data.attachment_ids: + ensure_not_blank(data.text) + if len(set(data.attachment_ids)) != len(data.attachment_ids): + raise bad_request("Duplicate attachment ids") + settings = request.app.state.settings + try: + message, replayed = storage_of(request).create_message( + chat_id, + user.id, + data.text, + key, + request_fingerprint(data), + settings.idempotency_ttl_seconds, + data.attachment_ids, + ) + except (NotChatMember, AttachmentNotFound): + # Чужое, неизвестное или ещё не принятое (pending) вложение — как и чужой чат: 404. + raise not_found() + except AttachmentFailed: + raise ApiError(409, "conflict", "Attachment processing failed; it cannot be attached") + except IdempotencyConflict: + raise ApiError(409, "conflict", "Idempotency-Key was used with a different request") + except IdempotencyRace: + raise ApiError(503, "unavailable", "Please retry with the same Idempotency-Key") + if replayed: + # Информационный заголовок (не часть контракта): ответ — повтор. + response.headers["Idempotency-Replayed"] = "true" + return message + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +def list_messages( + request: Request, + chat_id: UUID, + user: User = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None, min_length=1), +) -> MessagePage: + after = decode_cursor(cursor, chat_id) + result = storage_of(request).list_messages(chat_id, user.id, limit, after) + if result is None: + raise not_found() + items, has_more = result + next_cursor = None + if has_more and items: + next_cursor = encode_cursor(chat_id, items[-1].created_at, items[-1].id) + return MessagePage(items=items, next_cursor=next_cursor) + + +# attachments (lab4) + +ALLOWED_UPLOAD_TYPES = ("image/png", "image/jpeg") +UPLOAD_CHUNK = 64 * 1024 +KNOWN_VARIANTS = ("thumbnail", "crop", "optimized") + + +async def s3_call(store: ObjectStore, deadline: float, fn, *args): + """Вызов S3 из запроса с жёстким общим дедлайном. + + connect/read timeout boto3 не покрывают DNS: в Docker остановленный + контейнер s3 пропадает из DNS, и getaddrinfo висит секундами (замер на + Docker Desktop: 10–13 с, у скачивания это давало 504 от HAProxy). Дедлайн + ограничивает всё вместе; по истечении — 503 и открытый circuit breaker. + Поток с зависшим вызовом доработает в фоне; для PUT это может дать + объект-сироту, его уберёт сборщик (src/reconcile.py, проход B). + """ + try: + return await asyncio.wait_for(run_in_threadpool(fn, *args), timeout=deadline) + except asyncio.TimeoutError: + store.breaker.failure() + log.warning("s3 call exceeded %.1fs deadline", deadline) + raise ObjectStoreUnavailable() + + +def objects_of(request: Request) -> ObjectStore: + store = request.app.state.objects + if store is None: + raise ApiError(503, "unavailable", "Attachment storage is not available") + return store + + +def s3_unavailable() -> ApiError: + return ApiError(503, "unavailable", "Attachment storage is temporarily unavailable", + {"Retry-After": "2"}) + + +def validate_filename(name: Optional[str]) -> str: + """Отображаемое имя: 1–255 code points, без /, \\, NUL и управляющих символов. + + Имя никогда не становится путём или ключом S3; ограничения защищают + клиентов, которые покажут/сохранят файл под этим именем. + """ + if name is None or not name.strip() or len(name) > 255: + raise bad_request("Invalid filename") + if name in (".", ".."): + raise bad_request("Invalid filename") + for ch in name: + if ch in "/\\\x00" or unicodedata.category(ch) in ("Cc", "Cs") \ + or ch in "

‮‭‪‫‬⁦⁧⁨⁩": + raise bad_request("Invalid filename") + return name + + +def upload_media_type(request: Request) -> str: + raw = request.headers.get("content-type", "") + media = raw.split(";", 1)[0].strip().lower() + if media not in ALLOWED_UPLOAD_TYPES: + raise ApiError(415, "unsupported_media_type", "Only image/png and image/jpeg are accepted") + return media + + +def declared_length(request: Request) -> Optional[int]: + raw = request.headers.get("content-length") + if raw is None: + return None + if not raw.strip().isdigit(): + raise bad_request("Invalid Content-Length") + return int(raw) + + +async def drain(request: Request, limit: int) -> None: + """Дочитать и выбросить тело (не больше limit байт), чтобы клиент увидел 413, + а не обрыв соединения посреди отправки. Память — O(chunk).""" + seen = 0 + try: + async for chunk in request.stream(): + seen += len(chunk) + if seen > limit: + break + except ClientDisconnect: + pass + + +async def read_limited_body(request: Request, limit: int): + """Потоковое чтение тела кусками; не больше limit байт в памяти. + + Возвращает (bytes, md5, sha256). Превышение лимита -> 413 сразу, как только + счётчик перешёл limit (остаток тела отбрасывается, не накапливается). + """ + chunks = [] + size = 0 + md5 = hashlib.md5(usedforsecurity=False) + sha = hashlib.sha256() + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > limit: + chunks.clear() + await drain(request, limit) + raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % limit) + chunks.append(chunk) + md5.update(chunk) + sha.update(chunk) + # Одна склейка в bytes точного размера; список кусков сразу освобождается. + # Дальше bytes не копируются: BytesIO(bytes) и memoryview разделяют буфер. + data = b"".join(chunks) + chunks.clear() + return data, md5.digest(), sha.hexdigest() + + +def _validate_image(request: Request, data: bytes, declared: str): + settings = request.app.state.settings + with request.app.state.decode_slots: # потолок одновременных разборов на реплику + return inspect_image(data, declared, settings.max_image_pixels) + + +def _inject(settings, point: str) -> None: + """Внедрение отказов — ТОЛЬКО для экземпляра api-faults (профиль compose faults).""" + fault = settings.fault_injection + if not fault: + return + if fault == "crash_after_s3_put" and point == "after_put": + log.error("FAULT_INJECTION: process exit between S3 PUT and DB COMMIT") + os._exit(70) + if fault == "db_error_after_s3_put" and point == "after_put": + raise psycopg.OperationalError("injected: database failure after S3 PUT") + if fault == "s3_error_on_put" and point == "before_put": + raise ObjectStoreUnavailable() + + +@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=202) +async def upload_attachment( + request: Request, + background: BackgroundTasks, + chat_id: UUID, + filename: Optional[str] = Query(default=None), + user: User = Depends(current_user), +) -> Attachment: + """lab5: приём оригинала -> 202 queued; обработку выполняет worker. + + Порядок проверок (как в lab4): аутентификация -> filename (400) -> + Content-Type (415) -> Content-Length (413 без чтения тела) -> членство (404) -> + слот загрузки (503) -> потоковое чтение с лимитом (413) -> содержимое (415/413). + Затем durable handoff: + [1] INSERT attachments(state=pending, staging_key) COMMIT + [2] S3 PUT staging/ (Content-MD5, SHA-256 в метаданных) + [3] UPDATE pending->queued + INSERT attachment_jobs + job_outbox COMMIT -> 202 + В момент 202 устойчиво сохранены: все байты оригинала (S3 staging), метаданные и + намерение обработать (задание + outbox в PostgreSQL). Публикация в брокер — после + ответа (background task) и/или sweeper'ом; остановка API после 202 работу не теряет. + """ + settings = request.app.state.settings + storage = storage_of(request) + name = validate_filename(filename) + declared = upload_media_type(request) + length = declared_length(request) + if length is not None and length > settings.max_upload_bytes: + if length <= 2 * settings.max_upload_bytes: + await drain(request, length) + raise ApiError(413, "payload_too_large", "File exceeds %d bytes" % settings.max_upload_bytes, + {"Connection": "close"} if length > 2 * settings.max_upload_bytes else None) + if not await run_in_threadpool(storage.is_chat_member, chat_id, user.id): + raise not_found() + store = objects_of(request) + + slots: asyncio.Semaphore = request.app.state.upload_slots + try: + await asyncio.wait_for(slots.acquire(), timeout=settings.upload_queue_timeout_seconds) + except asyncio.TimeoutError: + raise ApiError(503, "unavailable", "Too many concurrent uploads", {"Retry-After": "1"}) + try: + try: + data, md5_digest, sha256_hex = await asyncio.wait_for( + read_limited_body(request, settings.max_upload_bytes), + timeout=settings.upload_read_timeout_seconds, + ) + except asyncio.TimeoutError: + raise bad_request("Upload was not completed in time") + except ClientDisconnect: + # Клиент оборвал загрузку: в БД и S3 ничего не записано. + log.info("upload aborted by client before completion") + raise bad_request("Upload was interrupted") + + try: + content_type, width, height = await run_in_threadpool(_validate_image, request, data, declared) + except ImageRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) + + attachment_id = uuid_mod.uuid4() + job_id = uuid_mod.uuid4() + # Ключи назначает сервер: не связаны с filename. staging — случайный; + # конечный ключ оригинала детерминирован от id (идемпотентная публикация worker'ом). + staging_key = "%s%s" % (STAGING_PREFIX, uuid_mod.uuid4().hex) + final_key = "final/%s/original" % attachment_id + # [1] намерение + created = await run_in_threadpool( + storage.create_pending_attachment, attachment_id, chat_id, user.id, name, + content_type, len(data), sha256_hex, width, height, final_key, staging_key, + settings.pipeline_version, + ) + if not created: + raise not_found() + # [2] байты в устойчивый staging + try: + _inject(settings, "before_put") + await s3_call(store, settings.s3_upload_deadline_seconds, + store.put, staging_key, data, content_type, md5_digest, sha256_hex) + except ObjectStoreUnavailable: + await _compensate(storage, store, attachment_id, None) + raise s3_unavailable() + finally: + del data + # [3] задание + outbox одной транзакцией + try: + _inject(settings, "after_put") + accepted = await run_in_threadpool( + storage.accept_attachment, attachment_id, job_id, + job_payload(job_id, attachment_id, settings.pipeline_version, 1), + settings.job_max_attempts, settings.relay_fast_path_grace_seconds, + ) + except (psycopg.Error, PoolTimeout): + await _compensate(storage, store, attachment_id, staging_key) + raise + if accepted is None: + # Строку уже забрал сборщик (загрузка шла дольше grace) — объект лишний. + await _compensate(storage, store, attachment_id, staging_key) + raise ApiError(503, "unavailable", "Upload expired, please retry") + attachment, outbox_id = accepted + relay = request.app.state.relay + if relay is not None: + # Выполняется ПОСЛЕ отправки ответа 202. + background.add_task(relay.publish_now, outbox_id) + return attachment + finally: + slots.release() + + +async def _compensate(storage: Storage, store: ObjectStore, attachment_id: UUID, object_key: Optional[str]) -> None: + """Best-effort откат в рамках запроса. Если не удалось — уберёт сборщик.""" + if object_key is not None: + try: + await s3_call(store, 5.0, store.delete, object_key) + except Exception as exc: + log.warning("compensation: object delete deferred to gc (%s)", type(exc).__name__) + try: + await run_in_threadpool(storage.delete_pending_attachment, attachment_id) + except Exception as exc: + log.warning("compensation: pending row left for gc (%s)", type(exc).__name__) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID, user: User = Depends(current_user)) -> Attachment: + found = storage_of(request).get_attachment_for_member(attachment_id, user.id) + if found is None: + raise not_found() + return found[0] + + +VARIANT_CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} + + +def content_disposition(filename: str) -> str: + from urllib.parse import quote + ascii_name = "".join(c if 32 <= ord(c) < 127 and c not in ('"', "\\", ";") else "_" for c in filename) + return "inline; filename=\"%s\"; filename*=UTF-8''%s" % (ascii_name, quote(filename, safe="")) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + request: Request, + attachment_id: UUID, + variant: Optional[str] = Query(default=None), + user: User = Depends(current_user), +): + """Байты оригинала или варианта через авторизованный API (без redirect), потоком из S3. + + lab5: queued/processing/failed -> 409 conflict; у ready нет такого варианта -> 404. + """ + if variant is not None and variant != "original" and variant not in KNOWN_VARIANTS: + raise bad_request("Invalid variant") + storage = storage_of(request) + found = await run_in_threadpool(storage.get_attachment_for_member, attachment_id, user.id) + if found is None: + raise not_found() + attachment, object_key, pipeline_version = found + if attachment.status != "ready": + raise ApiError(409, "conflict", "Attachment is not ready (status: %s)" % attachment.status) + if variant in KNOWN_VARIANTS: + meta = await run_in_threadpool(storage.variant_object, attachment_id, pipeline_version, variant) + if meta is None: + raise not_found() + object_key, media_type, size_bytes, etag = meta + disposition_name = "%s-%s.%s" % (variant, attachment.filename.rsplit(".", 1)[0][:200], + "webp" if media_type == "image/webp" else "jpg") + else: + media_type, size_bytes, etag = attachment.content_type, attachment.size_bytes, attachment.sha256 + disposition_name = attachment.filename + store = objects_of(request) + try: + chunks, length = await s3_call(store, request.app.state.settings.s3_download_deadline_seconds, + store.open, object_key) + except ObjectStoreUnavailable: + raise s3_unavailable() + except ObjectMissing: + log.error("ready attachment %s has no object in S3 (see python -m src.reconcile)", attachment.id) + raise ApiError(500, "internal_error", "Internal server error") + if length != size_bytes: + chunks.close() # закрыть поток S3 + log.error("attachment %s: object size differs from metadata", attachment.id) + raise ApiError(500, "internal_error", "Internal server error") + return StreamingResponse( + chunks, + media_type=media_type, + headers={ + "Content-Length": str(size_bytes), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "Content-Disposition": content_disposition(disposition_name), + "ETag": '"%s"' % etag, + }, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..0963b22 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,25 @@ +"""Однократный запуск миграций: python -m src.migrate + +lab3: это команда one-shot сервиса `migrate` в compose.yaml. Реплики API +миграции не применяют (RUN_MIGRATIONS_ON_START=false) и стартуют только после +успешного завершения `migrate` (depends_on: service_completed_successfully). +Повторный запуск ничего не меняет; одновременный запуск нескольких копий +сериализуется pg_advisory_lock (см. src/db.py). +""" +import logging + +from .config import load_settings +from .db import wait_and_migrate + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + applied = wait_and_migrate(load_settings()) + if applied: + print("applied migrations: " + ", ".join(applied)) + else: + print("schema is up to date; nothing applied") + + +if __name__ == "__main__": + main() diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..9794559 --- /dev/null +++ b/src/models.py @@ -0,0 +1,113 @@ +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class UserCreate(StrictModel): + username: str = Field( + min_length=3, + max_length=32, + pattern=r"^[a-z0-9_]{3,32}$", + ) + display_name: str = Field( + min_length=1, + max_length=100, + ) + # lab2: пароль обязателен; длина в Unicode code points (len() в Python). + password: str = Field( + min_length=12, + max_length=128, + ) + + +class User(StrictModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(StrictModel): + token: str + token_type: str = "Bearer" + expires_at: datetime + user: User + + +class ChatCreate(StrictModel): + title: str = Field( + min_length=1, + max_length=100, + ) + member_ids: List[UUID] = Field( + min_length=1, + max_length=100, + ) + + +class Chat(StrictModel): + id: UUID + title: str + member_ids: List[UUID] + created_at: datetime + + +class MessageCreate(StrictModel): + # lab4: text="" допустим, если есть хотя бы одно вложение (проверка в main.py). + text: str = Field( + min_length=0, + max_length=4000, + ) + # Отсутствующее поле == [] (контракт); значение по умолчанию попадает и в + # канонический JSON отпечатка идемпотентности. + attachment_ids: List[UUID] = Field(default_factory=list, max_length=10) + + +class Message(StrictModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: List[UUID] = Field(default_factory=list) + + +class Variant(StrictModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(StrictModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + # lab5: queued | processing | ready | failed (внутренний pending в API не виден). + status: str = "queued" + variants: List[Variant] = Field(default_factory=list) + error_code: Optional[str] = None + created_at: datetime + updated_at: datetime + + +class ChatPage(StrictModel): + items: List[Chat] + next_cursor: Optional[str] + + +class MessagePage(StrictModel): + items: List[Message] + next_cursor: Optional[str] diff --git a/src/objectstore.py b/src/objectstore.py new file mode 100644 index 0000000..10a6864 --- /dev/null +++ b/src/objectstore.py @@ -0,0 +1,229 @@ +"""Приватное S3-совместимое хранилище оригиналов (lab4: SeaweedFS). + +API работает с отдельными credentials, у которых есть только Read/Write/List +на один бакет (см. infra/seaweedfs/README.md): создать/удалить бакет или +прочитать другой бакет этими ключами нельзя. Административные ключи есть +только у one-shot сервиса s3-init (src/s3_bootstrap.py). + +Все вызовы ограничены по времени (connect/read timeout, не более +S3_MAX_ATTEMPTS попыток), поэтому недоступность S3 превращается в быстрый 503 +для операций с вложениями, а не в зависший запрос. +""" +import base64 +import hashlib +import io +import logging +import threading +import time +from datetime import datetime +from typing import Iterator, List, Optional, Tuple + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import Settings +from .tls import s3_verify + +log = logging.getLogger("messenger.s3") + +CHUNK = 64 * 1024 + + +class ObjectStoreUnavailable(Exception): + """S3 недоступен, таймаут или 5xx -> 503 для клиента.""" + + +class ObjectMissing(Exception): + """Объекта нет в бакете (404/NoSuchKey).""" + + +def _code(exc: ClientError) -> str: + return str(exc.response.get("Error", {}).get("Code", "")) + + +def _status(exc: ClientError) -> int: + return int(exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) or 0) + + +def make_client(endpoint: str, region: str, access_key: str, secret_key: str, + connect_timeout: float = 2.0, read_timeout: float = 10.0, max_attempts: int = 2): + config = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"max_attempts": max_attempts, "mode": "standard"}, + # Соединения на реплику: не больше, чем одновременных загрузок + скачиваний. + max_pool_connections=32, + ) + return boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=config, + # lab7: HTTPS с проверкой CA и hostname (без verify=False). + verify=s3_verify(), + ) + + +class CircuitBreaker: + """После отказа S3 следующие open_seconds операции сразу получают + ObjectStoreUnavailable, не тратя время на DNS/connect/повторы. Затем — + одна пробная попытка (half-open): успех закрывает, отказ снова открывает.""" + + def __init__(self, open_seconds: float) -> None: + self.open_seconds = open_seconds + self._lock = threading.Lock() + self._open_until = 0.0 + + def check(self) -> None: + with self._lock: + now = time.monotonic() + if now < self._open_until: + raise ObjectStoreUnavailable() + if self._open_until: + # half-open: пропускаем одну попытку, остальные пока отклоняем + self._open_until = now + self.open_seconds + + def failure(self) -> None: + with self._lock: + if not self._open_until or time.monotonic() >= self._open_until - self.open_seconds: + log.warning("s3 circuit opened for %.1fs", self.open_seconds) + self._open_until = time.monotonic() + self.open_seconds + + def success(self) -> None: + with self._lock: + self._open_until = 0.0 + + @property + def is_open(self) -> bool: + return time.monotonic() < self._open_until + + +class ObjectStore: + def __init__(self, settings: Settings) -> None: + self.bucket = settings.s3_bucket + self.breaker = CircuitBreaker(settings.s3_circuit_open_seconds) + self.client = make_client( + settings.s3_endpoint, + settings.s3_region, + settings.s3_access_key_id, + settings.s3_secret_access_key, + settings.s3_connect_timeout_seconds, + settings.s3_read_timeout_seconds, + settings.s3_max_attempts, + ) + + @staticmethod + def configured(settings: Settings) -> bool: + return bool(settings.s3_endpoint and settings.s3_access_key_id and settings.s3_secret_access_key) + + def _wrap(self, exc: Exception, op: str) -> Exception: + if isinstance(exc, ClientError): + code, status = _code(exc), _status(exc) + if status == 404 or code in ("NoSuchKey", "NotFound", "404"): + self.breaker.success() # S3 ответил — он доступен + return ObjectMissing() + log.warning("s3 %s failed: %s/%s", op, status, code) + else: + log.warning("s3 %s failed: %s", op, type(exc).__name__) + self.breaker.failure() + return ObjectStoreUnavailable() + + def put(self, key: str, data, content_type: str, md5_digest: bytes, sha256_hex: str) -> None: + """PUT с Content-MD5: S3 сам отклонит повреждённую по пути передачу (BadDigest). + + Тело передаётся как BytesIO поверх bytes (без копирования): botocore + отправляет файловый объект блоками. С bytes он склеивает заголовки и тело + в один буфер — лишняя копия размером с файл на каждую загрузку (замер в REPORT.md). + """ + self.breaker.check() + try: + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=io.BytesIO(data), + ContentType=content_type, + ContentMD5=base64.b64encode(md5_digest).decode("ascii"), + Metadata={"sha256": sha256_hex}, + ) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "put") + self.breaker.success() + + def open(self, key: str) -> Tuple[Iterator[bytes], int]: + """Потоковое чтение: (итератор кусков по 64 KiB, длина). Память — O(chunk).""" + self.breaker.check() + try: + obj = self.client.get_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "get") + self.breaker.success() + body = obj["Body"] + + def chunks() -> Iterator[bytes]: + try: + for chunk in body.iter_chunks(CHUNK): + yield chunk + finally: + body.close() + + return chunks(), int(obj.get("ContentLength", -1)) + + def read_all(self, key: str, limit: int) -> bytes: + """Для сверки/тестов: прочитать объект целиком (не больше limit байт).""" + it, _ = self.open(key) + out = bytearray() + for chunk in it: + out += chunk + if len(out) > limit: + raise ValueError("object exceeds limit") + return bytes(out) + + def delete(self, key: str) -> None: + """Идемпотентно: удаление отсутствующего объекта — не ошибка.""" + self.breaker.check() + try: + self.client.delete_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + err = self._wrap(exc, "delete") + if not isinstance(err, ObjectMissing): + raise err + + def head(self, key: str) -> Optional[dict]: + self.breaker.check() + try: + return self.client.head_object(Bucket=self.bucket, Key=key) + except (ClientError, BotoCoreError) as exc: + err = self._wrap(exc, "head") + if isinstance(err, ObjectMissing): + return None + raise err + + def list_page(self, prefix: str, token: Optional[str] = None, + page_size: int = 1000) -> Tuple[List[Tuple[str, datetime, int]], Optional[str]]: + kwargs = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size} + if token: + kwargs["ContinuationToken"] = token + self.breaker.check() + try: + resp = self.client.list_objects_v2(**kwargs) + except (ClientError, BotoCoreError) as exc: + raise self._wrap(exc, "list") + items = [(o["Key"], o["LastModified"], int(o.get("Size", 0))) for o in resp.get("Contents", [])] + next_token = resp.get("NextContinuationToken") if resp.get("IsTruncated") else None + return items, next_token + + def ping(self) -> bool: + try: + self.client.head_bucket(Bucket=self.bucket) + return True + except (ClientError, BotoCoreError): + return False + + +def md5_and_sha256(data: bytes) -> Tuple[bytes, str]: + return hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest() diff --git a/src/outbox.py b/src/outbox.py new file mode 100644 index 0000000..53635c6 --- /dev/null +++ b/src/outbox.py @@ -0,0 +1,187 @@ +"""Transactional outbox relay (lab5): PostgreSQL job_outbox -> брокер. + +Почему нет разрыва «COMMIT в БД прошёл, а publish не случился»: +API пишет задание и строку outbox в ОДНОЙ транзакции с переводом вложения в +queued. Публикация — отдельный шаг, который можно безопасно повторять: + + быстрый путь API сразу после ответа 202 (background task) публикует свою строку: + SELECT ... FOR UPDATE SKIP LOCKED -> XADD -> DELETE -> COMMIT. + sweeper Каждый процесс API и worker раз в RELAY_POLL_INTERVAL_SECONDS + публикует строки с available_at <= now(). У свежей строки + available_at = created_at + RELAY_FAST_PATH_GRACE_SECONDS, чтобы + не соревноваться с быстрым путём; у retry — время следующей попытки. + +Окна отказа: + * процесс умер после COMMIT до XADD -> строка осталась -> sweeper опубликует; + * умер после XADD до DELETE/COMMIT -> строка осталась -> повторная публикация, + дубль в брокере; worker идемпотентен (redelivery_noop); + * брокер недоступен -> ROLLBACK, publish_failures+1, повтор позже. +Строка блокируется FOR UPDATE SKIP LOCKED: несколько relay публикуют разные строки. +Успешный XADD = подтверждение приёма брокером (AOF appendfsync always). +""" +import logging +import os +import threading +import time +from typing import Optional + +import psycopg +from psycopg_pool import ConnectionPool, PoolTimeout + +from opentelemetry.trace import SpanKind, Status, StatusCode + +from . import telemetry +from .broker import Broker, BrokerUnavailable +from .jobs import JobStore + +log = logging.getLogger("messenger.outbox") + + +class Relay: + def __init__(self, pool: ConnectionPool, broker: Broker, settings, actor: str) -> None: + self.pool = pool + self.broker = broker + self.settings = settings + self.actor = actor + self.jobs = JobStore(pool, settings) + self.fault = settings.fault_injection + self.published = 0 + + def _publish_rows(self, conn, rows, fast: bool = False) -> int: + sent = [] + for outbox_id, job_id, payload in rows: + stream_id = self._publish_traced(payload) # BrokerUnavailable -> ROLLBACK всего пакета + sent.append((outbox_id, job_id, payload, stream_id)) + if fast and self.fault == "crash_after_publish": + log.error("FAULT_INJECTION: relay exits after XADD %s before outbox DELETE/COMMIT", stream_id) + os._exit(70) + conn.execute("DELETE FROM job_outbox WHERE id = ANY(%s)", ([s[0] for s in sent],)) + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO job_events (job_id, attachment_id, attempt, event, detail, actor) " + "VALUES (%s, %s, %s, 'published', %s, %s)", + [(job_id, payload.get("attachment_id"), payload.get("attempt"), "stream id " + stream_id, self.actor) + for _oid, job_id, payload, stream_id in sent], + ) + self.published += len(sent) + return len(sent) + + def _publish_traced(self, payload: dict) -> str: + """lab6: PRODUCER span — ребёнок span'а, сохранённого в строке outbox + (HTTP upload или consumer при retry). В брокер уходит traceparent уже + этого producer span: цепочка upload -> publish -> process.""" + parent = telemetry.extract(payload) + with telemetry.tracer().start_as_current_span( + "%s publish" % self.broker.stream, context=parent, kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "redis", + "messaging.operation.type": "publish", + "messaging.destination.name": self.broker.stream, + "messenger.job.type": payload.get("type", ""), + "messenger.job.attempt": int(payload.get("attempt") or 0), + "messenger.relay.actor": self.actor.split(":", 1)[0], + }, + ) as span: + message = {k: v for k, v in payload.items() if k not in ("traceparent", "tracestate")} + telemetry.inject(message) + try: + stream_id = self.broker.publish(message) + except BrokerUnavailable as exc: + span.set_status(Status(StatusCode.ERROR, "broker unavailable")) + span.record_exception(exc) + raise + span.set_attribute("messaging.message.id", stream_id) + telemetry.INSTRUMENTS.published.add(1, {"job.type": payload.get("type", "")}) + return stream_id + + def publish_now(self, outbox_id: int) -> bool: + """Быстрый путь для одной строки (вызывается после ответа 202). Ошибки не + пробрасываются: строку опубликует sweeper.""" + try: + with self.pool.connection() as conn: + with conn.transaction(): + rows = conn.execute( + "SELECT id, job_id, payload FROM job_outbox WHERE id = %s FOR UPDATE SKIP LOCKED", + (outbox_id,), + ).fetchall() + if not rows: + return False + if self.fault == "crash_after_commit": + # Внедрённый разрыв: COMMIT задания и outbox уже есть, 202 отправлен, + # процесс умирает до XADD. Восстановление — sweeper другого процесса. + self.jobs.event(rows[0][1], rows[0][2].get("attachment_id"), 1, "fault_injected", + "crash between COMMIT and publish", self.actor) + log.error("FAULT_INJECTION: process exit between COMMIT and publish (outbox %s)", outbox_id) + os._exit(70) + if self.fault == "crash_after_publish": + self.jobs.event(rows[0][1], rows[0][2].get("attachment_id"), 1, "fault_injected", + "crash after XADD before outbox commit", self.actor) + return self._publish_rows(conn, rows, fast=True) > 0 + except (BrokerUnavailable, psycopg.Error, PoolTimeout) as exc: + log.warning("fast publish deferred to sweeper: %s", type(exc).__name__) + return False + + def sweep_once(self) -> int: + """Опубликовать все готовые строки пакетами; вернуть число опубликованных.""" + total = 0 + while True: + with self.pool.connection() as conn: + try: + with conn.transaction(): + rows = conn.execute( + """ + SELECT id, job_id, payload FROM job_outbox + WHERE available_at <= clock_timestamp() + ORDER BY available_at, id + LIMIT %s + FOR UPDATE SKIP LOCKED + """, + (self.settings.relay_batch,), + ).fetchall() + if not rows: + return total + total += self._publish_rows(conn, rows) + except BrokerUnavailable: + # Транзакция откатилась (строки целы). Отметим неудачу отдельно. + with conn.transaction(): + conn.execute( + "UPDATE job_outbox SET publish_failures = publish_failures + 1 " + "WHERE id = ANY(%s)", ([r[0] for r in rows],)) + raise + if len(rows) < self.settings.relay_batch: + return total + + def loop(self, stop: threading.Event, wake: Optional[threading.Event] = None) -> None: + """Фоновый цикл: sweeper + сборщик просроченных аренд.""" + interval = self.settings.relay_poll_interval_seconds + last_reap = 0.0 + broker_down_logged = False + while not stop.is_set(): + try: + n = self.sweep_once() + if n: + log.info("relay %s: published %d outbox rows", self.actor, n) + broker_down_logged = False + if time.monotonic() - last_reap >= 5: + last_reap = time.monotonic() + for job_id, outcome in self.jobs.reap_expired(self.actor): + log.warning("lease expired: job %s %s", job_id, outcome) + except BrokerUnavailable: + if not broker_down_logged: + log.warning("relay %s: broker unavailable; outbox rows are kept", self.actor) + broker_down_logged = True + except (psycopg.Error, PoolTimeout) as exc: + log.warning("relay %s: database unavailable (%s)", self.actor, type(exc).__name__) + except Exception: + log.exception("relay %s: unexpected error", self.actor) + if wake is not None: + wake.wait(interval) + wake.clear() + else: + stop.wait(interval) + + +def start_relay_thread(pool, broker, settings, actor: str, stop: threading.Event) -> Relay: + relay = Relay(pool, broker, settings, actor) + threading.Thread(target=relay.loop, args=(stop,), name="outbox-relay", daemon=True).start() + return relay diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..3b1b1e2 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,299 @@ +"""Согласованность PostgreSQL (метаданные) и S3 (байты): сборка мусора и сверка. + +lab5: префиксы бакета + staging/ — принятый оригинал до обработки (API); + final//original — опубликованный оригинал (worker); + final//p<версия>/<вариант>. — варианты (worker); + originals/ — оригиналы lab4 (до перехода на lab5). +Строка pending ссылается на staging_key (объект появляется ПОСЛЕ строки). +Добавлен проход D — очистка staging: ready-вложения (staging больше не нужен, +результат опубликован) и failed старше STAGING_RETENTION_SECONDS (после этого +replay невозможен — нужен повторный upload). Текущую работу (queued/processing) +сборщик не трогает никогда. + +Почему неучтённых объектов «навсегда» не бывает (подробно — REPORT.md): + + загрузка: [1] INSERT attachments(state=pending, object_key) COMMIT + [2] S3 PUT object_key + [3] UPDATE state=ready WHERE state=pending COMMIT -> 201 + Ключ объекта записан в БД ДО того, как объект появится в бакете. + + Сбой/обрыв между шагами: + после [1] -> строка pending без объекта; + после [2] -> строка pending + объект (S3 прошёл, COMMIT [3] — нет); + ошибка S3 в [2] -> API сам компенсирует: DELETE pending; + ошибка БД в [3] -> API сам компенсирует: DELETE object (pending уберёт GC). + Процесс может умереть в любой точке, поэтому компенсация в запросе — + оптимизация, а гарантию даёт сборщик: + + A. stale pending: строки pending старше GRACE. Сначала строка удаляется + (DELETE ... WHERE state='pending' RETURNING object_key — атомарный захват: + если запрос всё-таки успел сделать [3], строка уже ready и не удаляется), + затем удаляется объект. Если удаление объекта не удалось — объект стал + «сиротой» и будет убран проходом B. + B. orphan objects: объекты в бакете (префикс originals/), для ключа которых + в БД нет строки, и LastModified старше GRACE. Строка всегда создаётся + раньше объекта и после удаления не возрождается (ключ одноразовый), + поэтому «нет строки» + «старше GRACE» = объект не принадлежит живой загрузке. + C. missing objects: строки ready без объекта (или с другим размером) — только + отчёт: восстановить байты нельзя, удалять метаданные автоматически опасно. + +GRACE (ATTACHMENT_GC_GRACE_SECONDS, по умолчанию 900 с, минимум 300 с) много +больше самой долгой загрузки: чтение тела <= UPLOAD_READ_TIMEOUT_SECONDS (60 с) ++ проверка + S3 PUT (connect 2 с + read 10 с) x S3_MAX_ATTEMPTS (2). +Строки ready и вложения, на которые ссылаются сообщения, сборщик не трогает +никогда (FK message_attachments -> attachments ON DELETE RESTRICT). + +CLI (по умолчанию dry-run, ничего не меняет): + python -m src.reconcile # отчёт + python -m src.reconcile --apply # исправить A и B + python -m src.reconcile --apply --grace-seconds 0 # только на учебном стенде +""" +import argparse +import json +import logging +import time +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional + +from psycopg_pool import ConnectionPool + +from .objectstore import ObjectStore, ObjectStoreUnavailable + +log = logging.getLogger("messenger.reconcile") + +OBJECT_PREFIX = "originals/" # lab4 +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "final/" +SCANNED_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, OBJECT_PREFIX) +GC_LOCK_ID = 2026_0004 +BATCH = 500 + + +def _try_lock(conn) -> bool: + return conn.execute("SELECT pg_try_advisory_lock(%s)", (GC_LOCK_ID,)).fetchone()[0] + + +def _unlock(conn) -> None: + conn.execute("SELECT pg_advisory_unlock(%s)", (GC_LOCK_ID,)) + + +def collect_stale_pending(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, + apply: bool, report: Dict) -> None: + """Проход A. dry-run: только подсчёт; apply: строка -> затем объект.""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, COALESCE(staging_key, object_key) FROM attachments + WHERE state = 'pending' AND created_at < clock_timestamp() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + """, + (grace_seconds, BATCH), + ).fetchall() + report["stale_pending"] = len(rows) + report["stale_pending_ids"] = [str(r[0]) for r in rows[:20]] + if not apply: + return + deleted_rows = deleted_objects = 0 + for att_id, _key in rows: + with pool.connection() as conn: + claimed = conn.execute( + """ + DELETE FROM attachments + WHERE id = %s AND state = 'pending' + AND created_at < clock_timestamp() - make_interval(secs => %s) + RETURNING COALESCE(staging_key, object_key) + """, + (att_id, grace_seconds), + ).fetchone() + if claimed is None: + continue # загрузка успела завершиться (ready) — не трогаем + deleted_rows += 1 + try: + store.delete(claimed[0]) + deleted_objects += 1 + except ObjectStoreUnavailable: + # Объект стал сиротой без строки; его уберёт проход B. + log.warning("gc: object delete deferred to orphan sweep") + report["stale_pending_rows_deleted"] = deleted_rows + report["stale_pending_object_deletes"] = deleted_objects + + +def known_keys(pool: ConnectionPool, keys: List[str]) -> set: + """Ключи, на которые ссылается БД: оригинал, staging или вариант.""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT object_key FROM attachments WHERE object_key = ANY(%s) + UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s) + UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s) + """, + (keys, keys, keys), + ).fetchall() + return {r[0] for r in rows} + + +def clean_staging(pool: ConnectionPool, store: ObjectStore, retention_seconds: int, + apply: bool, report: Dict) -> None: + """Проход D: staging, который больше не нужен. Строка обновляется ПОСЛЕ удаления + объекта; сбой между шагами безопасен (повторное удаление отсутствующего — no-op).""" + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, staging_key, state FROM attachments + WHERE staging_key IS NOT NULL + AND (state = 'ready' + OR (state = 'failed' AND updated_at < clock_timestamp() - make_interval(secs => %s))) + ORDER BY updated_at LIMIT %s + """, + (retention_seconds, BATCH), + ).fetchall() + report["staging_to_clean"] = len(rows) + if not apply: + return + cleaned = 0 + for att_id, key, state in rows: + try: + store.delete(key) + except ObjectStoreUnavailable: + log.warning("gc: staging delete deferred") + break + with pool.connection() as conn: + conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND staging_key = %s " + "AND state = %s", (att_id, key, state)) + cleaned += 1 + report["staging_cleaned"] = cleaned + + +def sweep_orphans(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, + apply: bool, report: Dict, verify_ready: bool = True) -> None: + """Проходы B и C: полный обход префикса бакета страницами по 1000.""" + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + token: Optional[str] = None + listed = orphans = orphans_young = deleted = 0 + seen_keys: List[str] = [] + orphan_sample: List[str] = [] + started = time.monotonic() + for prefix in SCANNED_PREFIXES: + token = None + while True: + items, token = store.list_page(prefix, token) + listed += len(items) + if items: + keys = [k for k, _, _ in items] + known = known_keys(pool, keys) + for key, modified, _size in items: + if key in known: + if verify_ready: + seen_keys.append(key) + continue + orphans += 1 + if modified > cutoff: + orphans_young += 1 # возможно, живая загрузка/обработка или ещё не истёк grace + continue + if len(orphan_sample) < 20: + orphan_sample.append(key) + if apply and not known_keys(pool, [key]): + # Повторная проверка непосредственно перед удалением. + store.delete(key) + deleted += 1 + if not token: + break + report["objects_listed"] = listed + report["orphan_objects"] = orphans + report["orphan_objects_within_grace"] = orphans_young + report["orphan_objects_sample"] = orphan_sample + if apply: + report["orphan_objects_deleted"] = deleted + report["bucket_scan_seconds"] = round(time.monotonic() - started, 3) + + if verify_ready: + # C: ready-строки, для которых объекта нет в листинге (или не тот размер). + present = set(seen_keys) + with pool.connection() as conn: + rows = conn.execute( + "SELECT id, object_key, size_bytes FROM attachments WHERE state = 'ready' " + "UNION ALL SELECT attachment_id, object_key, size_bytes FROM attachment_variants" + ).fetchall() + missing = [str(r[0]) for r in rows if r[1] not in present] + report["ready_rows"] = len(rows) + report["missing_objects"] = len(missing) + report["missing_object_ids"] = missing[:20] + + +def run_once(pool: ConnectionPool, store: ObjectStore, grace_seconds: int, apply: bool, + orphan_sweep: bool = True, staging_retention_seconds: int = 86400) -> Optional[Dict]: + """Один проход под advisory lock. None — сейчас работает другая реплика.""" + report: Dict = {"mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds} + with pool.connection() as lock_conn: + lock_conn.autocommit = True + if not _try_lock(lock_conn): + return None + try: + collect_stale_pending(pool, store, grace_seconds, apply, report) + clean_staging(pool, store, staging_retention_seconds, apply, report) + if orphan_sweep: + sweep_orphans(pool, store, grace_seconds, apply, report) + finally: + _unlock(lock_conn) + # Соединение вернётся в общий пул — режим транзакций как у остальных. + lock_conn.autocommit = False + return report + + +def gc_loop(pool: ConnectionPool, store: ObjectStore, settings, stop) -> None: + """Фон в каждой реплике: проход A каждые N с, проход B — реже. Lock — одна реплика.""" + import os + # lab7: у API нет права List и записи в final/ (least privilege S3) — полный обход + # бакета (проход B/C) выполняет только worker (ORPHAN_SWEEP_ENABLED=false у API). + sweep_enabled = os.environ.get("ORPHAN_SWEEP_ENABLED", "true").strip().lower() != "false" + last_sweep = time.monotonic() + while not stop.wait(settings.attachment_gc_interval_seconds): + try: + sweep = sweep_enabled and time.monotonic() - last_sweep >= settings.orphan_sweep_interval_seconds + report = run_once(pool, store, settings.attachment_gc_grace_seconds, apply=True, + orphan_sweep=sweep, staging_retention_seconds=settings.staging_retention_seconds) + if report is None: + continue + if sweep: + last_sweep = time.monotonic() + if report.get("stale_pending") or report.get("orphan_objects_deleted") or report.get("missing_objects") \ + or report.get("staging_cleaned"): + log.info("attachment gc: %s", json.dumps(report, ensure_ascii=False)) + except Exception as exc: # БД/S3 недоступны — следующий проход повторит + log.warning("attachment gc skipped: %s", type(exc).__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reconcile attachments metadata (PostgreSQL) with S3 objects") + parser.add_argument("--apply", action="store_true", help="delete stale pending rows and orphan objects") + parser.add_argument("--grace-seconds", type=int, default=None, + help="override ATTACHMENT_GC_GRACE_SECONDS (values < 300 only for lab tests)") + parser.add_argument("--no-orphan-sweep", action="store_true", help="skip bucket listing") + args = parser.parse_args() + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(name)s %(message)s") + + from .config import load_settings + from .db import create_pool + + settings = load_settings() + if not ObjectStore.configured(settings): + raise SystemExit("S3 is not configured") + grace = settings.attachment_gc_grace_seconds if args.grace_seconds is None else args.grace_seconds + pool = create_pool(settings) + pool.wait(timeout=15) + store = ObjectStore(settings) + try: + started = time.monotonic() + report = run_once(pool, store, grace, args.apply, orphan_sweep=not args.no_orphan_sweep, + staging_retention_seconds=settings.staging_retention_seconds) + if report is None: + raise SystemExit("another reconciler holds the lock; retry later") + report["total_seconds"] = round(time.monotonic() - started, 3) + print(json.dumps(report, ensure_ascii=False, indent=2)) + finally: + pool.close() + + +if __name__ == "__main__": + main() diff --git a/src/runtime.py b/src/runtime.py new file mode 100644 index 0000000..45361db --- /dev/null +++ b/src/runtime.py @@ -0,0 +1,53 @@ +"""Состояние процесса, общее для middleware, readiness и сервера. + +INSTANCE_ID — значение X-Instance-Id. lab7: случайный alias реплики +(`api-<8 hex>`), стабильный в пределах жизни процесса и НЕ раскрывающий +hostname/IP контейнера. Чтобы hook stop-one мог сопоставить alias +контейнеру, процесс записывает его в /tmp/instance-id (tmpfs контейнера); +scripts/ops.py читает файл через `docker exec`. INSTANCE_ID из окружения — +для запуска без Docker. + +DRAIN — фаза «осушения» после SIGTERM (см. src/server.py): процесс ещё +обслуживает запросы, но /health/ready уже отвечает 503. +""" +import os +import secrets +import threading +import time +from typing import Optional + +if os.environ.get("INSTANCE_ID_MODE", "").strip().lower() == "hostname": + # worker: имя consumer в Redis Streams должно переживать рестарт контейнера + # (тот же consumer дочитывает свой PEL, lab5). Наружу worker ничего не отдаёт. + import socket + INSTANCE_ID = socket.gethostname() +else: + INSTANCE_ID = os.environ.get("INSTANCE_ID", "").strip() or ( + os.environ.get("INSTANCE_ID_PREFIX", "api") + "-" + secrets.token_hex(4)) + +try: + with open(os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id"), "w", encoding="ascii") as _f: + _f.write(INSTANCE_ID + "\n") +except OSError: + pass + + +class Drain: + def __init__(self) -> None: + self._event = threading.Event() + self.started_at: Optional[float] = None + + def begin(self) -> bool: + """True, если осушение началось этим вызовом.""" + if self._event.is_set(): + return False + self.started_at = time.monotonic() + self._event.set() + return True + + @property + def active(self) -> bool: + return self._event.is_set() + + +DRAIN = Drain() diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 0000000..a6646b8 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,264 @@ +"""Bootstrap и проверка приватного бакета: python -m src.s3_bootstrap + +Запускается one-shot сервисом `s3-init` до старта API (depends_on +service_completed_successfully). Идемпотентно: повторный запуск не меняет +данные — BucketAlreadyOwnedByYou считается успехом. + +Проверки (любое нарушение -> exit 1, API не стартует): + * API-ключи могут PUT/GET/DELETE в staging/ (проба staging/_bootstrap-*, + который сборщик мусора не обходит); + * API-ключи НЕ могут создать бакет (нет прав администратора); + * анонимный GET объекта и анонимный листинг бакета -> 403; + * административные HTTP-порты SeaweedFS (master 9333, filer 8888, + volume 8080) не слушают в общей сети — доступен только S3 API. + + python -m src.s3_bootstrap # s3-init: создать бакет + проверки + python -m src.s3_bootstrap --check-only # только проверки (без admin-ключей) + python -m src.s3_bootstrap --check-only --sample-from-db + # + анонимный GET реального объекта последнего ready-вложения -> 403, + # GET тем же ключом с API-credentials -> 200 и совпадает SHA-256. + python -m src.s3_bootstrap --wait-ready 90 + # прогрев после рестарта: ждать, пока запись/чтение пробного объекта и + # чтение СУЩЕСТВУЮЩЕГО объекта занимают < 2 с. SeaweedFS отвечает /healthz + # сразу, но первые чтение/запись после рестарта блокируются на 15–30 с + # (выборы Raft-лидера master, heartbeat volume-сервера; замер в REPORT.md). +Значения ключей не печатаются. +""" +import argparse +import hashlib +import json +import os +import socket +import sys +import time +import urllib.error +import urllib.request +import uuid +from urllib.parse import quote, urlparse + +from botocore.exceptions import BotoCoreError, ClientError + +from .config import _read_secret +from .objectstore import make_client + +# lab7: + S3 API за gate (8334) и неаутентифицированный gRPC S3-шлюза (18333/18334). +ADMIN_PORTS = (9333, 8888, 8080, 19333, 18888, 8334, 18333, 18334) + + +def _tls_context(): + import ssl + # lab7: S3 только по HTTPS; проверка сертификата и hostname по CA стенда. + return ssl.create_default_context(cafile=os.environ.get("S3_CA_FILE") or os.environ.get("TLS_CA_FILE") or None) + + +def anonymous_status(url: str) -> int: + req = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(req, timeout=5, context=_tls_context()) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def port_open(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=1.5): + return True + except OSError: + return False + + +def wait_ready(timeout: float) -> int: + """Ждать, пока S3 быстро (< 2 с на операцию) выполняет запись, чтение и удаление + пробного объекта и чтение реального (самого старого ready) объекта, если он есть.""" + import psycopg + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + client = make_client(endpoint, os.environ.get("S3_REGION", "us-east-1"), _read_secret("S3_ACCESS_KEY_ID"), + _read_secret("S3_SECRET_ACCESS_KEY"), 1, 2, 1) + started = time.monotonic() + key = None + looked_up = False + while time.monotonic() - started < timeout: + try: + if not looked_up and _read_secret("DATABASE_URL"): + with psycopg.connect(_read_secret("DATABASE_URL"), connect_timeout=3) as conn: + row = conn.execute("SELECT object_key FROM attachments WHERE state = 'ready' " + "ORDER BY created_at LIMIT 1").fetchone() + key = row[0] if row else None + looked_up = True + t = time.monotonic() + probe = "staging/_bootstrap-warmup-%s" % uuid.uuid4().hex + client.put_object(Bucket=bucket, Key=probe, Body=b"warmup") + client.get_object(Bucket=bucket, Key=probe)["Body"].read() + client.delete_object(Bucket=bucket, Key=probe) + if key: + client.get_object(Bucket=bucket, Key=key)["Body"].read() + print(json.dumps({"s3_ready_after_s": round(time.monotonic() - started, 2), + "last_cycle_s": round(time.monotonic() - t, 3), + "existing_object_read": bool(key)})) + return 0 + except (ClientError, BotoCoreError, OSError, psycopg.Error): + time.sleep(0.5) + print("S3 did not become ready for reads/writes within %.0fs" % timeout, file=sys.stderr) + return 1 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check-only", action="store_true") + parser.add_argument("--sample-from-db", action="store_true") + parser.add_argument("--wait-ready", type=float, default=0.0, metavar="SECONDS") + args = parser.parse_args() + if args.wait_ready: + return wait_ready(args.wait_ready) + + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + wait_seconds = int(os.environ.get("S3_BOOTSTRAP_WAIT_SECONDS", "90")) + report = {"endpoint_host": urlparse(endpoint).hostname, "bucket": bucket, "checks": []} + ok = True + + def check(name, condition, detail=""): + nonlocal ok + ok = ok and bool(condition) + report["checks"].append({"check": name, "ok": bool(condition), "detail": detail}) + + api_key = _read_secret("S3_ACCESS_KEY_ID") + api_secret = _read_secret("S3_SECRET_ACCESS_KEY") + if not (api_key and api_secret): + print("S3 API credentials are not configured", file=sys.stderr) + return 2 + api = make_client(endpoint, region, api_key, api_secret, 2, 10, 2) + + if not args.check_only: + admin_key = _read_secret("S3_ADMIN_ACCESS_KEY_ID") + admin_secret = _read_secret("S3_ADMIN_SECRET_ACCESS_KEY") + if not (admin_key and admin_secret): + print("S3 admin credentials are not configured", file=sys.stderr) + return 2 + admin = make_client(endpoint, region, admin_key, admin_secret, 2, 10, 1) + deadline = time.monotonic() + wait_seconds + while True: + try: + admin.list_buckets() + break + except (ClientError, BotoCoreError) as exc: + if time.monotonic() > deadline: + print("S3 is not reachable: %s" % type(exc).__name__, file=sys.stderr) + return 1 + time.sleep(1) + try: + admin.create_bucket(Bucket=bucket) + report["bucket_action"] = "created" + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + report["bucket_action"] = "already exists (idempotent no-op)" + else: + print("create_bucket failed: %s" % code, file=sys.stderr) + return 1 + + def denied(fn): + try: + fn() + return False, "allowed" + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + return code in ("AccessDenied", "Forbidden", "403"), code + + # lab7: least privilege по префиксам. API пишет только staging/, читает final/; + # worker пишет final/ (и удаляет staging/ после публикации). Проба — реальными + # ключами каждой роли; объект пробы сразу удаляется. + probe_key = "staging/_bootstrap-probe-%s" % uuid.uuid4().hex + body = b"probe " + uuid.uuid4().bytes + try: + api.put_object(Bucket=bucket, Key=probe_key, Body=body) + got = api.get_object(Bucket=bucket, Key=probe_key)["Body"].read() + check("api credentials: put/get staging/", got == body) + except (ClientError, BotoCoreError) as exc: + check("api credentials: put/get staging/", False, type(exc).__name__) + got = None + ok_denied, code = denied(lambda: api.put_object(Bucket=bucket, Key="final/_bootstrap-forbidden", Body=b"x")) + check("api credentials cannot write final/ (worker prefix)", ok_denied, code) + ok_denied, code = denied(lambda: api.list_objects_v2(Bucket=bucket, MaxKeys=1)) + check("api credentials cannot list the bucket", ok_denied, code) + + worker_key = _read_secret("S3_WORKER_ACCESS_KEY_ID") + worker_secret = _read_secret("S3_WORKER_SECRET_ACCESS_KEY") + if worker_key and worker_secret: + worker = make_client(endpoint, region, worker_key, worker_secret, 2, 10, 2) + wkey = "final/_bootstrap-probe-%s" % uuid.uuid4().hex + try: + worker.put_object(Bucket=bucket, Key=wkey, Body=body) + wgot = worker.get_object(Bucket=bucket, Key=wkey)["Body"].read() + worker.delete_object(Bucket=bucket, Key=wkey) + check("worker credentials: put/get/delete final/", wgot == body) + except (ClientError, BotoCoreError) as exc: + check("worker credentials: put/get/delete final/", False, type(exc).__name__) + ok_denied, code = denied(lambda: worker.put_object(Bucket=bucket, Key="_bootstrap/forbidden", Body=b"x")) + check("worker credentials cannot write outside staging/ final/ originals/", ok_denied, code) + ok_denied, code = denied(lambda: worker.create_bucket(Bucket="forbidden-%s" % uuid.uuid4().hex[:12])) + check("worker credentials cannot create buckets", ok_denied, code) + + status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(probe_key))) + check("anonymous GET object is denied", status in (401, 403), "HTTP %d" % status) + status = anonymous_status("%s/%s?list-type=2" % (endpoint, bucket)) + check("anonymous list bucket is denied", status in (401, 403), "HTTP %d" % status) + status = anonymous_status(endpoint + "/") + check("anonymous list buckets is denied", status in (401, 403), "HTTP %d" % status) + + other = "forbidden-%s" % uuid.uuid4().hex[:12] + try: + api.create_bucket(Bucket=other) + check("api credentials cannot create buckets", False, "bucket was created") + try: + api.delete_bucket(Bucket=other) + except (ClientError, BotoCoreError): + pass + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + check("api credentials cannot create buckets", code == "AccessDenied", code) + try: + api.delete_bucket(Bucket=bucket) + check("api credentials cannot delete the bucket", False, "bucket deleted") + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + check("api credentials cannot delete the bucket", code in ("AccessDenied", "BucketNotEmpty"), code) + + try: + api.delete_object(Bucket=bucket, Key=probe_key) + except (ClientError, BotoCoreError): + pass + + host = urlparse(endpoint).hostname + exposed = [p for p in ADMIN_PORTS if port_open(host, p)] + check("SeaweedFS admin/UI ports are not reachable on the shared network", not exposed, + "open: %s" % exposed if exposed else "checked %s" % list(ADMIN_PORTS)) + + if args.sample_from_db: + import psycopg + dsn = _read_secret("DATABASE_URL") + with psycopg.connect(dsn, connect_timeout=5) as conn: + row = conn.execute( + "SELECT object_key, sha256 FROM attachments WHERE state = 'ready' " + "ORDER BY created_at DESC LIMIT 1" + ).fetchone() + if row is None: + check("sample ready attachment exists in DB", False, "upload an attachment first (make test)") + else: + key, sha = row + status = anonymous_status("%s/%s/%s" % (endpoint, bucket, quote(key))) + check("anonymous GET of a real attachment object is denied", status in (401, 403), "HTTP %d" % status) + data = api.get_object(Bucket=bucket, Key=key)["Body"].read() + check("same object is readable with API credentials and matches sha256", + hashlib.sha256(data).hexdigest() == sha, "%d bytes" % len(data)) + + report["ok"] = ok + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/security.py b/src/security.py new file mode 100644 index 0000000..7912e34 --- /dev/null +++ b/src/security.py @@ -0,0 +1,148 @@ +"""Пароли, разбор Basic/Bearer и генерация токенов.""" +import base64 +import binascii +import hashlib +import os +import re +import secrets +import threading +from typing import Optional, Tuple + +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + +# Argon2id (argon2-cffi). lab7: параметры из окружения, базовая линия OWASP — +# m=19 MiB, t=2, p=1 (ниже стартовать нельзя). Итоговые значения обоснованы замером +# student-tests/security/argon2_bench.py на стенде (REPORT.md). Пароль хешируется +# целиком в UTF-8. Hash в формате PHC хранит алгоритм, версию, параметры и salt +# ($argon2id$v=19$m=...,t=...,p=...$$), поэтому старые хеши проверяются +# со своими параметрами, а при успешном входе пересчитываются под текущие (rehash). +MIN_MEMORY_KIB = 19 * 1024 +MIN_ITERATIONS = 2 + + +def _int_env(name: str, default: int) -> int: + raw = os.environ.get(name, "").strip() + return int(raw) if raw else default + + +ARGON2_MEMORY_KIB = _int_env("ARGON2_MEMORY_KIB", MIN_MEMORY_KIB) +ARGON2_ITERATIONS = _int_env("ARGON2_ITERATIONS", MIN_ITERATIONS) +ARGON2_PARALLELISM = _int_env("ARGON2_PARALLELISM", 1) +if ARGON2_MEMORY_KIB < MIN_MEMORY_KIB or ARGON2_ITERATIONS < MIN_ITERATIONS or ARGON2_PARALLELISM < 1: + raise RuntimeError("Argon2id parameters below OWASP baseline (m>=19456 KiB, t>=2, p>=1)") + +_hasher = PasswordHasher(time_cost=ARGON2_ITERATIONS, memory_cost=ARGON2_MEMORY_KIB, + parallelism=ARGON2_PARALLELISM, type=Type.ID) + +# Хеш-«пустышка»: проверка для несуществующего пользователя стоит столько же, +# сколько для существующего, — время ответа не выдаёт наличие username. +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing-equalization") + +# 32 байта случайности = 256 бит энтропии; base64url без padding = 43 символа. +TOKEN_BYTES = 32 +MAX_TOKEN_LENGTH = 256 + + +class KdfBusy(Exception): + """Все слоты KDF заняты дольше KDF_QUEUE_TIMEOUT_SECONDS -> 503 + Retry-After.""" + + +# Один хеш занимает ARGON2_MEMORY_KIB памяти и ~1 ядро на время t. Семафор ограничивает +# число одновременных хешей на реплику (PASSWORD_HASH_CONCURRENCY), а ожидание слота — +# KDF_QUEUE_TIMEOUT_SECONDS: при потоке входов запросы не копятся в threadpool бесконечно, +# остальные endpoint продолжают обслуживаться (бюджет login DoS — REPORT.md). +_kdf_slots = threading.BoundedSemaphore(4) +_kdf_wait = 2.0 + + +def configure_kdf_concurrency(limit: int, wait_seconds: float = 2.0) -> None: + global _kdf_slots, _kdf_wait + _kdf_slots = threading.BoundedSemaphore(max(1, limit)) + _kdf_wait = max(0.0, wait_seconds) + + +def _acquire() -> None: + if not _kdf_slots.acquire(timeout=_kdf_wait): + raise KdfBusy() + + +def hash_password(password: str) -> str: + _acquire() + try: + return _hasher.hash(password) + finally: + _kdf_slots.release() + + +def verify_password(password_hash: Optional[str], password: str) -> bool: + _acquire() + try: + return _hasher.verify(password_hash or _DUMMY_HASH, password) and bool(password_hash) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + finally: + _kdf_slots.release() + + +def needs_rehash(password_hash: str) -> bool: + """True, если hash создан с другими (например, старыми более слабыми) параметрами.""" + try: + return _hasher.check_needs_rehash(password_hash) + except (InvalidHashError, ValueError): + return True + + +def hash_parameters(password_hash: str) -> str: + """'m=...,t=...,p=...' из PHC-строки — для отчётов/тестов; salt и hash не раскрываются.""" + try: + return password_hash.split("$")[3] + except IndexError: + return "unknown" + + +def new_token() -> str: + return secrets.token_urlsafe(TOKEN_BYTES) + + +def token_digest(token: str) -> bytes: + """Токен высокоэнтропийный, поэтому для поиска достаточно быстрого SHA-256. + + В отличие от пароля, перебор 256-битного токена невозможен, и KDF здесь + только замедлил бы каждый запрос. + """ + return hashlib.sha256(token.encode("ascii")).digest() + + +def parse_basic(header: Optional[str]) -> Optional[Tuple[str, str]]: + """Authorization: Basic base64(username:password) -> (username, password).""" + if not header: + return None + scheme, _, value = header.strip().partition(" ") + if scheme.lower() != "basic" or not value.strip(): + return None + try: + raw = base64.b64decode(value.strip(), validate=True) + decoded = raw.decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо + if not sep or not USERNAME_RE.match(username): + return None + if not 12 <= len(password) <= 128: + return None + return username, password + + +def parse_bearer(header: Optional[str]) -> Optional[str]: + if not header: + return None + scheme, _, value = header.strip().partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token or len(token) > MAX_TOKEN_LENGTH: + return None + if not token.isascii() or any(ch.isspace() for ch in token): + return None + return token diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..a275c30 --- /dev/null +++ b/src/server.py @@ -0,0 +1,82 @@ +"""Точка входа контейнера: Uvicorn (только HTTPS + mTLS) с фазой осушения перед остановкой. + +lab7 +---- +* Бизнес-порт APP_PORT (8443) — только TLS. Клиентский сертификат обязателен + (CERT_REQUIRED, CA = TLS_CA_FILE), после рукопожатия identity сверяется с + MTLS_ALLOWED_CLIENTS и CRL (src/tls.py). Plaintext HTTP на этом порту нет. +* Docker healthcheck не умеет mTLS, поэтому есть второй, минимальный listener + HEALTH_PORT (8081) на 127.0.0.1 ВНУТРИ контейнера: только GET /health/live и + /health/ready, без бизнес-API. Снаружи контейнера он недоступен (loopback). + LB проверяет реплики по mTLS через основной порт. + +Последовательность при SIGTERM (docker stop / rolling update / stop-one) — как в lab3: + t=0 SIGTERM -> /health/ready = 503, ответы с Connection: close; + t<=1.5 с HAProxy помечает реплику DOWN; + t=DRAIN_SECONDS -> Uvicorn закрывает listener, ждёт запросы (<= GRACEFUL_TIMEOUT_SECONDS). +""" +import logging +import os +import ssl +import threading + +import uvicorn +from uvicorn.protocols.http.h11_impl import H11Protocol + +from . import tls +from .runtime import DRAIN, INSTANCE_ID + +log = logging.getLogger("messenger.server") + + +class DrainingServer(uvicorn.Server): + drain_seconds = 3.0 + + def handle_exit(self, sig, frame): # вызывается из обработчика сигнала + if self.should_exit or not DRAIN.begin(): + return super().handle_exit(sig, frame) + log.info( + "signal %s: instance %s draining for %.1fs (readiness=503), then graceful stop", + sig, INSTANCE_ID, self.drain_seconds, + ) + timer = threading.Timer(self.drain_seconds, super().handle_exit, args=(sig, frame)) + timer.daemon = True + timer.start() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + cert = os.environ.get("TLS_CERT_FILE", "").strip() + key = os.environ.get("TLS_KEY_FILE", "").strip() + ca = os.environ.get("TLS_CA_FILE", "").strip() + plaintext_allowed = os.environ.get("ALLOW_PLAINTEXT_FOR_LOCAL_DEV", "").lower() == "true" + kwargs = {} + if cert and key and ca: + tls.POLICY = tls.policy_from_env() + kwargs = dict(ssl_certfile=cert, ssl_keyfile=key, ssl_ca_certs=ca, + ssl_cert_reqs=ssl.CERT_REQUIRED, http=tls.identity_checking_protocol(H11Protocol)) + log.info("HTTPS with mandatory client certificate; allowed identities=%s crl=%s", + sorted(tls.POLICY.allowed), "on" if tls.POLICY.crl_file else "off") + elif not plaintext_allowed: + # Нет «тихого» отката на HTTP: без сертификатов реплика не стартует. + raise SystemExit("TLS_CERT_FILE, TLS_KEY_FILE and TLS_CA_FILE are required (lab7: HTTPS + mTLS only)") + config = uvicorn.Config( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8443")), + timeout_keep_alive=int(os.environ.get("KEEP_ALIVE_SECONDS", "30")), + timeout_graceful_shutdown=int(os.environ.get("GRACEFUL_TIMEOUT_SECONDS", "10")), + access_log=False, + server_header=False, + log_config=None, + # Ограничение размера заголовков запроса (h11), байт. + h11_max_incomplete_event_size=int(os.environ.get("MAX_HEADER_BYTES", "16384")), + **kwargs, + ) + server = DrainingServer(config) + server.drain_seconds = float(os.environ.get("DRAIN_SECONDS", "3")) + server.run() + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 0000000..eff7b7e --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,243 @@ +"""Сессии: PostgreSQL — источник истины, Redis — кэш проверок токена. + +Роль Redis +---------- +Каждый защищённый запрос сначала ищет `sess:` в Redis. +* попадание -> пользователь берётся из кэша, PostgreSQL не трогаем; +* промах -> читаем sessions JOIN users в PostgreSQL и кладём результат + в Redis через SET NX с TTL = min(SESSION_CACHE_TTL_SECONDS, + время до expires_at). Запись кэша никогда не переживает сессию. + +Инвалидация (logout) +-------------------- +1. UPDATE sessions SET revoked_at (COMMIT в PostgreSQL — момент отзыва); +2. SET sess: = "revoked" PX=SESSION_CACHE_TTL (обычный SET, перезаписывает). +Заполнение кэша идёт через SET NX, поэтому параллельный запрос, прочитавший +сессию до отзыва, не может перезаписать «надгробие» валидной записью. + +Надёжность отзыва при сбое Redis +-------------------------------- +Отзыв в PostgreSQL и строка-«долг» в session_cache_invalidations пишутся +ОДНОЙ транзакцией (миграция 002). Строка удаляется только после успешной +записи «надгробия» в Redis. Если записать его не удалось (таймаут, обрыв +связи, Redis заморожен), на кэш больше нельзя полагаться: в нём могла +остаться положительная запись отозванного токена. Поэтому: +* пока долг не погашен, положительные попадания кэша НЕ принимаются — + каждый токен проверяется в PostgreSQL (отозванный -> 401); +* не чаще раза в секунду API пытается погасить долг: пишет надгробия в + Redis и удаляет строки; строки старше SESSION_CACHE_TTL + 5 с удаляются + без записи (любая положительная запись к этому моменту уже истекла); +* наличие долга перечитывается из PostgreSQL не реже раза в секунду, так что + и другие реплики (lab3) перестают доверять кэшу не позже чем через 1 с; + реплика, выполнившая logout, — сразу, до ответа 204. + +Отказ Redis +----------- +Любая ошибка Redis (таймаут 250 мс) -> кэш пропускается, проверка идёт в +PostgreSQL; на CACHE_RETRY_AFTER_SECONDS Redis выключается (простой +circuit breaker), чтобы не платить таймаут на каждом запросе. +Redis работает без persistence: после рестарта кэш пуст, и это безопасно — +следующие запросы просто заполнят его из PostgreSQL. +""" +import json +import logging +import threading +import time +from datetime import datetime, timezone +from typing import Optional, Tuple + +import redis + +from .config import Settings +from .models import User +from .security import new_token, token_digest +from .storage import Storage +from .tls import redis_tls_kwargs + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionCache: + def __init__(self, settings: Settings) -> None: + self.ttl_ms = settings.session_cache_ttl_seconds * 1000 + self.retry_after = settings.cache_retry_after_seconds + self._down_until = 0.0 + self._lock = threading.Lock() + self.client: Optional[redis.Redis] = None + if settings.cache_url: + self.client = redis.Redis.from_url( + settings.cache_url, + socket_timeout=settings.cache_timeout_seconds, + socket_connect_timeout=settings.cache_timeout_seconds, + retry_on_timeout=False, + health_check_interval=30, + max_connections=20, + decode_responses=True, + # lab7: rediss:// — TLS с проверкой CA и hostname + клиентский cert. + **redis_tls_kwargs("CACHE"), + ) + # Счётчики для наблюдения и тестов (GET /health/cache не нужен контракту, + # поэтому они только логируются). + self.hits = 0 + self.misses = 0 + self.errors = 0 + + @staticmethod + def key(digest: bytes) -> str: + return "sess:" + digest.hex() + + def _available(self) -> bool: + return self.client is not None and time.monotonic() >= self._down_until + + def _failed(self, exc: Exception) -> None: + with self._lock: + self.errors += 1 + if time.monotonic() >= self._down_until: + log.warning("redis unavailable (%s); bypassing cache for %.0fs", + type(exc).__name__, self.retry_after) + self._down_until = time.monotonic() + self.retry_after + + def get(self, digest: bytes): + """None — промах/кэш недоступен; REVOKED — отозвано; иначе (User, expires_at).""" + if not self._available(): + return None + try: + raw = self.client.get(self.key(digest)) + except redis.RedisError as exc: + self._failed(exc) + return None + if raw is None: + self.misses += 1 + return None + self.hits += 1 + if raw == REVOKED: + return REVOKED + try: + data = json.loads(raw) + expires_at = datetime.fromisoformat(data["expires_at"]) + return User(**data["user"]), expires_at + except (ValueError, KeyError, TypeError): + return None + + def put(self, digest: bytes, user: User, expires_at: datetime) -> None: + if not self._available(): + return + remaining_ms = int((expires_at - datetime.now(timezone.utc)).total_seconds() * 1000) + ttl = min(self.ttl_ms, remaining_ms) + if ttl <= 0: + return + value = json.dumps({"user": user.model_dump(mode="json"), "expires_at": expires_at.isoformat()}) + try: + self.client.set(self.key(digest), value, px=ttl, nx=True) + except redis.RedisError as exc: + self._failed(exc) + + def revoke(self, digest: bytes) -> bool: + """True — надгробие записано (или кэша нет вовсе); False — не удалось.""" + if self.client is None: + return True + if not self._available(): + return False + try: + self.client.set(self.key(digest), REVOKED, px=self.ttl_ms) + return True + except redis.RedisError as exc: + self._failed(exc) + return False + + def ping(self) -> bool: + if self.client is None: + return False + try: + return bool(self.client.ping()) + except redis.RedisError: + return False + + +class SessionService: + # Как часто перечитывать из PostgreSQL наличие непогашенных инвалидаций. + RECHECK_SECONDS = 1.0 + + def __init__(self, storage: Storage, cache: SessionCache, settings: Settings) -> None: + self.storage = storage + self.cache = cache + self.ttl = settings.session_ttl_seconds + # Строки старше этого возраста можно удалять без записи в Redis. + self.stale_after = settings.session_cache_ttl_seconds + 5 + self._lock = threading.Lock() + self._pending = True # до первой сверки с БД кэшу не доверяем + self._checked_at = float("-inf") + + def create(self, user: User) -> Tuple[str, datetime]: + token = new_token() + expires_at = self.storage.create_session(token_digest(token), user.id, self.ttl) + return token, expires_at + + def _reconcile(self) -> None: + """Погасить долг инвалидации; вызывается под self._lock.""" + try: + rows = self.storage.pending_invalidations() + done = [] + for digest, age in rows: + if age > self.stale_after or self.cache.revoke(digest): + done.append(digest) + else: + break # Redis всё ещё недоступен — попробуем позже + self.storage.delete_invalidations(done) + self._pending = len(done) < len(rows) or len(rows) >= 500 + if rows and not self._pending: + log.info("cache invalidation backlog cleared (%d)", len(done)) + except Exception as exc: # PostgreSQL недоступна: доверять кэшу нельзя + log.warning("cannot verify cache invalidations: %s", type(exc).__name__) + self._pending = True + # Пробрасываем сразу (-> 503): иначе запрос ждал бы пул второй раз + # в find_session и отвечал за 2 x DB_POOL_TIMEOUT_SECONDS. + raise + + def cache_trusted(self) -> bool: + now = time.monotonic() + if now - self._checked_at < self.RECHECK_SECONDS: + return not self._pending + with self._lock: + if now - self._checked_at >= self.RECHECK_SECONDS: + self._reconcile() + self._checked_at = time.monotonic() + return not self._pending + + def resolve(self, token: str) -> Optional[User]: + digest = token_digest(token) + # Сначала сверка (она может дописать надгробия в Redis), и только + # потом чтение кэша — иначе прочитанная до сверки запись устарела бы. + trusted = self.cache_trusted() + cached = self.cache.get(digest) + if cached == REVOKED: + return None + if cached is not None and trusted: + user, expires_at = cached + if expires_at > datetime.now(timezone.utc): + return user + found = self.storage.find_session(digest) # PostgreSQL недоступна -> исключение -> 503 + if found is None: + return None + user, expires_at = found + self.cache.put(digest, user, expires_at) + return user + + def revoke(self, token: str) -> bool: + digest = token_digest(token) + if not self.storage.revoke_session(digest): # COMMIT: отзыв + долг инвалидации + return False + if self.cache.revoke(digest): + try: + self.storage.delete_invalidations([digest]) + except Exception: + pass # строка останется и будет удалена при следующей сверке + else: + # Надгробие не записано: до погашения долга кэшу не доверяем. + with self._lock: + self._pending = True + self._checked_at = time.monotonic() + log.warning("logout: cache invalidation deferred; positive cache disabled") + return True diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..a058f38 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,707 @@ +"""Доступ к PostgreSQL. Все запросы параметризованы (%s), SQL не склеивается +со значениями пользователя. Каждая публичная операция — одна транзакция +(`with pool.connection()` делает COMMIT при успехе и ROLLBACK при исключении). +""" +from datetime import datetime +from typing import List, Optional, Sequence, Tuple +from uuid import UUID, uuid4 + +from psycopg import errors +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from .models import Attachment, Chat, Message, User, Variant + + +class UsernameTaken(Exception): + pass + + +class UnknownMember(Exception): + pass + + +class NotChatMember(Exception): + """Чата нет или пользователь не участник -> 404.""" + + +class IdempotencyConflict(Exception): + """Тот же ключ уже использован с другим телом -> 409.""" + + +class IdempotencyRace(Exception): + """Строка ключа исчезла между шагами -> 503, клиент повторяет с тем же ключом.""" + + +class AttachmentNotFound(Exception): + """Вложение не существует, ещё не принято (pending) или принадлежит другому чату -> 404.""" + + +class AttachmentFailed(Exception): + """lab5: вложение в состоянии failed нельзя прикрепить к сообщению -> 409.""" + + +# Одна реплика за раз выполняет сборку мусора вложений (src/reconcile.py). +ATTACHMENT_GC_LOCK_ID = 2026_0004 + +ATTACHMENT_COLUMNS = ( + "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, " + "a.sha256, a.state, a.error_code, a.pipeline_version, a.created_at, a.updated_at" +) + +# Внутреннее состояние -> публичный status. pending (приём ещё не завершён) наружу +# не попадает: такие строки запросы API не выбирают. +PUBLIC_STATES = ("queued", "processing", "ready", "failed") + +# Подзапрос: attachment_ids сообщения в исходном порядке (position). +MESSAGE_ATTACHMENT_IDS = ( + "COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) " + "FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[]) AS attachment_ids" +) + + +def _attachment(row, variants=()) -> Attachment: + state = row["state"] + return Attachment( + id=row["id"], + chat_id=row["chat_id"], + owner_id=row["owner_id"], + filename=row["filename"], + content_type=row["content_type"], + size_bytes=row["size_bytes"], + sha256=row["sha256"], + status=state, + # Варианты публикуются вместе со статусом ready (одна транзакция worker). + variants=list(variants) if state == "ready" else [], + error_code=row.get("error_code") if state == "failed" else None, + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +# Одна реплика за раз чистит истёкшие ключи идемпотентности. +IDEMPOTENCY_CLEANUP_LOCK_ID = 2026_0003 + + +Position = Optional[Tuple[datetime, UUID]] + + +def _user(row) -> User: + return User( + id=row["id"], + username=row["username"], + display_name=row["display_name"], + created_at=row["created_at"], + ) + + +class Storage: + def __init__(self, pool: ConnectionPool) -> None: + self.pool = pool + + def ping(self) -> None: + with self.pool.connection() as conn: + conn.execute("SELECT 1") + + # users + + def create_user(self, username: str, display_name: str, password_hash: str) -> User: + try: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid4(), username, display_name, password_hash), + ).fetchone() + except errors.UniqueViolation: + # Гонку двух одинаковых регистраций решает UNIQUE(username) в БД, + # а не проверка «SELECT, потом INSERT» в приложении. + raise UsernameTaken() + return _user(row) + + def get_credentials(self, username: str) -> Optional[Tuple[User, str]]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT id, username, display_name, created_at, password_hash + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + if row is None: + return None + return _user(row), row["password_hash"] + + # lab7: Argon2id rehash и общий для всех реплик rate limit входа + + def update_password_hash(self, user_id: UUID, old_hash: str, new_hash: str) -> bool: + """Compare-and-set: параллельный вход/смена пароля не затираются старым значением.""" + with self.pool.connection() as conn: + cur = conn.execute( + "UPDATE users SET password_hash = %s WHERE id = %s AND password_hash = %s", + (new_hash, user_id, old_hash), + ) + return cur.rowcount == 1 + + def rate_limit_hit(self, buckets, window_seconds: int): + """Фиксированное окно в PostgreSQL (общий счётчик всех реплик). + + buckets: [(key, limit)]. Одна транзакция атомарно увеличивает все счётчики + текущего окна (INSERT ... ON CONFLICT DO UPDATE ... RETURNING). Возвращает + None, если все в пределах лимита, иначе секунды до конца окна (Retry-After). + Ключи — sha256 от username/IP: таблица не хранит username/IP в открытом виде. + """ + retry_after = None + with self.pool.connection() as conn: + with conn.transaction(): + for key, limit in buckets: + row = conn.execute( + """ + INSERT INTO login_rate_limits (bucket, window_start, hits) + VALUES (%s, to_timestamp(floor(extract(epoch FROM clock_timestamp()) / %s) * %s), 1) + ON CONFLICT (bucket, window_start) DO UPDATE SET hits = login_rate_limits.hits + 1 + RETURNING hits, + greatest(1, ceil(extract(epoch FROM window_start) + %s + - extract(epoch FROM clock_timestamp())))::int + """, + (key, window_seconds, window_seconds, window_seconds), + ).fetchone() + if row[0] > limit: + retry_after = max(retry_after or 0, int(row[1])) + return retry_after + + def delete_old_rate_limits(self, older_than_seconds: int = 3600) -> int: + with self.pool.connection() as conn: + cur = conn.execute( + "DELETE FROM login_rate_limits WHERE window_start < clock_timestamp() - make_interval(secs => %s)", + (older_than_seconds,), + ) + return cur.rowcount + + # sessions (источник истины) + + def create_session(self, token_hash: bytes, user_id: UUID, ttl_seconds: int) -> datetime: + with self.pool.connection() as conn: + # Заодно убираем давно истёкшие сессии этого пользователя. + conn.execute( + "DELETE FROM sessions WHERE user_id = %s AND expires_at < now() - interval '1 day'", + (user_id,), + ) + row = conn.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) + RETURNING expires_at + """, + (token_hash, user_id, ttl_seconds), + ).fetchone() + return row[0] + + def find_session(self, token_hash: bytes) -> Optional[Tuple[User, datetime]]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s + AND s.revoked_at IS NULL + AND s.expires_at > clock_timestamp() + """, + (token_hash,), + ).fetchone() + if row is None: + return None + return _user(row), row["expires_at"] + + def revoke_session(self, token_hash: bytes) -> bool: + """Отзыв и долг «инвалидировать кэш» фиксируются одним COMMIT.""" + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = %s + AND revoked_at IS NULL + AND expires_at > clock_timestamp() + RETURNING 1 + """, + (token_hash,), + ).fetchone() + if row is not None: + conn.execute( + """ + INSERT INTO session_cache_invalidations (token_hash) + VALUES (%s) ON CONFLICT (token_hash) DO NOTHING + """, + (token_hash,), + ) + return row is not None + + # долг инвалидации кэша + + def has_pending_invalidations(self) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM session_cache_invalidations)" + ).fetchone()[0] + + def pending_invalidations(self, limit: int = 500) -> List[Tuple[bytes, float]]: + """[(token_hash, возраст в секундах)] — самые старые первыми.""" + with self.pool.connection() as conn: + rows = conn.execute( + """ + SELECT token_hash, EXTRACT(EPOCH FROM clock_timestamp() - revoked_at) + FROM session_cache_invalidations + ORDER BY revoked_at LIMIT %s + """, + (limit,), + ).fetchall() + return [(bytes(h), float(age)) for h, age in rows] + + def delete_invalidations(self, token_hashes: Sequence[bytes]) -> None: + if not token_hashes: + return + with self.pool.connection() as conn: + conn.execute( + "DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", + (list(token_hashes),), + ) + + # chats + + def _insert_members(self, conn, chat_id: UUID, member_ids: Sequence[UUID]) -> None: + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + [(chat_id, member_id, pos) for pos, member_id in enumerate(member_ids)], + ) + + def create_chat(self, title: str, member_ids: Sequence[UUID]) -> Chat: + """Чат и все участники создаются в одной транзакции. + + Существование участников проверяет внешний ключ chat_members.user_id: + если хоть один id неизвестен, INSERT падает, и откатывается вся + транзакция, включая уже вставленную строку chats и других участников. + """ + chat_id = uuid4() + try: + with self.pool.connection() as conn: + with conn.transaction(): + created_at = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING created_at", + (chat_id, title), + ).fetchone()[0] + self._insert_members(conn, chat_id, member_ids) + except errors.ForeignKeyViolation: + raise UnknownMember() + return Chat(id=chat_id, title=title, member_ids=list(member_ids), created_at=created_at) + + def _members_for(self, conn, chat_ids: List[UUID]) -> dict: + members = {chat_id: [] for chat_id in chat_ids} + if not chat_ids: + return members + rows = conn.execute( + """ + SELECT chat_id, user_id FROM chat_members + WHERE chat_id = ANY(%s) ORDER BY chat_id, position + """, + (chat_ids,), + ).fetchall() + for chat_id, user_id in rows: + members[chat_id].append(user_id) + return members + + def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> Optional[Chat]: + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s + WHERE c.id = %s + """, + (user_id, chat_id), + ).fetchone() + if row is None: + return None + members = self._members_for(conn, [row["id"]]) + return Chat(id=row["id"], title=row["title"], member_ids=members[row["id"]], created_at=row["created_at"]) + + def list_user_chats(self, user_id: UUID, limit: int, after: Position) -> Tuple[List[Chat], bool]: + params: list = [user_id] + keyset = "" + if after is not None: + keyset = "AND (c.created_at, c.id) > (%s, %s)" + params.extend(after) + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT c.id, c.title, c.created_at + FROM chat_members m JOIN chats c ON c.id = m.chat_id + WHERE m.user_id = %s {keyset} + ORDER BY c.created_at, c.id + LIMIT %s + """.format(keyset=keyset), + params, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + members = self._members_for(conn, [r["id"] for r in rows]) + chats = [ + Chat(id=r["id"], title=r["title"], member_ids=members[r["id"]], created_at=r["created_at"]) + for r in rows + ] + return chats, has_more + + # messages + + @staticmethod + def _insert_message(conn, message_id: UUID, chat_id: UUID, sender_id: UUID, text: str) -> Optional[Message]: + """Проверка членства и вставка — один оператор, без окна гонки.""" + row = conn.cursor(row_factory=dict_row).execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT %s, %s, %s, %s + WHERE EXISTS ( + SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s + ) + RETURNING id, chat_id, sender_id, text, created_at + """, + (message_id, chat_id, sender_id, text, chat_id, sender_id), + ).fetchone() + return Message(**row) if row is not None else None + + @staticmethod + def _link_attachments(conn, message: Message, attachment_ids: Sequence[UUID]) -> None: + """Привязка вложений в той же транзакции, что и сообщение. + + lab5: можно ссылаться на queued/processing/ready вложения ЭТОГО чата. + Неизвестное/чужое/ещё не принятое (pending) -> AttachmentNotFound (404); + failed -> AttachmentFailed (409). Любое исключение -> ROLLBACK сообщения и + ключа идемпотентности. FOR KEY SHARE не даёт удалить строку до COMMIT; + составной FK (attachment_id, chat_id) дублирует проверку чата в БД. + """ + if not attachment_ids: + return + ids = list(attachment_ids) + rows = conn.execute( + """ + SELECT id, state FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state <> 'pending' + FOR KEY SHARE + """, + (ids, message.chat_id), + ).fetchall() + if len(rows) != len(ids): + raise AttachmentNotFound() + if any(state == "failed" for _id, state in rows): + raise AttachmentFailed() + with conn.cursor() as cur: + cur.executemany( + """ + INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) + VALUES (%s, %s, %s, %s) + """, + [(message.id, aid, message.chat_id, pos) for pos, aid in enumerate(ids)], + ) + message.attachment_ids = ids + + def create_message( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + idem_key: str, + request_hash: bytes, + idem_ttl_seconds: int, + attachment_ids: Sequence[UUID] = (), + ) -> Tuple[Message, bool]: + """Идемпотентная отправка. Возвращает (сообщение, replayed). + + Всё — одна транзакция READ COMMITTED (см. migrations/003): + 1. INSERT строки ключа. Если такой ключ уже есть и он не истёк, + ON CONFLICT ... WHERE ничего не меняет и RETURNING пуст. Если + конкурентная транзакция с тем же ключом ещё не завершилась, INSERT + ЖДЁТ её COMMIT/ROLLBACK на уникальном индексе — поэтому одновременные + повторы через разные реплики создают ровно одно сообщение. + 2a. Ключ наш -> INSERT сообщения (с проверкой членства). Не участник -> + исключение -> ROLLBACK: ключ не «сгорает» на 404. + 2b. Ключ чужой (уже подтверждён) -> снова проверяем доступ, сравниваем + hash тела: совпал -> то же сообщение; нет -> 409. + Ошибка/обрыв на любом шаге откатывает и ключ, и сообщение. + """ + new_id = uuid4() + with self.pool.connection() as conn: + with conn.transaction(): + owned = conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, chat_id, idem_key, request_hash, message_id, expires_at) + VALUES (%s, 'POST', %s, %s, %s, %s, + clock_timestamp() + make_interval(secs => %s)) + ON CONFLICT (user_id, method, chat_id, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, + message_id = EXCLUDED.message_id, + created_at = clock_timestamp(), + expires_at = EXCLUDED.expires_at + -- истёкший (ещё не удалённый очисткой) ключ можно занять заново + WHERE idempotency_keys.expires_at <= clock_timestamp() + RETURNING message_id + """, + (sender_id, chat_id, idem_key, request_hash, new_id, idem_ttl_seconds), + ).fetchone() + if owned is not None: + message = self._insert_message(conn, new_id, chat_id, sender_id, text) + if message is None: + raise NotChatMember() # -> ROLLBACK вместе со строкой ключа + # lab4: чужое/неготовое вложение -> исключение -> ROLLBACK всего. + self._link_attachments(conn, message, attachment_ids) + return message, False + + # Ключ уже подтверждён другой (возможно, конкурентной) транзакцией. + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if member is None: + raise NotChatMember() + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {attachments} + FROM idempotency_keys k + JOIN messages m ON m.id = k.message_id + WHERE k.user_id = %s AND k.method = 'POST' + AND k.chat_id = %s AND k.idem_key = %s + """.format(attachments=MESSAGE_ATTACHMENT_IDS), + (sender_id, chat_id, idem_key), + ).fetchone() + if row is None: + # Теоретически: строка удалена между шагами (очистка/каскад). + # Отдаём 503-подобную транзиентную ошибку, клиент повторит с тем же ключом. + raise IdempotencyRace() + if bytes(row.pop("request_hash")) != request_hash: + raise IdempotencyConflict() + return Message(**row), True + + def delete_expired_idempotency_keys(self, batch: int = 1000) -> Optional[int]: + """Удаляет истёкшие ключи пачкой. None — очистку сейчас выполняет другая реплика.""" + with self.pool.connection() as conn: + with conn.transaction(): + got = conn.execute( + "SELECT pg_try_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,) + ).fetchone()[0] + if not got: + return None + cur = conn.execute( + """ + DELETE FROM idempotency_keys + WHERE ctid IN ( + SELECT ctid FROM idempotency_keys + WHERE expires_at <= clock_timestamp() + LIMIT %s + ) + """, + (batch,), + ) + return cur.rowcount + + def list_messages( + self, chat_id: UUID, user_id: UUID, limit: int, after: Position + ) -> Optional[Tuple[List[Message], bool]]: + params: list = [chat_id] + keyset = "" + if after is not None: + keyset = "AND (created_at, id) > (%s, %s)" + params.extend(after) + params.append(limit + 1) + with self.pool.connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if member is None: + return None + # Keyset-пагинация по индексу messages_chat_created_id_idx: + # Index Scan сразу с позиции курсора, читается limit+1 строк. + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {attachments} + FROM messages m + WHERE m.chat_id = %s {keyset} + ORDER BY m.created_at, m.id + LIMIT %s + """.format(keyset=keyset.replace("(created_at, id)", "(m.created_at, m.id)"), + attachments=MESSAGE_ATTACHMENT_IDS), + params, + ).fetchall() + has_more = len(rows) > limit + return [Message(**r) for r in rows[:limit]], has_more + + # attachments (lab4) + + def is_chat_member(self, chat_id: UUID, user_id: UUID) -> bool: + with self.pool.connection() as conn: + return conn.execute( + "SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)", + (chat_id, user_id), + ).fetchone()[0] + + def create_pending_attachment( + self, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + width: int, + height: int, + object_key: str, + staging_key: Optional[str] = None, + pipeline_version: int = 1, + ) -> bool: + """Шаг 1 из 3: намерение (state=pending) COMMIT до S3 PUT в staging. + + У любого объекта, который API кладёт в бакет, уже есть строка с его ключом + (staging_key): объект без строки может появиться только после удаления + строки сборщиком и никогда не «потеряется» навсегда (src/reconcile.py). + object_key — будущий конечный ключ оригинала, его запишет worker. + Проверка членства и вставка — один оператор. False -> не участник. + """ + with self.pool.connection() as conn: + row = conn.execute( + """ + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, width, height, object_key, + staging_key, pipeline_version, state) + SELECT %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending' + WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s) + RETURNING 1 + """, + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, width, height, object_key, staging_key, pipeline_version, chat_id, owner_id), + ).fetchone() + return row is not None + + def accept_attachment( + self, + attachment_id: UUID, + job_id: UUID, + payload: dict, + max_attempts: int, + outbox_delay_seconds: float, + ) -> Optional[Tuple[Attachment, int]]: + """Шаг 3 из 3 (lab5): после успешного PUT в staging — ОДНА транзакция: + pending -> queued, запись задания и строки outbox. COMMIT => можно 202. + + Возвращает (вложение, outbox_id) или None, если строку уже забрал сборщик. + """ + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.cursor(row_factory=dict_row).execute( + """ + UPDATE attachments a SET state = 'queued', updated_at = clock_timestamp() + WHERE a.id = %s AND a.state = 'pending' + RETURNING {cols} + """.format(cols=ATTACHMENT_COLUMNS), + (attachment_id,), + ).fetchone() + if row is None: + return None + conn.execute( + """ + INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, + attempt, max_attempts) + VALUES (%s, %s, %s, 'queued', 1, %s) + """, + (job_id, attachment_id, row["pipeline_version"], max_attempts), + ) + outbox_id = conn.execute( + """ + INSERT INTO job_outbox (job_id, payload, available_at) + VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) + RETURNING id + """, + (job_id, Jsonb(payload), outbox_delay_seconds), + ).fetchone()[0] + conn.execute( + "INSERT INTO job_events (job_id, attachment_id, attempt, event, actor) " + "VALUES (%s, %s, 1, 'accepted', 'api')", + (job_id, attachment_id), + ) + return _attachment(row), outbox_id + + def mark_attachment_ready(self, attachment_id: UUID) -> Optional[Attachment]: + """lab4-совместимость (синхронная загрузка без worker). В lab5 не используется.""" + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + UPDATE attachments a SET state = 'ready', updated_at = clock_timestamp() + WHERE a.id = %s AND a.state = 'pending' + RETURNING {cols} + """.format(cols=ATTACHMENT_COLUMNS), + (attachment_id,), + ).fetchone() + return _attachment(row) if row is not None else None + + def delete_pending_attachment(self, attachment_id: UUID) -> bool: + """Компенсация при ошибке S3 PUT: удалить только своё незавершённое намерение.""" + with self.pool.connection() as conn: + cur = conn.execute( + "DELETE FROM attachments WHERE id = %s AND state = 'pending'", + (attachment_id,), + ) + return cur.rowcount == 1 + + @staticmethod + def _variants(conn, attachment_id: UUID, pipeline_version: int) -> List[Variant]: + rows = conn.cursor(row_factory=dict_row).execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY CASE name WHEN 'thumbnail' THEN 1 WHEN 'crop' THEN 2 ELSE 3 END + """, + (attachment_id, pipeline_version), + ).fetchall() + return [Variant(**r) for r in rows] + + def get_attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> Optional[Tuple[Attachment, str, int]]: + """(метаданные, object_key оригинала, pipeline_version) принятого вложения, если user — участник чата. + + pending (приём не завершён) — как несуществующее. object_key читаем только в ready. + """ + with self.pool.connection() as conn: + row = conn.cursor(row_factory=dict_row).execute( + """ + SELECT {cols}, a.object_key + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s AND a.state <> 'pending' + """.format(cols=ATTACHMENT_COLUMNS), + (user_id, attachment_id), + ).fetchone() + if row is None: + return None + variants = self._variants(conn, attachment_id, row["pipeline_version"]) if row["state"] == "ready" else [] + return _attachment(row, variants), row["object_key"], row["pipeline_version"] + + def variant_object(self, attachment_id: UUID, pipeline_version: int, name: str) -> Optional[Tuple[str, str, int, str]]: + """(object_key, content_type, size_bytes, sha256) варианта или None.""" + with self.pool.connection() as conn: + row = conn.execute( + """ + SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (attachment_id, pipeline_version, name), + ).fetchone() + return tuple(row) if row else None diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..98a4abf --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,397 @@ +"""lab6: OpenTelemetry для API и worker — traces, metrics, logs по OTLP в Collector. + +Что настраивается в setup(): + * Resource: service.name (messenger-api / messenger-worker), service.version, + service.instance.id (= X-Instance-Id), deployment.environment.name. + * Traces: BatchSpanProcessor (очередь ограничена OTEL_BSP_MAX_QUEUE_SIZE, при + переполнении новые spans отбрасываются, процесс не растёт). Sampler отбрасывает + «сиротские» CLIENT-spans без родителя: это SQL/S3 фоновых циклов (outbox + sweeper, GC, readiness, callbacks метрик) — иначе в Tempo тысячи пустых трасс. + Всё, что происходит внутри HTTP server / producer / consumer span, сохраняется. + * Metrics: PeriodicExportingMetricReader (10 с). Корзины гистограмм заданы + явно в секундах (View), лишние метрики размера тела отброшены. + * Logs: JSON в stdout (для docker logs) + OTLP LoggingHandler. В каждой записи + timestamp, severity, service.name, event, trace_id/span_id (если есть span). + * Автоинструментация: psycopg (SQL без параметров), botocore (S3). + Redis НЕ инструментируется: команды AUTH/кэш сессий могли бы утечь в spans. + +Что никогда не экспортируется: тела сообщений, байты изображений, заголовки +Authorization/Cookie (FastAPI-инструментация заголовки не пишет без явной +настройки), параметры SQL, URL с подписью. Метрики не имеют меток user/chat/message. +Отключение: OTEL_SDK_DISABLED=true (остаются только JSON-логи в stdout). +""" +import json +import logging +import os +import resource +import sys +import time +from datetime import datetime, timezone +from typing import Callable, Dict, Iterable, Optional + +from opentelemetry import metrics, trace +from opentelemetry.context import Context +from opentelemetry.metrics import CallbackOptions, Observation +from opentelemetry.trace import SpanKind +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +from .runtime import INSTANCE_ID + +log = logging.getLogger("messenger.telemetry") + +# Только W3C traceparent/tracestate: baggage в сообщение брокера не попадает +# (jobs.schema.json: additionalProperties=false). +PROPAGATOR = TraceContextTextMapPropagator() + +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0) +JOB_BUCKETS = (0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0) +READY_BUCKETS = (0.1, 0.25, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 900.0) + +_STATE = {"service": "messenger", "enabled": False, "configured": False} + + +def enabled() -> bool: + return _STATE["enabled"] + + +def tracer(): + return trace.get_tracer("messenger") + + +def meter(): + return metrics.get_meter("messenger") + + +# ------------------------------------------------------------------ logging + +_STD_ATTRS = set(logging.LogRecord("", 0, "", 0, "", (), None).__dict__) | {"message", "asctime", "event"} + + +class _EventFilter(logging.Filter): + """У каждой записи есть поле event: явное (extra={"event": ...}) или имя логгера.""" + + def filter(self, record: logging.LogRecord) -> bool: + if not getattr(record, "event", None): + record.event = record.name + return True + + +class JsonFormatter(logging.Formatter): + def __init__(self, compact: bool = False) -> None: + super().__init__() + self.compact = compact + + def format(self, record: logging.LogRecord) -> str: + out = { + "timestamp": datetime.fromtimestamp(record.created, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "severity": record.levelname, + "service.name": _STATE["service"], + "service.instance.id": INSTANCE_ID, + "event": getattr(record, "event", record.name), + "logger": record.name, + "message": record.getMessage(), + } + ctx = trace.get_current_span().get_span_context() + if ctx.is_valid: + out["trace_id"] = format(ctx.trace_id, "032x") + out["span_id"] = format(ctx.span_id, "016x") + for key, value in record.__dict__.items(): + if key not in _STD_ATTRS and not key.startswith("_") and key not in out: + out[key] = value if isinstance(value, (str, int, float, bool)) or value is None else str(value) + if record.exc_info: + if self.compact: + out["error"] = type(record.exc_info[1]).__name__ + else: + out["exception"] = self.formatException(record.exc_info) + return json.dumps(out, ensure_ascii=False) + + +def _configure_logging(otel_handler: Optional[logging.Handler]) -> None: + root = logging.getLogger() + for h in list(root.handlers): + root.removeHandler(h) + stdout = logging.StreamHandler(sys.stdout) + stdout.setFormatter(JsonFormatter()) + stdout.addFilter(_EventFilter()) + root.addHandler(stdout) + if otel_handler is not None: + otel_handler.addFilter(_EventFilter()) + root.addHandler(otel_handler) + root.setLevel(logging.INFO) + # Шумные библиотеки: их DEBUG/INFO не нужны ни в stdout, ни в Loki. + for name in ("botocore", "boto3", "urllib3"): + logging.getLogger(name).setLevel(logging.WARNING) + # Ошибки самого SDK (Collector недоступен): только в stdout, одной строкой, + # без трейсбека и НЕ в OTLP — иначе петля «лог об ошибке экспорта логов». + otel = logging.getLogger("opentelemetry") + for h in list(otel.handlers): + otel.removeHandler(h) + sdk_handler = logging.StreamHandler(sys.stdout) + sdk_handler.setFormatter(JsonFormatter(compact=True)) + sdk_handler.addFilter(_EventFilter()) + otel.addHandler(sdk_handler) + otel.propagate = False + otel.setLevel(logging.WARNING) + + +# ------------------------------------------------------------------ setup + +def _sampler(): + from opentelemetry.sdk.trace.sampling import ALWAYS_ON, Decision, ParentBased, Sampler, SamplingResult + + class DropOrphanClientSpans(Sampler): + """CLIENT-span без родителя (фоновый SQL/S3) не сэмплируется; остальное — ParentBased(AlwaysOn).""" + + _base = ParentBased(ALWAYS_ON) + + def should_sample(self, parent_context, trace_id, name, kind=None, attributes=None, links=None, + trace_state=None): + parent = trace.get_current_span(parent_context).get_span_context() + if not parent.is_valid and kind == SpanKind.CLIENT: + return SamplingResult(Decision.DROP) + return self._base.should_sample(parent_context, trace_id, name, kind, attributes, links, trace_state) + + def get_description(self) -> str: + return "DropOrphanClientSpans{ParentBased{AlwaysOn}}" + + return DropOrphanClientSpans() + + +def setup(service_name: str) -> None: + """Вызывать один раз в процессе ДО создания пула БД и клиента S3.""" + if _STATE["configured"]: + return + _STATE["configured"] = True + _STATE["service"] = service_name + if os.environ.get("OTEL_SDK_DISABLED", "false").strip().lower() == "true": + _configure_logging(None) + log.info("OpenTelemetry disabled (OTEL_SDK_DISABLED=true)", extra={"event": "telemetry.disabled"}) + return + # Стабильные HTTP semconv: http.server.request.duration в секундах, + # атрибуты http.request.method / http.route / http.response.status_code. + os.environ.setdefault("OTEL_SEMCONV_STABILITY_OPT_IN", "http") + + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.metrics.view import DropAggregation, ExplicitBucketHistogramAggregation, View + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + # Resource.create добавляет OTEL_RESOURCE_ATTRIBUTES из окружения. + res = Resource.create({ + "service.name": service_name, + "service.version": os.environ.get("APP_VERSION", "lab7"), + "service.instance.id": INSTANCE_ID, + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + }) + + # Размеры очередей/таймауты — OTEL_BSP_* / OTEL_BLRP_* / OTEL_EXPORTER_OTLP_TIMEOUT из окружения. + tp = TracerProvider(resource=res, sampler=_sampler()) + tp.add_span_processor(BatchSpanProcessor(OTLPSpanExporter())) + trace.set_tracer_provider(tp) + + views = [ + View(instrument_name="http.server.request.duration", + aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.job.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), + View(instrument_name="http.server.request.body.size", aggregation=DropAggregation()), + View(instrument_name="http.server.response.body.size", aggregation=DropAggregation()), + ] + interval = int(os.environ.get("OTEL_METRIC_EXPORT_INTERVAL", "10000")) + reader = PeriodicExportingMetricReader(OTLPMetricExporter(), export_interval_millis=interval) + metrics.set_meter_provider(MeterProvider(resource=res, metric_readers=[reader], views=views)) + + lp = LoggerProvider(resource=res) + lp.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + set_logger_provider(lp) + _configure_logging(LoggingHandler(level=logging.INFO, logger_provider=lp)) + + from opentelemetry.instrumentation.botocore import BotocoreInstrumentor + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor + + PsycopgInstrumentor().instrument(skip_dep_check=True) + BotocoreInstrumentor().instrument(skip_dep_check=True) + _register_process_metrics() + _STATE["enabled"] = True + log.info("OpenTelemetry enabled: endpoint=%s", os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "default"), + extra={"event": "telemetry.enabled"}) + + +def shutdown() -> None: + """Сбросить буферы при штатной остановке (ограничено таймаутами экспортёров).""" + if not _STATE["enabled"]: + return + from opentelemetry._logs import get_logger_provider + + for provider in (trace.get_tracer_provider(), metrics.get_meter_provider(), get_logger_provider()): + try: + provider.shutdown() + except Exception: + pass + + +def instrument_fastapi(app) -> None: + if not _STATE["enabled"]: + return + from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor + + # Health-пробы (HAProxy каждую секунду) не входят ни в трассы, ни в метрики + # HTTP: доля ошибок считается по бизнес-запросам. + FastAPIInstrumentor.instrument_app(app, excluded_urls="/health/live,/health/ready", + exclude_spans=["receive", "send"]) + + +# ------------------------------------------------------------------ messaging context + +def inject(carrier: Dict) -> Dict: + """Записать traceparent/tracestate текущего span в словарь (сообщение задания).""" + PROPAGATOR.inject(carrier) + return carrier + + +def extract(carrier: Optional[Dict]) -> Context: + if not isinstance(carrier, dict): + return Context() + return PROPAGATOR.extract({k: v for k, v in carrier.items() if k in ("traceparent", "tracestate") + and isinstance(v, str)}) + + +# ------------------------------------------------------------------ app metrics + +class _Lazy: + """Инструменты создаются при первом обращении (после setup()).""" + + def __init__(self) -> None: + self._made = False + + def _make(self) -> None: + if self._made: + return + m = meter() + self.jobs_processed = m.create_counter( + "messenger.jobs.processed", unit="{job}", + description="Обработанные сообщения заданий по исходу (succeeded/retry/failed/noop/...)") + self.job_duration = m.create_histogram( + "messenger.job.duration", unit="s", description="Длительность обработки задания worker'ом") + self.time_to_ready = m.create_histogram( + "messenger.attachment.time_to_ready", unit="s", + description="От durable acceptance (COMMIT задания, ответ 202) до публикации ready") + self.published = m.create_counter( + "messenger.jobs.published", unit="{job}", description="Сообщения, опубликованные outbox relay в брокер") + self._made = True + + def __getattr__(self, item): + if item.startswith("_"): + raise AttributeError(item) + self._make() + return self.__dict__[item] + + +INSTRUMENTS = _Lazy() + + +def _safe(callback: Callable[[], Iterable[Observation]]) -> Callable[[CallbackOptions], Iterable[Observation]]: + last = {"t": 0.0} + + def wrapped(_options: CallbackOptions) -> Iterable[Observation]: + try: + return list(callback()) + except Exception as exc: # БД недоступна: точку пропускаем, а не роняем экспорт + if time.monotonic() - last["t"] > 60: # не чаще раза в минуту + last["t"] = time.monotonic() + log.warning("metric callback skipped: %s", type(exc).__name__, + extra={"event": "telemetry.callback_failed"}) + return [] + return wrapped + + +def _register_process_metrics() -> None: + m = meter() + page = os.sysconf("SC_PAGE_SIZE") + + def rss(): + with open("/proc/self/statm") as fh: + yield Observation(int(fh.read().split()[1]) * page) + + def cpu(): + r = resource.getrusage(resource.RUSAGE_SELF) + c = resource.getrusage(resource.RUSAGE_CHILDREN) # подпроцессы обработки изображений + yield Observation(r.ru_utime + r.ru_stime + c.ru_utime + c.ru_stime) + + m.create_observable_gauge("process.memory.rss", [_safe(rss)], unit="By", description="RSS процесса") + m.create_observable_counter("process.cpu.time", [_safe(cpu)], unit="s", + description="CPU-время процесса и его подпроцессов") + + +def register_pool_metrics(pool) -> None: + """DB pool: занято/свободно/ожидающих/максимум (psycopg_pool.get_stats).""" + m = meter() + + def conns(): + st = pool.get_stats() + size, idle = st.get("pool_size", 0), st.get("pool_available", 0) + yield Observation(size - idle, {"state": "used"}) + yield Observation(idle, {"state": "idle"}) + + m.create_observable_gauge("messenger.db.pool.connections", [_safe(conns)], unit="{connection}") + m.create_observable_gauge("messenger.db.pool.waiting", + [_safe(lambda: [Observation(pool.get_stats().get("requests_waiting", 0))])], + unit="{request}", description="Запросы, ждущие свободное соединение") + m.create_observable_gauge("messenger.db.pool.max", [_safe(lambda: [Observation(pool.max_size)])], + unit="{connection}") + + +def register_queue_metrics(pool, queue: str) -> None: + """Глубина очереди и возраст старейшего задания — из PostgreSQL (источник истины). + + Публикуется API-репликами: метрика должна жить, когда worker остановлен + (именно в этот момент она важнее всего). Несколько реплик дают одинаковые + значения — в PromQL берётся max(). + """ + m = meter() + cache = {"t": 0.0, "row": None} + + def row(): + if time.monotonic() - cache["t"] > 5 or cache["row"] is None: + with pool.connection(timeout=1) as conn: + cache["row"] = conn.execute( + """ + SELECT count(*) FILTER (WHERE status = 'queued'), + count(*) FILTER (WHERE status = 'processing'), + COALESCE(EXTRACT(EPOCH FROM clock_timestamp() - min(created_at)), 0), + (SELECT count(*) FROM job_outbox) + FROM attachment_jobs WHERE status IN ('queued', 'processing') + """ + ).fetchone() + cache["t"] = time.monotonic() + return cache["row"] + + attrs = {"queue": queue} + m.create_observable_gauge( + "messenger.queue.depth", + [_safe(lambda: [Observation(row()[0], dict(attrs, state="queued")), + Observation(row()[1], dict(attrs, state="processing"))])], + unit="{job}", description="Незавершённые задания") + m.create_observable_gauge( + "messenger.queue.oldest_age", [_safe(lambda: [Observation(float(row()[2]), attrs)])], + unit="s", description="Возраст старейшего незавершённого задания") + m.create_observable_gauge( + "messenger.outbox.pending", [_safe(lambda: [Observation(row()[3], attrs)])], + unit="{job}", description="Строки outbox, ещё не опубликованные в брокер") + + +def register_worker_metrics(busy: Callable[[], int], capacity: int) -> None: + m = meter() + m.create_observable_gauge("messenger.worker.busy", [_safe(lambda: [Observation(busy())])], unit="{slot}", + description="Занятые consumer-потоки") + m.create_observable_gauge("messenger.worker.capacity", [_safe(lambda: [Observation(capacity)])], + unit="{slot}", description="WORKER_CONCURRENCY") diff --git a/src/tls.py b/src/tls.py new file mode 100644 index 0000000..aa71289 --- /dev/null +++ b/src/tls.py @@ -0,0 +1,193 @@ +"""lab7: TLS/mTLS для API и TLS-клиентов (PostgreSQL, Redis, broker, S3, OTLP). + +Внутреннее mTLS LB -> API (обязательное на «3») +----------------------------------------------- +Uvicorn слушает только HTTPS (`ssl_cert_reqs=CERT_REQUIRED`): без клиентского +сертификата или с сертификатом чужого CA рукопожатие TLS не завершается — это +проверяет OpenSSL (цепочка, срок, EKU clientAuth). + +Цепочки мало: любой сертификат нашего CA (api, worker, intruder...) прошёл бы +рукопожатие. Поэтому после рукопожатия, ДО разбора первого HTTP-запроса, +`IdentityCheckingH11Protocol.connection_made` сверяет identity клиента: + + * SAN DNS (или CN) должен входить в MTLS_ALLOWED_CLIENTS (по умолчанию `lb`); + * serial не должен быть в CRL (MTLS_CRL_FILE). CRL проверяется подписью CA, + перечитывается при изменении файла (не чаще раза в MTLS_CRL_RELOAD_SECONDS), + просроченный или битый CRL -> отказ всем (fail closed). + +Отказ = немедленный abort соединения: HTTP-обработчики, middleware и сессии +никогда не видят запрос от неразрешённой identity. Событие пишется в лог +(`mtls.rejected`, причина и CN, без содержимого сертификата). + +Разделение ответственности: TLS — шифрование и подлинность сервера; mTLS — +подлинность клиента (это LB); пользователь по-прежнему определяется только +Bearer-сессией. mTLS не заменяет авторизацию пользователя. +""" +import asyncio +import logging +import os +import ssl +import threading +import time +from typing import Iterable, Optional, Set, Tuple + +from cryptography import x509 +from cryptography.x509.oid import NameOID + +log = logging.getLogger("messenger.tls") + + +def _env_path(name: str) -> Optional[str]: + value = os.environ.get(name, "").strip() + return value or None + + +class ClientIdentityPolicy: + """Allowlist identity + отзыв по CRL. Потокобезопасно, дёшево на соединение.""" + + def __init__(self, allowed: Iterable[str], ca_file: Optional[str], crl_file: Optional[str], + reload_seconds: float = 5.0) -> None: + self.allowed: Set[str] = {a.strip() for a in allowed if a.strip()} + self.ca_file = ca_file + self.crl_file = crl_file + self.reload_seconds = reload_seconds + self._lock = threading.Lock() + self._revoked: Set[int] = set() + self._crl_ok = crl_file is None + self._crl_mtime: Optional[float] = None + self._checked_at = float("-inf") + self._next_update = None + self.accepted = 0 + self.rejected = 0 + if crl_file: + self._reload(force=True) + + # ------------------------------------------------------------------ CRL + def _reload(self, force: bool = False) -> None: + now = time.monotonic() + if not force and now - self._checked_at < self.reload_seconds: + return + self._checked_at = now + try: + mtime = os.stat(self.crl_file).st_mtime + if not force and mtime == self._crl_mtime: + return + with open(self.crl_file, "rb") as handle: + crl = x509.load_pem_x509_crl(handle.read()) + with open(self.ca_file, "rb") as handle: + ca = x509.load_pem_x509_certificate(handle.read()) + if not crl.is_signature_valid(ca.public_key()): + raise ValueError("CRL signature is not valid for CA") + self._revoked = {r.serial_number for r in crl} + self._next_update = crl.next_update_utc + self._crl_mtime = mtime + self._crl_ok = True + log.info("mTLS CRL loaded: %d revoked serial(s), next update %s", len(self._revoked), + self._next_update.isoformat() if self._next_update else "-", + extra={"event": "mtls.crl_loaded"}) + except Exception as exc: # fail closed: без валидного CRL не пускаем никого + self._crl_ok = False + log.error("mTLS CRL unusable (%s): rejecting all clients", type(exc).__name__, + extra={"event": "mtls.crl_error"}) + + def _crl_valid_now(self) -> bool: + if self.crl_file is None: + return True + with self._lock: + self._reload() + if not self._crl_ok: + return False + if self._next_update is not None: + from datetime import datetime, timezone + if datetime.now(timezone.utc) > self._next_update: + return False + return True + + # ------------------------------------------------------------- identity + @staticmethod + def identities(cert: x509.Certificate) -> Tuple[str, Set[str]]: + try: + cn = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + except IndexError: + cn = "" + names: Set[str] = set() + try: + san = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value + names.update(san.get_values_for_type(x509.DNSName)) + except x509.ExtensionNotFound: + pass + if not names and cn: + names.add(cn) # без SAN — только CN + return cn, names + + def check(self, der: Optional[bytes]) -> Tuple[bool, str, str]: + """-> (разрешено, identity для лога, причина).""" + if not der: + return False, "-", "no_client_certificate" + cert = x509.load_der_x509_certificate(der) + cn, names = self.identities(cert) + if not self._crl_valid_now(): + return False, cn, "crl_unavailable_or_expired" + if cert.serial_number in self._revoked: + return False, cn, "certificate_revoked" + if not names & self.allowed: + return False, cn, "identity_not_allowed" + return True, cn, "ok" + + +POLICY: Optional[ClientIdentityPolicy] = None + + +def policy_from_env() -> ClientIdentityPolicy: + allowed = os.environ.get("MTLS_ALLOWED_CLIENTS", "lb").split(",") + return ClientIdentityPolicy(allowed, _env_path("TLS_CA_FILE"), _env_path("MTLS_CRL_FILE"), + float(os.environ.get("MTLS_CRL_RELOAD_SECONDS", "5"))) + + +def identity_checking_protocol(base): + """Подкласс HTTP-протокола uvicorn: проверка identity сразу после рукопожатия.""" + + class IdentityCheckingProtocol(base): + def connection_made(self, transport: asyncio.Transport) -> None: # type: ignore[override] + super().connection_made(transport) + ssl_object = transport.get_extra_info("ssl_object") + if ssl_object is None: # plaintext сюда не доходит: сервер только TLS + transport.abort() + return + policy = POLICY + der = ssl_object.getpeercert(binary_form=True) + ok, ident, reason = policy.check(der) if policy else (False, "-", "no_policy") + if ok: + policy.accepted += 1 + return + if policy: + policy.rejected += 1 + peer = transport.get_extra_info("peername") + log.warning("mTLS client rejected: identity=%s reason=%s peer=%s", ident, reason, + peer[0] if peer else "-", extra={"event": "mtls.rejected", "reason": reason, + "client_identity": ident}) + transport.abort() + + IdentityCheckingProtocol.__name__ = "IdentityChecking" + base.__name__ + return IdentityCheckingProtocol + + +# ---------------------------------------------------------------- клиенты + +def redis_tls_kwargs(prefix: str) -> dict: + """TLS-параметры redis-py: CA (обязательно для rediss://) и клиентский сертификат (mTLS).""" + ca = _env_path(prefix + "_TLS_CA_FILE") or _env_path("TLS_CA_FILE") + cert = _env_path(prefix + "_TLS_CERT_FILE") or _env_path("TLS_CLIENT_CERT_FILE") + key = _env_path(prefix + "_TLS_KEY_FILE") or _env_path("TLS_CLIENT_KEY_FILE") + kwargs = {} + if ca: + kwargs.update(ssl_ca_certs=ca, ssl_cert_reqs="required", ssl_check_hostname=True, + ssl_min_version=ssl.TLSVersion.TLSv1_2) + if cert and key: + kwargs.update(ssl_certfile=cert, ssl_keyfile=key) + return kwargs + + +def s3_verify() -> object: + """boto3 verify=: проверка сертификата S3 и hostname, без отключения проверки.""" + return _env_path("S3_CA_FILE") or _env_path("TLS_CA_FILE") or True diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..b8768ce --- /dev/null +++ b/src/worker.py @@ -0,0 +1,417 @@ +"""Worker обработки вложений (lab5): python -m src.worker + +Отдельный процесс/контейнер. WORKER_CONCURRENCY потоков; у каждого свой consumer +в группе Redis Streams и prefetch = 1 (XREADGROUP COUNT 1), поэтому одновременно +в работе не больше WORKER_CONCURRENCY сообщений на процесс. + +Обработка одного сообщения: + 1. Проверка схемы (contracts/jobs.schema.json). Мусор -> ack + лог (poison). + 2. claim(): аренда в PostgreSQL (lease_token, JOB_TIMEOUT_SECONDS). Нельзя + захватить (готово/упало/устарело/арендовано живым) -> no-op, ack. + 3. Источник: staging-объект (или уже опубликованный оригинал при повторе); + ключи берутся из БД, не из сообщения. Размер <= MAX_UPLOAD_BYTES, SHA-256 + должен совпасть с метаданными. + 4. Оригинал -> final//original (если там ещё нет того же SHA-256). + 5. Варианты в изолированном подпроцессе (src/imaging.py) -> детерминированные + ключи final//p<версия>/<вариант>.: повтор перезаписывает те же байты. + 6. complete(): ОДНА транзакция, fenced по lease_token: варианты + ready. + 7. XACK — только после COMMIT (consumer ack после устойчивого эффекта). + 8. Удаление staging (не удалось — уберёт сборщик, src/reconcile.py, проход D). + +Ошибки: временные (S3/БД недоступны, таймаут обработки) -> retry через outbox с +backoff+jitter, не больше JOB_MAX_ATTEMPTS; постоянные (битое изображение, лимит +памяти, нет источника) -> failed + dead_letters сразу. Если недоступна сама БД, +сообщение НЕ подтверждается и вернётся через PEL. + +SIGTERM: потоки перестают брать новые сообщения и дорабатывают текущее +(stop_grace_period в compose больше JOB_TIMEOUT_SECONDS). +""" +import hashlib +import logging +import os +import signal +import threading +import time +from pathlib import Path +from typing import Dict, List, Optional + +import psycopg +from opentelemetry.trace import SpanKind, Status, StatusCode +from psycopg_pool import PoolTimeout + +from . import telemetry +from .broker import Broker, BrokerUnavailable, Delivery +from .config import load_settings +from .db import create_pool +from .imaging import VARIANT_EXT, ImageFailure, ImageTimeout, Sandbox, sha256_hex +from .jobs import JOB_TYPE, Claim, JobStore, validate_payload +from .objectstore import ObjectMissing, ObjectStore, ObjectStoreUnavailable +from .outbox import Relay +from .reconcile import gc_loop +from .runtime import INSTANCE_ID + +log = logging.getLogger("messenger.worker") + +HEARTBEAT = Path("/tmp/worker-heartbeat") + + +class Transient(Exception): + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class Permanent(Exception): + def __init__(self, code: str, detail: str = "") -> None: + super().__init__(code) + self.code = code + self.detail = detail + + +class InjectedCrash(Exception): + pass + + +def final_original_key(attachment_id) -> str: + return "final/%s/original" % attachment_id + + +def variant_key(attachment_id, pipeline_version: int, name: str) -> str: + return "final/%s/p%d/%s.%s" % (attachment_id, pipeline_version, name, VARIANT_EXT[name]) + + +class Worker: + def __init__(self) -> None: + self.settings = load_settings() + s = self.settings + if not s.broker_url: + raise RuntimeError("BROKER_URL or BROKER_URL_FILE must be set") + if not ObjectStore.configured(s): + raise RuntimeError("S3 must be configured for the worker") + self.pool = create_pool(s) + self.jobs = JobStore(self.pool, s) + self.store = ObjectStore(s) + self.sandbox = Sandbox(s.worker_image_memory_mb, s.worker_image_timeout_seconds, s.max_image_pixels) + self.stop = threading.Event() + self.in_flight = 0 + self.lock = threading.Lock() + self.processed = 0 + self.beats: Dict[int, float] = {} + # lab6: DB pool, занятость worker'а (busy/capacity) + telemetry.register_pool_metrics(self.pool) + telemetry.register_worker_metrics(lambda: self.in_flight, s.worker_concurrency) + + # ---------------------------------------------------------------- loop + + def consumer_name(self, index: int) -> str: + # Стабильно при рестарте контейнера (hostname = короткий ID контейнера): + # новая инкарнация сразу видит свой неподтверждённый PEL. + return "%s-%d" % (INSTANCE_ID, index) + + def run_thread(self, index: int) -> None: + name = self.consumer_name(index) + broker = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group, + self.settings.broker_timeout_seconds, block_ms=2000) + recovered_own = False + last_autoclaim = 0.0 + while not self.stop.is_set(): + self.beats[index] = time.monotonic() + try: + deliveries: List[Delivery] = [] + if not recovered_own: + deliveries = broker.read(name, own_pending=True) + if not deliveries: + recovered_own = True + else: + log.info("%s: recovering own unacked message %s", name, deliveries[0].message_id) + if not deliveries and time.monotonic() - last_autoclaim > 2: + last_autoclaim = time.monotonic() + deliveries = broker.autoclaim(name, self.settings.job_claim_idle_seconds * 1000) + for d in deliveries: + log.info("%s: claimed idle message %s (deliveries=%d)", name, d.message_id, d.deliveries) + if not deliveries: + deliveries = broker.read(name) + for d in deliveries: + self.handle(broker, name, d) + except BrokerUnavailable: + log.warning("%s: broker unavailable, retrying", name) + self.stop.wait(1.0) + except (psycopg.Error, PoolTimeout) as exc: + # БД недоступна до/после обработки: сообщение не подтверждено, вернётся. + log.warning("%s: database unavailable (%s); message stays pending", name, type(exc).__name__) + self.stop.wait(1.0) + except Exception: + log.exception("%s: unexpected error in loop", name) + self.stop.wait(1.0) + broker.close() + log.info("%s: stopped", name) + + # ---------------------------------------------------------------- one message + + def handle(self, broker: Broker, consumer: str, d: Delivery) -> None: + """lab6: CONSUMER span — ребёнок PRODUCER span из traceparent сообщения. + Внутри него — SQL (psycopg) и S3 (botocore) spans этой попытки; retry + пишет в outbox traceparent этого span, следующая попытка — в той же трассе.""" + raw = d.payload() + started = time.monotonic() + with telemetry.tracer().start_as_current_span( + "%s process" % broker.stream, context=telemetry.extract(raw), kind=SpanKind.CONSUMER, + attributes={ + "messaging.system": "redis", + "messaging.operation.type": "process", + "messaging.destination.name": broker.stream, + "messaging.consumer.group.name": broker.group, + "messaging.message.id": d.message_id, + "messaging.message.delivery_count": d.deliveries, + "messenger.job.type": JOB_TYPE, + }, + ) as span: + outcome = "error" + try: + outcome = self._handle(broker, consumer, d, raw, span) + finally: + kind = outcome.split(":", 1)[0].split(" ", 1)[0] + span.set_attribute("messenger.job.outcome", kind) + if kind in ("failed", "poison", "error"): + span.set_status(Status(StatusCode.ERROR, outcome)) + attrs = {"job.type": JOB_TYPE, "outcome": kind} + telemetry.INSTRUMENTS.jobs_processed.add(1, attrs) + if kind not in ("noop", "poison"): + telemetry.INSTRUMENTS.job_duration.record(time.monotonic() - started, attrs) + log.info("%s: message %s -> %s in %.2fs", consumer, d.message_id, outcome, + time.monotonic() - started, + extra={"event": "job.processed", "outcome": kind, "job_type": JOB_TYPE, + "duration_s": round(time.monotonic() - started, 3)}) + + def _handle(self, broker: Broker, consumer: str, d: Delivery, raw, span) -> str: + msg = validate_payload(raw) + if msg is None: + log.error("%s: poison message %s (invalid schema), acking", consumer, d.message_id) + broker.ack(d.message_id) + return "poison" + span.set_attribute("messenger.job.id", str(msg["job_id"])) + span.set_attribute("messenger.attachment.id", str(msg["attachment_id"])) + span.set_attribute("messenger.job.attempt", msg["attempt"]) + if d.deliveries > self.settings.job_max_deliveries: + # Сообщение многократно «убивало» worker (или его не могли подтвердить). + self.jobs.fail_poison(msg["job_id"], "poison_message", consumer) + log.error("%s: job %s delivered %d times, marked failed", consumer, msg["job_id"], d.deliveries) + broker.ack(d.message_id) + return "failed:poison_message" + claim, reason = self.jobs.claim(msg, consumer) + if claim is None: + log.info("%s: job %s attempt %d: no-op (%s), ack", consumer, msg["job_id"], msg["attempt"], reason) + broker.ack(d.message_id) + return "noop:" + reason + with self.lock: + self.in_flight += 1 + started = time.monotonic() + try: + outcome = self.process(claim, consumer) + finally: + with self.lock: + self.in_flight -= 1 + # Эффект (ready/retry/failed) уже зафиксирован в БД -> теперь можно ack. + if outcome == "ready" and self.settings.worker_faults and self._marker(claim) == "crash-before-ack" \ + and not self.jobs.has_event(claim.job_id, "fault_injected"): + self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "fault_injected", + "crash after COMMIT before XACK", consumer) + log.error("FAULT_INJECTION: worker exits after COMMIT before XACK (job %s)", claim.job_id) + os._exit(70) + broker.ack(d.message_id) + self.processed += 1 + log.info("%s: job %s attempt %d -> %s in %.2fs", consumer, claim.job_id, claim.attempt, outcome, + time.monotonic() - started) + if outcome == "ready": + self.cleanup_staging(claim) + return "succeeded" + return outcome + + def process(self, claim: Claim, consumer: str) -> str: + try: + variants, original_key = self.produce(claim, consumer) + age = self.jobs.complete(claim, variants, original_key, consumer) + if age is not None: + telemetry.INSTRUMENTS.time_to_ready.record(age, {"job.type": JOB_TYPE}) + else: + log.warning("%s: lease lost for job %s; result not published (fenced)", consumer, claim.job_id) + self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "lease_lost", + "result not published: lease token no longer valid", consumer) + return "lease_lost" + return "ready" + except Permanent as exc: + self.jobs.fail(claim, exc.code, consumer, exc.detail or None) + return "failed:" + exc.code + except Transient as exc: + result, delay = self.jobs.retry_or_fail(claim, exc.code, consumer) + return "retry in %.1fs (%s)" % (delay, exc.code) if result == "retry" else result + + def _marker(self, claim: Claim) -> str: + """Маркеры внедрения отказов в filename — ТОЛЬКО при WORKER_FAULTS=true + (compose.faults.yaml). В обычном стенде filename ни на что не влияет.""" + if not self.settings.worker_faults: + return "" + src = self.jobs.attachment_source(claim.attachment_id) or {} + name = src.get("filename") or "" + for marker in ("crash-after-effects", "crash-before-ack", "transient-until-3", + "transient-until-replay", "permanent", "slow"): + if name.startswith("fault-" + marker): + return marker + return "" + + def produce(self, claim: Claim, consumer: str): + """Шаги 3–5: прочитать источник, записать оригинал и варианты в final/.""" + src = self.jobs.attachment_source(claim.attachment_id) + if src is None: + raise Permanent("processing_failed", "attachment row disappeared") + marker = self._marker(claim) + if marker == "transient-until-3" and claim.attempt < 3: + raise Transient("injected_transient") + if marker == "transient-until-replay" and claim.replay_count == 0: + raise Transient("injected_transient") + if marker == "permanent": + raise Permanent("invalid_image", "injected permanent failure") + if marker == "slow" and claim.attempt == 1: + # «Зависание» заметно дольше аренды + grace + период сборщика (5 с). + time.sleep(self.settings.job_timeout_seconds + 20) + + original_key = final_original_key(claim.attachment_id) + data = self.read_source(src, original_key) + try: + # 4. Оригинал в конечный префикс (неизменен: те же байты, тот же SHA-256). + try: + head = self.store.head(original_key) + except ObjectStoreUnavailable: + raise Transient("storage_unavailable") + if head is None or (head.get("Metadata") or {}).get("sha256") != src["sha256"]: + self._put(original_key, data, src["content_type"]) + # 5. Варианты. + try: + rendered = self.sandbox.render(data) + log.info("job %s: variants rendered, subprocess peak RSS %.0f MiB", + claim.job_id, self.sandbox.last_peak_mb) + except ImageTimeout: + raise Transient("processing_timeout") + except ImageFailure as exc: + raise Permanent(exc.code if exc.code in ("invalid_image", "resource_limit") else "processing_failed", + str(exc)) + finally: + del data + variants: List[Dict] = [] + for name in ("thumbnail", "crop", "optimized"): + body, width, height, ctype = rendered[name] + key = variant_key(claim.attachment_id, claim.pipeline_version, name) + self._put(key, body, ctype) + variants.append({"name": name, "content_type": ctype, "size_bytes": len(body), "width": width, + "height": height, "sha256": sha256_hex(body), "object_key": key}) + if marker == "crash-after-effects" and not self.jobs.has_event(claim.job_id, "fault_injected"): + self.jobs.event(claim.job_id, claim.attachment_id, claim.attempt, "fault_injected", + "crash after S3 writes before COMMIT", consumer) + log.error("FAULT_INJECTION: worker exits after S3 writes before COMMIT (job %s)", claim.job_id) + os._exit(70) + return variants, original_key + + def read_source(self, src: Dict, original_key: str) -> bytes: + limit = self.settings.max_upload_bytes + keys = [k for k in (src["staging_key"], original_key) if k] + for key in keys: + try: + data = self.store.read_all(key, limit) + except ObjectMissing: + continue + except ValueError: + raise Permanent("invalid_image", "source exceeds MAX_UPLOAD_BYTES") + except ObjectStoreUnavailable: + raise Transient("storage_unavailable") + if len(data) != src["size_bytes"] or sha256_hex(data) != src["sha256"]: + if key == original_key: + continue # частично/чужой объект в final — перезапишем из staging + raise Permanent("processing_failed", "staging checksum mismatch") + return data + raise Permanent("processing_failed", "source object is missing") + + def _put(self, key: str, data: bytes, content_type: str) -> None: + try: + self.store.put(key, data, content_type, hashlib.md5(data, usedforsecurity=False).digest(), + sha256_hex(data)) + except ObjectStoreUnavailable: + raise Transient("storage_unavailable") + + def cleanup_staging(self, claim: Claim) -> None: + """Шаг 8: staging больше не нужен — результат опубликован и подтверждён.""" + try: + src = self.jobs.attachment_source(claim.attachment_id) + if not src or src["state"] != "ready" or not src["staging_key"]: + return + self.store.delete(src["staging_key"]) + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND state = 'ready' " + "AND staging_key = %s", (claim.attachment_id, src["staging_key"])) + except Exception as exc: + log.warning("staging cleanup deferred to gc: %s", type(exc).__name__) + + # ---------------------------------------------------------------- housekeeping + + def housekeeping(self) -> None: + broker = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group, + self.settings.broker_timeout_seconds) + while not self.stop.wait(30): + try: + trimmed = broker.trim_acknowledged() + if trimmed: + log.info("broker: trimmed %d acknowledged entries", trimmed) + except BrokerUnavailable: + pass + broker.close() + + def run(self) -> None: + s = self.settings + log.info("worker %s: concurrency=%d max_attempts=%d lease=%ss claim_idle=%ss image_timeout=%ss " + "image_memory=%dMiB retry=%.1f..%.1fs faults=%s", + INSTANCE_ID, s.worker_concurrency, s.job_max_attempts, s.job_timeout_seconds, + s.job_claim_idle_seconds, s.worker_image_timeout_seconds, s.worker_image_memory_mb, + s.job_retry_base_seconds, s.job_retry_max_seconds, "on" if s.worker_faults else "off") + # Прогрев forkserver до приёма сообщений. + self.sandbox.ctx.Process(target=time.sleep, args=(0,)).start() + HEARTBEAT.touch() + + def on_signal(sig, _frame): + log.info("signal %s: finishing in-flight jobs (%d) and stopping", sig, self.in_flight) + self.stop.set() + + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + + threads = [threading.Thread(target=self.run_thread, args=(i,), name="consumer-%d" % i) + for i in range(s.worker_concurrency)] + aux_stop = threading.Event() + relay_broker = Broker(s.broker_url, s.broker_stream, s.broker_group, s.broker_timeout_seconds) + relay = Relay(self.pool, relay_broker, s, "worker:" + INSTANCE_ID) + aux = [threading.Thread(target=relay.loop, args=(aux_stop,), name="outbox-relay", daemon=True), + threading.Thread(target=self.housekeeping, name="housekeeping", daemon=True), + threading.Thread(target=gc_loop, args=(self.pool, self.store, s, aux_stop), name="attachment-gc", + daemon=True)] + for t in threads + aux: + t.start() + # Heartbeat для healthcheck: пишется, только пока КАЖДЫЙ consumer-поток + # проходит цикл (зависание дольше аренды -> контейнер unhealthy). + limit = s.job_timeout_seconds + 15 + while any(t.is_alive() for t in threads): + now = time.monotonic() + if self.beats and all(now - b < limit for b in self.beats.values()): + HEARTBEAT.touch() + for t in threads: + t.join(timeout=2) + aux_stop.set() + self.pool.close() + log.info("worker %s stopped; processed %d messages", INSTANCE_ID, self.processed) + telemetry.shutdown() + + +def main() -> None: + telemetry.setup("messenger-worker") + Worker().run() + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..d14d4e5 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,128 @@ +"""Общие HTTP-помощники собственных тестов (только стандартная библиотека).""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import base64 +import json +import os +import uuid +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +BASE_URL = os.environ.get("BASE_URL", "https://localhost:8443").rstrip("/") + + +class Resp: + def __init__(self, status, data, headers, elapsed=0.0): + self.status = status + self.json = data + self.headers = headers + self.elapsed = elapsed + + +def request(method, path, token=None, payload=None, raw=None, headers=None, + base_url=None, timeout=10): + import time + h = {"Accept": "application/json"} + if token is not None: + h["Authorization"] = "Bearer " + token + body = None + if payload is not None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + elif raw is not None: + body = raw + h["Content-Type"] = "application/json" + h.update(headers or {}) + # Переход 2 -> 3: POST сообщения требует Idempotency-Key. Старые тесты lab2 + # его не передают — добавляем уникальный ключ, если тест не задал свой. + # Явное значение None в headers означает «не отправлять заголовок вовсе». + if method == "POST" and path.split("?")[0].endswith("/messages") \ + and not any(k.lower() == "idempotency-key" for k in h): + h["Idempotency-Key"] = uuid.uuid4().hex + h = {k: v for k, v in h.items() if v is not None} + req = Request((base_url or BASE_URL) + path, data=body, headers=h, method=method) + started = time.monotonic() + try: + response = urlopen(req, timeout=timeout) + except HTTPError as error: + response = error + with response: + content = response.read() + status = response.status if hasattr(response, "status") else response.code + rh = response.headers + try: + data = json.loads(content.decode("utf-8")) if content else None + except (UnicodeDecodeError, json.JSONDecodeError): + data = None + return Resp(status, data, rh, time.monotonic() - started) + + +def basic(username, password): + raw = (username + ":" + password).encode("utf-8") + return "Basic " + base64.b64encode(raw).decode("ascii") + + +def new_password(): + return "Pw-" + uuid.uuid4().hex + + +def register(prefix="t", password=None, base_url=None): + username = (prefix + "_" + uuid.uuid4().hex[:16])[:32] + password = password or new_password() + r = request("POST", "/api/v1/users", base_url=base_url, payload={ + "username": username, + "display_name": "Test " + prefix, + "password": password, + }) + assert r.status == 201, (r.status, r.json) + user = r.json + user["_password"] = password + return user + + +def login(user, base_url=None): + r = request("POST", "/api/v1/auth/sessions", base_url=base_url, + headers={"Authorization": basic(user["username"], user["_password"])}) + assert r.status == 201, (r.status, r.json) + return r.json["token"] + + +def actor(prefix="t", base_url=None): + user = register(prefix, base_url=base_url) + user["_token"] = login(user, base_url=base_url) + return user + + +def create_chat(owner, *members, base_url=None): + r = request("POST", "/api/v1/chats", token=owner["_token"], base_url=base_url, payload={ + "title": "Test chat " + uuid.uuid4().hex[:8], + "member_ids": [owner["id"]] + [m["id"] for m in members], + }) + assert r.status == 201, (r.status, r.json) + return r.json + + +def reachable(base_url): + try: + urlopen(base_url.rstrip("/") + "/health/live", timeout=2).read() + return True + except (URLError, OSError, HTTPError): + return False + + +def wait_ready(token, attachment_id, timeout=60.0, allow_failed=False): + """lab5: опрашивать метаданные до конечного состояния (ready/failed).""" + import time + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + r = request("GET", "/api/v1/attachments/" + attachment_id, token=token) + last = r + if r.status == 200 and r.json["status"] == "ready": + return r.json + if r.status == 200 and r.json["status"] == "failed": + if allow_failed: + return r.json + raise AssertionError("attachment failed: %s" % r.json.get("error_code")) + time.sleep(0.2) + raise AssertionError("attachment %s not ready in %.0fs (last: %s)" % ( + attachment_id, timeout, last.json if last else None)) diff --git a/student-tests/dependency_failure.py b/student-tests/dependency_failure.py new file mode 100644 index 0000000..9d17f86 --- /dev/null +++ b/student-tests/dependency_failure.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Автоматическая проверка недоступности Redis и PostgreSQL. + +Сценарий явно останавливает зависимости, поэтому не входит в discovery +unittest. По умолчанию использует docker compose из корня репозитория: + + python student-tests/dependency_failure.py --base-url https://localhost:8443 + +Команды остановки/запуска можно переопределить (--stop-redis и т.д.), например +для стенда без Docker. Результат — JSON-сводка (без токенов) в stdout и, +при --output, в файл. + +Ожидаемое поведение: +* Redis недоступен -> API обходит кэш, всё работает через PostgreSQL, + readiness = 200, задержка ограничена таймаутом Redis (250 мс) и + circuit breaker'ом; logout по-прежнему отзывает сессию. +* Разрыв сети API <-> Redis БЕЗ рестарта Redis (данные кэша живы) и logout + во время разрыва -> logout 204, и отозванный токен получает 401 и во время + разрыва, и сразу после восстановления связи, и после закрытия circuit + breaker (регрессия: раньше доверяли старой положительной записи ~60 с). +* PostgreSQL недоступна -> /health/live = 200, /health/ready = 503, + запросы к данным -> 503 unavailable за ограниченное время; анонимный + запрос -> 401; неизвестный токен не принимается (503, а не 200). +* После возврата зависимости тот же токен снова работает, данные на месте. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import json +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from common import actor, basic, create_chat, login, request # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +MAX_FAIL_SECONDS = 6.0 # верхняя граница ответа при отказе зависимости +MAX_DEGRADED_SECONDS = 1.5 # верхняя граница ответа при отказе Redis + + +class Check: + def __init__(self): + self.results = [] + + def __call__(self, name, condition, detail=""): + self.results.append({"check": name, "ok": bool(condition), "detail": detail}) + print(("PASS " if condition else "FAIL ") + name + (" — " + detail if detail else ""), flush=True) + return condition + + @property + def ok(self): + return all(r["ok"] for r in self.results) + + +def run(cmd): + print("$ " + cmd, flush=True) + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def wait_ready(timeout=90): + deadline = time.monotonic() + timeout + started = time.monotonic() + while time.monotonic() < deadline: + try: + if request("GET", "/health/ready", timeout=3).status == 200: + return time.monotonic() - started + except OSError: + pass + time.sleep(0.5) + return None + + +def is_unavailable(r): + return (r.status == 503 and r.json and r.json["error"]["code"] == "unavailable" + and r.json["error"]["request_id"] == r.headers.get("X-Request-Id")) + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--base-url", default="https://localhost:8443") + p.add_argument("--stop-redis", default="docker compose stop redis") + p.add_argument("--start-redis", default="docker compose up -d --wait redis") + p.add_argument("--partition-redis", + default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1", + help="cut API<->Redis network WITHOUT restarting Redis") + p.add_argument("--heal-redis", + default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1") + p.add_argument("--stop-postgres", default="docker compose stop postgres") + p.add_argument("--start-postgres", default="docker compose up -d --wait postgres") + p.add_argument("--output") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + + check = Check() + summary = {"base_url": common.BASE_URL, "timings": {}} + + alice, bob = actor("dep"), actor("dep") + chat = create_chat(alice, bob) + path = "/api/v1/chats/%s/messages" % chat["id"] + first = request("POST", path, token=alice["_token"], payload={"text": "до отказов"}).json + request("GET", "/api/v1/users/me", token=alice["_token"]) # прогрев кэша + + # 1. Redis недоступен + run(args.stop_redis) + try: + r = request("GET", "/api/v1/users/me", token=alice["_token"]) + check("redis down: existing token still valid via PostgreSQL", r.status == 200, "%.3fs" % r.elapsed) + summary["timings"]["redis_down_first_request_s"] = round(r.elapsed, 3) + timings = [] + for i in range(10): + r = request("POST", path, token=bob["_token"], payload={"text": "redis down %d" % i}) + timings.append(r.elapsed) + if r.status != 201: + break + check("redis down: writes succeed", r.status == 201) + check("redis down: bounded latency", max(timings) < MAX_DEGRADED_SECONDS, + "max %.3fs over %d requests" % (max(timings), len(timings))) + summary["timings"]["redis_down_max_write_s"] = round(max(timings), 3) + check("redis down: readiness stays 200 (cache is not critical)", + request("GET", "/health/ready").status == 200) + token = login(bob) + check("redis down: login works", bool(token)) + check("redis down: logout works", + request("DELETE", "/api/v1/auth/sessions/current", token=token).status == 204) + check("redis down: revoked token rejected", + request("GET", "/api/v1/users/me", token=token).status == 401) + check("redis down: anonymous rejected", request("GET", "/api/v1/users/me").status == 401) + check("redis down: unknown token rejected", + request("GET", "/api/v1/users/me", token="x" * 43).status == 401) + finally: + run(args.start_redis) + time.sleep(6) # больше CACHE_RETRY_AFTER_SECONDS: API снова пробует Redis + check("redis back: requests ok", + request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200) + + # 2. Разрыв связи с живым Redis + logout (замечание ревью №1) + carol = actor("dep") + for _ in range(2): # промах -> запись в кэш, затем попадание + request("GET", "/api/v1/users/me", token=carol["_token"]) + run(args.partition_redis) + try: + r = request("DELETE", "/api/v1/auth/sessions/current", token=carol["_token"]) + check("redis partition: logout -> 204", r.status == 204, "status %d, %.3fs" % (r.status, r.elapsed)) + check("redis partition: revoked token -> 401", + request("GET", "/api/v1/users/me", token=carol["_token"]).status == 401) + finally: + run(args.heal_redis) + # Опрашиваем дольше CACHE_RETRY_AFTER_SECONDS (5 с): после закрытия circuit + # breaker API снова читает Redis, где жила положительная запись этого токена. + statuses = set() + deadline = time.monotonic() + 8 + while time.monotonic() < deadline: + statuses.add(request("GET", "/api/v1/users/me", token=carol["_token"]).status) + statuses.add(request("POST", path, token=carol["_token"], payload={"text": "revoked"}).status) + time.sleep(0.25) + check("redis healed (same data): revoked token stays 401 for 8 s", statuses == {401}, str(sorted(statuses))) + check("redis healed: valid token works", request("GET", "/api/v1/users/me", token=alice["_token"]).status == 200) + + # 3. PostgreSQL недоступна + run(args.stop_postgres) + try: + check("postgres down: liveness 200", request("GET", "/health/live").status == 200) + r = request("GET", "/health/ready") + check("postgres down: readiness 503 unavailable", is_unavailable(r), "%.3fs" % r.elapsed) + r = request("GET", "/api/v1/users/me") + check("postgres down: anonymous -> 401 (not trusted)", r.status == 401, "status %d" % r.status) + r = request("GET", "/api/v1/users/me", token="y" * 43) + check("postgres down: unknown token is not accepted (503)", is_unavailable(r), + "status %d, %.3fs" % (r.status, r.elapsed)) + r = request("POST", path, token=alice["_token"], payload={"text": "должно не записаться"}) + check("postgres down: write -> 503 within bound", is_unavailable(r) and r.elapsed < MAX_FAIL_SECONDS, + "status %d, %.3fs" % (r.status, r.elapsed)) + summary["timings"]["postgres_down_write_s"] = round(r.elapsed, 3) + r = request("GET", path, token=alice["_token"]) + check("postgres down: history read -> 503", is_unavailable(r), "status %d" % r.status) + r = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(alice["username"], alice["_password"])}) + check("postgres down: login -> 503 (not 401, not 201)", is_unavailable(r), "status %d" % r.status) + body = json.dumps(r.json, ensure_ascii=False) + check("postgres down: error hides internals", + not any(s in body.lower() for s in ("psycopg", "postgres", "select", "traceback", "5432"))) + finally: + run(args.start_postgres) + recovered = wait_ready() + check("postgres back: readiness recovered", recovered is not None, + "%.1fs" % recovered if recovered is not None else "timeout") + summary["timings"]["postgres_recovery_s"] = round(recovered or -1, 2) + r = request("GET", path + "?limit=100", token=alice["_token"]) + ok = r.status == 200 and r.json["items"][0] == first + texts = [m["text"] for m in r.json["items"]] if r.status == 200 else [] + check("postgres back: same token reads intact history", ok) + check("postgres back: rejected write was not stored", "должно не записаться" not in texts) + + summary["checks"] = check.results + summary["passed"] = check.ok + if args.output: + Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(("PASS" if check.ok else "FAIL") + ": dependency failure scenario") + sys.exit(0 if check.ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100644 index 0000000..b48b072 --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Уровень 4: последовательная остановка КАЖДОЙ API-реплики под нагрузкой. + +Для каждой реплики (по X-Instance-Id): нагрузка -> stop-one -> 15 с работы без неё +-> restore -> возврат реплики в ротацию. Собираются RPS, ошибки, задержки по +окнам, время обнаружения отказа балансировщиком (по странице HAProxy) и время +восстановления. Перед следующей репликой предыдущая уже восстановлена. + + python student-tests/failover_series.py --output evidence/failover_series.json + python student-tests/failover_series.py --mode kill --output evidence/failover_series_kill.json + +Остановка/восстановление по умолчанию — scripts/ops.py (docker compose); +для другого стенда их можно переопределить --stop-cmd / --restore-cmd. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from common import request # noqa: E402 +from loadgen import Load # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + + +def observe_instances(token, n=40): + seen = set() + for _ in range(n): + r = request("GET", "/api/v1/users/me", token=token, timeout=3) + seen.add(r.headers.get("X-Instance-Id")) + return sorted(x for x in seen if x) + + +def run_hook(cmd, env_extra, timeout): + env = dict(os.environ, **env_extra) + started = time.monotonic() + proc = subprocess.run(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=timeout) + return proc.returncode, time.monotonic() - started, proc.stdout[-2000:] + + +def one_round(args, target): + load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() + time.sleep(args.pre) + up_before = load.lb[-1][1] if load.lb else () + t_stop = load.now() + rc, stop_s, out = run_hook(args.stop_cmd, {"TARGET_INSTANCE_ID": target, "STOP_MODE": args.mode}, 60) + if rc != 0: + load.stop() + raise SystemExit("stop hook failed for %s:\n%s" % (target, out)) + time.sleep(max(0.0, args.post - stop_s)) + t_restore = load.now() + rc, restore_s, out = run_hook(args.restore_cmd, {}, 180) + if rc != 0: + load.stop() + raise SystemExit("restore hook failed:\n%s" % out) + # ждём, пока восстановленная реплика снова обслужит запрос + back = None + deadline = time.monotonic() + 30 + while back is None and time.monotonic() < deadline: + for e in reversed(load.events): + if e[0] >= t_restore and e[2] and e[4] == target: + back = e[0] + break + time.sleep(0.2) + time.sleep(args.tail) + load.stop() + t_end = load.now() + + detect = next((t - t_stop for t, up in load.lb if t >= t_stop and len(up) < len(up_before)), None) + lb_back = next((t - t_restore for t, up in load.lb if t >= t_restore and len(up) >= len(up_before)), None) + errs = load.errors_between(t_stop, t_restore) + # Сколько ещё после SIGTERM реплика отвечала (фаза осушения), до restore. + served_after = [e[0] - t_stop for e in load.events if t_stop <= e[0] < t_restore and e[2] and e[4] == target] + # После возврата hook (реплика остановлена) и до restore она не должна отвечать. + after_stop_ok = [e for e in load.events if t_stop + stop_s <= e[0] < t_restore and e[2]] + return { + "target_instance": target, + "mode": args.mode, + "lb_up_before": list(up_before), + "stop_hook_seconds": round(stop_s, 2), + "lb_detect_down_seconds": round(detect, 2) if detect is not None else None, + "target_last_served_after_stop_seconds": round(max(served_after), 2) if served_after else None, + "target_served_after_hook_returned": any(e[4] == target for e in after_stop_ok), + "errors_during_outage": len(errs), + "error_kinds": sorted({str(e[3]) for e in errs}), + "error_window_seconds": round(max(e[0] for e in errs) - t_stop, 2) if errs else 0.0, + "restore_hook_seconds": round(restore_s, 2), + "lb_back_up_seconds": round(lb_back, 2) if lb_back is not None else None, + "target_back_in_rotation_seconds": round(back - t_restore, 2) if back is not None else None, + "windows": { + "before": load.window(0, t_stop), + "outage_first_10s": load.window(t_stop, t_stop + 10), + "outage_rest": load.window(t_stop + 10, t_restore), + "after_restore": load.window(t_restore, t_end), + }, + "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], + "data": load.verify(), + } + + +def main(): + py = sys.executable or "python" + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) + p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) + p.add_argument("--stop-cmd", default=None, help="default: python scripts/ops.py stop-one") + p.add_argument("--restore-cmd", default=None, help="default: python scripts/ops.py restore") + p.add_argument("--mode", choices=["stop", "kill"], default="stop", help="stop = SIGTERM (graceful), kill = SIGKILL") + p.add_argument("--writers", type=int, default=4) + p.add_argument("--readers", type=int, default=2) + p.add_argument("--pre", type=float, default=5) + p.add_argument("--post", type=float, default=15) + p.add_argument("--tail", type=float, default=3) + p.add_argument("--output") + args = p.parse_args() + args.stop_cmd = shlex.split(args.stop_cmd) if args.stop_cmd else [py, "scripts/ops.py", "stop-one"] + args.restore_cmd = shlex.split(args.restore_cmd) if args.restore_cmd else [py, "scripts/ops.py", "restore"] + common.BASE_URL = args.base_url.rstrip("/") + + probe = common.actor("probe") + instances = observe_instances(probe["_token"]) + if len(instances) < 2: + raise SystemExit("need >= 2 replicas behind LB, observed %s" % instances) + print("replicas observed through LB: %s" % instances, flush=True) + rounds = [] + for target in instances: + print("-- stopping %s (mode=%s) under load" % (target, args.mode), flush=True) + r = one_round(args, target) + rounds.append(r) + w = r["windows"] + print(" hook %.2fs | LB DOWN after %s s | errors %d (%s) | RPS before %.1f, outage %.1f, after restore %.1f" + " | p95 outage %s ms | back in rotation %s s | data ok=%s" + % (r["stop_hook_seconds"], r["lb_detect_down_seconds"], r["errors_during_outage"], + ",".join(r["error_kinds"]) or "-", w["before"]["ok_rps"], w["outage_first_10s"]["ok_rps"], + w["after_restore"]["ok_rps"], w["outage_first_10s"]["latency_ms_ok"]["p95"], + r["target_back_in_rotation_seconds"], r["data"]["ok"]), flush=True) + report = { + "scenario": "stop each API replica in turn under load, restore before the next", + "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0, + "write_retry": "same Idempotency-Key and body until 201"}, + "replicas": instances, + "rounds": rounds, + } + ok = all(r["data"]["ok"] and r["target_back_in_rotation_seconds"] is not None + and not r["target_served_after_hook_returned"] for r in rounds) + report["pass"] = ok + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + "\n", encoding="utf-8") + print(("PASS" if ok else "FAIL") + ": every replica stopped and restored; confirmed data intact, no duplicates") + sys.exit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/orient6.jpg b/student-tests/fixtures/orient6.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c842b15ebae2ae63ebf1f66468dad2a47dc14fb6 GIT binary patch literal 4812 zcmex=BGmU}4*Ox(yfq{X8fr*iyfeFas1!6Tu zAqHk3&B(w8B!M8fGOr{fwInmyIWZTg`!<6!12Yp72mt{L5V3Nwva+zSa@^NzuhzSS^i-?Mf^6-mGh>1uDiHM4T3}Ivj>S1B!WM$Nsy6Qkn#T!26>>%SsB3q;(Q>V3Ft;Pb`DN1ZlHp#0zmIF zGc&O;gIo=itp&<6un4jWDH=Mm2?r*!D;0_uHBMZ}q3pErplHy=4=Tnkx|JhscGpMnOVgprDf$6l~v6xt!?cc zon4bAPnkMx`iz;g7A;<~blLJ1D_3pWyk+aQ?K^hvI&}ER(PPI?oIG{u@|COCuHU$M z>*1rvPo6$|{^I4UkDoq&`TFhqkDtFl{$gZchIk7|LVSkiFF~Ncm{?etS=d4TVq_`@ zd0dc%Rnd@5$T5&Tu~1masF6d&Y2w0-2RW6EgFc8R6KQ!#m{`Vr(cu+NC|SQA-3 zgL@3&uUiZ}%#6UuV-{qvXZU&9`9H%1z4`wc;y(Uoc%igDRF?mr!>;U*T4(^8J=)2 z|IcuJPyH{B^?vhzx5j@w{GY*dMdjuH47JPuasDgN4|ElMCf5<@D#GZF-{QD=ng9MX zbo^(K+5cr(<;_u>M?-rwg^lK_(UN4e^u$qBjn){Wwd81!ZP zx%S_0)jyd3oark>u}GkU2x@OrBz!r%XF9Je_TN+IKZ1WIK)RQxeOL{WyP~5Gf6)pI Iaryr@0p-Ck8vp}PN6CDsgmOm_ElhjerxNC*6buqMbA($gI~cC4-*_@k$%H%@=@czy7~O`0@evLSBD z6hoY$q0uMPO^uArO$-gE&X{U$VS&fvjZCeqtt_plTjDJTjp%?!^~dQOj2~}cX>4e0 z`ES3lGRSZ|^hD=#T^%cEjG>OMp$_%{!h?D0>1bZi&`W2G?pQF(@e?La0$(Wl7#gFa zt2+kFA1nZTI|lqeG}cgW%8Ugo#u@o;)whZ;b~v1Las13LZj_s>s}Wi|Zu|Dggo&S+ zPBk;PnPoeBj@?41MT?g#b#`6p=I*h|lkB&CgFhu;@X_NZl}~Hy>Yu;h zzHDf0X???QYwzgn>h2SX`z6u=nOrfrE*(f$yDacmyRZ@KG6d@y1CC{^{@}WF#zcX) zuHjg{84JcuS>da{HNwcs;qZ9lFOn|aD4#IXah=d)+qX3nKe1leV$(OcG|kHX-olRj zM=Sfs!hT;@BQ#l82OOTRAw+`2R~5a|gBX-dx`;tdkI__5suhI`7jRWcY8?8S)SpRK z8^x1;n~B=U9ot0VZxEm7)A52x403p^F2x{$c{3b>$O#I(`X1c1jKQOGsp7eC6A!J) z*Oz_=3?o{@DU}<)MEI*QsDf1AWQeLtDV;HUspBZGBZWKm?BS_O<@B5_JEe5^qNMn( z5`(PcR$@?bqI!06iBBUJ-N94>V~tL~p&UfCYM+lX8iQ(&6OO8XhTAuSFCyx!py+V9 zInw7PakfP}OHDiXzy%njVmvb%T*d{mqL%HYW?~ROKa?r$!=Otd%8S&3-hpEn6ekHT z!=R58%!!m2cnIbvhayrQ29d2T6x5lgh&7c8bO7VhpO8LV4~aDjq(tumIo6@)1uXe#**4$HJ9! zY7j81#AJ5B@Xtk$z=j;msDIgt?DCPPlLS2&w3A{av`5L8Phrq*4C-IWXrk9M#q;4N zE?S#!+e}q4yr-c}7}R;3U?^F})TMgKJXERA+BaEccPm+T@`I+fY+Wu~3pR zs9G0&4NDm&aO}X;W^kKA#Od7;JUxVec_&E<_H|`x%p9td>JtE}Gnc@wgsZ1W`m^WS zm#ICN@I4O|DUYbf%EzD(mVBlbk1H7>o0ur&a<~%zE13Tq&7H1^&QjNsTB#`{VkMHG zCavEC7jacpc+S1O197-F_YXv3Zu%? zU_mbig>s{mL~yHT)R45Z%Quo10Z@gYufb`YifBRjyD;dXeSIfWnb}XhSdMZqNJLqv z#s2|94_R(6g~5j0&V+&YN~xr7TE{_N&qVRh6LnRKyJ>T%0W^TVWY0{nVKDF}Y#-J& zA~=LfYYmW6JvZOx2~Am@)k5a9{w3j!2)cu%1=W3(E_kp&`=Fss6txIeRZ*pq!Cm@Z zrVp-2h@18rTal}97-TijWnbS&eMrn_{gC%=lWf;*2WGKI?R^#RPtyaIe6*JEHsHb( zQB+m1)>MgX50uaHHr-9=FkPGE5w;y=i4K{eyTkx0;!q4mE4f>{)p49c+ zr@Y|{hOWt8atjrYgxL%)_u~!@_W7I0j`}U(3ZHUH-F+D*Jni1omuJQBQ09{kV14 z;ZQ|fbBHSWnHHaN!U^>iQv2qiOP^E5k=taDG~P)>F;%XGtM3BqBqg~(`Sp6zYkIh3 z3F|OVw0<9~{TWtHPdje-K$lMhxW~u9I%RD3IoS-bOk@zM8h0X3_|s4<;)qU>sLdXs zbtJL9)|uoxHm9pn1vAkmxbqC5Kz$i*2kv?hmq8)r_VPK6T3kB#`?NX%cU?mAr03K* z^_9jXQK!{YI@f^1vL^z>7mo_!{UkjS|IAMOAZ|T8gQ5fk4EhYGJOI4)`VA=VRd6i8 z4)Ygk?qk{rJkz+sK$QGdipIfsacfhrBf1jH7_<*BM6by{Vc%6>K=_GTe1d@VgdyR* z1$WXKb_FXKjF*U8YvKUdk>m8$D&;KHrC;q^j*c3YHxG;+%+n^JD>*7hwf7BtZ`E^p z9M?&>Bc=lcZT*ihsEU`~9ofzd=}_PU>7xea&Ah(|=8t~>n~;Y=#)w}g{O&{zJx*Uq z0?-W7dIL{B>t_NVX}}>BQ{x0uCcOQ3m@j!wGr0!t9u>LVeV8)(LIY$Rx{& z?CUaJup=o$nDVvR-vO6xL9!h0)53pkL4Ze*ry*K+O%mcz@-IWru4Md1e~v-od4nOX zCf~FfuVS1x9SSsaNvadPONif2*jRCgZO*zK)^e5ql+LS?EV>f2QD};K5{Jyt7!~Mq zs(2o_YlCLqG~26^W3*=EggEt8q&-lBwydCjgon>ld(XU4p&AD~1S<;$KFZK0#*a#% z4Il7YfZFwA(1L>2i3(=3hbsBJHq^=qZSf-pg2IK|+@HM@2vQIv&g>neCyWTv7~s{f zh7S3T&BX#wz$-cid4*jo9eAh8^%^1o=t+(Wm$E2M$QE}y%QD$q5WXi`kjvFs-T1>y zk=H^EFthmi_R^n#^(alm+RxM_@zX+q#|EAWuokLd1+d<$z6YF`9Y=6uEe0KsBlWFp zeYn!M1`H1JAU2bAnmBwQd->x;)Cw%qu~nhi+Z?A#Rs)CvLLn!#I}abQGf;74UflkW zX-dE;s}ZdiDd8NUiu?)_m4KgQZqMgP&wx3?nmI-=$0Gb3Eg)A`ZX_M!;?QQbje@zF z`$!^nqlNoYFD^Az-2*hVmNzPPH@s9CgM->}2?o*E!X|=O$WoP&G{nG{A|pj4;81sX zDLuzxIEEQwYI(SPPKJ*r+}l&h>+Y*?^#=x&58w(#LGz}6&B0CIb8zw7v$EbgiHH8^ zaXE@NdOY_Ju$LQ=gdEAD2QXs4L4gkf!Fu34%`9m8V&K&1)Ld>x43pUzv-_{&auhsj zJhz;Ny0cX4)N24>@7TL3rsDyB0*`wU538D4IkVLuos0L$kq6cx=lRkF7RL|uS*o(Hk;AbOHBp$~EWK@D6;`=%jC z*_V;z1ZT~YfB;7-Dd2VhMgbz2dkOUrEb`w+9g1Mr0h6$&C#$`)Res&wvR&*p*0Fs` zm!H3o5(u+5eRn)?mzBg;VZgrt<t(&YNrp0|>T*(0KM2ONsWbz55B z{dISLosN>!xNIQ#^RazmBe-2L_M;?Bm3`c;G){O0Fo$4zxsEQQ=lEMLV}XR>i-56x~l&?JPXs82cu zdER*IsuOqQ<`H?1{2WL1=@ww|CkndS3otASwYiERd=$Gov&lWA=oaq!lb1iHecNF@ zIWoAZHqY1B>-gSXYkCZAyX8oKUPAQTWzWx3&wYsM8EXlMXI%)Sw%A$7f*6Fj;58h0!n9e+XPUy3@ z4!<{9b|gbORn`b-H$6{6jp*P$m4B&De|0dM=1UE23@VSpoelpq$?;^u<1VTxrL%a_ zdEexww7?aO&P(O&6E$0UfB7x(+S5Ir_TkGF@M&K+6M6Z&?~Q#w*Bg49?mHo$JO71S zaNCVBZapP%Vm7S~-nn<{fqCco^CtH#;<|U^B$n!)&J1zyiALk_uQCMp$TuI9&tEll zevVFnxPH398F}1R>QBP?bYaQ&hp(wC1#RLgdcws947ilL6jo;|S;-5HD9*$Tn{x#VcY$2S%56TsNxskt?z0k=-XHFl)L#}HBf zL_o5KJr&!e<^@wj)2up?_Mjt8Cm*acT*vL&7% zcQ4xLpKF&AwfmNOqr!SymWj#DiL)~cX3x%yhu**KT&`H%DN1YH$1{Es99e0$?lX6f zB2Tv^l@BkU&(IBSZYxZq8uTldk_&(I+G}L8Y)bJI-NbKzZu)yxz9p|^qrbPs$GWE* zQaiXy(ad{!f8h48+m6#n{?^=9=So{~81;dCylcwTH|;=?h?iM_o95y1UIS6k*F>qoK(NG zCe_}%w$El+tl`vmUG>zX35_4k4Vv~2S@>PHu`l&$N9PJRYgY6RA%T3!l`pK6pYD zob-@p06bP!mPT>}={$-0*~pPCuLc%8#vsFk@R`1tn59Bj)RUg{UiTJTj2zLYhp22o z-Qm=zQWYg+^hEd?^Fde~2JHdKj?d+=HwTqg?8R4Dzn(jRZlQs|6h3;ac}t%NS2_Us zyckVLF6nKgpkLxOyK*H@g)e!mk3lm@=McYykwHh!YA#l#KC>MZ*wR_qbGFj;yO(Km zyp6(nf0Cq=2t7zc54vhdTK2V}pLmFW2liu%>Wp0Kt%~YWWW}oUk@f(P^2fF{i+(5AJ+U-`3rk; zzQHsQg%SJuo@Yc3r=NIB&4l?sc4}%vKv$cf1Q}&@0SL!H_`ZuY44|F{Af*Mux3A3} zW)VwOMbZl9i9idRNkQk(b3lgsn`x%n05?LBrc{osM5PVMdC~Y5#IJ=sBD)pEjFhwl zzhKZIP^SnvD&t+O_@hQ;)a;O_qSPeSJ6tarYxtu_10Tdo76`}xv@8|yt7n^lSgZDK zQ~5Q8e;O8cdw0{)Q}O(h+xIUS=vmA4Y*pIdjxooe8?*HJ@MSvV8CN8YN~@ap=$F{0 z$WAFx2>g15jOPdKVE77Phy3$1{)i**v{kLg+ZZpF)Oa~AC|m2C{N2RJ2+}8!$o0;i z?DARj?TKjJ(+AG|OQ$7TjGwuOe{ne5?(V^kC1M^w@lil8aCsLMswK(->Fb`b| z=f5&P@Xd4rvGpg;EK}m4FX<*_^xq3Y`E{f6MCz~gU5^+S^|lvoIVQK8FUw>1{%c*i zJEtW0gRcBRSN{Lhm2U>r23(jk&bK}fw>IoC4c9uG=8>8GVw+b98hCmwGwF7D$GMLk zoCnOT&81{n&Wh)~h2G@lu0HA8bIbJ^k?*d##9|P4>Zj(iuRFePVQ$@?DeyOcs2(^W zf@8UN=sCN*TL3AoVdB%baj(WumiPVWuuDKU zODNm+nMs?gq2)&Vo;5ws@PXt)!iqHpoq&73aRN;A(8Sr7H@#}`Zk@FJqj$9~O=ZuF zqpbyN11>o4F>pND%ypkODbQrYx-8Q<#1jf7?+e={bth8KC(rAW-rVxrCBs-T{`BGV zS+`l|B&Xav{OddF=RZAj`RiLXhoGqt?sEwLeN1dlK;X+Sf)D#te_Q>kASUdMotkwt zdjC@IrR(!T`~%rlH`Q>nrOE237u)OU#xbs=myiF-TX;(w+Cet+L8AU3Q4dzo{<|gW zAM#S>C}B`&z9g8?VeBbVihCieH%!3hU_h;qWX>#9I@1m znBgR<#h~v(awoSJZn%`PEFGO%_xs22PCB1x)k2 zrnqHfvmKooL{i4~gFY5V>NIYHNnfcO{V1O*61ORsUbNBbV3godn_k?g&FV)>NOCh( zo8ny5@QN7a89cnx#cPs~-KMIdh!2Mxtsm($ae9Be=W}Wy8UJE z$?VL(UHX}QZDdDDJmH^nbqY?o0j>tUErVSt@q`QM$~O`?_tO3)@FAXPL;p~VCF^Qv zW?HanLl+tx^1wtwDod2xIBPpIMZRufr%-c=Q|^t2^ig`SXJ-y?<@y_aT*m(J?)W$* z=NXz~B6LB?35i-odL^S@ZM^Qno6!X8RNu8x?eKljVYh@gI-fKi*-dG@{%^HX8vj*i MRjTiBo`^O451jGYxBvhE literal 0 HcmV?d00001 diff --git a/student-tests/fixtures/sample.png b/student-tests/fixtures/sample.png new file mode 100644 index 0000000000000000000000000000000000000000..e7c920789e90d16586ecff5029a87d1fc247b086 GIT binary patch literal 111817 zcmX8adpwi>|3C0bDCg50hY&IhVa-}PAeB?jayAv^IER`~Iyjp-)LWvQLWRh=OeBYm zoGEFQvV&768SgUXUB9RA@1NcF=XKp~yIqgH?$_f%K51_)Dx@g1Wy==Pw!jr)^g?I<%@C1k??w5p=(q(7=5 z`O)3FZPp38c>W(UJfUrpsC1dH4|d6t6J5HOg!bOh-6L~LXiXxo=aMmpV`1f-lBN^-U^$ZC)OrursoPQL4nqDJjNOl@^j=pB-ybDDfPPX79G`*_N^9O*vMc%u6}|94&9p<2h~vcw#jcs+B+ z9RC-(0KXC<)T!ZAe6>F%@sZGTDL>tfqpVh;=h8E}8;4n#l!F#7o}7ks&}Vq+H1}nwTAdz zJ?%HQ6(uesSdI6XQ$vX(>|9kIG~FKTb)7_(v5z6FtAo*=Rl(KlV8kQK5JZ>@vZDME z){Ra?y{is3z*Q|)Z9`?t6}!Z^O;}mE6B7reT}$q*oF--rI=hxUScxH~4u+%JYw*rR z_g6ym7OL>>MU^Y@d2&@>)zB_+HwH^km&g=X-}0u`40E;<;jW>8|wQ>v%}NXE9kt7HAQ>0s*7r)G6t&&cLqP9&Q?*ar)${eh_xEh zp_B9^l!}v)^RN@QA;a1Qk5X|q@}eUVITUS|VOJvudM;ZB;ZQ?5N{`Po6?QSNSux8S zqsX~*R;_5}3Dec{3~J)7=4#lcc@)>qx)o9GgF#JH{=;~ijz>|q2QRvG>V9-?^rj0T zbm=HWjcZ3eo3HUGJq=~<#BJnR(koHs&fGTM5xOVhi%YGXl1r#d<8k^~?)?mX)RTuB zC*mKiDCNyO+^~aygr zUO(t_UcVt#(KYG8VYyfnc|}{%TMed@ubIJ-DV1d<0)7^|j)>k!{a`{EXvIQB+D#Ra|??I2ITFD2k7h?qZK|x)sGINL9fP zxrSLICMtDh^E*VX&tp$mL{A*Cuo1Nm3D)!s39ewti-yq~6|-NB+gn_kIBa1pIv&}v zc(1HE6|ML*L)xUIin}{Yo8oLz@{ntiRY@^6x$uhN!@14Y%L)p?TXWjj_^gGHjzfXs z#Iin$$zO%$b>m>OTFG3+h&QCP zzMX9Qtc&M5yf~Id$2q67oK5TzuT1RU7+UlmVEc-$oum5o@+B&&OfFwfy-m24nr4bG zXE|~UQr{}h2a~+Z8n;L34lhh)tJZ@6(TvM*&F z44!_#wdUsbm74x3=elzX`@Sh&eM7a_DC0V0T|KwqwNXK=?7M2Z^zwW{>|_Y3Z zcRb)Tu5(In-P@ zx`cxq`h@pdFvCF(eZu=K@Z%tdVtwy5SB^ps73VL0eN+KCWbLP%|7kOaI46|dKS2(C zp?dajQHC7C)x9=4)(<(9t%j?6Lxda(EXY=h-OQmyWOek-QOF@(=pDCE#+Bd-4w z_oIL;bICB@;$|K_lCW$3Dl~FF+0z8+?vGz5qG2Fog4) zSce>1AQabs+03D6^5}gW&GcW4k;_ehQ4fs915glKf*x{ec3p=(29c``m%9&0aFY)6!WKcRjwFv zC?@>PdiyBkQ1R8z8#W7&L&f`J|I|e1`wEBNJ!*PaVcbfYFLmpyYnq{av7%T3Zho?s5tTh+dr1x579uLBJBs*Wf8(Syr7E+Yt{Vc`d&j+7498z~O$d0Brv%=~ zWBU$y5o;4TA}z+_zA=X56I(=(1~`Sq(7Tj`I`;M?ZAJV5bATPZJge@e+IbgyOeMBml8p%3ly{5tp1C*ayL`NR zN05J80z$wo!&Sgdi8RwCNXSYM7mYv74CIm%97N;KGNZXE36BM)6g&(t@{C~r+=LOf z4WE}ts3|Xx_l~(w;JGQUnD>z>F3_s5;ka>nIVkD5{D$ks*~s?p8PO$)tMVJKn0^9( zW&g@gpIi1zYR~lamo|O4b0iu4xxmo2Kua&6uqvd-h$S>;c|f|8d?;Bdw<}7<(a0=fdh6J2~oSHGfg)J`pU@z&+L{0qx>362mZ5D@n1Paygra* zr$SgcIWajKpjCJqKW#Z8sg-R)yh)xfPa)sZ%EnBbXu(YM(Tud-WdzzGgUXA^Kg*NI z*;@W=(!_vekhG6&3_`mAKQVh;%R#oX0RMXSoR))JP{G%;P3;55^ix`6n7HhLXnLI1 z*uJ=g0TuRAnknHf zImAw6FR@rYl#ovTVmG}j^^W{m_EG8Hzx|S0)_0I+RohV=9Hpc81tr>}kJKlt->Va0;d#(K2@%^*y>;uw6cD06$PPKa*-_d?+ znM;J&#s{-Cq*W;nw(;lL#?qw}1KV2D#!=e;8oOynr3-CCW&Q3j2!xt}dfFSU=SJ&~ z2ENnQw5}Sqjm+k-wQ~;^ZqR4z+52+e+x-cTO5^p;mTLXoyWYy{pS92ZduRI0)Hbgw zNBv;g7{~s^fbJImo$rtM2xs%VlVm@iTGrpL-Xd{|F>*vtO1L`ZgZ^pt){`E3`b%mk z$`AdebBzhX!9)W}mL5jdPH!oUs7m>r@t@vO1a}KXFe6D%C6p^cNy~U3E8ff)X0v3^ zXE#LNOR3N^k<-^}X^Ct}aVvkEf|1Q`9Ct&8u&~14^jhSU^iSB4Ksgo|O_fta7v*QK9f%1$- zmo;c)yg#B!tWN2aMK?#bQG0swWb2y79sX^Ne4MgXwoZ~y-#aAWdde4?zp#a#cLYno zXtKvpc%P^IpM%*A*q7`*lzrqv{f`8$8>NCSPYKk|ZYCAVM{qAtE_xL0T52KX)7$m! zWpefGpTMQL7>%%FgpKv4HI^PnmZtpH$L|Ve9TpZpRiJbB6p7KJA*+JOIaLrAke$*` zmzJIPEC^y9=iW@I)W-(~bkgte%a%Uo710;y!m@rXynpB^gd6mH{Z0bcm3xOym-TDq zrIKE;r6@M~otht?P>0$3WoHoIPkDp~q@?5#@={*Qer#P%@CamC2&40Y5%N|)_Fa0Z;olkgK^a%Yc+MTVuBzlDjo{JWDd|Mk4?NBIcX;FCTZRqLd#$ZBp zMB#77wENef=&OZ-4E#yt=~e&2GzQ+qmPQ!zOhsF8K*jy6im(=` zM`10E#LTQGY!9XWx4}Ub&ldeFDnuDr-CEglo4l4;<^(K~#tcLmeH>Xj@=dCc8-hI_XKZzrce8l`pw+Z|AG4XcR z5FLT3Ya!-kT`7N_RTC!CK+KX4=6EV4g?eiXyq70%{-?Ary$qB3mO9b5{Fj(Kasy-i zw&VD8=&EdC>pP@=9|IW@T{QBZMk;K5-_bk2ome^29{MpXy1d|Ybkm3qrZVFkrt*D3 z07r#bpY=O*I)bzHu#)&Y(u+P@qoKZDj7lT7iZt98suUfHN0&4hz3Ia%Ib>9Z;=?!! zN~>X0dJXHfBMYJC2EGmJtr23BcqOYGVtyKDsH{7?t7FG%gt8H%d4Q*(g z=+javq~~LvFc{}ob-9m6B0{f(tnTMN8&M4nZ(gt94a|>Wt~RdM@J8k@6h41Ht@fv3 zy>jGv=pX$*jcu>ypJOh*ecUB>-bL^ZpU))GA8V-W(R1PCn*>f5q9o~mV)23rJCrTO zE<{UEXoo-hUlhFkLRm)4Chb!CV8G?HOSsToin!^a0FAUPE9taL`=|R^orHsFzxU(2 zSPAT1X@dLlG*(%8b#E^c6HTHf-O)fIKQZ?Rj#vo`>MJ9^-OUUBb~h_25ZQw0$6-eg z?-%@LB`Bz@+||u3&rKQ?dx~8|*+-JfdvlQ`diF%p!`@fO67*;!VCxV87^yY)#!NN;pqWaRy%8sxvy|0#Qa z^S_o9LOUe*1}8tu#2GQVB?RB>rydrGH5~5F5=;G?qk+th!49(Q*k{vPu^q2j!A5;7 zk2DVQqgLPkjt?wlG41sI${**J!_%bcyVG)&?TJ`d+i2{EZV|z?^h?Uq3(PBm;yZAp z@oo#Tvl*wvR4y#1Ct1@2#m??8;4q2YIEL9*N+|{*1CV=151pX)_YL-&)jkZH>bB)>~Ab} zsG9$RW5v^PZ=e?8-jq*3EtIEdq+H#y_hdAyeTU_udQ@W}#bYIt*cvciG2#ev?`# zx<1}>KrP?d=jHgkfm)tYr|7%nfLhS0RO-h~YM}(6!(%q7g%x#g$Po?H(%wOzdxQpR zLH9pJE&#QBoco6U9tqU)X$75su^p(Tt>|l1#t=};&?od+Dp1Q8wz=F|JL%9kJqfK6 z1=JG7ZJ=0t1GPjOy{9APfLgqVV~xf(sYTAAozzE<&o>qJHvh6>mJih8-MP4;nGe*G zZ_pl3-=vm&icja#CbbM{qVt~uwS0@ZJ#^6&-)c z-aswUZ6D^oqk&qy|9o08$OmfiUW!?-AMy#;Z%9*ooiq>Bf|pmZ6~EPRVDbl(6q&M6 zRw6h9)FOT(?E{{s&b|IW#s{Dl@s$3TKrJgCGn^6?pca+26~+so78>WH${H7_WsLJj zMSK~kN=^wyZe~tpe17FInX7&H-w{mwe`$#wzodYELn_r*g*PfLfSbs~m$kpq9A!w;Wv+pq4neo7fznmIG)mb#jwha)$7o zoK0#;n^SQk0ky<=d}Cb}2Wn9<|9AOD8c>Uh`H$t>X+SM1Q~&c=0BT_dRHWId0JR(l zHns3bq%P6#dXNJ8VA(EAy)QZJ+QQL zKH>6YmAQ%P(<)Bya=@>K|Zk9d*k<>_*>>c zEn~mT*Zu`+`L`{9?5H_Vi}~8mwwy7b7W1{WxNBoTE#~4s-hHzX68UG>-d=kesiFf%f7SCm196HrTM(C-+)>st^HK< z*MVACoD-_<>p(5Psqgx?r~G57*LCO z1UIf`45($S(Y${hsAVm#eQbX`P|L55p@k3TKrQClzl+#FE%VV&a}>;hT7FT-`u{Um zk@%N;HzzF~s3ji%l_RMN)Dl1Z)2M%wT2!Mi53d5X4EDb=FX0-^0JW5k{fZymq?S^H zU-1J#EiV>^%>B4PEh7tr()!;(EuW&vWB1L0T7G-auPT=UwZ!YKHTD3t4ECQ_{r(%M zh086=$%zMQ;c}~U#^QlmmK&$~ajHNqs!EqbSAbeR1yXVznFF=_-soFsH3w?>z0tjZ zDFteY|MPWKt`w*xK74JxeGI6j^y;sTp-pNj-GBMdm*{*yVdUMTqIdbnt%|oC-C}%A z^XryaKI~R+7Jrpom$*c_;UO9S)-8?4w%yx$LHs%ye@L=H1gK@(lsGl0OQ1ZU>)AGp z5Kzlb>cy_h+kjexsaL!BwgI&Wks`Y+wgI&W;Pw2QucZlAnW=Z(*qSW_)UsoIZ`Z_D zpq8!gwqvz`T0*aJwzUWYwd@!-+OiF(g?|xw*XCLsJNcTn0A2^E<>s|Y0lYp?OXjr? z+b)O$wdgY6ZXMeR)MCI~-)gV}s6~Fc{MrFNpccW7-TvRV0JUuWB|y~!YRTB*EV=p*tNr3fm--Pc1B*lrbHXx(z|V1ev?|%nJ=zYYy)Z$BiVF$@d34L z%NE6=fm-6b-t&Ew1Zpv6`fU9!2Gk;iHDlhq=1gz7_KI(K2T;qtuFG4O#Fwx6UlZiR z%K)_?n5->1+kskS{V!ZI+X~dOy+Fy|_Zm>k))KL0+iUAvLu9wdW0=uffm*i58!>IR z0<~=Q6XTue0&3YRCm4SKsAc4u%{H_sP|JR1)3xv0fm-DJP>a>I$6KfPfm*gR?qO1GV_Q_|HN`0;uKWizEvbDWDdU*#Nb|6g(cNMJ-!|=#oBPZk2vZEn5tz zMQlQgW~2txVu6$^_euX*Zjqj?CIQrb3uMa@qLs6}W! z>V^9epcdga?Ccjcpq5l`si<==fLhe@Q{uIOTAoP^S=8h&`zr+0qCGo${cq|tVrrXpm7_+mWQ?PBqGWfb|IYnKKJ2jH z?oN{YY_+VhT|HC6ilKi*@1Gs#l0RsiR?j(Usi(1|_Avd2#*zY1i#)MDJxd*^MSV$` zSeg#hqQ0ccy_OEtqOOAEUQY*Vkrbye^w=!P^V#W-O6K=F_PJ2 zpcW+-b;mb#pceI*%xgPYHYHkLk3WbCc9eFfdbT`L1<0?paIm9LApST zRtIXKkgm{#)qz@$cI+XHr31A{8ju$yCf1xpVSg3oe0=1a*H0;=Sbdy@rv3uE7=|C+SAEmfc=@s;X^gxYl3Q&tG_h?j7P*KrL1zY>$Q{P>WT8 za)5m@P>baJ(E>T37MJ8o4ZL)~71|ws$vMWB`o!)J_U z$UrRyeJk?11fUi|Z8rBxra0}6ylNkB4^T_4d}$v~52%GA9}-wg0cuHaH0paG57c7x zgvL+=YKa|=AY4%bYPmeDN(iR_wdl-`8C@jTYD_S-L{ zWp)lFUrI*>{Fi)53#dhFS~s8~Im;?H`I7duHc-p+lNMD@*SJYH>~+-T8Dk zP|GnAp(j@gsO326e9tQ>pq66{{eZP(pcbi;;^m@bpcbi;(&fZtpq8BmCm6;+EyNxh zsh{>hEqCrlBo0XdwH%8)lUO4K)FSO&?0+aRgx0nFjh6gPpcZWR_3dx8fm-Yg^}4fm zZc>X>wk;L~)H0vky1T;^s6~s#NajcZwP<%7U@7keYLWh!x4e`rO~03%D{X%qsKo|* zpgVIrP>b|*KJ(9Z@f}(u!)}Y6KrK5}?3a%v0=4Wst6h)_)MA;~wsT$+s3nj2Y5Umj zg3^YwhkJsg&GUg;j`YY$ciseQxw$--T)!Krt!-fi!|DXXWs+V zvh$X<#|@yCJpb6lHEE!h-8{o?>zzO?yLl$vK|6t3c6${1`y~D#SR`ic#ApJw*fwZ( zPbCAjNMFfaexD4~BK_xjq*o76%g(FzJewY%mYuEI8wY?|ta}bh|H%Vtap)=D`PYuB zTg`9pSn*)oX|dd1NF(KTkJN>Xr#eM4b<0kZHmu4WmIdl38oi*^qVrOwz4C=+N%^r; z4}e;9zEn;d1GN~=`RXX$2Wq)Dy?@SJ2dL%#^uf8UIzTP=@z^;v9iSFxm0L}x278?+ zkvG^cWUTK4wcHE7&0dswWcfuV%n6wb)PkWCb>7_vYPq*quuTW3#VN*qLKCRv$e^@y z$xWb^6NApqCAWZDjt++Fv{wSPIAe}JO@ANoV^Fs zQpoONWG5~6+mT{;gY0)=snhHCa z7p}D31Zr{WELgdJQwQKtqkIsvuVG~($Q zGP?AKGBwT}r9drO^fVpw6F@Dy>6JR>Hb5=v^mj5}oPb)KLTwv$fLhLE=<7Ur0Mt^v zQh0Ob0Z>cnO4`l$PCzXu)}sggoPb(v+78Tp*8ysA`g41w{w7e1(~|9a#GubP{m4{B z)1+HKEqmn^hDB~Q7*CEcPe-QYm-z}DwAj@n;+XnjFHL>m`dP*S3#}CqtG>NJErT8h zIVB1}EedOcj6FatyEz{f)&_uD3^;!j#9sil?BR4Phz|g@+~GE4j_g$_DVt9hu|A7M z0kx=G42f6=0ks4L=dub!fLau?hsM!BEov71A}@eiZk9!-q7{Hz_LdZI?_~nD>@B&& zZOH^`*?VD#aR8{LHZv#)uMO1Fk+~4mu?wgrppUZm=P*!95%-_WXU0G+XBP`uO(H-o zd%bV79*X>D`B%g}Nd7udi(&6$1%Wd_Eu)6Jdw(bZwHS*GMFwuRTGBH$jI#%UTFf~5 z3K1hfEdy8#hnfl0q7X4gvW8a6Xy(N;9s7Y=bU3q_&U=Ac_S%mbY5=u-6j?h%JqXmI zpfbFC5?U>8scFXee}Gy%Qr{}fp9X5V5fGjlrvTJ)hJ@=iR+tZ}J$0S?HgoI@P|I~L zJ=5R}P|KNkGmfqTP|F#(n;)4#Eym~pstHibx6Glvow-0QzNvExZl{4-&Uh5F{u2Re zQ7|7~cHC^WD435dpW1A-C`_I8$Ome<9uSynrvTJq96Hco3e@r?bLR902B_tC=E2j` zw}4ul`$~=f`~%eD)%Q)|>KIVV4enUx)iXdXxx}EptHw)%=M$nQtIS%EPpfi!Uj)oI zBo|kNi)p9nb|8O5@yxbK&ZkFxy`>?0Jlmvy$!zb9g*{QX%+@4aM}L{E4Fk0dx4DlV zH3MofTN`b2*le|!t&PPU+HAF$iI2t|gI3F{qO`ftTdT`RQ|npm-hM|jpcXUFc`Od7 z1!rA|?4Jc{86kRSUoZn|F~iLtf>uiL+4yfgy)ybmDIiQvs&0eER zWmZ;%TBW<7-GBP@PcnGKk)&I(@WWeZP&`4s@*nB)1s3oPyU_M?4sAbQ>keS~A zP>cQoyQqEvsO5I_@zMKcKrIX2w^oabfLfyT#v0XuT2TF;k>3}9S_ZfQ**Q@_Ed$(h z*<(>aEiW1|0MK9PsqnWebP%ZJcA!o6BQv0ug$=ERRx_ZMg$=cZCq+OlQGbe7^NWC5 zqQb}4FF~s%r|9bZhAvP`Y0>`ZKZVimzQW#jkLuq=0JUs8y0saoWm~K;mfxF&sB-Tz zkVtPhqJKL}kg0BWgWzU3d&0cxpduJar0 z0c!DBE>Aol4AdgmanJwzHlP;%U$WHNO={WZE=zp~)bhORHNSuiP>WtUt~NiF5f7l{=@KrLvJO}CdYP>WEuKDKU?TDsp0e{=+DX=eKHe@6qg zps;Psn~6XziLZo*_W-rDc3tLQG5~5x5ERC{0JXR>S=(}CfLh%AFC>`p1GUH$c=`Ko zwp#d0(95<7>--_IGVxEC(fmLyGVzT}8-Ab`em^wt#Ad67UrsLm*(S9l+6bZbfm+&` zO^M%SfLh%B;}Q`3KrQ@ka-`>*)RG`B9PhG8EeQ_7@ot;clJJ;+3IWt2!|?EThE~ha zHZP%2eV~@7%zgaN^?_PCnd1Dd2%r|1<%PuOGC(bEk(at>gn?RQHkz4!{6H--)2Eha z65BI8nNp_bcRm7YxoLQ!KvORuzsjpehb2^R`Aq5xv|2RFrP}k-t(?^L8!thtMKj}{ zGtg>rZMrgWzy_$rX34e55n3(B2cFq1Id4+Sz<)L>CpW2OAjw9>38eh5=v(=(GhK{q}Y_({P8N?liR!h%6a(Q1*0ku$PH8n5g1GVT+ zw9?LK0=3+vHql%)fm$@{&_;JRsf8S3BZ2~I@g^K2f3X2-xh3z$?vvW27ESA0NY_nj z8Spys(G#e}IVyC3s0q|!^U-%D`yZed&F`qVi_mIu8W_;5yb08jSL91Ly4h;cT+$_8 zg;t9V-h*8x1=J!Xo=>{BNi9;%Hp4nVE#QZ>Okpo2?ej{Ji+eO=^)EvZ>tz)Pib!O8cz| z)N&%;gIys7)N&%;lie%@)N-O$zp)OerHs}mRd@oZGC>2&hHqT*?RC)9UpnUG#K;S~7m?l;L==-9NYB^pblRjAIvwFx57g4k z{wKpm7pSFU1ml~AcGP>W+gSV}Ac zs0D#>W4HjdH1*sP^3nxrspHwvBXogU>UkFQFa%JG>qZ;f00GqEI_=9{%K&OY{H^7= zZ?;+x7u_~m*>w+PspjV=e~o{(kc<7?TP5CXt{3pe`opit&%FrW%ab+Y&*3G}l(oy> zgV~>YucjyD@bRQ4_P+aFr}UwCbA>YfLHzifhN#uC-psFGl|`G?P1mI#jF@x~3U9-Tf!FHzmY9HMne6l4#(+x%V$$s z_jI^UKZq>q;RU>yYc+12tA249giyi8;NuRIq|;Gl6!-#p8N8(L_tKYgf61islgM%KID-juc60U zrVjB%rp}#um<8>;lpNjoBdnv1$62{TrB1vS(ot5kP^mMogLIg6_Wy<_9komZzn4rX z;=N4AL#8SJd%75Zy)Mb3yO6EVAE8UK?zV5V>24Rgpi6S-sAux=tD<F+Y`J?m4iPze5Rpw94 zq=1`aO?JjJm#_MtA2O`zov6>`Uns?@;HvT)3fJH9r8SCa}QFU z7yPI^Yw!u`J^3dJk@s*=?1*!9jfu;h@`@8S_aD8K6s(SM{U7SGWYyPudtDUB9%ss% zng?C19@$#evHcM#dt6;m=UtHMVy?u33^Yni&c_v5R$iE~RGyZh;d+SfSYDY?;rg%B zgx9~0Z{6pNP`}B4P&uw~MT1+}q&)2^yyM&<;y;6jQGzu{_0)&>Q*$~f2SjBx-e*n$ zl}0vpEx8{>&1j~Zp+;SaB^j%9eiYip=m_14O+e&VQC;X~*ii`gDykFR6(%Ncs$gCc zOSqTu#dZ2FVe%^0b51@l(Dmb z**=JN*ADHE?%ae7ao70W=5AbahJ))BEpzwvltBYmct*LcXAMTU!ZXT!J!w$Y_0R4r zZtIjmKi5B6SKQar1`nYA%3Q79@R~b_`s*6P7gFnL&GA)SlZ`c9sv!Dh{jj{8C2E41 zWE}fiNxYgR7>qw@GR3eAbj}*FjLy{*isF)K=kd0I za#?AVI>nrEhF##r2@B2*#WL=m31`lP;-zz6^?wD!C`A7MwjllIvG%=Nh#jfYCPh^N zcd^HNcZnvMibz_R7S*z3L@P61nH1HrL_~XOw-rZU8hUfyvXNN}=kX4lb@sL_x%0>} zt~#-?&&I^}A$Kpatxwm)x0;J0KJE)K@vY(FjGQ<>6eGs5pAX7E3dl@tR1BBdcaD0h zS6?w4(Rxn4GN2=MC!sv`y{YZX$TX^R?`ctI6I;Z^a|I6qo~9}g9;BX93<~M+nyw&b z_gyj_MNDEHIPqD+w9ATe=e_R{8$HU2)qTHB(b7Yv=6AV=vvg@EvT97~rg+y=Jwr_J zwJb@|5IRoui)rZiauvz8m!nuR-ryMea`|3rr3qfYWMa8Cb-@I0;5*)Ms+XeZD`gN8 zZ^e<#@}gt1>d(j9aV|7kaK0$cjPo9lY&h=~XC|o5UtjX-`aUSmNPiF6IK)X)JSe3a zvSG=oR5X_^4dET>%~$M{@(bY|?QK=;l%5IU9q!FhydrflglE-TuXsiJc?i#VU!pw4txdGmkN&T`b5sDQ^f`3*&sS~FNBLq@_9%V{ zEpM(KJzL&33MJdmp_-mwLMtjBWnUZ&a~3E!`qk);+_w8z-KGV>zLD{hYcsJ}E}X1(wXw;=Sns4}{9 zl%H*0EF6n|+2)})(dM%7A1n0XaXmvpkwI`iM|C>K@zq zAiAAVwi;U;u^8{%aepd^j)kJku?Oveug3&g-A#W}{rU!g%1%ihITw{`5A!8X2Pkz@gNNu^^VF_`O$ zwRsA|qAH;c68F0Y{j<6)Xk`N8#tb@>RbJDrA#%Z(;q$K{>E2;^{E2@}NG;vI0+%S} zFbaXu$>tWMQ6Ssc@DMVBxt&j2QGSp8Uc(n%{(Kg&1<_W*(z~kJCVZ7j*xoKf;)8@{ z5s`hQC)j>wo4}Ycsgv5pOcAh_?>Ocix-6BHuh?Ph9lk7-ryw8UpI8vgyug<$GOe~`V)(X8hR>Pq zBk(|>AlyH-;GF-h?zaNpW!L130+)T0qWMKfjj=P#ZN$><6ahH}Z@pXc-shLEBn{D` zlEmfX$C*NnFPW&M8d3Apcc&Ej#Fa?2F3-l%uK)2ViGUazFH;k=MM7l?MeygCI(!b4 zAQ5~B(~J+CSR@p)T%M3m8xrs{qSBdejU&t&0Y77^&sRg<)9!sDojM=)@}6{G6j_pK z7wHU%Z0%m7{T4WA$Sdp4q4f(KG~!h!jL{YZIt`caC&0l(7p@&F>L7XDrFn+T~*@scRfEE=AZlcbHVh`?8oxDfy=f_Ax$M!5lsmL zsai_p|7@2cR<`5{X7p(l7KEN}x*Z(elrnHj%Zogv^}ZlDq`Wvd;)qe%oyA=y(oc8x zChOqtG}tn9)E~k2knX=Nzbktm zF_A27CnBSLTmCemfviTTBInu_5!f!oJLFe(MFE5Y>?_(St<*xiS5e@KckaS%yq~xK z%8^_RTi@MiOdNT@#?Ds@y)Q0hK-bP!vknuNH4tLwt9|rN$H!SwE$jaTE9^3jjHYN= zY%S@1g;YB25gQ}TvCZ6PblBE}*iJsKwIp%zPC+OkgIvu;hoD6W2>Em3RXc2PuB0(ETmNG`<|Eo)4OEl37P{tNcwpJ?=>xgeNp;q0V9y@ zz|AK=x7!G0`!!=4=$^59OJ|4?j&IoJvTKh$941>CT?7kC7VV1cS}W-qozE(eVyB0B z%&;|SW%LkiDA2*Bgz2?B9=C64z3XvOKStsX{|8v4$SCW3$c74MV_zhObT>08T}QNq z%TkWfWTk|Q_3z1soRN*D2zu#Ro@;FK2;@o-tLVLY7aB>_#q-=9ln;6rM*^lG1G4swkq@yW zJ)dMp8w_oW&aw1_d-Y57M;l3X^i{nQ19Staj4ml_ASLJ3L1W*SHIUBt>gZwT$<|A` zdv)}&AIsKDYv}t%a0Mt8f08C-`#j`BSVxJuDTVse1eP1IFoh`F=T)#v%ai({XSeXh z^!@s_PYmfjF2wwlbFvS-3e?Ve7MvkIp|{fy$S!zvykUDErmX6Xwo(sG2NBEYRBN@IgtZF9rN9sywhV?BI0Yp1Xh zT}#-(lQh`#imf0#=1CgufkvKip?*g&OOx9~ch?JVq`HPS^B&UQ=!MJe*Xs=Dic%sn zTJ<`^xXP6MbTs9==X4nN0JoDau6HGvtHEugJLp|G&obt=r5NbNpJTyVASFUCK7<8p zffQA}jbN5Gw>ia6Z{s{`AGbZ_px)QUjkXYPWw9`0vI`9SoB24* zn4BD=V|;!`p&XeK=1W+0D$FMjh4}`oo+xxD*M#{7u6h=}A>Y8%x5g!nIR5PpRtgNw zd`zsM4)l#+!W)S-)R8`{5+}@7LON_ZJn)~a{j_2XJXr~W52-_aCQ7fuh9!){hMUaZ z&wpU66h?=B42iD0I}(Rkc<;Syskv=@UcHb)mnyUg`xp^@+ar8+YoYi%Z#B1f-e;m8 zkEDf8dIoX=lyVpzx>IjGA_9}L;4L>srInkUl|{(S(hkF);hf`UW(9@egE%lF$Qr@; zws8L;y`9&^v@)hqPu|AAo)5vaQj{@%t?Q{H`{-VnnJ4SnBgKULk&9tVPu5FFGxLI& zgDvYfc#L^%%)!Up+q~&MIZS5@7bcE<`!Jo4xiE3;6USU>;pXz*_8DNVJmwbiKJ=+# zelmE@8)rBXN`D!=H+7<48{CxBE}2bxO~z7ldM_(aN&isBL^8$$G#X2L|5H|BE+3j; z1W2dZDPyEZl%GU0UIst~H$X`+Ou0q+oAMqp7eVFZ`9l|BkYZ&mc#H3Sf`(WW6Ss`y zsYi|v9A4Zd)tE#(93)uX>v_9(n$;~P?1-yHPGRwu$j{4$X+B65^iv$ZCtzP1I7^x} zF1Z(+CCz&O*SUt)MGj0t0{?Fbg8ah73WBqU*+)i}CxNpFCLu*G7+Q+NMm8lmBa2Y> z(UDJ+jFBSJK{)JCcVrqk3$o0Zs66?cJ9j9lRX9ZiaRM+Ww|Dq(Wm{vMjEc~M3KsPvxV6L() z(a?2z&i_u*C$Y2Xx5dty6@6OZWb$vUWV;s_gO&$9O-z+*JVVS7cw029@+VM*)$EDC9Qscd@l=}B`soqd_T*+ zafsC}c5#2aL&x`JzBEr^v566SC08>eD2FHW308>!#AdYu@ z<1156@HviWx3R<&7rcbr(D@U!+?sS1x$$LE;KHez?4f7gOZuOmPWR7sqQ8)Tp_Qni zi%mBeV{?Dd;*oK8gTFSBhaR8{$@6G0YUrmX*bOyhQJ$dRk$<9PV_Eq2nc~i?ix6l4OPzyS$k)dFIjZnWYu~{FFUDZs-ti6?L)LxPl=x zT3pV@M}^f?I0Vizq~X&u2hK87;qx=w{#t}J)*J=`+=^2P9+=r6wMK+L*OjvvDmnhA#fJ7L5x|?95_pUKDA?V z4xA<5o!aqo4xAa~^#3t+ zC;m{c@BhHF@4K-MLTKzmX=oZnQrRkm>?DfHpc*w~HyB1C%AhDz3TYW9$ueV*$Z1nK znZ;JAW@s7Z)bEidA8nSMrRhfTELO*7t zXo9n7c7%)`PrXND&LAtl|3$T!PpW;iY^_Q98DgE%IXDy5PM_IR`Tc%~>ACyFo|$ax z08)i@KxfFuMMKRyFKC1bhOzaF!eiDi?oY5d;4DSVP%JyDVPcxIL(`{|un)`S1Zj?S z5+Fqo)lsAw+O;O^_;TSRE9-lOhpoepEnj#9&hqGGXg7NioaNEI&~9#0)5OeTXyyIT zZnL*1mNOqIg{H5;9bYbZ+~rk~*F7j3fsrGC0o@P^@R{p(n~*r^tOCO1qUoMqD|E}heJS896f5lAFW@ZSCI(!CmB3k)3Zn}b`D$3|JUGix|7xY|P2en>s0(a$C2$rj^%GlO z37kcLbkTNF=q!}Elh`q#vrsfoBH2P`VR~T0-V2?DX@L!UCv+BOJT~kDILqJMulh%M zLTC9HNzQNA2WLs8z3R8p2WLsunv3kp2WQEDv&nN2oaJf%ekETlI7@2PY~)}0;4Jw; z`mv|LS-AKT=BZ7ih%cBiE|CJxqOW-}WEz}>gTKoBxe1(Q6OP5-LIG!?T*u!0CMeC{ z7mvj*uvL`6S(H9(O5lOBjP!H$YkA--i@eMEO;~UiKCdNz4hzo0YkSpSqz}$=ipbz< zP{3IzttW|(!CCsbAM~eq;4F{0Z}g}5;4F)Ro6LXo!C6kN^lQT@&=9b^ZkpFKY>|rAqKTd;o>-_W$Co7M&Hf|<`;~=23ahf|B0OdW~r+f z)2RbyNzjH1sdE=B%mTBpm$ujOfLY!x#nn9^0<#dme8rO~z$}#LG~^sG%Uh-l z&ie~6%XD-pr56Xxg7f}{FQfpo;Jm-%^C-Y92C>UppM=aZmYh~QCuEi}xwH}t+DVlGe>hE*b-784F13o&{!MFCD8pDgb7gSvp+zSpdwk zOnfC zfLYR3*wGXUFbnS2H$H`O`rp?*BJs~GqDbs1shS6VI>C4B6e5ii32bTE<<&*0x*ln7l@xmN5n`)YP)EPVyR?Hg~W)MKC(?YrpG+Os|T1xL`iPK1ehhh$5`~Z959PCVgF`v zU>12oN>7*wFbf1iAaEkgfwYU1$N{q$uyR(JNdmLT+OF+kt#XANh?x||1eoRaswOFn zF)+)4j1EzGC14gaR;S3A95BmjR<4M!2gfm!5j9ec`G0keoSN>u3s zvuH=`?Rg~v%py)zr-6K#Q6>{!<_ zu8ZN+FTS|2N=b^a2AIXV=f^5tQD7E19wEa_6qrR6tIfK)3YbL{tIO(K1*hm!X*evskgNi-gMovq%e0_G}OZW|0*f?>Qj~%p%&Yn|PuJm_@`{S`gU-%p&3} zBRH4=%p%e)&5OwZW)bO@;T_HZW)Vr3=0#@!vxub2@ZvLoSwwzGx0wO6w5~cWB{1uG zxaBab^v?U7Nef_>BZFx!pLBs)3~p^1%rgXLartDhm{AJMVpyG%paaaJf6Hy~oFOoa z;hUT|U0@ai7nw_W^Q&(wpOiKRX31-FnJ`4+cS&!^(d^UQLh?53Q$9m-U1R8!zuoo4 zu*7RbZC&(<+AdbAefMuy$))fLZdePWxL+AGp;ir@124 zfLRPU6Uxk7U=}633rWotm?c+BJt~o87!^kRv+N^1b)~8Rv)I#v4D<4VS=1IM zOXqTdS(F$1N{wBBSrk)u40^Z%vnZvy4O+MYvnZx|4aU0yv*dN*rWFi@%;HLxK|27m zG%9b%rhQKVj#EsJ%4S`t^w4!hjWgBekVxNm%;*yWm}LV8 zr}DiRn1wR2I;x)p%%bw0LfA|fbK{b{j2w!AStt`mQIANle@i-Sbh{Xsg>rH0F5?vC zed3!SjiRn+9H_D&U4mJgrQ2J?Dq8K*BpsQvovw0Ly2kjjN_>pU5=qM4a;tWGVW1i? zOB}u5=y(Aziw>Ve`o{>E#YkUUMFp6}SYNL_fd01L2l610Hxb4_5^}gkz%1?7>60~^GHHwQ5o2(r3=jBn^SE(sS3;zoP$>xa0h1D zM%&bPSsIwd*t5je1ehft=b%cXJ21<(Dg^#G9hgOBZfn;DPDn3l$!L@U%%X{pq`RpA zv#3;V>v9HW!E(e@49LJNWSTkWJxR%}`HpE{zlweVFiTMzt}jm-n8gUI&n}SGWi(3r zY$KQhvz(V6+eWYkX33SVHtx~_W=Wz47#%49W+@UJqQ5i(W-;p4=NFTJS&X{%_%)=B zjGLs-+a~pZS<>lBM(J9>EO@$)QMwK=OKOgd5m6hM5ZELSPo%dvjI)RF9Hje2qrqon|tW!-Ef`aT_)r8sr#$PqL!3z|YM zr~+m|Q%YW00JE5Fo6=&^fmxUshXFhqm?fI3%%y7qv!EpzT1a3P9qzPN6CIdEgZo7* zmJZB9#_uJw(ZDQp?1tG?G%yRj%XXGj1ZZ6);P5n99;{6);P5nA+0oDqt3L zD<#!+1ej%BD>{l?49tQWc#J+$0L)@FK-1zz1G7X=>PPP(1GAt_nN_O5EEWT|wVp-; zvqX;+I&K7JsjW(mdaIj!Gh!qaJ;wxQDeAPHt*4QdYF<|k zFpFwfRpQn@U=|sh9fX5@z$`MY+X*p!z$`MYI|zsSKF|kqUYd0o3YjG(3oSEX3d~Z@ zniapd2AJhsL`W822ADtA_m}OMFOwDr#FiSYz$`a%j+zn8fLScq_hn9)0khQcvU3bffmx`$vYa|oV3t}|k$AW%FiQ=q zT71eBn5B-DFYas#%tB>Via#_3W~pUS#80ULv+U%Z$U3dMQvFQQJLtOi@Pj?TERB<9 zGuu^RUa*Dky``IawXB1fFi0~o-a4@R4Kr?ovlwUTo-f0GNyKsRi3-ncP}b_4={^Y zYT&Rn3Yf)L|IDr6;ZODM!^XZwz$~%D4Zh!Y0<-KG-@rhlfLTx_Uigqg5spCJ>@o%o z%(AC&3reyHn57Zp$UvchS*~CwsAn%L6WV*#Wb>FMoNt%K?~W|L_Ojx1~a6abi@dNBaV^?8GM*2JZo8@jzQM1nO#Z zy86AQs&ZhK*uo)H<7Hr$ooFk@7j=6&tak4i^#*1!o1vC}Z=_mOC4~(yqkviVSbKJ^ z1!h?@^LP38CSaB(;_8`f6fg@a;7Ul%;wAMvFD?@>3}X~93+hGFL@6*!U?I~Ndj^=L zoHL~Ea|M{CjPpu;>9h%oBSpZO?IZc17>p zF02}u#g6+``{68{Z`t*EHPS@kwjQ$iOPFp?>P+2V&6Y4?`sFl z5>s_9@+7RY9LNjOj*SIoDZ_gYoU$K9j9_fID`a36ZOsEA<-jaw@dpQf+5@xL<4XCx zWDEK*`MTZBQNb=)XR*Ul*=OYA8Gp%tuum8SX0hz&YS#_|vkde0=QY^@vkdVP^5*P- zSq9r2`-`-JSz?JM+;ik$#$9sj0iqQ!i!t|u_S7ISiv{MDF5-Tw7uayU8 zv0JHO*UA4ra3?eQ#kI2JRw^)y?{R2U>3yWLzajHFpHc6YT`C9OBcKfZY2lIB6k<{FpDK2r8i6hm<7Th5H=BJK-wirEPz?USvjI+8o(^( zw#_{(QD7EPGfm7DA+v}!X<{0L%#zt5D{lkL(!}bN7_$IoxyZ_ua90Or3E`d1s?-2x z3F8%J1!w@Xgz_$Az0?3^u_UDR>dFe4MN`W+B0;##B5}kz(L=b+A~Ar_3I}EhWPKAY zlLcn6wC(IE7X@aKXhc+n0kZ@}?CpIe0nDOK_2s?IN|ultwZJ?SGD~l+geC%*MLlF6 z?-wlpE@mmI6Wf4UZn3UqU6;f80keoIX%a32v)t_YA*w42%woYKWSYqWv&dqDSXV`X zS!A)ntWHs27TE-AUX*Z~MIszUy8_H|gLPdZ+ya9$u23cShbHVZ66SBZ8vfaUn zCxqK963%M{k-}{j31^hxpm3W-qI)ebM!3x)(T(CA7H+dhq_5>g3%6M$(owv4;WmrJ zueEJWLS_*?ttn{gdFXbSb@tBt^2zH$W*JJ``6(EfCH&Twp*#~{mYtu%7c=Q!MhZwc>uFiX!%7Y!Zu4>L74}2kC0gko_bJs1G99{ zgG}-&fmwVPC(q7R0JH2`>^p0`6PU$1b;pp02QZ6Gs@sr-2QZ6us@G7w2QbUIuI_0C z6CtyBkWr982WC;-P)WNYWENHKPO=R!izWV4!44B(mUC4hQ9EIq<)-S>ot`$pESBi& z(+g*TS$2-1Qjfwmi^*ICH5{0wb-M3t(N18Noz_;Vr(v5Vbx3L_E)bX{D9L~5xru=} zFiSzc2X+rIOF&dwfr1GzO9i2eu4V$vVp40#_Y-cjnABSFLxkHbCJvQ^b|JHzZ7rYN z1I)5_@%h=-3SgFU;)7{j6JVAK;{EB3CcrEvQ&ye#h0Jo+pnRoG$Sh~;%2#d+nMF0c zTyRUsEUHuGg1bUyQFSgCvl3Yn$gp~s3ZXX=(iB5bpmfAR-r zxhH(S{su5h>%_&V{z70D#P?QME(T^P^fGt20nE}mQ6KfF5SRtFgm2#fW@)|Xx61;U zMdAS@QPlM;0uc+cC79-{;NI)3qKhs~vXQXOf;cN%V{sV~AA?vblnSui8q|I*&{xPT zjDGXuSAkjf@kxdMm;j$YIfLSc`L)!yjo5csRAdoc?MnDpBx#qwu0hmjD1qfgk zbNOqwG+>t7eM1QEtH3Nb`raW%n}Jz^*e$aC7Qif52}c-9=D;k?gh+-80+=PB>oVsG z0+=PBtDaMV0A>m3YUJEO0JB(xh4b?Yh0KB&@Y4zwGD~i?#pE6#v*h9t0|CG+dub2) zF3SS5Sa{yFy&`0m+=Gb50AQBARsQ(n!fh7BoL|>HV3xuq^U+pdmH>Pt!wmtq+TEZ5t*`|@OgS;hR}7FnOYglj@( zksaGhxG7{7*=mcfKq0d*0?dzG1!lP}IK+5q4$NZS9mX#%1ZFYs4&~Pr0<#o;-a8p8 zWEO^!d3vCbSr|U%>HCDtA`A!w37IALs5voM$Sk=A<|~0hW)Uv^?-Mc$?D{WU1!lS4 zH--3hm3P!%mzHm>NUYQO6x{)2}s&J6DyvK+pqeB<-| zm&fk^mZQ8m_TXlCnCyF0bL+*-xaNyzvIb-?kQ!w#pyF;`%#FKxu_Eh*?CZ5Hao5{w z`o76-B$dc+MER*7*SO~5b?IhXk*)d#S*zT;w9S_qlx0Klmt1@+69Z7 zdnbI@>t$OJ|02Sgcqbv1q9A+Jk5JZE&9IU^>QAWdd&AhAdtz;uFMBmQkgc5SigPZZp@q!rHA$%mWPZQC7nHPh0XNTmrt|j_! zVZ_UZ1@Q8+Qlq-FBxK+E@y-!~dtD?xuO$R@HL=np4Aus&(Xa@tkGPrzTYFguS)5|<}styWJrcCc94%RV9d zP}6zs%5Jug>|N3g+4Q}ne`B{Vtm{}5YT&C z_8aMe>^IazD=TA@Zg((h>Uq#bZOejN}wcoZ{zl#lqU}-PJGtl16^`GlkpTEL9Pq{+G?|&Rk#X7t&I%?r(0rd)&+I1paEF zzPYt8zH|O+ePyAA-xrO5YqXti1{G8NZ6L!Z|;0U&qiH z*bbkutmyEECt-Fi*AG|vbz9ChQ?D-~Dq62K6TBE7%-`-p`-}v%Z)Q}Rzg0bYJ+PUh zhk$sspRj}kQACeIc=%cRw?l-f-31y+AVO>Iyr*;D47|fz5$~6NE$K#JQ__RsF~4+Z zvsu0}-0F8kIsE!$b5hyxQTl;Dtt2zwIZ_qQ*H(5gG}`?cW9I3*+r1J3_}2@k{8p@g z?cz5T>X{!e3^o6?hkvi|H2nzT_cehJLm&QM;1ECkk-IEJ;#-;<`O3x58JBdFo)TyS z##qGZ#mqxkZ!VuBUTk)#)=_QVA8Ky8)SU-a19{tYbmHTTE*y$$ywsVeQGX?Gua2;rq`m&ez;T^7 z@e8Y+4lP{XJw6fCth2uOuL0}6Cvk^vuX_z4kdke2DU4j5O~u^qONms@DT z{k96<<7fesP3&|9r8q6?ZE;#xBg^xo7_K_o;~O>`hk9Y@&1 z-CA5m_tmL;As`P7W$84l?-HY^7`(2 zzP|#rkq|=rb#-P7mUT4Z2)_OBM5cn=`mP4Hu#M#Vdg8^%Gl4CU1$kF=&J_sDNd&w4 zp>cxit`iWy5=gAa7Z3QXhp^kan=b|D2ZZ$`51$Dt1{9!;M0fp>zXin;E&Sl(se>zF zD;N3h#m)yOjP4#11ohkK+%5R1lm47noj1iW(E0VO&9xsgZvEjpzh3Yt6F1o53cn9d z7=Jr7(G>YGPi&t_L=ED3ihg~^Ul_l+NGGM2sX5KUV2ArV>^@|}QeX}99X9MwTgduj zFI3ImwcJ?RU(-bLJyzSuP0@=b*&m}`;U?%EqdPi{KF8OVEV+e3!^yLk)@|ByGlUL0- z@8FP&Pse3yZqjx&H9dW)P1?nDb28Fv-=324{LFi3VQP4SZ!g*7_l~yJeS_XVj4Mtj z=V_kInfj)ZFsGYO#)Ctya?SP9QobPqJ}fjX>D0U|{N>d8H1)x#2uHGJ!~3osbB{S1 zHTT|kdCh%{x(K}^Z?7hoYPBSjMlU&ic9K5vL93CgulK$1XUb$NS46L$A+2Y8jQltM zB4Il-PA@uzj8OVO*vf2h(pTfAkYA#=2p`1M^LbBgPCiT6Irr45Zrm|w*PMEd?=hT3 z?Ndjuxs)2%v#RMx8GQlC>Me6cPT_wq->PazzBw9@(tWLgvtFLzuF92Jhj~k9RBE=ML3C#yaKZ`0~|J z*pwLk>g4=%8G4WADxDtw-MNK$Be`MJ%juy?^|AEpOB-vZiVRYIH7}Xz1=IKHIe!qi zF&)}NnXOLgPxiBDuGn6PZoXB0gh9M^StEl4e@O z5NsrGhnSjzr>c)O;dPI_q^X=vRaC^a%veV9|H+(WzLErS(ISlZcu0n6B+HMi=M`oK zDD3Szz&nspG&u{4QfRMxWbp`RBAr?QXX`}S7rjjIG&&fDpP^2i>bMg=bMeY@4 zntrmyHx+CVhEq)QF3>)#etS5nYUq^bTZ>QjkN}bLRMdn|)$1IGr?WYRWer{>Xve+S zghiDy$_1h)+1~N(w0IdjEaizi7*$`egM(7@sU{52_is`2_4HiNZy}7**Qq`Af=`xS zBX;OFblCD&v$;SRqCbi%Dd;o}L-Vf<6~6TBKC-A#wnQhE$$54kT!gxSps~}l9J+mg-ed|N9ZX+T3r6VBAD|X~ zK`>00QQM2p~{dn2K3oDPOKXa7I zPG6Xegl`Qoi#BDat5@6?9F|*$etUjWIpy`qzH@3?q(ikWxA2gLV*9R8Xo5VtwufjY9=xme1}c-pci*U)Au2z7U2z{k+DQuq(uggDjfc zhaKZA*WYBE)c6n#FW>D;*x%mHFs;AJIG|A-oM5Hqa2h`fuUmcTNG#5ccRMZj zGKluJZxuc}SJ!HCKl=c_C^y(@@&Nld`bnRj#;4o1+vV@t())JFozSD-w^HmNP#C8e zX&RsJ2{IV$s2s)_4fi18F6@5%Z3bF{S(tByi{kGtq7>jYn8DNtK9mD6%&a_b+wQ=| zun)?O1y)(cMHc2>)u=^`1yvo0{3{pMH*~DGn6#aZ}Z9+R#Dd( zbymNk@YNGX_yI)^gH}RUBKbZ=8!Z6@Eq}%Ej~4wV{iE^wj-ZaQDXJBI65oTd5&oig zckg~YVCZnI$n`Mt05?(BouuPXavNV>+`vFPl-$8ni@DGf;(dGf(`{Dxv&F9+yzk-* zG0z7MI{4g6&BHvM4RRQ}8{|%fu8``vSJ?1pcY?+ar2T0xDNb;>`)cdDJFi9#Bv)H) z^{Bp)aQE)3fo*QVx*rp6-y7eOa(jFzFRGzq*Woya6y4CA$o|`rui6+A4oKZLcA?y1 zf&)=EQ~Ge&e*0V6Mf3v!)mI;TPCLj|Yre5`t9a~@aJ=JHXvm?C{Kt{t3W6y3JH_b^H{0qwczVb6B-^MMYIqwdoL&wGG>x1p;fK^Gu5 zp(~DSqH9y+=rH@5z!>x=PRa-i2-L~BL7@mT0(0ol~&O*Ua_2)M$J+nsb z6SKlV3VfhCe6wx&(qkyGDW(XUNuV=eKnDs*{tKgU?RDX8m;~ZMu7$2H{Uv06CiZi6 z+g}abbLj5iKM?c|P#n^4b-E?oS?+ioo^V9%&Ea2%mNryBEOJixbuc;qPX+}_Ne(X= zw7-E;lHVPj?hAK1$7a*3r51MfD4Jh$#L%}K!#8L5Baxq2(!2Y6&nT88CEmsi^tLOO zBqw%ZdV32L4>>iPcqi?8(>tW-tP;R~{(KxW!rr9#8S&k*YlY5&1bWs~>blu`b1SyccvOyGakrjkt9&8>QIjMA(a+`#*V; zs*fztm26GDu4z^bn%Ns7)ruiJk7UFvhMy4R^@FbTODImIz#CI<*e>~>G4RIJ94{^3 z028uCV`B&>@oGQlNVqC}R#>H$ZKFq=iO3eb z?%gI;jU*a7Y!tlcJ)zjG>5LJK_xdQ_Bi;C~j0AM0KV0wEEDv-A%1AyX!WYaWL1q6# zyf3m)qjOnKybJHoFC1)O(^ozuHN(W|ahtC!57m;7>zQv8%$&e)=s#LW;YI z5P~;kiQtq;%3adrp1?H`0k#tMXR;*-XqaPps%^Rxp&_#0R@sQzXm08aV<94r;!OyM z2`TuSko+O?auwWmdXDvc?OBBvOEgYYOYERY&=52Q7+=Ldt@)zwu3v!Z(#fZSvDuR>o6o&&NqbV~s zV<@XfJ;I#VLQt%d^pt{Cm>$ar2Ut`QA*c{!;NObKtZ3`1)+|J8W^4A*446L>MFghs zyVnho?iPb*F2$y#oL9j=6qaft3f5z!0x}G;U`%BhN37?y3&aD(h_bL5lkv4Dwg<)z zmIuoMe&sq7(g~v7D%F`sGe2j*|B)U_j}J?u9kHUlA_3EB_QR})nQ$=lIQQzYyxZkg ztnC_XJV}3 zmk%+c)Z9lPBb5Q23*7ov+G{sw&6hOV-Y+NcHANpn<&`(kXyXho>mwTXXtd{y!`4SM z#X=|99B)mK5lz<+`D1waD|jSpHKLQ!CcE6A0CbbYI zW{j;3jxASd%_Vg`lCd_>^q~>|W^C028L_TC&LQhNMF*`L@5ASSzp7@U z?iPWJScj$X&lZ70Bz;T3=~8#+^zTC7X%VHQN=|*q43uc(sF`-z6fN*x;uqmQ3Nsq z6=4ZU{NjlPPO#=wryy*F#c|hk?wmA&BAQn-HrDqFK3bnXwp{%PWW@S+N@xEJq|#=> zt$!b1rc8WX46pp%IcfauugSTjhmXXLm_#Tco@0@mOMhXW^+)O}eoxG4X0x{|xu;_G zA-U{8C1xrnjLKqfRa(F4nW|Z8^GEi6B{&&2QD^xn6p)ckwKMz#3do55*jGX=6J&() z75m~lw2Xfdw$o5RMxcVm;X51;UnYj5<0*%+ez(603ulx-MwDv6F`eXuEjZtwxczCt zkqQoB_3{{Uwti~o5!#u1{8GE<(Hkg0Bl@QZ zT_aqe5oVZvD{mq=1hPPsH0&2dz^VB;wvG}U7Wy}r5+CFPjVNt6NgL`18d17;lJ>YC zXe56lcJf_hsn$XyDgToGCnwU0Q^;AqJ_Tq5`}Q+mgaS13rv+sFMWNcx*p+#<83kyB;(Tg_!F6c=huf-udP#6!(9f;YKS6+{hSZfu+$sIvD)%@2 z`oMRl0F6+7Vn2N+^jsHTXsEwHaV5e(kRO;4CL0GdLPWmft~CG}AtFC;F$O>*MCyD) zBL!&0;7%HKfeSQ3c}lGPv;;IV)UHn1 zu>Qhc=B0@neMvntHW+KES{Ituh_gTQ(~=mr(4h{}&42Yc7I=P4A7`_a+ByHOqjJmw zf<^&9x`ty741nixKbC0&OyGIS2V%(Q=nPsfvlw?|5#K>$FmK?FEaC5K&2tqEPW;eX zBl%N{6WvU?ugLiam%rQv!5@hGeE&)8nM*|X&26u1AmY~QFFL-S)2aiW#~JXKo5$dv zOdU7Po3{)_n{#D2(;ro9Y8UypDX~OPjW1{h)6W2S{{N(k3_8+;2{C>>rIOfXIV-68 zz(p9e{%C=snGd)ro}i>#n=9m5}n(6+$BvuDJU*|!z{@Uqj`MtAmE*}?A z=#RrKw!dZq&l|)OiSMIt1kOc2_+Rr3{|cp*n4lc~b7DTai~>AwF!e()$^@P_aQ-0} zn$u{1J~xIt&F}O|_`ZC7Y>N2H`WJ7xX-p3X->GrG7nkpi0ngX{{;@JPr{Df^4p;Xx zO`m5WBGufi!75z^JTIasmq^Gk-deoMT@qss^G4MwrXOIzhH#{3Nfd?-$rjDF^gs19+eCDw5{xcLCTO&!_Gr|{ z^z0Q0mP$~&SbsR#72%Ian8UMB5 zu-rr%GN1+LxI&fW$X5L}X#&1ykDmn8%-qX>b4BEoc$Xz>ral%zMy=9tu4wbFWu%G1 zxgs^M$+{pMLPT!LLCuT^CvVjTNgAr>G3VT>dy+J(o{v!x85>0>wIfQkkP+K6E{O=w z6)B_!uQmhD716f_yv&S#1}Vc>tV9;hm9?z-45*_)C`l35uo^_*ToEl%!WgpPVcZNK z$ttagvjp>=ry{j7wn4VSM3ESrD{{70JzrMA&gH5O(Gf-A{%lW?NSG{~D{2u(G6F=q zblNy~GFqjG#w;w_nU$4sR1(e=y~Ice{lK{*CrIy65`}X`%vy%Fjs>{@R$+#LB%CV} zg0(&GSHZa=+9Ja{m;qZ;8DB+ka&WE~^;C*<%fh*$7ZIJ|B?9M)7*UsXe-)f7qAOag zri@_viwtL}6%&?2`zBVa$Z0t^SBzP8A}7S*T#;C@U`>g@xgz#Um-jH^6#Z?+Pti|G zgw@UB$ol$yBv%(WSMo5HbhaU!D=tWFPO~zcD=tV~4ow-(6&I>j{Q=TG&Sm90u2dvP z9e;@Q)TLH`Q67I_u*0QR?^X%MY`W8MOb*T!-CJRUKMdhqG1SdXK;9~d)43HiIBN*! zieYeWg5E7JQsG_0*ST>9_1W`U^}f<3TVpTfJ#m>g{0lNi+I_NcuDD>>y;J4^AzFQM z?0o61T8s6iy9Wc;7~aqGbUk3C6qM72*^j#B5~g}9c<>YF@6t^!VH%4`#2JE(xPFntdy(SZek;GsrQztS@UA3X*GYBG0lWt(*mY8aa~y3;*Mqbedb_+L zdJX-Fvd~6c73ETW2Ej%QKPxS79%M&t9Xw;`o(ndjo|H0(HiX&L6{os5UJ5p1$jqlI zFT!j7gJ!Ot^5`8{1m~UdSWcCtmM~Fd2sV;ir4eLE?DTn8RjVQLG>5a-@BZl1y{7@3i zw1(;OeBVJUD50ry{iT{zJ)yt&vh-mt*oeyFSm{O=uo0JEh{eyPU?YaV6_&#XH$@#C zOf&qRk5%3)TC!`mRiB{>*oZMwgUyqMYm7<>Ingf%Y(%A`I58v#Y(&MIyvyFj3cn}k zwTgF1Vj*UI-$4~03Ly_;&k0f)D}lR=8h?^B=LR-X9B32uFA3^Kgv~T16I<66Pgq7R zk*u~t(#Ys;$%Ltk+Z$oo4CTp|6!H^Pmg=RBUC@B@MLIM`*nr~(Hljk6Cb0_Tj0q|$ z(wWl6Ve05Yt)ewiFG&aKp3^EtBu#fqH-EZ^(eWa<*yyyo(;pdujci*$@C!*`BPy+h z6J{>q_+vTgDmM!fZ_FF8|B`;`4mLtcjLxYx0vjnvjL&&v1U9mbX53e!0yi9$yscU~ zY_O3N^hlM@1%y_b4d-7K0}>{L{9hpr*oaX{3C5HY$WNrZ8{u^LXtXs)KvFXT8zBky z=M<>GRmTXc$38DzCSmMZV7tDrgVQPf)X0-!>pDWVwdh-r1{*Pgyw4SBuo38?A(-@C zSFs2y5}c%OFajGfwk8P<(7{GjG`A7WIPgp^>9hOFYK~Dm=1=_$*og5>k|2^UY@MMP zwUofkXHDM)>AG!TBW{5P><($L5n&YQx-{6xpVk?+Zo6aOJ(cbv-UESQA4Mg-kmi<9 zEJ&t@8-a}&|I*>#Bf%BO=$AI$oesef`a`2%dVGWSjhu%nzl&(@3&rr0_(hVc-`RsU zzsA6<*>(&cNA8b8>T{)c8xNd8mlWb}V@wCy(IrLrE{x$o0s7DY+>e3F!_^mE%}CJi$c@wP@E>^r zMBN;x+Yky#Vz3dk6Ju9YipC*%2plyO*+x}C;OH)Gb)K3SIs_jn|6c>nNKw=_bo3ee zf{uOh`2y?#l_gHqmF7aIm=R`-JfgpmQiDbwxs6plhze8bxJR>_^+*4t!#%iU+-6`s z8f+vwL=!4zz(&wV$ar5e*a-SaF+Q05hS8&Sf=;s^SdGRElxn#~3vHyt(Q(TH6!LeIpQE8YgdV8IeX0d( z-dZ1`i5gsZkeh51O+-XzLl$U2t2zp7geh*7SW{cY$@%%N-){0(< zfm!`(NW(*GFm}DWWlA)dYcLMIc$pGSvo#p>mdE0-c_MSj4CB~~;ye>?GQnfycnlGSP}dBoitA|K>H;7D0P&0*&RCQ@yDU1UcN6+If7YZ6sgXIfZ$vP^ht;zpi?-F<5#EzjnJ zsx`{kpE0vk4WqWH`$O^vYG_X25BBNH*mw{+=g+qV^jWCFXlvSK9$6<2qpi#j6Pju7 zo%+*RA5=r?A~I-(P%m?2XJQA_oH?doZU#6-L{g7$!;q>@pcsQy6V` z@=~&5RXsI4ZB2TgiNk2C>gi#-sTVf6dt1ai)L^vT&Wp;bRE5!YO;t6kLcGo2ZKL%{oe-6$XN=d%tu31cFd(jrR5I zF`3glXkH0Eg805ERTyoz3u629WMH(F`MpzcqVF{QQ_gSG6&tpG`v!J*&QCS{?dBp) z&CTaHrRp%+q7*&wghCi?)!iF0<_s8Z)tQYLYX*$A>Sm42=j&m#b^EAp)&!&N10}I9l-E5lx`D@wv7qrZlw;xXgj_Y;4;DqpiBP?Bzs8E~-mCy>N35%4Vnb z6|&peTk*r|VoVpy-G+;N&()jHn4s=o0blWYWjqsu0$+iAQL5YUTh4HKz?FqE@D(Sw zhw3jcgRl7DLkmCng0Ebrp&4E%@RiH7^^C(P@D<<5Gn`TNY&uKbb&tH4&{sCjw3UCq z48Bsi_^Vv$GWg167U#UzY4$GjxS1uFn7<6^;6;!P^ z-?MNiOBg1)0KO7Zco5at1isQ(RS|W(5PSs{=E>hvSP3;W$-Z4@InT=xtyhSfW^_^D zD=5v&M0@a+kL7ZZ_W@rC8~*N_UJAZKP1-Sh1qHr>3b?#t3%+8`a7XQJ0$({B8)DsVoi>-r|_1YeyfG}o}F<-u3%spt4#$lxpXwH5pYGWd%2 z*keN2z>?cPjZXJYc(0Tqfa+)1ngJS0^*%##D8l-*mQDRyiT!Z8{ z9H60qSI)`bJ3zAnUWv%tXg7Htc!e6dJ@1mX@ER10Jjbsk!!^k6Z5=<84A-DPO*F!5 z5O`KNkhQKQYWS;|7RlqCe{ZF-Phu|92UoP)!57(eawpRUL zqdvoM4Jv~IngF}53+{se=iT$AiI_!-oZS$2IYO##yt{VgM=xdVYmj> z{HdFPYmhy$ntfj$u0eJyRqQ4*T!YBYu`4FPE1S5j+NU20uR+{8?GyXq8Wgi)!JX2E zYmoh~YW_nqT!Y9z?LLhWR$mvVHq`GcaZQ42&@9H1$##TmP!jSyw;2i7pd@59mxhFE zP!hGO;XnyogOGQUsZ`*VLnTj>YHNX44vcmr)z$#7n9X%Mjy-~FP<2Nbw9>#e$WeDD zfeO5Gp(AK?))B5jj=?htH630hg?Ala&&1UMuT(YoDw+VV%nGwca1F{o4_Tukxg?BZ zX+J`9miJ%vP1oAka(j&(qc6SHy0fh zGRk)}n+C%_vowTMONPyp8G;kj-qBQUw(4y3p%U0UacrEas*TR8Iv>5ag1q>;z`oh1Rud%Mp43HiY0$y`Y3(qk%Yo+Mk3EPIc_=85Cf*z%T9wP43w7`xT;0T*1$q6IvDCUN! zHA}^>arZVe@2q#eS}Y3X9#;(=#g@=bWQX zjGj*ZRhdkgs7?+a{nt@xx*|i)&(9)pd**x26VZGGrmIJ#ep{xo9MYc^hxg2kmO}=> z=Ny+zHMwV&yF93V7msI_$${`&CGP8eDhI-EN!Z)lAqT>*j=jf9l>p(Fdtnjy5WeZO zMKokV#7MbuzVZ>u&8z#eG!bOk0VCAJorrz5YN@^TEQW-~Zhr{v(prN+W; z-mNTHJdrqs=&FYgJTbl3VGa06KuGAt@XUBwSUl0B-R$`)0gETHlYy*8QCK{Yo!rN| zD+-Gz5}(#4#%6AbO3XBr$kPxmo`fa__MVc3FFh87)4fu%uy`U{Vu`uN%H~|kYLF{I zVfJD}cxSRG8nAd`f$3)DWWnNzEG{S_Dsu-1Qazf4#-9DXqQ0`8R`AK^X3uR=D5O!g zmm4v+z25^1`Jy%&gsVM|qP#OZCBjg=L%oGBWxK<8#aT=Al`J{g?jRnNCqlkx3MIHn zXk`ggKD6zY*0js4zyAb{EM(II6R!mNvl>LN%faFas;#l7Ndgv6BwlC=THp&%k!ZMv z;98F)|5#?RgaZN=PppMmAX!*E`I7~bg~b!OU;mG(J8_44|NqB-PBqln3WIEwLdIUw z&}6G9l-&Qe*jD1c~qi8`XODIHHXUr1Ha$1a%%&4?TGxUzm>D~D~Ue5dT zyS{(Ixvtyw?BRZ1_qhkcx6`jU-!pque%_h-Dq>`&t4w&1ZV7zEjhX4H@V|mgmQJ$% zLBoe9GINIQJyRK#ilp4UC|!92k04EdQIjH_2_K$pB_GMHa)u9249GFL)6VeW$yPEk z_p&a0c;fubgdCR(AD-wQGE6zu4fVTs)uLZBbpP{>@p*KfY^yhp#^7Tu1Et}YQeriV*9+aX+W4)(kk&=29t`Q6UJ zz7Vc7L$!!d0^y2Lici0dE`%#C-weqIb0J(&9J|oojf5+?8ZHywNVt+4;4YB}*+3t~7A|QoMEn!W9qEpkl}c2vXeT%}CC@xqXc8%b*b}f%==FrTrKs) zDy(G#QE9X#?@6toGY?W0?EPBeXYBnNh*_{Lbq;@H;f1LisX;qqtYuo!{+MrA_=^bh z80+`XOsUP!9<`siGyiyc^gACtviI1?mTiBgwJwSu#3#QdC7)>Tn42?n-cnJYbI;jS z{n*Qj14s6Le~gVaH1u3L-L-)+(<83GPXBP&z7g$ie{%FzW;f6v6us??3YDGtDt?G0K(o*K0Ek4b==AYNtM-c|K;rfnKb$=!QE}5C0(%v&IOx z8ny=KE&61=xEg&@#S2^RFEqx$WJI0#JNoNw(&hGMb_cdRSO`DTfsXXXHM0RLW)jLG z82r`TWJ72;>T6E2A*C^M(QwI*>c?`zPy<=|qNiw^x~T=UWv(f9>0V|dcFCXW z#d?pv?=8L~@Mqme-zT*1nXM-gvN{Z5S0qHJfcfPobR)WD*%FpqvS3#Y`?HJPh55DV zS3PX=r2DjoXxY-Dc83)z=g0-X!!k1gAeu4 z1`66sK)qH@AdKw!<~`!_yYe@c)5_m`M%?Vv&FzGC$O4VZ9%XqSoGbjD>>%8qRizAe z(3q@RwoBIXB#x%B>%WsYCSJ%S&XOn&m67RgzlDe*Y~8PXL&wX^(Z1pO+MAhJ6afox zJV~vYWHc{7TG`bdnWg#Fmy2#hHfQ9B_t+YkNtmQD-zW6`-z<$d30g+wdUM_sCalmv z^>u4`1*sZQgZ{caGs87PHDmBMLW{0v(lPkT(1y}mp-2}9DvJh`K!}S zx#BCP4xro&DE?usk;*o8Vymn{?gBc*eW{WSKVHZ7K@$uEH^0DA4+sa*W4Q$7?pjh8 zdyMlL{lGX<<1jlK};4OYDG6UOb|jbp(| zWff90(zmX`g&<=w%BnRV>R~JftYBVyClE%#%BTwpN@Lu}!HT!M zkCp2?u<~yIg(`2bvb=}`Ry+yEZ-bQ!7o7ucZZ`4ySo_TztY9+o-1luE!u-sBZ?IzS zxUZpd?ae$ahER)GA-Cp{v0%m9)t!7N53z!6Bv>7I2UZ3&d?s(qgB2(MwcSQ0jOLRr zB;Q7?q%a3QV!_Jo@`||X0kDGoNjY3Pk~I9UTqLQ;V|Y{RMPMaw*qhow1S?!77WTHl3O521;0;!41=Ylk z0kARRitc*}9dBqlB zB?`Z9x<(zWSX^z8$QFW?LgO9x|3u!E@aV!=3$S8we@tLMJ+2N`Dyp*McYu}SxR5@S zI#@aWpSe8U>dCOGAse_f{l)yd*Ie1s( zTr7;9BkzjvfSlWLihp=(q=ikL#HtXisE3R!Rr0_J&j(6DU?qxrV0us;tXOo{lDl|d zMI9NOfl`pM+{tLcZ6R36UbN_J6vDfb&9I=>3&BeEum!at46Go7Gk+rQN?(LrfCX5I z5>$tE@W2W$NIf|ku_BQ!w2k_CV0Lt=nHORKR-P_(@VeB&O4P(HiD@BNDNK+4=``Kr zpBSq3Jn92(zY|z-iq(?-j9Bp?IPKN~E7tN)$?Zj8#k%OvQ9C|ZDRQ({|BP6f#$Dlm zMy#Y%4izVWl~4G6Gd0#=CE;qrg6tKra>Y3D{-4OZ5*d9ZHUX?8+#egapBc9XD^*oR z@rhvNBQAuGvIZ+3Kh#eXwZMwj>x6_ccvqf{x=1I06;K31B5b{<)Pa ze1_mKv;Rs}0$8!eT&W!bD_&#AKY$gadUOk{eEjq+0jy|c6uCQXA!-pm>`wqI=8jGc zRcmh+VYLXK5G&-?BC<7DNpKw@-zh?@ST~MY9hd%k3k@Ke9D4!KQA(>Izt~4jK7T`Ty_KfR#_w12coxUBrI7+E&fE6ed z(fAr*#f(zPDwYK+>Ui!{jTBf}E)>zg3Nl)=4c?W(&}dq$ELf4fzf-`P8kefQ5`MiZ zBVG=ytjE3SMM;5`^&euUh%3R$%Ga_9JK5$-#Wseu(p4PRZ~Y_k1>`ZrmyvN9uM#BK|5B|&Y! z>sUbLq+$$N2vYjW9dnJP*ON~M9m5gCo>It}4*u6}t z?rrd{@OmRw2FQXHbwNaE2MesQf~1l)z{*LnG|g5Wte7t~vqEIS%J!uWR+ki5QJ;tu zo6^7vEnVX$f2zkZF|_!3R4M+^5wLP3wph9h4^|G39obzBR`e9Mr?mHj6+Onb@!q#! zrQcCcqYMvL=5b2=vSVO{T$x+!T5fdtGLFls(Stb{*YXo0StYQdWPGq?8yT$ZjaG_v zZNYqTZP_`>;*9InT?xNe)gOQGs=oCl{F}EZJyT`XOS3Wa#A2|5l!11RgO%~YEw1pp zK(60+j1M0HD;zr|nyah|d{?l;{4octaP})zxd!TB^^}!rbHR$k&ahH7V^^h%YhIlN zE5gfS*S|z@#Tor0N45})38k*C)jHd)ct=j$TYIw~TTHmD306{C`^kFz{AkxhW+`|2 zKmVfT>NV^%@8^J(T#bVhM?_$SygxnMAlD$xOPUtObuAkn>3SD=Yo~ulVuaf z=D|w;qHFibd9c#Ya7~Mu2P^%uUeC54VC8K@ae!-@;r+`K5pf;4U?n$5 zFR2WElJ2BPTFLhE5?GNm=Z3ic1S^t`+%COeYtt_aB1NW3U_~jtY>Ch5(fhS%oBuq@ z5C4b=SctJO6a!f9`i&8H=K~fMg=R`S7qF->?g@HDfQ9R*q7i^#k>F(b0e}VR6crl* z7JnR>&#|W?(eQy;E69Eg`PKIU#Sd3&_%^tG>3wyszl@VZp z?on+PV6krt^#d$27g4V=H=8t1`HPJJOMV7-gt&zWog!DGYMo{rk9gwl+M8T#KEWTb z+@rK|q5hK}ZFH!Ka)*mxQE6y1>t_R&|Fw+FVRA;M%@4sMOp;0V11v2hX4Xd)z~WaL z7FL}FSXAbak`Z9ZZ#K3$LN2<_McPGVz(VfkqErCOVd{j)As?{hyBL|YwgVQ)fqb{a z)Rzvekw*Avp_L3^Q3-AmS7rg0EFTr$0Kno;>1PkB02ZUpsIaaqz@jqQB(BZ^Ecqt` zCXPt}3wP0|`=kW0a2ZBvF%rPS9X3ikK?W=$rb^GfCBPzz$PX|AEdCP_#1D3nu2_&$A=@p3eQf176VpdWLwQ2^S}ykzeZIISg};rs4WC5 zd)mSTU`6Ai+^cM`B77;10V`!0PevlP5X%Sx*BG#3#fv!6y!Pf3Y#HGtVkM>Z3E2{? z#2ji$x$^|EV%gAU-p>Oog&MIFN2cLn;dyJc2@osONgBxl#0r^N_|X!q2uj1ks|&%3 zfKlf{_slM7(w2_q&gW&x}`8IDOi5f4`0Gc9}WArH&@2|7ly>Z!_qA2DD1KXD=!6+)213=MI-&y5}((Tlo(2X9)-g{3Ii)) zv2^KOV1?{A7PgxXR^%1pDeZk=MV=8S=oNyMK1X?tU0~(27$?o&1y+z%m|{(^VvXan zYUII+X8GKdtTb4WHr8s1L*5npXz5r@u%g+LJi=m)%hz3*ZmQ~w*8(e6_%}k7JXo=s zorOXWSfRhxOiG4#B|$K_NfWGSeoqt(hk+HA9Wp(mvPV-k(d;n`tg!Y=S80M3d1dKZ znoHL577keXcMHcP9@)YHEA))Mk+3a9IsxaZS*;U~nv41QHYGXTS=sR2W)K11s`#%lk51 zBYLxz&0A8@^**|mjfm7xIzvM0Kw!m+IstVcutImyG)aVaMJPT%ceA3tv~P{n#DAW$ zk_IdC!6b1d4Xn_7CaAYfv7nXm)-I?V^Dd`AL$vng&*2ZD!(w7_9UyYIeiY zjHIHEp_v9tGm@IVVa>ET5?B#3<$L0gcSRThqcvc~dg48)g9cV;LGnqvz{>y4&&-m= z&9o3ruo5rspmoWE6>GuFl&LgWkxt*W#Ao#!N(^N_k7~z1Ito^f#xkWl5G#krj_ziH z6%&Q{l=eZeV#0_U?|lbW1|3Z_IuI-KI8}ZJVuf6pUmOTl9^$y%8WXS*SUx9`RRt@m z#z8G{$h)#HS~WHhtOT|skFvPqCSc`W)nI%OSb2bd^A1*GlvN+h&dw8=V1@ZQFew?{ zm4xv@-9WGs_&sra_$XN6+Nsh4!3s1#%pb$grLejCRjUHQ3Y^BZ`C!E%IjkM5s9uzL zm9^O)# z<{+E5aIg&0gKR`_u*}4D528%K$^+`eI|n9MVY&pGB*MEQoIk*Hdq91;uQf6d|5;?E z3RX;lNs`KZu#)cseG#zokkZc`GyyAtou7zZ`C!Flk|e2yzKAjNWXHq@O_^Oq)XvmUm2(o;c)Pc^AP92m~t+C*H?(^b?KzPc%6J|{!9NNIE8(#W=~S>H@f!=@ z%>XOf3Ja8WK3LIa%nN#_!3y6|TO$>*BEc!}Q^AUuR7o$!gOyV_ZeNWySowDi2COI; z?{1k#-jzMk3bA;wf^S(IVfBq`gO$5g{P^8qB?bRx8l??ZQf6l*LGM`-qaapq<_=~E0j`UST!B2XwNM!o#BHOM)Pi) zw{WlU>AP)2aIegeyZI<>u#!TZn08=*6^0AmIPfEDdv zvAB{BR_H$3zNuj46s5m!P#dg3^(3r|4py`$#o}r@SYez@o%n#&a%$o|xq}W?=t0^^sfZP!w1Vv^up$vR(?jrJ zWkK9Q@6rYmX8^0{f*4Pa%1@#&U% z7X@<;U#o0`VGHx%QfxvVbLlqUQ*eGwPWq_4Ei(yM(Wy3{8EW(aIY+H(b#~MAJmB%hySMH zo=}5(R1z+!mh%OBEo#LD5FN4ow1E1MK#QYeqW$|eSSd?N%bVUHX)Y0wcX^H4RRBUZ@7 zoMH#Cl8w{l)NBGP4&~aSg0)~}t?`}~G`uT@dq1y@bpR_4EefNXIOChZ%DwPM@q55Z z7T)$PY7w5ZU6yG-2d7q=DM6`YunBuRz%;|CZ9#D zkO?^-H-VM2rR$(70#-K7A>|XW@<;O?8|Ap7&__t?1nw2pxU@&8O<*O9ibv*XIHEt0 zIU2ZE*3I|-;g&_UhJeKZuOWJ{7OZRvUM(T!fR!AdO}=!nl1(w-3~mA|4xQ>SPy<#r zO|F(iCinrVaOs&RgcD9yGiwdT-zV z5#Rt;vL{sHQggscPSB<#I$}jcUu&Ck;h;qORvPHvyccnmJsG|c`biNk~Od(&{$nITD(*Y|wvV~?w zkgw>lAFZy^0W0fDR!5x$EB1`gySMOLiQvlGD7|Mwwg`%As$zQG|O%m7K;wtph9j zsCW@O2dw0{=$I5D?~0^9$88_g8uAq#yoT_>YOt~{I7du83s%nhtn+mOD~=Qc_TW0O zqSL7!cKR$>SvQ#@jyMZea!xu;sKLE*g0ra8y#el(Fb+c}O%v{wo7TVSLe8dWwSWrv^D?FSouf`m#5X-fv3)H}hnsInb9`dd@epZVmf)!#5V{{X5 z+#IYlhd+)F2P<5>?R%6tSmDlULQw>)l)fe=G2mS(7&p`-f)(QT!f{i`S2pr`)o4Vp zLX<5uH-dachxbUWiU?NBOVpwYz{(y*7zeDV`O5Bsd?jDV6BEHosr_TKh%KU00>_mI zR;=_Qw42u+eOy~g;2~C05+9Sz!3yzEPD<=!#EN+X!`uM!mAC?p@CjwOS9E#L)Y>?R zmFe$LKtZgK2~ZOOE1c4G;SmL3#e9zO!|pLyDQyn7QI0PPeM}FxQG|O%H6E#nfE6wk z4>b|6QtCo9DMa3th5k}EF4Y?H6(U|^`hgl)F%Qm>5DUOcfseT_AFS{w2E0LYutMxq z4?kT1R?H`JBoPH*rSv3!LJjVf6OR{(-5cOu346>SrfI^xa{TcyF-;rp6`l7?^PW89 zU3ve$G=K)Noj1<8nB{E z#S7X0Pd5aAPTYUiO_yp7`HCXGZ0f-(u(D>kZ~|5`eb)HugB3lB0c&s#SW)b}42==6 zvSxC>I3g3Qz|71;Z9nW{4tKaS^9xcC7&`dIIKD<0`PSDqVj`pVf09ajN8b z)D8Sr$X5)HekqZ@g;+Vf^JrHISTRs=NudmY6$6Iz_(sTA{;i21R_5E4`L_@&WMW>i zCs?V*>2hlfz=~)2RZ)R5SW!0iYH>#1m3^O;V?Du&XN%kDChoWaSh*KI5bp(6ZsKj< z!90wz>do2mc~J>iDS7Rg$st=Cs^_PesJ6r@|BI;US*mmSb^q;xe??mI^0Lf zRi0pFc>@QmIJkw~04vJA=v|PnxCpDoo?xZKe!%SL77+~2xO##WE4`yv;a(Z2Eg@7R zR#FlN$Od4=^U(g3*a5_fL4%vQ0pu%jc^Y05C6KS^a-S(f2L(opMBkN@Zy;94guIUi zVC6>XIbuW}STUG$`(X#IllqcoFPjp$R|e=_Hs|49VaBBmpbWsuO)CB!y9C~q5*JUC zgYd4Lo9{1iyGgZ%e8m%ACVHR@Rt$poONe=3CC>-?B4DMOV!$0V04tuImx-tIz>2}- zen|xMMedZGyfsmZ+$#f%p551wdu4#(nO2V6D+9xxX;+bZ z6BpxB^T0}8kU`Qd#EOWnY+DWQ$^l7yUWg}RMUtA=WdK&H1x!(+GFVYgzqO>#?Kz$} zRq#CO2!1Q%D~91;3Zx^!O26OE@U8-|vR+}3Lg9gx^^5_*M#xvfc#i8eA`vT+c3FNT zSb0PuW*2V*D-k%|zMA!5<=+_?up(=`y=4G-SN43Cjok)TwzUk8Z0Z|d4_5Am^WwLI zm7{puY1DeKa&)#_A}RnY1+TXy4a2+gSYWt$8(7)){fWR7@|BH!y|T1zU}c-^6Eh>o zS9JOw$yRLxE9*;Sqq4z@{cz|JumY`=U68K~PDO~fft3P#o>}-7Q32tI>$Ym0L7ZOr zRk&ApwFQI-urfeN(wIT4 zEI`*}2C+gW78S>VmG?MZevK7ai7UT4U7!h8G>zk129S5f>9b~R99W5K86MrlAGZQ4 z&Ed#2jK1|dyzLCi3aq@FEng6ogO&2vaY@7Qu0TUXKMt(KeSb1;3i--Ley=7i4y?q< zJ~20fd_{-28B>b0W0P9LuQd%MCF9Hu5n<+ zN-y#%+$%%1<%IW$m6XIGvK3f~JM<_eb_lV8v`-8mUx_Qyh@U7y?iKzs&9=9QmFe%A z$!`%WWJ1wLE3opm^jvsE5m>RB8~$N81Xf^##-;@Bl_7e(&3U+2mPcr;z{)!+euiBR zR?1!COr9X`%0hp++dHZ?F>1o&n@tnR#Cx z5C>M?Ph5;oEdnb=K~_mKh?Qx&rtN#giX^@$Bo476NiFKK0xRzY%;`o=u%ekhv!u`O zNlBc_d>*BYzYY0HUFergX;rXd;MW$~l?hg09fm?-ffW=3E4Ys2D=bHphALPI5Vx=3 ztAdrSBw|J}8my=w6EG;Sf-Vo9Dp&zlRv62)VBuY1}kV;W3zh5 zS1NlSt*AnS6;#QJs0^1ZUpv!KWw5fs_xBFSR}80A#AvXRY0qkcd?k~h?24||F~n7e z1~2d5!1xSU!BP@gWE5CIAKFTZWg%8j4W?!_NWPMxAv56(_sY%QXDiy2!HO@8&m=2@ z6(bTM<0A^JD3^MKMr42$)ST&(9Sf{vHp8F{+$$`)jEyJUD+E#+3xxtJ7;5JfI}@yA zx}Z&r;a%|%_h-6cs1G4u`FC0dte}Fo!n6!n$?!q>s)Ci}`UwiGpga9TPiKG?)Z|ui zLda&@WR8PIn3_ybwl?h)` zY6e)z2tp;Pf|Zad`U+bWuwo#N&j>+-6+>}qMi&aKs0avCjVr*)|BcTiPMv=qRf4|_ z`AXf9FXyG1h?T=_N4m~~6y6IVrQ^& z5qA@+CSb+6JXlnq2v!u0U0Sg4uGH=QtQhMIR-9W*M=Lnvx?tsAcz?VLSh;}jd5h8o zD;H)1=SAni%K6vMNv80w7>(C$aRw{S-;Ku`Az!)9=~bjTgB534WAl2*S1LJ=6sw%U z3e-}f-~$$42h*?;u%hU@rUUX7L*YfSGgvuq-`@oJ3e-(poxzG#^^st>SNdzu6D}fF zQWE>28Im9Ed}wP*?D7XJx(%l0HAuelzXFQ+P0lmLwi3jO=(}Qa31Wpz$o;4bR!T}e zpe+JcbmvT=e*#v{H@nz)!@bgv6i?t@A;hKiqjbT_1!^aCB~DoSvhJQ&`_R4=xSX zV4l?yzhJckxqJ3r!M?ad?80heKM^ENSnXis6BZwQZJt~$rRA6(*U={?kQ>gq>w|AV z^9dEXxC*qsukZ#I#P|wJpe~UWA!>nwMAr7&k-(PQ4uja1TJ|MTJ+`I6p$rRXYDc=! zTD7Dk45;B2`QY2~W^xbt;Cu2^y(JE056mutM1)OR7gQ&>llQs~9+`S#BepURhKsP@ z?+|;i-&I%Enl+(mwPsKXVUJ*C>mYBbL@!rR(zhZN6z6kN&K)KD_qznp{$QT-{61OL zWjoiPv0OvhwRpEuaz3w8IWNyr?uf;)uQe81FZ}24Q?}Z}>B8~l+KJMC4b|LPy=eF) z4~%S0=rL3uM~rs#HoKSUK}Snh?9=3JR5$el9ejIg-`&nk z45rhYH7L4(4!$$hiVjvi>D_cM(+AUOFj+hG1wC04+Un;ee2IQ@hjhg+M3{}<>AQ55 z(!*B3{%m3&B*tfL@L3{fUCVJZ{8`T)HvH)TD>ej7T99z3=oPxBPW(4IZLO`Z*b8M# zXxjJFlhufcxibY@HmG!OaU==~a_C>RQ(w_ZIqBG?w#yMV-s^v)A!N6)PV>$9Zfmt%ngPjF+;jkJ&+1g83fSX>=qO z)DV`=u%$6JTW*u?v+0MZwUKa_5~XEmR#> zJu4d1T?b$qNi=pL8b%rof7%J@=$*Mw(6*jSciC>7TWDLar3Y+N4inwuxzx%&$!SCP zcrA5hZs5Sbh@Rr+OkLhNL&2tB9?+uvM|oDcLwVM> z$%z$wQ?F@2{qLHIkfX-rGXpDqUicn%(!6o6VSwfX!>!0f%_;2Xd?~B~A?55qwrKwa zry9KfJ9ClW1%3@Kn7c-_7gOk6enWI*)Q@X`y?px^_K?r0EB83|a=r@IzZMs?&ybv* zx80}wrYHtBYhGZ7e8^V^)Mx5C?cQZ!O%+w^Vyfm9FEH~4y@MO>-5E#{p)Y*#Npc_e z2&}nR?0kFH=;7_z+IwXKb$gRBgt~UeeF{kb!yDa1R$;pH?DlQdtIhG|U=`2RmO?q@T)qSM z28(fg#KE+=1c(DH+5wqjsZHg~99(ytGDE2Brl9{KkmE`3S zd^)Q|i7@GP0qa{s@#?%Oip6}r?PvVrc3|Cn(1p*OQx`P6Pk6(wbjUkttgZKh12Mio z&J8ep(sXQmeq0(D_DS2+?fv7zK%-CE=5C)KR|dZN{M?x4Eh)~s<`c87=k}5h*B1k0 zjXpmsL~a=P&38V;j@=M$k<-uT zsUI30*}KP!T$>%Ce&{Lu12;0_+4oW1KX<#uzq2*@@fIyFmS<~T;6kQR>ZZ>q=i2?k zH42%>I=us{C2@tZJMJqHj>RV*};`<**5BbMZtD?wA%Hs>)G&&{@I(d%)p=2x`VC_g z8qyQla^eLgpBJ!WQt!kVh0fcxV^VkSh2JPHxJt*n@G>ve;?uvQIFlU;F>*yx1e(6y zG4YO7>JH`}dv0uc@kfq#-xj$`*@4RHdeK{VcypEIymq)gH4Az?Lb}Ggq#l|3_YRxu z68l1hHBsevBw>Z{uaWwO=S!u$2gIv9{tlZ8a<|`g$$>(*kD>QS`%Nalm1v=w-9;-c3Ts*tDCP5{Z*MU)R(AT(OiM^ zobj+`yhtg)`Oc87i^j;AI1i`ITKXSxhb>-^^NRGWFXwqBT0A9RE_z>?Rdmt%(AcPR zdBwD=R^&%qKzUVFE7B*q)y(^0Z90-tF$8}U}Gxpl+n=$^p3X=|GL?z5v-wmFTx}uQ?v(jTfGKLfqPbk6w z&5ecdD{!$K;0efTFu z$y(_jmb{inYC^1ky|}yQXVr9w);}MSks86Ba7Izrse_@k9pAUN_ptm$ztN=C>ydSt zN1;J9X$_c~j}n{F{v-QBRZdp>gm^Vgj(bjuv5nHgg5z0=v7OYJp~T6RQeThj@9nR= zo8cvEVLswr9(DZ^?A2JP8q2m=kRE3Eiey(ts^bQG$3>be@5q%$PvL1hhIUdiSTb_3 zy=f`nP3cORVwmeYCqe^gUpN@4R~oaSO;zoGhU(MI1cwvM1Kk-C!<0tJ69;>sHS#2v_vg7v>@ZI6hHT~ z++?WOkp{P}+$2%#OKZ!0B{vx+K155-?U3`8E!uf|_4DfyVr|+(Gug{h#Zi>6B#oF~ba$9tEnDyI5Y*fw4=$CR{>0{3A^U=8>_j9~Ob(0E zXbNi7k*PhIQjr=|!IY*Xhvg(^69IprM~cZbH#N9&)u?|?c}c0}-IB8jC!J$)rJ{_? z#q}98vPPIeS^UW<)JoI5cT&D;l>Xj95oTqhI^{|4XOY}WL%I5G;%drZ?<=WE^Ctg# z^Y#~09Z)*?kJL`JiDTm3l5UoQY}zqWu^>jgnYKaW=U!@*csuQy1{}D@#fG#Hjh~Lx zXz?!E8;vFVpU0;3SK4X_bkfYlbs2Z0U|w2dsjv5%6pUX<{n{pO%g_>SSox3o(r_>Q z#B=Y7m8r6`JEB&U)!)6=|KEcz6s1e?BTz0$ZY^!@Q{Zf78z3yY7^Q2;@ z(wMD97AYnDRR^^<7wt$X&z(M4zVVR7=p)Vny+bO62g_lTCAV7St@OmTJfiZ!v9QXU zxla$aC`Ojzo^lTB4XMnO9*Y$PD$QJ~CwKSDxCS4a*CF5Sx70H=PpN5t{PvyRtIIL^ z4QAt$N2H6jxEJ(xT{_!8RGHu3px1G!(>Y-$xfKRxzO5a*+&;9|O{r_|Nu|@S-*$|; zecSnO?}QVowA?#!mEHMo?}T$qDcrSSht;9Llk-x=J1=)0n24EIDQ>$Ae;1vYmoHAf z-03nAdyg2`d0^jdy&c7irO&d${sk0ud$c!3XI)M1 z0X2djXo`GKV<CNOgx!Od{ zn-|A(@x_cXD*G+-=jKKp#77cq{31jtN~Wb052KxT z*&G+WV3o#F%5zS%9{fF2>UW%YA$KP?KyT7gQk46gSlHjH_h!d~L-ji!u--Zr!>aVb zw9xrI#bum`;tiL63f}6Ayo#?~`Z@7dRpe3}a_Q&Gw>w1v#cwYCoa88ooRn-YEsb+j zB>A}^2V-_f%9dVpHt7AbfKk1j5@v4C!P&L{lrD{N()D0e4@xM4aAM}PUzeaN> zVIQwol%WzGP`_FJp5K^=r2x8<$o+fXz{;hN%@J23%EY^C6%Fjr29BNR7__4al^#I z1>@jQi9^0>ZocvTyOdn63-_psX`A2$aThGEd{^1!H@mf=MbNKfl%x`5l+-GCBJ!#1 z$f`3A46mTKzhEb-obvymdn({VctvAYT4hhx?@*^2jh?%H24xyWrm5leQ5M^@r8@t~7VPW^O{ðy=(TIZ6WiN zD=V{jxcoTzV*4A>N|g;QOMhJFj;n0Ak8{E=Z`T|yH++8>nk7mqGx;c$Yb{G9+$_=Y z{9(VqBjmeT>Ms8_OI*0yWQGGMEOtKeK^8_O_@4M4h0Cr}32qiYr1Y_!W$KLJZ|8gB z`&r3G^(xUu4XsNJ+(cp*S59T8Kh>=h+9mlL{3o_p2K;;^;;Y!MyXOD%v1mr6XKjf8 z&jHbxO3%7C{y&FAiz@Jkv;R^*dsyXuPKeRZsIa;$SeI7$)g-=^1#=QvJw`uc!{|tv zMCDgAjCI~?dd1e3NxgHq*D`80GCW$pT=*S zsta%vymgl9d7EQ3k4_)c;OB(LYDO+l%Jg%TVlD5boPSchFTCuQAcojngL@vaz3kR_ zjETP>=J-NF*&WV$%i)%ki%*)lR+a&;@MG^&EThU#POs2Vec8T0Fs7=o&?P1iYtl@9 zP) zJfYO~YI%%iPBHn_ZGqa!J?m=ap73^CCWHyhV*M=VUeoU%k^ z#d+nrH!A-8lgAMz=(6J2i8oNN-dDE6xo6JLunj|+0Z^alXyS<|Dcsd&Su`m3H3x~^w#{;n!$ElO= zWh_IKmzQVGKVfnWEkjy=Tz)df)vzSI3_QAUP;&7}TWoiDe5+r0_?^P!*m|YUu`x|| zM|r8D(;6EDOCvlzn4od8w7Dd}N(|ndlYNA2LA}=5BK5Tz_w3j86FCvY&Ykr*CD($oj%CMiO zaCH7IynQ*cJyWcC)ku98o*Sk_hl~YAXz)+M;z!wGF7#WwVswW{lp~?52i9%9XyW@~OF#ust*Tx#`-6ucR}58S>J(EO|Ok^Ao95ZH{!d@0`3- z?(&Gt=g{)@N!GuEH0|T8e+OyWU$eC2k&a0FD|;@BBAw^1bxeMad|jq5<700e>8nkP zmbH8k{c7pdNU|DEmb;gJu1^rA<4G^pjF6A_z?IK7CHP^yG!G@hJWqmopYXJNgmtOz zT(bBKO(w1V=IwOwAt%5S!b zr$`^RrVa@`?>-hT$cK=YD*fh2S$#~+ZlhrGQf=RuW;c`sU~C6Vhm1wD`-@nM4jGGd zv{vWm&?|mlLus@^`AMa(yOw%cba~6%C-OFN;=44+SmbTu#SdtZvB-DFiCbxqvB-DF zi@W+DW06mfTWaoujD_A~J@H8l8B3q7=C6;WGQphKl%Bq8V$(13Qe)pgn(#A3nsZ6* zd|~h{y{3HizaI$R?1PLYe`V0KBo8Oex0Ma~v>;6#`9yMQ{wb4B`BEnG$>g*7kg=#b zJ;1%>LdK%Xe}JES2N?^qF!1Wz`6Hu(cLpY9?J?LxiQ|vo9XBbpY61gfLG8U5)WO7|TWGp6gB=W6%$XHCUCcF;!v!5K| z$qbnwH4`YOblDrWa!@6oM#4+{c$CQ9cXmcQK=e<`2+?g;bdU~Q== zu6FRa$*uhLCST*tpK>8%$(Ihqe-Lf5S8Wc$&k~C$9B!3~cDy;^alE8GUqA4%s!SmM zqsTxNG8U7oW$!?)O;~qzV~}szzW;g5)aY4OiO4CgJXZm?X)$d`1ve$0iY( zFA1V>I?lxI`xFcitGr@4*FYTz@m3Yk$75C|ou_z?&)|?eW#zODI zKU;7Qu@IBm>6`Hu%1)pG79|4Z*w+dg9WoZ!GU?qDIm+EbP8CXHvQssM9;yVehuiX=R zCl*O``mWs*x^v&%C1#T}9dCup^i=!@wm9@u6MmvYA(m0(G*+drNBOo}N_&s_X!u+6 z@1;43Z`I|1Tq$F z8?m^84jBubfv3(Qzv4k3##85nkg?Dk@zgIu$XMuK@zk#p$XNK-@V{os`Ab~bD``9R z^We`pi2@`26#Q&r>hfMma!xvad9MUA7DmVJIZ69i-J01dUYgDKMGqIvscAcyBYCz? z`p50h)1ez6V>zw)zUnFiG8RVPX-#2O1@2G2hfUFUasloIpKQZeOv%KJMsB{Q{{w&6 z;(JQou%1m>p4b1!)V=sK-T!d|56Ss7XNHi&9L5NvwaT%a&xb5Bg-T_prdG+>FbPpa zQBmaBVag?kZH_r~RJm$4hfY^*bk&rr?)UoLkH>xg2R&+iKJWMQ`+a?1C+(NhE>3{4 zOq|F2eShg$J-h5(8^3~!tmby!5k^WD>sc#!G>>!zVmVCwD*>G`2*#pU@iXCL{Uv(8 z#|b?hN~%iOk9Tuoc&7q1{{D}53tY3QzZak3Tlp_t2p^Z7YB!(!hWP8`H~z~P@mr?e zF0QE+J1LLIJSoq2e7b0jnOT&qH9Sc})SsmB9iI<^u@q}lX5MkZSc)SlGatBMEXA8B zGw-=zEXCAfK3+vwnKHw5QO{9$k~`0Rb28y)D&;W)3s*!>;Rkv9-vFQ2zAdd`!u|Tbf#Zvope322p_TQE22{4w4#gjekWw-X3Md@0r6-q=m zXE~&{b%nC4dp70TL%~p@umU?gBu`!pI<>zUT&JgfvfJd%cmA1Tg_DVh zs*{vCZmb>{%Sp-tH$@MOarq<@j1K7X0adyPhqAvDBUx ztUg(UwwuLh{O@mFF00(-=-S^a@Hy6%nSWA>C;mF+!(L!NNhhVIgRzLvWXP0WFcuMQaU6<0Tu15+ z61A6_4zAW}i|BnKYOii7dRLZwyf>6nm>$0lXU=}fsZPHoR-?`9USl_|khCpbMvSll zEk9}8Yb1hJtq`Z%BE#EF!J)WS?GL&a3nn z;-56zD&Fa|`StdQ{LKF)BC!^!kp!J1N!xp`h~(zaig>I=Y9^Iw=p=dfwu`XxKZ?yL zvRP}uSj1)!?B{F1Sj1+O*yC%!Sj2)w3l%O)+NGoy}k4x3`B4KMOC^kGtOmAIBv1EVb6s4PswMYxDlg-($Mczu$cQ#A4+4nwx z+{wQp_G_m#*qiyM(iKE08oNuUk?i_4dJ+OxiWa+bO|68$nWD*VS`#fHaQ|OziNxx* zRSjMPRF=#boRSh8=)EO!TKS>Gs<5&IKJzT{TUv0eSBF!Z{(If36|}9Hvp=Vw5dAE- zUo^L|Jv#3{m!}AcEUdNbKZ;;1`X6$zScX1lQZYBXQN6AwL6O5#^vFsurOA?l^XgrY zhBP@6VTkKev))(srRm%beLuA#ml{OJj-l(Er)M8I*Qj-LB`9_59eUz&7ZIO@vz?33 zpHN%OO4M1JIlGwMNII1#<4n-t7?4PL_WEck(ha%=V_yGNPJ&5`pcq}@jmO~EHCHi zEvw*YFgJ9zp7HK%tTRmx<4e%vxuM8J#)=ks|5V zsDE4Wd)8#OQon3WHvO31EB#An4{i7JmVBQp>zqR*4w>#j+9>TC+M(Zhw#(GnS3V-A znTpbH$f;IKzF(=d)T3zYGKTn`OJ2{&)YnDGVm2`b^cV6xoN-#5cEuECw_W5 zU(YDjuaR!|$LVk`C>A(d$VbHP7>aknt0&(GS6f;+Yn4R_uu$kSnDf$)R%^|o*v%c# zk5)gNMbJzR83JR`e=EP@JM@g`lXpk|ja<8LvqHN*gQW<@;zCj63@BDH78Lce1y}}r z(ok4mf@7%vsV@;YG2nr-;@fNieolcicy>$ns^gp*-)kt^<&fM;m0%hB)~dtYX?^&7y#BG=8Rg`|;mTO&qW=s&>0LRGUEhC3X}?FKp*+>!p}$W_ zUMbfdYvKHt$-@_Y>J zRU6zB)%dsb;7yr4OT!x7_HwKt=YrA%m3$X#Lj0Jw=#H~;E?PDuD9-jvg0UFh&QIEr z1;(O;E}pb^-b8ZC`ljUM`p&ckx=ch3Qq^`Dq!vwD_J`D+U^E#F_gB!ZN{h#`_0N=hE?NF#R-q%!&4z6@ezYk|Y zm=m$92$ zWkYu*ZeIR&Ltkp8+nAdYDVbqI4Drs#+Hr2@Jz*B+eRr$Wj5$x=(l4f@TgW@RJL{#A zF3s8lhnSs`EOx6jUvMMi`jwRy3dEIa3VFd<*_`*16RzPZF(ihTmjPaz-<9XZ?8uw& zpy+c7h-nN}rD&?4gKWfUQi?7R+$S4yM)H~rdQfkuf}@OM1}$m_44R90hq8_kPiDzM zFA3SNTW>Z~QqO&QgT05qw0}?uuT<+Uu;^b@>QRh#7nt?GQ0h^7>n=d|e^)xKxW#=H z#hFmLmltjDON(@X0mh>AJ5LbLfLe+_g(ak1fyw_0OQ4p5zKK(lccT7cFJ}KI_1NDU zU@REQ|DPh1^Vv(xccN$6p+#i?_$7M{$x7z;)>kH5PxcWvw#jh|HL$SKh%W|1wx zSgcrFOidmbOCEnuVFMA2Wsq#ls?#WleIAZjepxxe1Y==R@Cz~+FqYVsyp_N)FcwU7 z(aO#-FcwT0i=Ya|5;}G-wnrI^MH%quSS%PzY+_zJ28`uD4Lydy8jQt}<$=M|z*si2 z;xYJQFcuqD7^a5?#`34Pq!^3^v)~7vC15Ow@$0!B{ZA zi&mn?pzdbuZ_IbbjP{GQzW%Wf*ZwpCV%eUwJL8oU5DVg|Pf}6_5Q~(&m%kMd%lf`q zDcW`*mJ960wH_uwELCiJ)}RRxOJ!R?e<=cp#RO**Qh#AdlFacuO zPLAvEL;$hun0A`&lA7@H=dP|*-$1tpV&OpWgm4asB_d-~`L+)bOUdPv8S+v1!{!Rjl2moazO{Qp5X@^hGya>cn zljPc8A_c@EHMjv;1;m2ylao5+om5J9?6Z_QBd`lfA0eS)=Af8_+J z&pR&57@oP%>nS7_Whbe>vuY~OPWowVkYzHm0{XEr29L8!;nUM80UN5T+q{8T5WYTG zOCT1**^^Rn#y~87ysa4$%D4@cyN_n%Dg&`>AaCr8l>%as()HuHW~>mmWtb@A%7Iv% z`fR1tjKZtiSm&}Pj45a!7K_1U`Q0}e*VyWHjl2IfD#9|^?-V_ixl3AmzDvjV>$5B8mD0x4UD>a8Rq~dW6 z>jcCytulBGhX-O|lobWmlmW3|=TWGtt3WJ1FD>Txp@3LWk=JpXfmqx}Y=dgbfmnP; zo}(sw(xRwZiw*Sk&`1Kr(vVQfIi~`|5=f{KHkoj!D*ptfZUhxTDnKl#L)%HQR3H}Ap&g_HR3H}A=k26;Di90G48>~j&${UBMe>v0rGbKc3KfXu z(&HWPuYYA^m7N+{55#i4Fpz~(F1cP^X5)3Cz+ir^3?eF(=aTo2>_Y*upuTuvY=Kx# zQzZg%l^p#(mChhsH4w`NW<5&R3y8&wPo|m#0Fa1;Y6TN|0wlF17cyAyUUw#0!I0%TvN7!Hz1jHh2Flh|}V)5bcqJ9YiV)0%H8mSEe zV!66%H~+j0h$ZM($F={E|*GBj<&eXj|`VpDU0-&_d9Vsm$#e2BGd^Ax*fKwJ|& z2E>AOsgkYO1H^KEc1iZ2EfC9|gxV`EIW*1))S0XYVli@gA^W!t5KDZw6W2}?h{ZOl zq8<84>W;DSnm^-!SmKdqNxNCl55Usb%#8zLiAR=@_ORg9N)|?QFb;@iBFROV7TIuU zJjp93(iVtiBFQ6XlPwU-xc{&Nyc4P}-U2HEL_AZ{T)n){X+Pj0z|$bsHd- zctQ#PS|JdNP0vL>G@2AR*tCq%&45^hu@Z_25R37^E7>n`KrHdoXQ7$|h(!}W#EZ#+ z_bGFVZ79`1Ec z;psRamiRxNB|t1Te>zK`nWFH4?dM^-Lz7fB5R2ot2_TlU++L_80b-eNtm3kfKrAW9 z^Yu}*A9dSk11ZQ7AQqd!07u$55R3gF$+36>hy@d0Uu`lE#4?`dId1F-#FAn^nznO% zq0VjmyrbU)5KGDOSatk75KC$25n-yN0*J*CJw%G30kJr~nxk(3V(CCuEGAR|u^4## zMCv31u`IkR6LSP&alF6q?mQ67=EY}@t;0YpRm=CQH>UuxqWvCWih>4ECq-q zMSj?EBM=L2F|XP%1&AeuHtc8*#B!3Rojg+s#F9dbOrE(2#BzeRDS74s5X&)|Cpe6X zur_6;)J5H0-Zr;fNGyw~$@d*MWAqzpZHItZ9P6t|-flhk38}CM#4^lJpbaJiu_WVaxcSIsW(Lx4foulE zl1htDwis#00>Uqr1Y((| zY+Bsp2*lzD;S&cS7W`liQac5RWrDwrrjP=}l0qrta*;qRDU|cvWh4+w3Z@AtY_@yCM$6BJQDri~wTUmE_&mE)K*Z zGvm)@i2||6%mlKZivqF8%mlEJ z9EgPgbtZc1KrB1iouaksKr9jLCedhhAeKn>1JSqYKrG?BhKw!hKrAT1fxcVfKrHI3 z_3T=4AQti8W`bjVKr9)*)mB@9Sen?MGfqfSvxh{b{woL}z+#9~Pz40FAJSS(&P&h0P)V)3d8?AS33#B%PD zXH6&&OF+lo;U``|EM>UPxdAVli?m-xC>b=A#z83D1BQSQtgFalyJBSa*J3*sSgnK&KlB1(DOWKPA3_46H#Ty3X0*n-E7%Td{Y~tB zsxV%H44dbNz|W<*BcR+QB&{QXFpmncztM36duJprq$YH4%2eIR`H-3@;e=ZrADR|g z4pYm`#9i}UAvIxcMaxa}%T$>r*qsS(Rc7cL9N33cE5fGk3TIriS7B3Es4%yfIfa}t zgS#js^;TV^Dx5DNQM=~tLf|(OnzpNaa}!WY*vp2Io)B1{n~?tVy=Jfyg~;DXPAG>l z*UTf>v10^Am-5(~$VevrcS`Ep)7bLU$iatN1_nC=JvHV2arP`Y9)F;I0`R-Dkp4QnR z8M`uq`V;#QGl*_Pl+r1wZBj8(CFDC<^sK8{!F!ClL!G0X^+$}xP%($3mOYy*qd6If zXe+)P`%lg)0m?0$o~6?=M}tq!Huk1uWA2&_x2lE zkBkyge0BbT|63r$kIdiXITOKwvL(cC3w|r&x7q{GnO&S3#CNJNz{RgcDCEKaEi?XG z#BcKu(NO;Yw>~9lnu-SsyK`HJ_+Z|C>6_-*&~0)7L473Y{isGuTA07bwq4=?HmcQE z*?Ytr@{UXUhCmKxA15_iPNn7w&wI*$HbQzLbb9;7&}rA0pzOuG6xEsr9wqw`LtNS_ zm|W6dz%-J!3Ly(qB*BC$m%B35q<2DuqPm@ra7f28W0>%60+-2P3fZ?3gfy=!U9?SG*9#_+gV zci1d6h^Q-=Qv7FKq+xTH)(aKnWq7#hiJw&c8Z;R#>u>Gfw_ZGxsVMHEDjQ@?DD>x5 zLJ~z+yjcdjCBYbgt+*5i#Kt8^D9teA`=|dxMWbSUf>lRb#uw4yrQX*?(CMLY8iZ6CG2LxH4YSr6g)R; z2@s}6AO>>Cf6Co0l2lKv3f{_mRCkH`&CIFsovG7}Pt6?#BP|@Y@_RSCedhn+$d@0x z+3mI0VQSZ2h4N!f-5y_q(^^M<2EVb=^k{b;3H6%QilvInUc(Wfj-!;@d)VNH% zOxP}R7h9IynXp==3%pAjWaiv zj`4FxwuJn;wsg4NYyMFAiQvzwKZ8GqtrU-laW0RznEeh}sog0R?&78@=aGO7L|&Us zkPERx8s6O3*fJX~XXol8H@L6y+LgQ5C>I^MBX-s6eV-SuS#y~zJ)H*@()QO;eCyPU(m^qeZUO>+O(rA|%; zq&}XGh*dAz#kABq{ES@1dd<|)`XX!(IaK$6rK|PjIl1`V4SrP7_kG>bxIg)heQ^67 zpoDV`p)=(Fb)O7umCM`+vE7Zb&#b-S(-F=Cv0Co?$%Mkk);k9@iRa$Q!)BU-xIXXoJ8_~}NEo46~qR7jx#|ke@jX+|=0zXCf<|Y@_QYSVx zPqqaPEOKPzqW23t2VQWr<(f7&Kj+;N_zry6IyKqm-u$f1ci_0(7V4iENoGzFu{7tA z*4>0vx7C}ZCe}f%dj)Sdex9TUalddBicare_2RxIRmK|hVAo)~)Be1RPP8Y&CKFOO&B^J*2Vt0z zyFtg~Bi8Nd6VlD`A5MPC4IdLPLqkbCHOFbF$GfoLZD$e57yl(AGJ~ zEOI*ZSVYI6;HcB9kMAzm*T1{k**Oj|6YBq>A(!qIY)XA|V;OzvZ`Wg&HmAP1wX8c8 zwD_0PmnUft$y*mQo%9|92R>hE8}~@Xnf-Y8uBpD3_KTQKJL9zOy@o~3JB^#Mm9$x6 z7Hu%K@F}0JaV_>Xl(4g?P+{`1@^UOEruu41$GAzVuhO3=$P*`+TaNfXaon}o$#J|i z@sYA^v4CTA$?9YEmCg++P@8hW$?nR>jj3agW2C)uZw*XgBK>ol2A=S#yd!I!oOMdb zAs)?!QzcaI!2?doI@zWlE9+xhXlYL73m*xWmycXJoab<)=UU2@aTr;g)ZY`Dmv`}N z#*LgRK2V}5mzP~G>AhbKr_`lf7@v3AqFVbtT$nb2GGeEmr~GnSE7QR#^nLg(ZY2K_ z?aPtX%dth_O|h3~|2myg+45cxF}O}gn`x|5btG|Wr9cp}7<%b->LK-q?`N*0G>k(# zvD5FTD;47pndnOoj(nc#3j6e=D|&H@6AUX(3h!6L7Ed2H+AXFUB8!Y+Zzq+Vsh113 zODb%PyL|sltlU#YJ2_L~V?R8jpYFYK(myrhD0% z>n2p>5kZ?d`}k> zyJU~5LVL(F_Q7?B)sJa(KVk1$_W&X%Cp5ZWvp3f@^qr7<6VE%vPw$hE!|#Em{U9$U zvsi;1{1r3#V(S*M|CnpU#N=dcAkmGne%fm-Ho%k=E7sK5$0lJ$dK1KAWeZigvV{)T zeZ7nUC@$F}%z_xyY!sCBK@P<5x>j{u()~W@kzH5oAjs%bXL_xRw#N?a-eI~fB-2)I zzcaL^RPOCnIWGdal-+kPi@ZpWXRFR?OUh=gGvMDmGrg6#UiHG67qWBfqyy}&2YU%( z%d(+k4)k6&bde|?)}oBEoih&?9Fs-c(5KlCIGZ!QWYKnl9zicVcik2n7;YJCo6J$U zQ)YPEw*FpEvDyMFS;{`Pxma|;VcBMzwpYFPIGe;EJJQ#y)&9KKN$jKCs_7{kUgsIP zb@o)Bb@tlRM4mdosqfahdzv%oQ?~R0wr}Rcf-SPY_CZ0(=>lPu$xLig>vVk5!@kq9 zx{4BVx;BE=zTbtOy~buYE3y5I5{{{|M^)o~jz+*E*CWb<7aR5WU&M~PS}NaR@{K=K zHXh(WFhJN&lE_gBn1FVYk5xFI>O&o8hHbrTP+AvpqjN8fwewlVRh5bRmyo2E;brfd z!yOmsy>r>dRyJwvbjI8@V=LRVu6(Mw_eO`QepfHjwfs=y78}wX2%KmsLtq3FAe?o7 z`b``z(udc2&se;=kZ`yzGoMyhIy`Rtpz@R1Axv7+@N4fk=+D+^4Z{z-k1GG|^UXT# z-tY-mztZm)yC>%Ulg?CS4a?bEMfA5>d@fV+F9eQky8O=gjDBqPDWF&5bWXc4?yx&2o zQqsqj6QbOn&u}%fQEt63J}KBx99#q#rq_ZnAJtp-!#h}rq&#?VDvDR|5XGln&&GJG%b zPX1x$G2>f&j;|BHTjdsoIJ0{gNbRacW8WcmCa`6Bjj zqF6zEI4)@8ZSr|$r=>lj$kM)j`YFeeLnv5`z`Z5flFl$?ETi`Di6lsmSVqP1lVw!H z!V?Z%$<98{@szn`LD)OHf9h4gom$JCq)YTS{c>ur!V}C7-A*bM52r`m?wow%EFq)g zW>Rp;@@V^X$PAGmT5vC-d)G`1-?!jc#3$qXk)LAthYG&u)u?^BL#}21Wd1MO!Gza1 z|MZ5;222e)N6Y+Y0lTw>>}~6z{@oZxzrW0Yg{-=v1uib;YW5)0RHojtu-!Veu+#cY z|5_Pk1@4A|QcHXciAsLV2~w-QD=4hoPrAr_uLg^A1mz7!MlD*!DuVKoBU@l}B|O-2 z{C1<%-*quD=f#7<)e>cB*)KEH2r@OLuYuT+96w99Wwvqn*GH-_yZ5KMMDU0sU$DvY7y4J5pn>_&;+N&oh?#?=V&-4OTIPRhzuU=Mk+;c1 zoQG<^JIzF1H#QQvlG+~B`b6Zx`kO_uu6Ekb4k8cK=QQr)VzuuiK2;lh-56WnQUm~O z@7qD@C<=1jtljXc`S9{a()~y4>?hjNt|ufcTU5orPTZjNS{N;Pl)~t;pXf~Mc@)PH ztx|v0J+#2&AgU@}b&oB~+ncqtP0hH=Xj?MN?RUPUkFcKA-6=YdNI1ZCBK=jQop`%_ zDs(1+YpmTy>P(2Q|Eoye-ihjKzv=ZSVrLVbxp2$=aA(>B`oO{^`!5|YeNyke99a0< z{)^J)gKrKj&T8GXMnde_qCR3?(G~lg zml|mIX~=wR*9K*=cYm!B8{4cAAKP6dWpAv@+F@_WMw3Q5`ECk2LMQoZD}_s@@l*eBWn)Wubk&Epgf&sog1lkZzCDZk2rN zzT5D&QhwM$hCM?2#!zB+oZ-EIypAGA`-NAP@ts6rqe=9=l^pVug-Cn#SCq)^*p!k- zOnc}ien}vv+<5ezDPh0m_Dabks3Nwncu6?49K^p+q-r1f3aW@>xL(@PFQND~HnzOz znSGDy+n0hjECu`Ld3WrezYIUj_28El)e5600f`-uI8m#5m=}HU(4DSF3rG1?k96#h zy@GDym}S4J(=S&eKXr7yUeK*N!NgbnJoqbiIUSp@eD=}lgTLa#M34FhFyXB!fE7pV zt8F0?04wr0H({RxtUT!>$?tarSUKiDxNVmNz=}@w9D9!hz>2)@k?^-dSjm+`*31H| zOwLBgPaFkUIXcaV3Caan(WzPFQE~xRWO47Y0Ukf+083m8m8Vup%rp zDbEC0k$8cnKNrG^gabB#3$QZP=OfPsSQ+1TLP7@#u!1~#bebuIm4Qf|Hw!{o8Q7%r zW*%VWb6f7f108@Br|uV!?EqN8QYL3z{-jRyjtcTJ0aheV>U>QBSV^*;>{~AZup;52 zBbx%SlEkabERhFTK^7jhW&x~>L&8KEDTEcI?@>Y40Km$hS`i_v4D6F{*aWb$$VVCCavoRW0N@&GHDz1!XatT^TAY>*Eu zaU?(5_8DMBA_X~i#Cif?C07R7^H~Th1NV^VRDhMiS%cgH9e|Y-I%7647hnZ>>K)H8 z7hna6PNB~TVMW3m*_;Zn(%<*L#E5{^XC~341F!OsQ@bjv$gWSlL1!dlkN>{mIqjo|GsH;5MX6;RzLT* z)9S>Hwceq@`o-&uRXj>ON<)U>OzCh+vxsncx}n(cKnct#dvrR`@bs|+myf%Eb9-v& zXY@fCw&eVE3{A#`I(?EF$$o14M&pZ$KKXEzFwz(+uC^o*l0Hd^)K1T$m(dI9)y?#b zMBlsXBdO-Yp>rW~&+6>b{Q_Z49+ zX53;NWB{OWd^za4s&~F$*MBztth71*ME=h_xMdib3>R0csVZ_MIg1kfRsZL?|Au}M z*EpU-HX2>vLzv1j9t_wg&UqRSP`Mk^8Hg4b(Q)2)AY^U2i{i;Wg z0R}V@cHK-zDdUyIL5(QT?Li zm3NH+_X)>^W5|i+OSKn<`%=vChGA#ThhdwUtC$O$gS|Idwd>+)D8=KH->&%R)eM!}@B?ATyqIrAKH2Rd`QN>O9u*5KG8PSZlMD_NnX_rr`KlA=$ zY+;Cy&SG6xy;eO|znn&!-=yRE=V2Fo3zmFT=YgqyR+USq; z=8sD-m5!DYYCbK&R62hH{ow)Dr}}f8g z>R>7rhJ=#gpPj+HhDN85x<;wy1p8wz`}e|BIvR>@g>IOHf0nc=Ufl2YC45<0S8U2hm8ntEM9dbe(A^@g{ib)91S@lv;Sp(2s8gT`$I@o{(INgB+|VCm`c^Z-=|M7 zVJcNuxKAE(gQ=7`fYv3gG`{0@w2q!a|11zeOWa4!CAVpCk{d^3(76RlXuTB2$MZ0i zj!vNm=}2={7xGJT#%O+S%(;SLbWIY*?N8Rn?*aT0j$j@!Q`KQACE&hr#MNOcC3Jrx z<`kUsCp4nt?oVcY?oVi0hikko}R&ix?d-n&MqZjDjZVO9x z6Z;4TERy*53&2ItZ<5<5ncyPo1q6$u{1);A6QS<4tJ?~n#J^Hd zOrTicN$q3IWOZ;6LQ7KD<9TqAis<{RBlGb4sAwXz99_937@z-+KAqI%-h8BMeExU^ zxCr6br{v60aFNkR=zDa5+v*2mQ^7&>-GaA-Pf6`hnBXGnr|%0!ncyM?f1|(Ar_H~0 zO@HC&se_BC=Mq*wa<<$#i3IkntOB@*?)XyD-CS^yTw`76sPU-&IEzzQ zJdt3G<|SRpEx-{h(EOwuxuZD3=XQ>C@uSUmdDDS}_w7(Mjf=9}d7_)ey`$Tz{NhA+ zU#xa9xQOoICCOrN5#84(KC#>D-_2d=%thm7i?8O6iL$)f|#3b5_sESB1)2VmP(@WNUA z%Y$uqAXgW^#Di^jz(!$oGn^#L^nR{qF>JfIQ}6la#jx$-T9)XqxUlUOzdfs0)4^Dom=OM=cpIb8>;kFLXs8O|(hyScY?@20_#vfMT>;Hv=JE^hTbw9vq|3w<@y z+?dws#h8b=r*TL{iOt9*UTf~}ZqHu4`Aq??e-X}=T92R2AC}=B@E--uAD7`=nJA^yd|HNcW$_07!%yM4Qh%n{mICL>59}y3&cL~XzbHufz=d#2}n2!uZaR2b?Q|19)A&@LcI^oj8Sua|JK)138xh=SoWBL<}Cz6};ZEFO%gc!-Esl8Y?8jmH;K)tUaoql zd)XuXf!K`U@pW{EUZ{vs|6hDZ1U@+@qASna-s|C3n|@O4o6K~ytR&W&eK|doc{bfe zOkM^{D3lb0_m+t0sn?6rk{NpoH~1Djn!ImK(`8eevPeYSr%feoV9Jb@z*u(@b};oBYs=VNhH2tq~e9#J8`&# zL^9K3dbf9LS<6?h^>$O^-APaDbuJSdm%=)*yVs~Li&;qWj;F&VB=SbO&ARtB=hB)2 zQ44uq3)!mo+!`-2xP&BmG3mvka0yAZS@bTgflElVMUrmAhDXHd(d$;t*e^I$=}n?7 z@`76eH2b^AX-T?!vm71G{*exskmxUDl1pz9F{t;E$i20232E&2(70-nYcC^uI;Et%XZSd|HoPyGDZ1wn^owO8%=a85QHGrKKjy$Cq_K4MtVu3hLNaOl^M2^VC6o)XqC!Qu zg!IRyNk{YC>qvRV`cP8iQLWACR$R=5OGuIwlULvjmyi@`U*4!QLAQfLJ^RSzuH?%= zf@Vj^P?IxULb=_>b9eON63Xq?-&1uKE+PHH()nlM64HO2`$-YzRh7DwJhVPsLe4W< zoV$u}2|3Sba#|GscKxon*?C5XqdR5G_)Gt@4C&$j;ycpt$vJs`5~O$F63Y6d-4P9g z>(-$&`V!f231uO*lEa5)iLOI~&NSH-x`t|U>JUbMkP0<5xz&ova0#hEH;suiTtYcy zGsY8rxP)B1WiXo=olqh(kxSNx;q{uLRc@tH-3DPb&ISFAS8}bL$EEj4n_ty=Eaez! zeO3Fh%nFs~vQB^CYzf&8!begUs1}d~T0Wa?np>@$d}t_5-)tc_T#@syBH+ABw<<9v z4{%;zEjt{^*~O2_GtxhwLyrCv3W7jTI0ZqB$eY*4tJY={l#=%j!8oh`PdR*f92lo#jM#aqAqKii{OzP* zM!jKZ5xKBYt^GJ7*6_KaouRIJdnn@-N04{K;CT@lUZmvgQ1YXZsg3IVr;`5~RvY^k zPp!{6v}x~ z5>iB}|4aTT9a8^a@<-`@{%57*uJld)o9p7UP8iOp&z|CM&5|(0sa%23;_R$su`8@N z`O^->hk5@o)APh28HEj(&Bt!!wB^AYD0#(hNKHNoW5#)x7i&OMh6vHBdw0F=|r zN&2}~8vIH6G#KIxue(>O#T?IqHAktYnCF}YIIk2%vo>_zVXE|0($+A+8JcmbRi{)! zT{qP!lkVBP4IoSeUBB#bKttC~f(3jQLFl&S0U0 z(^%;-*sKNPie=otm4hr8S6Dc7+^tF&S1=zL&Dt=o5XVZ1HAom&tQKxyJ`BRRqFcYM z(3S|}3JYrvg)=a&U@kHev|wC0RJcs6)`M|nJu4f7roySA>zE7c3Z8^p7(6ZJY>#%Ogxh2xm}3>a7PNe2pF z6JcD*BgGd!Ai}s3`!$brtnh^EgvNIk-HP=e=ZuB|i)`u+(IVCWMmHbE73I#uW2Z1M zu3#isNEH}Y5Ge^`F&G$EFnR@$G2#+~$EGlY3>a6KNWB#2F&I|@3x|bKA2*yP_l5?< zeB8Z@FqW*Z8ZfR9aeCZr4H#F5-N;z?!gKz_Mob)QGD~+L&(Zuks<{$kM+=F4m502o z7R>vLH5sz8?wn_pq_GWgW*wES%&}OtI%JCPSQloI*;uI;yRGm=Y`6ZKLVjprHs+1m z7e=^BEU6H19&?@%uEjmc-&Uv)OHtv*@^=*O#hf1$1XI*f+{Vf=ThyZ&t2PT^n1^a_ z83fgoBV&N`n1{tHE@O7YgJVZ9F9ufwR#lc4E47J-RH}%FXkA7uE6xjz$k@b!_Fb!L zu^xq&FsIZ{GX%D*Xv{CQ2gFabb`w@qyA|t~#%~68y*vx6!hNgpn^~T&92&fX?Ai}F zFIDUtZa0gm+u8p?s@Nyo<*7HB(7#jaKmk|zITY2*Lhd8W#0cj&i>iz24^n>Wmvj-k zd0Xb%_zl>gjowM28J)@-5k<-ycT8^VbL7mdT{OaO*=9@HmLa1|C}R^zJ2JeK3Fp~M zS*l?h5^LJr`pzoM!E$T7Afu==fPVsn;74%@28#gPb>XN8I+B#>eDl z>9{qly`-GFV=`ps9Q*m&dq(tKGbQZtwa1L!8Q(W~SHk|h7I0qbvrkfV2C#{gt~bv= z0|Ee$-NBvX1x%U5yv_O#0bg-O$a`Z=XLn;Z%uZ$vAmr5z5%}{-d-?~Njal`|v>n*c zLN6?SHY-b6Vc_c8Qe|ApZYtTPFG#A!msePMaraJWqp4AyFe2moWTc{0tc=KxeT8d{ zu59pE9`n8|wfRg*QmAuKxYR#czWpyVAb1qjAC^^Xyer5>tw`@A)4qpNzr4&u8k!r=E#;K~&IjsV>exRL>w2f`S&(j3M^58~C3@MN zz~^d%4b7z;heiPBQGT9j$s<9of0Z>{Z>;5PBps)&Lyd1IT~DawG^@lnBy7-XNH|xw zj|!rO8uucdqQ-GDRMfB2O_@w#IOsavf;o>eZ`uA5x3{1Sa2^$Tt=nw=S>2J51A$Rh zOHQPdBiey?cf1URzLo1`+tWG|DmqS%$fGtOQ_GD zQ$DGFQ-;jHQ2=Z~GgX`(6~K96a_5Q)yr!$7&_LcWUti}p(u4YW?UU8ItH{b^|B*i8 z_K_>7+-pd*`&Ej?{7@O-Jj$a1S(@CeQISj;kwOWP4Q1RwHejmo_fQ)GahEun7(3=| zl_xWVtzDXKUs$`5uRTV^Cm?KufYyfnjA9n3_Ua?-Jt9DnNRpw)^{(Q(*;xJsI^1MS5*#Xc*99Z0h&9%mm5 zI}n$od0;Q>Koj96v40oB4rE(X#-A&M9Z1vfk^hzf*nwo_Y^qCUVFwzB&>SC!9cX-V zgUb_H*nw=TD`r<^VFwbXg93732hv18N{WCl`h*fn+S6ESny>@e-W?%_uwVx&e6gq7 zo(nrrp~IfU(stN^xIUUCeAt0ve-`T4zz$@SIzDOS0y~f}^mBd|b|7Ksr~FFQyP7K= zIS*`M2a2OR3_v}My22i^^?-{e>_D2j!dkg7S7l{vx@%_PJ=vUlHWp*NyJVArI@to-Zu2*zZ5sw2 z%L2~ZP%3BZWdY}HEJoXK0}Y(cLUYIw3A+CO;+dO;O}3}ZdqxE(S$Z}gwYDvzbUe41 z(@?l2e&#Hxe!7g5!m`vnrSvdv`eICE&Oe-cvYTu^jqx4`3)xYuE_q56IeB^t?sPz95+nXMbSpmQpayP7ArkV4xLC#9M`Ig8|NlJ zABl)W)^_@i-*<#C*#Zu=IPB=B^lx&_#rjh;cd7alWI;Ts2w4c)qe4QaM;sUDs;g3>Xw(%)wK?+UL2b?>V_ zSuN)tZZ{wNj69JHxS#xKg6~HY|7sZ45~JVwB5z_?7oz7L`E3-AP4r9%G4NqP{rNZ`kyE~&ym?^n&f`Q)@?vE zUKRUbtKk%VYpJM@Ots;-Wm43cWtol2>MHcMUN>>HT7?SzcJE&3m=7np^(}};swARX zEZ{5gA>yrSw`8J%cn}4GY!7^t-ly*_hn3-=>e^T3_8{XQnWLuD0jXw_U0W+e;qi|6 zATp7EAF92ArzWn@3Sj2;2r!7u8glKmT&ECX5H1REPg%DLQzJH2C~7bj&j#N$;rW`|6o-$6G) zIAo4g0anB_Gh+I-cki@DSE_ish4SuXr1d$M$&910_t@Q{s>?DKro7{409M4`n6ukc?c58~N<~D&*!{i2AItJp8$UEGL9Js!qm9ulxPxr8%1G{-U@Lp8Ls&2O z!e~315a|BI22Pgqq$thBiYae`KP!7LA(#aNiw)Y6+hK#Y>ij=^oryn`i`)NC3$hJj z#x_Wul(D6vAyiZ4z;L_MH9LxM3)td)=H$#{Z+K@m zSW4VEb`wSx;*6blX%GBjc8=wamLP9J%7xu?Cl14dNcl+)#Z?&uOBbynt)&a*+uXsj zw3cqbi!N_s=e!ONUYnI=as&>_FtzMd*XP~X2|t>y&#petcf!k%H%d3m;TAlLc2aXp zs(Zk(+{(;0dDH_V?oR7mllwhDm|MK=0*4uRbFNJ5^j#*qv70nD$xu^zxHso?un+ z%)T?jlM;zZC1|NCk4TX%j29Az2|J#~8Il7;csM2Z_VtGKaYxa%H6AH>r`sDP?z`-; z>Z$f{PH506*fQgcukBbt>&sU2-$=Mu@gez~5bP{;PLO1lN+`MrFDBS9UP|N=XwNz# z4&LiPqjBg5=+^_d0iDPvYpwFoM09m^8|Ju3kjNR41@Y%`cjD^emJc}{ARRby0Ow4& zPPp#kkn{$fgLa07h-f0`w#2YA?Ra%9zf59Z4N{Agpo&9^MVN}{7^GMvNWtBOR@u>u z4w7@#8F%LFLAL&@CcjBEx$xtK^>M{GVKXvCqgS2V7tpJk->WBUCBD~qP=)1jS9FB6 zad4MGq(x`sWc$Q1yEcjQpUXq8BtRNT;=GQT1LpwOBjJLAdMV$5lGi~;tC!T(;7@QI zB{H2s>6G({32g0ZgAA9*xH!mz{CeDZV1K)|#Bp@FHr}mWS)xzDSKF^*GC(AqVIl#Y zBQ6KzDMfKhXwscKm45dj_aTWU-K}FWB@7%RD?2dUoe8-*&n!9FTwH=4y6HsZW50)W z=^afHwQ|F_XU8=an&gLd0&45xJ5(9z5~euX(e~cbtT%8NT?a!hSkzEX1kp_i6XS{U}0dLQNYzLH1Jm4E8Km=N$ zwUVy~KboK6j)x)1#0Md_I;JJw%b^o=6mSv*5u@EAPvaucN*zuf+QiB>OvYN5*z!qp zj?M;!YZ4QRH=yC9bQ8J?X*dx{AU>>fA`*nLLI(~~HBux02_18n?}gORpo1KQw~*h9 z#B(QMc1Jq*d_ulLkTdO20D)%5*&_jKFWSO0?H5HL8;f4OJLxL|gEV^e{$zy6m85;> z)q9h{B5=e*;yZy~3_q4!s5bk$eWdF3`szol7~HZ8QUma)2^vP!e?>vZ$h0O-22HIr zBTdWUKBa6SsNY-k%VsD2hRY;G6?Fa0jm+Lwfp=sc<(@@LI~A02ohY|cUJ(jPxVDs> zltf&g^Wc$!veNpLFv4!R*-E@SQv`QA#Rqr0isryPggZ~FMoL8<1Ze7?jB4)EaXO=2 zb7J&_w@x+^1{tdes9IE^JnZ^(wsGh;32=eG)){ULiaEI^@`t0MPL$8$BaWvZeNdTR(O zy4X$@HH+edF5dgH?H@!R`_l!9cR0aT)5(=*U5ZYv+eIs|6`|L=a=*5BIw4JV3nA@<;c2Flp)(VY<`HaVKFXS-egt5&4>z_c$8oKD5+iD?(yo|IZb(23?#)0lcB_wl~s-;~JMfA@^QctjB7 zr$EPu=65GSZpw24TSt@;R1`ZaKbE2gc{rqsZ&-vAGETt9CRa5F&HoMO} zs84`E!?a56K;%8zF)y3+hr8h`!uf!*`iHFZjK?aqufEzHAA?v()9}avo;1Wl)QraO za(o7l()Cn=-VB@?zh9D?IA%1i`P6^zKIgdfJEoh;{7^_{;-y0}(#z-ODnSW9FbP8FNH5H zB2UqER7B4!ZHC&C`<}E!IWp9is2o18w1wrbRPOmKQJh?wcof=3@;2Z1&%5vWG*Ov+ zKk<}`?|FD0`4^N^T3I4-D(-KDw^?xEN9Fx%VGb*Se1pYQc|SxusQcz;pkpB^g@sl@ zr$0zkxh?8OZ&8u*^sj?H5f4f0_^S3-{Gy)~C(;Y4N35zf2d}3PRCXdElIkI%f3-h6 ziYh4Stj8))2%xg>rLdj_mtIu%_^sOVU=JeQ-GB8M@37!8%T8s_`SHz7{^R!rs?wl$ zc1PEGwj{#o7nPWo{Dwq$4ynYv;LMr*r)2UCD}r13O!uCW@$s=1?COY%$HG(eO?F63)vsd zj{9DwJnNGWM?xW#M;W4~#|J47@OJDErhNlY$Y*11rRMv|@7KP;K_a`^^mfuxctI_1 zt(x7TT@d~tRq!?!8tD&3CHL7}Xp+tS@QgHzHmy}~`4CXqklhE1bA^H7O>d&oY0k43 z{%&oL+zag>UrqPUJv%U7QC60bA3m;Y`hj+c{m|4jbu2u5kkXQ|_n?c~yYTVKvKE>R z`J7&61kqV6$a;>B%>aRQmZPjuzvP}axb0HTZO(*1A zGxl3o@Gi@JNV@Q=>52Md*7vszJ0`s`wJ9VNY9z^As2OZ_{#i9*ZMtu0C0XbMog>8` zKW-r_hywgg;`+{*8(R3=86gj%<6=m?8N11T7<`F~b5Xw2b_J z8%Ekh&wus}xe^aEi|Bb(vw=yNS)kTMos7&Z?R05%PncO=PCANa8pF)u%O}ROm0@Pt z6B!pPFAXz`@jzRfwkXUj%J_F}%Azo{DEs+M2CRdbMN~xSQW^kqU7uGEEdQTdsI|Q709BJ=i~7-7RVBN3hG6GEGh-QA+7BqaiZ=< z!rSdY7SZ>+g*oj&7N`|@ZwzEHqV-IM#sgW}(W2=12eB%**Su?M5tZ^(sgtVS?IFp& zdcJKJkVSgEsO7qa)ED~$HW>n0M0c)hjH_${$Omj9(; z#2Cnu^(`OBvg6d)RIC<|CBNEhQ8o+6VyhmsxFHM3qLsNbsu5N%&9m6I&HBy)Svuuy z3;ei1mSj<_)Wm9ms2N8pLdqqS$Hi%#ISSLuGM4l zkYU@@{;YW0)<&5##)6dOb}e`_(n`Fg6hywJbnZWyk)(R9|71q;{K&s2cWT{Fs?@rF zD=&zgn0Jnx(>ba&o7Ai|d#eCa9(Ouj>{@-0o7{`7;ONdC!;#PTDy7D@9BOQjE+kD$++oI)TlLr{zHt&?nKVo|qnnrpK z29?TI+2v?eY3-BMxJ~j3eb@>85uKT~F#%BiS=lMAwQtu7Ea^_=YCYfC>=+#wn$_8> z^(5(`){|SSEC%da$-mDZ3F5*r=Hz1A)c`K6Rwci;T@B*O2+EV6+pY$Zp*Di?Tlfl@Txp{B??%y>pS<`dD3!7btL_)oQn+kaJyIQDbv zF_{fScWkg=z2vsj_@e~@C9w%7bqjFE8$zBY?9wgJ?t3&a&@qZpNeAbVhIHFB{kZ=xj-!)Iu=X9PorHN@mu3<)DW?e~}`~-y5+EZ}l$5UXJ8hnbh9dn8l zAP7vgVVw9+vLmB{Nb+>7uQ**%a%w>D+4*+a`y$=YDbf_8OT(B+E{G#FCyo5pBwYq( zqhuw~8}7BbZfE$X-eW#s$}p&yVIt`~rH~jb*pjN@ij)>n;+g%r@GYuNc#m1D3$M?v z<7X)iOg(ZH{F^~i z!Vg`jTh#sBRLAZ(Q>EpRZ}hdH`FSriUFAOxRQFrjM5*gFHI}K5s=)WmaEvqSPPt&u zEp5-Y9y2@qx*@8%mlrUk$R0CmXF)n-N9!sK>vh)n&phuL3xwAZs>XWx~x@CHhG`aD-(HDZj$X8lo0P5&0~XCv&#G*OU?tr{OFC1({V)nGWz z+Fv>2Y6%Q#SV>QZZ{1KeQvZ$xEk>&Q zMunsBY1>8h`Ka(6OM`2#dSz5N#)2w<^ecg@zFy@c-d<1A&5gglHCnS5ny%>os6o>e zzWNJ)KUFL_UTugHnMAi@dZJ_^BCpY(a*t50P0ZfHNA4oJmzw%)T`PWqBaS*A9`ZYW z^uRi?ay*_;Ff$o0=8MOlESSDi65pVWc`8jlW-dD}Dy0A0Bdp zuA;d28G~~V<(kxuJu}ajpt__~VVlB7x;T&`q8p>eOrISled3$?zJKfq6E6KLYA+rp zT*y1e=VM{QHK8%vAEIEw#fDQtTH;~C6?=yr|8S>;*4_RNszO3)=QxuVV&-b_DM4*~ zVR8)>CS0t=XTBU2CR~(8n2<(?377g0*5U(Sd+Z6m2B(rM=;?|JL+d%j7qDRT_onRn_A;&3<#-07aN^^;KH%!-rtPOeotxuoDS~SG zPgMx07DYGG_&lhVH;48pMmz=85=w!{00~qJjn;q2UJ+D_;&Mx58=_irWm=*ch-&F( zlPu;zwG4aj%e8+BswHBjvD=pfswHgYS+^nyR0|C|7ka%LQ7!ssD25}TT3$1EDAL_P zwYa5)8omY9GQ`9y^4vhRxY>ppj)H1=!`!X-$_-SDTTiIT7^oIV0pu>Z>C1+%Jm?-K zfocg^x!?VS1geFke{rR$8&pfTw|;I5s214zucSG-y&fRV=}%nn-0&)4ntV^v6|YZH zEJ|Izwy-%w?kT7ieKK6GcB&4&(fq;|u?>C5Q$vsc1xd5sqV&z76rW-W4UTAKg z_SFZ~LR)<~?V=B=#Vshbc@$L3n`z_R_cTx~v)oIHV{TCC8ZO96Ep`LdazT)v`rZvx zOSm96^|>3U7MgIFc|{*oi~hxy5gxhLRlbg`@-1roJ)&CJZ`Eb~1l2O)r3Tl?LAA^X zLNcu*LACrDd6Qjc4yq-}Y=}ou2i0O;G|4$A1FFRwI!IEepjy{DvPx zR11}*-uD-%mRU};jAA6HmMFh@j-`xZ6sQ)qjoF!xpjvu()9ST+P%T|vHkl89rUn9d z^$*#gTGVTozP^)q0`Y->h5C3lNL+Q(-)Nn5khsv(HvX}sf!1rKDD&hJNZh{$dg+Dg zAaQ?&24q0uLTF&%l@)A&)BjOd=Yzz3!K1^Oagewl1%{cKmq6nFHPX+^QU{4^Zt+E+ zoCy-w%wjArj8j-ogibhTjnV>!d$82Pmd_#hjvVc*l_$T$DDI%gV{B8khs(lE3PKG z%IvIqs7wO!OZ}h3zd+&+@KiF-n}ftPkC_s9WrD;t58?}Nu|eXhKmSvh%?62^nQs>K z6(sIov|ipGb&$AG!U_6C8IZWH9%9QpKS{r~70DQn7HcNzOD>Z>E0t!aTmQ&;fga4P&`%9hL6X`Y`3NL-7IzbFNn zAaOOe3C#v)LE;umM;CFIK;rhlx?BX6B9xr|M2r5fqeq}tq#kNTjOAum%-+w6Km~~p z1S6jf%pS7vCw=B^)X4qYPZJ3bfPN=w*DMacwARr8i3`mmexGle8>-&t~_JTKD5}IIu?CV5Y^HT&!OUX(GwpU;5jt=4*tKR;5oEp@oyK& z#KIC}2U4ifW}kU-8t@#lP+t?Mmcd_k$wq}QFy z0~TlAQVqUEGiN=D;W<=Hk47z4H(lzT>49}rrUUbH?$pPCxO*&`K7Q=I>_*L}r zWfQBOKTW!!WCWf=8cWeV%k}#Q*i{y1mIUw|V&_{-EY>?9|dOd~1K3@7&U5xs|lbS{5hkyL^9U4c=HB z(;#TTbLeulpnm9Q;fS?`#aFo2{TiA_idP`t%Hi2)xUY_6D8!T=^B)H zNCjTEjk6|WgM$lJd9vacYcc%c*ODOtS9wr9!WP48PA0Fh-?&XIK$d^=DvY>mi`4nh zDU#B#)>ta?V%uTp6-kF)k-B8O!Q{#sgAF52yCp}=CKJ{eNI3AgzvQl`?cOg*LDB;x zxyfS`!quELr$ylU22_tg;|Pqn>qv^ftO3=su2E({vrU@Ix~jHrLIDB<+LL-~f~7}J zjbrL;+Q!yRXg(GFjN$iP1=X@<0TpTAcIi;&)#ZQ7Nci8cLU_QzOktAqOXm8yE$Y&I z;#KxKGtIw6=M*O)2mn1LMJT@QRgk!Aie&hfR~4C8;WCGm1plQ)w8^B-8Zj~K=0s5} zCM2>=j(oPwagDVYb_+`ss~8g9CQc@|9bL0ctYGtf(E?1!Z*9tCAIJ;%h~;gW5zSNd zh`A~fx8{sA|MXQ*Eo&yE_|8`o$Vj`$1gh?!uJq5v|AnMuSJ7*3{^H-Ja(ch{WXl?9 zPtnG;HBx?(+wQ0iND7Pi>qOVkuP&r+_rUHD&r2b!X+XjT)hGwi3Q^&H=oYcN`uJz# zh=l~7aurkyRF4RC+u(EeFYcm3T-#y6g{yXJjKxMaH;Im5CiAa?YFQhj5pt;wRLhzx z62h=HP%Ue&ND42tfol0*ibhb5Ue9H}QQ^k>2FJaOSWVPMF=~Z8_qr^o7E7!;H%S&$ zi)MxdB|ORN;2*N=M7;jN%arIOdCMZD#a}6*FGMx%#iJhJ^`<^(_HT%~kJmX%zwS#7 z__a@b6hGChnV+;|S){QTcirw#1B3+5C@q%C8tJ1bN0V%<8f9}6Xl4wwW{rYNf}e6J zAqf_&WCJZ5r77t&W5xwd*-}{FJorsk&2oHGkkqK*l$~ZUrqFU?x|yi=VNAOfTCwPZm(=`0&)A#B)}J0`2H;0ZK*3G$f&qRl(Q5sNvtkOcx~0H`cl^Ch|W~k z^`;QZr<#9UUl8ASR(7Y@ln=0$v+9+oi&H95mm{BL=A44 ztOgY+qaj-*^;<^kw{8%4C)HX8D7aHc6v$yodgSv-fts!2kbu;mTGo6*McQc=%dhXF z!XYS?#V6OFQ&+84l^1KTU!jg`eivT}e(}Q+ZoFI0qnfu(P1I>kZZP_qT6g6doY&p( zcnemE17NvkDu+r&z|z5y%Sb}N!l~F`Z4F>a4M8eK04&z9QlBgbV6hI6g^CdX%jP0= zL0S@kWwUWwOx$EOHAOk@gG#3p9UxR<4J@fZ%(Q%s~V!Ka&FhmUKSx8hZTtmy<+~ChpIepNa+af3JdAMFz_LYDj-CWyA!7}b0W5M2Nl^^T z>I{d&6^9Y9c-mciEC+XXkw5|E8h}O4_#jd-!cE=aj#P{QSgzU08CwHb4v%0s`AGnl z%`qC3OAG*u+!Z8D0AP{3k}A9eIq71#A1T5J2AtPT`f+&GfC~*q-1($So9EO0*Jc1% zl)kY6EJ==gS+Rxy7IuX^@46Czg^X3_B`E<|3^P(F;b{OCB{mDMe;B}$CQmL>S^}{2 zh#J}_BVd{NVA!98fW?QN=gS7LB#+`}nhgOgUi&7xDF&zO5>jE^509X!xQ&J-XSkgufXY33CEaZueOG9}8 z7Pw*i%zOsG(i3F3kcNO|X+Ce60bogra_=cJ1h5z`WUlN_1F#sXX9$j@0a%nWnUQ8J z083hoVNn_a7TyP?eg*;--ind|1Hh7GxPx^^2?2{&>&CE<9smoHE?@&#SXgvecn|Eh z^iT}h?1D{NtOA9w&>jFw&pE?#Hh_gyfcCnW24G2xGMr%pSa@%gYMBUFc)d!&OaKd$ zh~<4z0GeuA9(;4AK6`L02VUK zFe?whvMco#8^EIEo#vkgU@;tG5LLYZEW9!$4K@N6FRQeEGJqvX;GI@W2C%S46uiRH z04!;NhONm6SZ0z>>TY-1CDBU?IIt;!MR+M-j06EEoZ>{NI9+*ES&malDNHmTh=VPO>^IumW@S4{IqKTmW{@?krz7vEZ5TIi}I0)g|i}Va0I|&Gh)U`5H~mmU|FC0 zD{c1y083oAJn059vBc$wpWX^!IX<9uU~D~rMV^%Vi#UKqzR`X_vmL;4O-+8{CNi;b z^u&Yh04zCmHXUQ~02Zy!IsCqB02c9u%?*j}0I+l%72mcEz@k-<6Y^UJfTiPXXi}-{s-#HGz62}yuI1XU3tJ7^?Tn}K0LyO;B@86+v+R~iUA};Nz)wr?77Qj-H zKVSo3krx%GUjwjs7VJ0*U=eS)7S&-{ecVB-LJPo>=V2H3SRBCe-vuMazY2BR0W9m? zwS>5K085;mxbZds3*-bQ^Z!j1gk1XX#t{IE_!S!=w2d%|#eW(b@o(dw#b{2`K;;g}@Vyb_iIQ7qqV<6U(7*3Th_+ zEViSD({|bbmJ<`dFAm)VuxPj3eP%ulVCf3dUa$qQWG~L&T)qonITGdGRiq7I(O$4$ z*`ESn(N;e$IFbTjQOLX(X~qPwq{L_!*#cOKxgQkz?;`8n+!Y0by8xCWhC7&d6cDhS zZ2dhnqzk~(<*7|A2e917W`~A%0a&^y+U#-w3v`c!hIRp1kjjyA0L%S?Z1Tkv082`g z_Dnf(Qy%w?LhU^O%MEU?LhwBR%ROQN_lp96MWOqIu04R|F7XDEI;bwW_xfaOK1?jf zbf&v*Hieu(G6w(_By#{@fvywC91ydi>!cjOqTrq4p8{afMoLFuVkzd9DQJ`fSZm6H8)_@@a1XOW%N&)tD53 zWfv)TjWU2`S7ZNxCLO?%rnYP1CNi<`^pt~p5wO(Tu*P-)SbRR`2>Q|hEXoUgk@hSA zOWLyc=UXtb6binlL52Xp0>vMhX8|lo@duQCX#VXpmy48sB!5@_S^5EBQ7-D|TP6Zn zq}J_{>iP*5o{qAZFM!1dn?s3Y0a#d$%GSOB79VUbC7K0bVI76q5de!%K@R0N7Jvm} z13tb07N5La&zMA!IOQ|F{L_g57UhW^zH=ggC6TE-(Fb7Zt<$A1NB(aJZa{8=ie z51aE^l%+j=8X*t>U@6NV>;|wv*9kohz~WV4*a=`!Zb*w_SyuNs_*D2HVDYp|e5?#$ zQMTyfQxaifQ8qp+)P;MesZ#DfLL42yl4z%F>i3MO${?R3jU;$Xte)y~!@Bl2l`Ls(r=X?Fv_5oP1-(~?UFC6!>V@&`o zvlUsq>sSCwD7J`~gaxpeWOPx&GXN}D_6%O%8o-hvA6k@$Oe}q(Cia~OSY|(%^uIvB z;!7{^odvLTj^byVO#m#RMa4^T1ptdtJ-`uw+D;%*+Bkz5wLh!W%P#vSY8OcGipNtEVCmwy}~j8EE$0&t(^#1W|vK# z%pzb}DlgbKi-2Y6Nx}131T0In1y^PPES)RCeLq40ETQv#&D&-to@xEhrjf^zSRyj9 z#F~nrv;i#DZ5)w|?EscjlND>N6#*{0S^3|aR3&HqBDFb z6-jB3FxHK{*bZQcONU|+m{@8jS40f916XuMo=ql*7~lXbYmpQI;#gw4B}tCR#1fk$ zavBX_f$kBg909OMl3acf0kB9m8Vr;mjwMb_a>5BFmdZ&zkzhRli*ub#`$CMek<@ z0KgK97P-0Bzg^|@fj=f&M5H}2jS@Awev;*PTnBUjERv!k^tgr83J=l_Z2*f%1Cl$a z)_1^EU;r#c54+gMKPyKpaC}NEfCaiogw=>+S?i7wK0q8xteuFlB7g-m;yjrj2Vjwi zaSyrF4qy?vf)j?d16V|0JNr^QfaQN_8qo)^Z2XoBVA<~2!ie1pV9BjG$-TZ2z;X3dtp~qtR67P>(H*UyvfBz^IW}=(ap-pd%hr~g&;CFhOJ~s51zi9O zVR8QV<(mMO?NRQXMOy(ZTNeyg_9p{awyNt3jwAzEHfCl=K0_Q!a?I8uT>y(a_ru2i zn*bIU?#f1kn*f&WhINcP8xgSBw4MkJ=>)KJdTyoW0$6gegwXI#081xjD?1m!l2hOm z8rlh9={&c!JQu){Q$QeJOa`zdM{S+SC2k^e-)yYS2Cz7Cdp8DW!^DzJByqoN1h8!E zKBj8`V7WQWZPV%s6U))6sjlCfLXJJ%`p56E;eXa)r!Iiy7;9_R@0CQt zPMur10G5s3$^OXzmaRiKiLNlQxO2-ui33F=};jKRmpXr5Ybc@%0V>zS_oJIbC7`D@{vFZ zK(nEI1nbTeClnEL-w^}y_80l}M&>#uoq17F_7A{4oL%Tvx| z&$>h%$c7&bF5e}}Gh311$m1gQA}?{Lle!Eu10$jy)gYuEL&d5(D z<KV7zm2!L^I1!N*-ego#t$5fqo~ z2$v}PZ1_Gzm~ip#$HA|LFyTJEj$pQ3{Q;MQQ?D$|jBHF{KZtymvhCn|+}{MjSu`+) zVCKf$MS#p}6?X-9rhoBLFxhb-eqBEcCC{M4H%3>#Kgu=IM{XOUaI3 zuHKfy`$x$LYIO3^xT>aBHBXspc54QxPAs1Rn zM>;xaW4_M#363keRjALfE zLSAaEjxJ7Dr#dAVH}1-Lf{xkNTFa+OT&d=76NDu|%;0W=B1}D33Z$Di}LPKJa@`cdKJY!Zj&J!c{u}!XkcE z{M`xVI%ZY{m34(34GKjM@32fH`UoKqJ7J1Q=sShnh(p>(^a%rJ4v0~1#dV`eCGfP% zj>~|C5r0a0+3lBpd@YL_S9joXG!JHdZ_1%H~Ol;DNVkmvnB5fpI=_Y=ZL9WIP23HOu2 zrya@U-#hj|BMBaYCV0E{e2M$1$`U@fynArEJI+L62A8*OMu&LEqa^MSxjt^IL=KLq zm7_x}@wgX9AU}>Xk{H4f%j;ObC8#O{Inj;=Kp2GMD)AlzKbtbXTf#zepu~F@VQj}2 zGPOen4V@r2ghY;hLh<%-y|3GaP6rOfZgvtW)*eup>u~VE2}!YA14J_73>0V`Cnq7E z60L11awCq|?y}kOI_`zS3?xnxpxc5FT^3T?5sds=T!ir*8tCu8#yAU8JKWLG8tWpY zc3hVDp>Rce)s?eP;s?QhZB>P%4KoJ!RZ*-0@4!5OLt!2hUOQ5fQea00hjO7@OWAVB z4gHtXYnQ?OyPXD)FdcCj7!x=4U?x-O$u-uBtB(2#xE=__H>{!bT^ z{nPui~$K~{;vee{7K656fq~@ z*9k*QCLO0Pm-!&lmdV2vL59Svl6*6z{5yRR$F9Iug%&8)E}qNouYmo$^2MlZgNkTJ zTZfh3yK)Jwo6itdZg&k6jw@t2wU&fFg7@+VUFUEUPKIth4u&V0y66TGLXV@N3)9Td zow*h*NzmPxO3-zzcVh0WCov_^I)v(tRfKBCdS|9-w$V`4_?^(QE<18Vmmco@iIt|2 z2a9Ui3lF0015U2AcF8yeDeiTOsR%9a%4D!{5tXY4r-SOPndLYfPNC$1&^F<|rDCGQ zN#-n07b!W3F6lrQNw^=n!?lrnkmSU`wQhuf%Axx^72T_JopNwhT-EGLgqW(uy@Efw zG@QOyEN&Atbh$fyf3Ubm(9*TfY5wF&xzJ|%J?@9&3gkw85IqSq zLSiSB*hEPq(p?VR^iH%fY8)OoK)TNfm3Hxi=iI%LGEqO;>7U&rg}cKs!{r*o zDRRjlDG>QwR$t4CDSggbqax;y{k3KoPvl9fP?J=RChk%ySe{&;bh^<*DaV2%8W_LIT1lJSgRMdy~Ns&&_HljYvzot~lIVxSY+w3omj3NAC z9xPG9hgcHvzj=d+fLV(w5{MoI)$>3fZ0ezh2)+3>ch zM5LOlSiEr~{dC9fJQa&!4{_J{GLEEl4SzRMapIohV0Bk>h@X9A_g$5p z>CM8^UM2OfiRkx!<5dFJ^!LdNM&IuUtkd@;7aM&q6}Y6sQ^@Fh38k2|kLj-RV@R0E zy7KsTo2mcnm-WRN|1j5>qFz(3XT%>0RJ#z4E#<*FRR;Y6wuE=Y(Z$+a?XL^FWu-6d zeo48RQN}bh&3F@4(8ptno7%sQ`n`|EJzH2l7f`z7H0b2JtFRY>BYGE_UXGsjRULGm zJ!fh^9F>Sa>ASbk_k)?Mb$VEMK^4zTZ8lXh9EqF2nNgc?(u1-;GEm%|YPp|imTcIs zQd7IIpZ$ibqBi_-?rvEF?YOUEq1y$x{(s@?!01U|2r&9;6kfgnRcjYe111i$n;?hM zZW=sba&Ww2NvqK60!n_*Cv43UM(s(;SonAi52Y57(h@%Ykf*M;oboLkiJhn+v6Hj` zL0yJ5#7@*-31}JH7)R9XKPuhhRsINR>@zfFf5dKMeTqIkyUP@gyMKs2Gb?H8t{@-Y zdUwgYur|r%LhF5K2$5(0S{OW6P#u<%_aN+c#-0ndio3#*pb0zRfEs*pcw=baGhDx) zZK;;ZAgkGg>%OVC)g7+4WFJ$jV)(0_53hbxf800yFVj79;}v09Bh_K4eNVz+ZuuB( zJ=Ua%qd8zR%5c-(a)r=RrYbMhfasg{j4>k`Dt3CG^*V zW|Yl{DOSyWE+nCxJ}fsUJaNr*vs!&d?ZMJ5Z*F|(VGf&07T)W7AUr&Krttm8)#K5% zOJarlKCT{JtzJ?reEtzyV{4XV3crQJ#PV|GN#B+5?^VO=jUCEzV*io)Y=GJkawj%k z3ajg{TH!S*yAIy4q_9Dg8`fE;JQe+@ToHUH*3a+Gjo30NtX_s;NOe0;O5Nc2j>yvX zEGc8mHdQmGhkSf_%6-v%<$$2N1iy22+3iE3V@XY-_PZjVw(nxx5*<@E>z*7H^)K+6 zB(0lO#=Da|CN0+~D$}5XWb(wiFVf+%xyrQhHnb?baw*UG{UTa}8CIeV?ccVpka7E6^j+pKQH&x>VCn`s_aiG10% zeVzS3)H3M|L;TPr-f&{_m1wacvdTHRB7joHC99vqeA=@qe>)uRK|Lfn`661+HPsYzqO|Ks=m~^`?3Af#17+0C1J-^9kX?qWH2`({)*DL zFW+8}+x}eoyBFWROsg(g<+1H&U-A{o)#P<{*t1*@$FYNI^3*)M6V`_tB5$3CKk?eg zvH##@d1;7^q}yV>xqg(iWG_2)-^EBuT&KKU(b-4!_|YkCXc#dg>^2(hIUAOxchDy5 zp>31#=;21AsN0>d?20t|c8~T=ozm(zcI08@XK&;C87q)EQM^ri!GN{^`P|z`p$M&q zF|Adzg958~5GC0`c}!z59&4H2n$xY>HOf5ek_AV3v&?P3?uKe2<5Bim_pE;oUddW# zi`s2+gcKZF*QsiYlG}6pS?82P7M;t3q^5Ny`c7YGqPW}~BN`9AJ&rUV>yiu`%I>RqS2{(xA>p4;QU8t0Fb z7xl7&Qy*%*H(aUdTxJ+&E#$AZOjI@t%l9R zQ%|&>L)2uoeJWGy%Eo>9tIVldEohC+U!_l3WL-&lZu>oOQ6uZeZKGJd*A?EL`y{`m zuG7W(aW+YcrM|^{B;pyclDtEb>Wc5^xKUb@@Ehi%tKTMQGCRxJE@_?WrHef;z^AfP z<#mhv1iz-HGdnT%@Q=W)EUjevVDeo9G*3fRr2a9}xbz|Om2OdxK#Fpoc@fk9D6%O* zWtNB^?uf$lDJl|a109Dja65!Zdu@41QVqf;`cFa<5|FTo>FKKpRZOISWT4a1mr}^o z$F!XOPp?RQ`Og*+3(U9E&{Xb<^`P8g9^n3#8m~9uK}n(wa_unG(^##bCj2;84-qvk<-bSJkP<1SZwI=Ywp39~KbDCT_~ev}Kl zz1-cB!Kd@|eTaGIJaZDDZe)g-ciyuoFW56SApwI)W@2)PM1Rj)#30Xvglvp+@*B($ zk$B#-kg4)XFQ^Xw^jh_!TnxleqFn1%V6HqC<_q6(H89JkR!^>a3xZRB z5LbsEhzFD%)>Hj9I^vG?;O$ox8^se}H#xFdn#QoCIf@@=$r)!TTnK#K;Q0RVy1{|N zTU4z_@u#s~JYqh3)a*=w$I`+4ZKDCY$D5*3dbN!0(RP8x4Wr{el7laObn+htHpvAB z7WpiN=ldq@3T$eK%IR$ge2r(bnvMG!D6O=%pHUO!+%xN*@!PSvP5KR#=5(C^*xGzF zI4BxG(yOVbJWM}sY?}Hh;9~vg8=pOc?|fwP%>$uHG4N~s=&;Ye!S^%2ucy?~`dR-NtJeu&)t9wF zRc0*mpqW3oKfXSamOj{H(V;(86m8a&5h{fvp}(hYZ_i zY>eHTR#GD0%$$ds5EDn+M=OcFoyO%!mB!_xk*`?chIE##YC}K|rqPhiBB?GhqE#gW zbQ|lB=r+|eSSM7w8KJ5=0o9H5$Eus^S*+8lp8|%6!C~>e>Bf7KdySurKn!J!T-6&9 z(0YE!yuNqloAJK7X1C9ED!#``vEjE#FQ$>+B))m4k%R*X1Y^XKhSy)EsgWNfQi zJXUutkX|oHkameQj#iCLH#p?C<&oT&@Yt`$Csc3slA;3}>Q21{$r3RCc;#X50%O_u zO5I9XZ!x3a_)7gsRquPog7J^K=Kh&tu0j5lu~qiW^NV&KVn+TZBmIbC5fd}Tzd#gA zyeSGr0mahO&Ov2_fMWS{rzrk!6et#yiV408Q7mF?Q?ouqu~4^}nsp+Ig?iP*+HAz!rT0#e(X)5KzCH}lxcqk~A*}7DE4GI(s>Jt9*2T&|u>N4rAC{QdYF+6q- z6w3t1gSrO~iX}WUi(Y~P#iE9ah)j(K#exco2%*MO;p>&Vpj0s$zq}L9tw zgd-e%YB3%Z3tss802368Sh2|fcCI7D!&Ep9ibX6V-T+JM2%Fry0Sk&n>_+?uC>F|f z(-&A!EYt-0OHeGlkXm}M8Yq?sVLV+!4HU~oVJh8S4HQd+kV?Od0>xsoI>ys^SPi{a?6{|-XV%5_Y0|LdNDxJ zsE_iWh6)4y5K2jEDhU(|g;JUNRUZ@!i8RlJBRVNnr10=HilA6Xq)$ly1lKgzjdbw> zD3*v(y0?8UD3;vIq^}o1u|$}(P4CYI#iB3P@~j41H}@u@_To#8z`0wE4RCc-9WLBS~;PQx5k*dXEt zTUW!NSl%<&DoVP6VsX=@)V~JBGR~Ax)NupF;s)V~A*N}r(NN3yWN2A8D3)$L{r91u zSUxVQzGBYm z>w30?!1^~R7RmRACPA@0O(HS$L9wL1J~RW0rA3#NqYsKD_mW%8Cq%JyYmmOrA&RBj zo%H=9qFB23k>228!jkfSLMcHYgUh zv%2dgP%OWh_3}QbgJStDU{;`;>3azj3pLCfiJMSjsgmZTDMYbQZOj{&L9xtmLSH3Yb#q_{nBrsST23$)z`8?v9N#UKz^-R#*gr% z)j_eCPfSrN*q~T4>E;t$M6poE%*Q#1VxhK}kMj`4Lj7iLHi;+}s+_v|S5Pdk>0avU z%b-|3aE?NF1W_zs@hl!O6BJ9P{iSFYqF6HfFGaT_iiMqP*7_MyEKn%XIuD9vN+6w? zWe$qP{P+_8J{1%Tv{Lv)<>}3NW>A~eL9wV~B8hXLSU!6uQ}3yRVo`UFBz^$J^2IZq z`dED^s=9uHCz`2h4vNJ*NFdB%gJQ{Ch>RXa6bpOVJZ2IU3rAp;xz`*N%U4=AZ<{(O z7Illi90YeZyrKK5LoQ{9S)RJZvWNIsJdYzOQ^PkCs;`xCPZ_Y9()NGeNPKf1easXM$oe|IQV-WP)Nb|IVQlvq7<_|M)7*WMBE?cAJI& z>&5!wOi(Nqs6Q#!GeNOvT!_YU5yg^8zl`O8Vi~qJ*ZAu)D3;5+7b!P0L9sx*B&whv z6pMxZpHaW}gJLNz7Y0DQWbn1`uHxRypja+9Esnkg#WFN|&cc2f6w52$y~VyOW@DgO zq69eOM3X+2{gDZZMI-kctpiak8fur@5XGV~{P)}xD3(57#bUS1 zpja+{<&VAs#WLorQGEF_D3;48zR3WhSo+&7g886W=9aXItuBLN`G%ce!f1eEi5{Ql zp)^3TM2~;rscV2@i5_3z{U>jd#uwCOf@1kMZ^AgDVgI+%^iRdoZ)m~(8x+g8|5MhT zxI?}8e;?1jFJmx_F&ZK}6EP%}CB>jfQj-`8QPyglrm`FRRuRflNywIjrZC37PMuRu zsTf-aWrmKXIGy{g@Av-Q*Y&&m54tYbc+Au1{drN}&YM_6V);(}FfU;ZiG})NX7zLl zB$g{L=2kD1Kw>EgR++YQfZ+zZ*s%u~?h)CO$!8S&B>>eM*JI@`Ktt{|rt`-cesHuL_hv zVkuGkG4a=CVi`SYy@v~lh0A@Il`3OjX>Gk)_I)gIs_QGaRR&&gj8Qwq3rn2u`i9^^ zVzE|NYsjjzV{81K^Boe4wM5DN(JEfgym!gRACOppePN^FdiVll~yc`68rMR9U%%OycbEDDofwtN$W#Dbdqx#d6SCO&pC$pVmAr2czu zvR|08qoZ=mW?~V@6eE~HVzHil#aAK*iN$bomG6`&Bo;q6ttn zbGHP==tE+WJ|?Iy0Eva~oA8DiB$oAmrpMWh^e#PhTF8|xv$q8`WsJOz(c1W;t`(4V*$p4>gDZ8%h z3>&P%N?)`SJ#W1ycuCm_YC_m161Jt;!nM4az>UX_98`hYOM8 zv{$ypxDiQ~A=3*tBJDL@oIRgpWi<1h9hZ$tqHE8eiWfc3HNDjWuQk z!3~5p3Ty0&7=Fi09?l_=y08bHIVaK@a6Ktp+#P8ykp_tccSnY6Lql~Z$QmV5ESP%# zxft2aw8QH#Rd8C#@Ya{40>zc#>{s3C*(SCl;;=+9Q=A=XJAz!bH56?%o>6Zg%;aV} z*%~4TZEHnacg*NESk4q?2iw+49QM0Hlfs!Ljc+qc9+zW3l7+;A`yj?WPy5DfmR(Nf zZDn3;(10_Ni%Gl#R1DdZ=4!K~c-iJEx>3#zakg(1z7k*XO)#~+ihU_}@xb;O6z=3y z;#Zi!XqWfx)Aq-#=;b63ZO`B$xEE;ZcFN_tFT@(mnNM*qkROwIIA#{^s@$Gro+h&n zcNN*5%+p}LkcGrzvjNZG$^M(f`}nW=>q>?2b{aGmb*vJ%lm?9jv1n(tja>#S4KO*S z&dw=6&-@Ls=n%3My+1^rTS|< z?)d&dg|Z6?X65G+Ua1#jrzFGABwR#{q=sE&qUY_b%-PY1Q54FKZp^kvz$(2R-ITme zgT{g|%A(jap|K!#?YCn>V?oqqjO&mC!cQg`BHH%D>oXlbA<6<0;AQX$)(E%_+5W>p zN!HgVVITX4XuQURtS_wv&u~U0?N`z|pJ{BxfD;sjr}p&SY*oav{3Uz(F7|H3vcfle z`X06b0^R~+k3n#!XwXm+FEuL9mXKS8&S8PVgM5oTB!#SA85uF40-%cMiVV`f3e{iyJF?=qs%|!5c77$ zD&cnu)WY{M6A`GCZbZHv2_1gx7&hFVc?02@@(1FL9Z3cLgXl$#5L!K+q19l)xG5TH zPdO2i!Es5rq(R@_Y){$45J5DjsUob?>rEK&e1hmRb*h~2({UvC73~Hs^^nF@#lvZp z5*p>o+*R7mdJRiNJMELWQMnqP#=9@7u^%x_JC(AHjK-&Br`2i5BL+l0xXuaB5ic@0 zw|UkK8N}6eP9D#KVT8Dv!71YHW~d?7?0N1R3RBCX|Jw1)tHdym_dhCF@18h?kzMU5 z5b3r$_Isw+_Bk9L-e%ZsWi^MuXUSjid!6RB=ZpZ$_{`k`hiQg1iGyI6Lb zskFU2P~Rmp+%a7X@1o%BXPU;c+AqP{vAQGcfCtn4b!5dhxEwsdc)K|@mgeWgW8nnb3-opU1^Kt zs1(-D{qQE|{aH%8{orK_eqU2q=KJ@s_VmOk!4$aMn_eZ%bdnM@7X0~CylIbdcx3t% z9v*3`ZZVkqj9;cjKWNdLdyijEiLPrgn)?T@M*Ch$(Vn9f$n;X}tkA0s1zEjRoAmVW z{_qFlk{>erdzFxWA?e{{YWf2|E95l1s5gDyk1jiHkH_^m;k`4tsg#{P!Fca1{9$4$ zcR%b_z#D?_72Q;mxhTBG+)%+|@`z%(#rC<0f>q{5fdttwz4d^8X6v@Omj#2&l>$9- zZF;L?O(twbzb#lK|9(sj4ni{UW+}t?W+p|i=N;UETq4s|rgt|O^sM8pX$>crXk1@d zjm65^&QpSCSqe&eYM4h7(5sINjuTHR;fD)KdYckdU!{DvCwfxw{}k*SGO|}P zNmcIMaNLG}Rp6CQqN?7d9qy@24J9vSaL=VfW5HjgasASvvEZ+!aD&sKvEYBwV2q-K zU8i)dm$$vAJ=}RRVdq)R38Sf>?3e@*Ei3J*&hm>4OEH;)6nOZ1YDz5Y0Oej~QN1_g zhnNa2NlZo4N_UFIE=cIrA{cgZ*$IN%wFo-<{K6yR+~rIUhE)X?mU_j_i5;hH9*G3r zF3kumnrsq$u$^FAe5>?i++(@o+x0ybVx!I!Zw53LF(V@7FasKkn2|FqPKbniF$R&| zIUIOslaVQw!#Nx7%m_wabR730<(E3e)r!HIjo5G7KHVyX2fIsM<6emUfU^+cZAt|` z*)7(clY-{kl+@1?!`$P_#FA2qkdm%pP7G))Vxg`S2L?2j|K6Q+UlCh&&XHckz> zvF)l02bL<38(LRgISz4G#D2Bdsrt+vG<;pS$DWjCTjaV0>1m84mvH* zy*Ri0V5+PX57t!tQQm_K^H%x8fV@rRzIK4o06JfMXLQ z!voR218HvF`i2Lig9g@gcF_jiyzi`}UWdlwc9EEJ9U2QPR5ZE|jIi6}Pr5}p|7Uo@ ze)QMO#1*%%&MeojuK2Cp9rDmvbc7$Y;JRhQ{ZfnEB%SfZiDEJ(^@Q68-SbY-mj`ya zeSqbP=t~2a-9G4jaf-e+AmRou405zkpY767Lu6IJCdIT)iL8dZ^{t2_0}p=2vvhjy zDHV~6mHxlO6XF4`TWY>LUESN6dpz}xJ5fH*jh;h}NQH+rQ-gFq5-m;+9k4KgSqj*r z7&&NRK5K>FGaGkZ!rf3V&E4=|tLf~H2CLcB>w50Ba@Fp&2V2c%+vIn?elVU*_Dgl% zJOR=9;K*$rFJ#}AU#_6IhB$Grrpma5pmw=Y3&;Vf&{%XX-sf%WKHso?_JNL`jzU?* zhW`DVZm9i^CbRQ82EGrX+^*)lRPc6$qm&gVL`6CrrL@Z(956k7Qyq2cfwtRqoho*b z{AG7)5qlT+N~)UMMgbchkV`$~wo%A7=Uz_T0Bz zOFLofbgZ(>0Hwb2c7scg?GCK2^dN=Oqp<_4vv8&+zu{_+aJg^Ke&wtIN)thAxAa&@xA08h&9dku#q!fhN0pBU1ZuoJLon$2OL;^#;7nlt zRE=anKux$yPo&A6d@eP7Uyp=|_jQ=2KiK1B(mXo8k93o}lJ4nGcO%_s@Z0ZCKTgWy zzD>Wq81 zo=tiKI}*O-1=puCB=bz*Wl|=@LEf33zsUQ0ZYc+xE;cBD|8A@De9Cuals~B;Jm*+3 z{BQ3o=cmjnd-{`Zh2J=KJKVYFiSnD%l?~QiNs=liA#z0~A^mWH^0fQ4xSOm1^vF9Y4amabQP<@?*(Xo~3P0+0Y?{kN)FDQ|~09v6yURPxVSd zV=>vtoq8h)jm2am$M;kcG#2HbBOKSHYmW;%EP{U@XgxIojm1J{DEiPyy#43m3lZ4d zRd}3tq$?7evwGmLdBF4Hza!0Lr6SD+qK}L~W3k8@4ku+oW3g}<3U|xwVb5S5%mn66 zA7Pm~JjM*ey@)|%(*VoTL4{@38)V@!8lFTjad?Iam?i8A$cT)%Q?yDihQ<=Hd1X=z zjU~dn@bQ%qX?A0A{ybfM{sz0MST$08{ttFx@te7&>yIC|7&_!*PDa8`R^-<{mWczb zVmXvxu0(!C{uBA@3}kEgmCR0y^L==`iGt}0jBTVO(tKX&&h*Ve`Ao{_b{O|1;u`;1%-acN6aH#P$m&0 zLl%1TA)Uo=buw(>IyBn4b)HcS58xI%MHmjZ67@%0)!}%gnz>LM98o*mYUq&hw}siv zc!6Kuc(I>T=FwaUA z3P&kbJgwdjn2&Qj7i!6LGG@;l@8Cx9bUvnij_15_XF3bB9&dD)oqcE%cVwEaajW_FF787NNHa zgzAnzg@TLSITLa73Rb5`hd7Kllmh)2$&K@Jb9O>DUx7}aR1ks}VJa*glZBm%H}dpm zp9gW!ScKZ-KOKizPw!HP*gHb-l0u;$@}EwWmV7$pvo{tV5rO*?Ny9MPSZFN#<_b#3 zeF|DyJ2?DL<>?+1M>%V8m*na06CRu|ao^I+G==)v*S=W1YPc z*0}wO)W*rmIM;tf?n=R2XXbP5Tf6iM3@Q6#sppiXEk zoeDypiX_dWD3W{Fxy}nhXKRe=%Q|}ap|S9v5T|=go@;p0u}kQK0S6w{p8SVjO@7?) zqCBs@V}u_Xi^P&H=U^-}mQGiRqvEGzv_sUAJ(W zzPi4WLb}9N&D^-4&YHc3FANXKYfBvIHWyk`mD z4UMH@=85L68_-zvSoiJT$ib|q+-}FO`p{SmSr69G0Tu2#w`t z%R}av95fcqr2Llqn~f#+X+Hh#W@E{{luv)K*;sPFvGn&kT6;rd zX`IQ^bhxqESWc3ODGi#?STye#a)~L>STye#acxqdvB(+SptNo_md%Ti=g?RlGF3FS z@}aTZEI7I{oC}R5*TiW=*WyI|OmS|c6EqelL+#djXe=!=xw%eG&{&*mwObpZv9!$; z<_0^}>Oo^kkQN64jPN*2OaMD6lg4R%lW)gXe@W(jO5}?Xe=bMXNs#M zG#1CJ#~a`^-8`Xe{}>yUZ7I&{!NdPO?kn{=K;~wz}qU z^}s(>@U{8SY^(?N$Siynx^bb(>fR=AJ+R)h2oE63QLnmzA0F^)s5<1x>|qc1XnD9Y zuA5oC`QxIn{b;zG0#p`{VmJFdUHIl)9eC7!6|O~YdNfs)Ur+$y>bg|3zv_Zoj6HWUeaOT6_~~_BFm4r{KA=|xZt*zXt9y7Bei!N1 zJ!*8n0iQ1(Q4h<572sz^_qQXg$ElE76s}Z#I$^(u)t<`dAz4GtZh4W~=`nF=ler4! z9{8qIpo?x2texC|Uq@B+=L5T{Ha}u!-+O#;pM@VA-Q~KKSY4h*b$r&( z!`gcmz7^eYJ!(<)OS&HE56K}4pcmacwd`BE@Egg4OI0vB!i7(nZYK{qsT+QBbiJ$D zf9pbU@t_~>p3#Lb(H_L=m0?}@V&pMW6LIv$W1;F|_t^n?VrbuQ&%Kkh>}J9c(S(gX(C6J^7s1MB&wy@i6iwkj!kV>Phs2>&(k3qx@i+18sA{J z+Mi7flCSV6aG_QhKW9jao1_xOP4xQ8w^sL6Y<(7o5w}KJD9pHJpV)`Z@!V%E^j$$- zR!D(bR+=5hD5u4hi>K;n8#-!hKP~qucoxVyz*XsBco2? zsz#M|O;vp%W1&2WQ7B$kf3xyYeI}z#d@%)%ShRUZ_t~-z?DG?HmN!)xmN)erv)Sjw za@-dzbWXlW;j?^`=h(h|4y=Rw0)$S=M=89QkMiW)W-QywB<@yNa~nUgQOa>h+@rJM zwNd`R4p5Y8!f3zlPNu}KdGLxDAOB6>BcFcA*sW6=f4oyk$WH;jgDVgZaUR6AGe3&| zJZyZpa~s~PQ(nkjfrxyiFm%Y+y;C25q7#!CsQ{lv3WkSxb&SL~ag+|JChY(2k$BQz zT}OxXC@kYxb=Ymj-{R)93~_UviJGvqxRyua%{t>SB{Tle=hrLNJvwfjDu#|m5LCRi3A;ZJ;Fud8< zokg;jP%yC9Q+VXjbeMIt$(`kGFQTAozg^+BN7P}KN0dA3WcMZAcGT`8Ye)WcIvvW5 zA5io7d2c0vn>JvryIutk=2c@#BT5JMk?RL=y2(`-t)%J=kJ(V&YyYbKJOno-2Ndoy z0~P#^qa1F@Xka}ylpp8;W0BrA@azG2Qb*P zb?qDc9I_rT+n?}J+alSYT{nR=Aj6?xoW(E{47s1kREO_VC*BLxgMu1n%02BKEnwfnNd3(+G9y4IDn-zh#B;bNv*zw>eq+`~8Y zWbM4Fo*PK4!r$tdQBUl-j&={)ICAfG!G6Ev8GkVY(654rO7VC5?%^LN>7yG{L(q-E zePvUTIqtTGQms2@5YanlUS*%Mklw0U`IPZhd4sK)SYV9i4WIymJmdNHu3y5UY`U0G~W z=uT7l3{%tBV|KRgETV0sl!0lJ{3Fw**JJx_Pq18UFH5PKM#;l@$?GwDTThm=?Io#? zK^xv*$Hv|Lj53~g%b>6Q&dcj5={7>I2QF1Nlzgg#9hO+MNJ=LP8*M=)TEF*iE?>6VeQhR%M^dIOH#^Y z=aAgvokJ{w=QJD=P4j2`dzPeK8NN9_0Z>j zmEn0k!|11hw0Bj$MML|Ub15^^k|OVIyCQ~o-@MywPSIh+~nbizhz#UyrugB@&2lN&hY^B?Vt-4$dbQ^ zbtds6>FRkSLza!7`g9xr>Z{dA&K$LOQEiQyd7a}h^GA-W!&OxwyM1t`pndOdy>|KC z%1p9lYMPv-dbSN&5Pr>iwooT{hns-Cf8+Nv0Ph(Y3K#Q zz3KW)4R&RwpJnDE0q3$nxIf*LX~M1RffGf4GnKYEeiEN~AWOtdDZj(V1`ZDFgpn#6!kYvS9fXEr+=be(m01;sml|C{+#;9|rw5eugE>y|01@Fql^P#_h<|-!XPp#4M5*R%ePJg+ zq_a@i7a(%3KC`n;7$72yR^q+hYs>1}>nGqW2@sJqH6LT_bz%+d4Hh^j2@sKNG9T;Q z>%i*Y8z68}5+EWOWzM_2!lTMop#YIUPHyL*FhE4OPMLllAQHs6+4)WwAR@f1Ob-Q!1afjZdxZfa z!X?Vo^8k^c$s4h6gaIOa-7#kYBC$_};jE-yl5XOo5|*$zdGaB~h+m*ARz_G|13T_d z;?EOv28i4uhOGGI0YtR&3W;X`BECEB_aD&$h&Wf^Z@u;hi1dHeGARIv+!_i|PfG!a zq%3Jg6aYj_a|Z3sBLO1F@5C8%fQTtKF~vp;AflC8sC@@aw!6c6j}wW(E9-*{Z3j&MF5dP`k9rset<~-39Zl~fJh-dVC6+W zK&0P6ORWeXQb;|s(wYJgN%7S3E&_;X$s?=8oIh!OC^RM(Fm`Z963zic#73EykuOAv z30l?#v>}R5TDX|LRue$PAK%cOWdIO4Mm&qZ)eR8ozOLx*1rTX2=pi3Z1BfWTIx*w} z5b?)9PSjTfh$uFm=<@}Lys|NrXf*(c7|cA*{%FH@c#MHk6!!v%9Me8qR*(n~Nz7Gr z^wK^D5b17IY;OXH1T@gQ#T5Y}ih3tD-m(EA-BpU!UI39}H9-iNK6wj^CpEzUkx*DO z2{izSz;?+P^S_fN2KoRI{if!z9-9-ak_$#te z8bIXNtwT3JL~;GtQZPUyG(4s|RuLegn0yR#79et->zUT>wF!|fhNy&w0YJncy}|f3 z1J2ACf)dw`14PbEslksXK*WI9VEhNe6#t$fDPdv&5HT2PAiQD2Uq$L{Zv{5CRZ6J5`W4tq2fNbnzmb1Be83i@R490V0Yu-(C0YrwGW_y~yryL-gJRJ5n1BjTpza^dq zh`c?6E($sa5c!iBY7##T5E&jaYa9oN{MmP2BRK;gqS6{N(+m)Kk~6yhstQ0v<Zqsnf4zt0x8!zVoBEu_YdgB0*KWjo(gNpznX4P+N&HzMw zA6Gb!14Q1{_&u&228d(`o4t4f5NTwM??~53Gl0nWlHcRHVSvamypH%SK;#{U99cOG z5E(vo?&q0b5K++xNpAv(G~;`Ej;R1dRFEMXL7NcCxE4Ze0*Ex@yL*nP07T#qIYUi& zr=ezi?;NQJAW|d|@-rA9GHh>#`hOcD0Fhx2GtYMbk+-CP$mhcVkzr>u^LGF0ElGxnPt8Mh`c47o^Hwjhzv)XjlKhjyruiE zHVy+shJ(%O-T_43(oe584+BJoFPkmD1Bkq(`mZ(&14M=c%u3z?MBY+PPd~{hp!nG~ zl)W8GI6wVJsw+uHUu8LD+zd1ImJ>pMI+fP3KX5ueqc2nBB0$8~CK(_ijoh^{zF9=N z?n&oa07MAHzl?8m0Ys96FclX7B6G&iyCkInBGM+i`q%)GxvkIQG134L)tQLwcThy$ z?6Vg6t_l!QrB0P*#{)#-%cWB-03rnKze=yizm%RHAm}e{E?uNVNDuC+nFNT;)Hin3 zsRBd@H4F8HT>z1;Lg^`h$mjaZt~P0ah%}nOdksaTZ=autGX@}nF}*a#fFd%mFIeOp z1|WiIx-`}aMWlaUfXGP9tsb*+=wiy2c5c$g|KmNTmKty`YVtg4O@@Xl05c!L9v+JESKty`kg1!t8`NYZT>XinFNS9bpzW_x3 zn!FMJMj9X@&>gz~5Q%>(Eh$n_kD*8U>MBBtWE< zxU}Mz4-hfTYa`A9M5cD!A3S0R5II>9e(UvS5gGhyXwm`@X&qWpPfG)cq%9dnv;ah+ zat7_sqW~hP@1&Va0FfweVw#O1K*TV$O?wU?GF5&r&CSs9S-ReIW6hw^Nq|VJ{&Hj7 zAV6erz_9%yK!n{`HF(w#AYvHYmM{nq8MHK9Yy*fq3!G%1KmkNh87ED{0V0vC33~|? zKm_&ZWK%dmB$D;PUIYaYf#dk7aDYf8>wWhnLx2crE$Yv~%_1_WcJk-UCPW6U4cFTM zBF`|B5v79wkwKhcavMP8*~TBUp@y3f34;@m11Nw9%8!(O0U&Y--`QPa2oQP3>+QBf z0Yp&lB;o~t$R&JNHxUI8K}C{AE&xO>wRX(58=}P9NHY-)|1%Z>5NRXLMm!k=hzwpZ zeDe$-(l#*@v3b|QZf+Z&BR@+6h@}16?9f~5Fj#m!Z7q1K%|X6zw%-bATsD+sP+sX(ng(I zX-xx&q8KLdyu%A=|zoMF18&6rfch~SPSeg=q0jCNl}y$~fO7+SZ`CMZ5x;bK-= zUjZUB_=euB-2f2^aRGm;7a-Dm9qS$q5Lqke@jIRc5W&8>JTwUqnZZ9!*2e-wu#K1d zrZypBYbevY8z8cKW;OdG6p^tm6jnSMAVSezC@V+?h$QD?9iz2B14Mcoe@$9w&o|I} z#jyYptls5~x0^+zw+dSw4G^K!{Dr^*M6ipLnni%fa#?J0=x%_BmFe0T6N<=?ZKRBW z6+pzQX>F_ripYrVWf@f~fQVJp+E_Ofkzw0QG9RM>BAoG8enwdU5$v_Eyu4n32zH&a zvP7-E@A;9*wASDXMo5e_fGFR79fIE zi>59CL>9QWdgrkK5v+GK=mjqn_howDlYexYhS;E#Y{sxGw zu*MybY5)E$U24{Svd+289jC7=iD!dsA*8szXC+o@IAfoT5Bdn z4N2YjYZD?_*Qms=0FgC(ckdB3fQZ_!6Aq`LHGJ-2WdX?`6xhS)Y;m6eS!@T$vS750FfWmnbn3-fXHZob;&wFYxSc9%V-my@=zoU>30T5BG zcj;*31BmdU1$kyrM6~x#Z*k@ai13@rjHyEr(cQba5bVV1Q3y{ckOt=2StQWL=d9{MFbOeprecriU{8|Dc*^W zO^EQPiyH5MB4XK5u*FXRAR<5%E01EZ*3@MQ`SumBO|$s;kZ_y8h&qe50{01+jQ zTgMit_D07Z>yyF9i@GT3`)c_(&oWmWxd;k%?5+N!UAfi0!5%Y$R%y!sn&i*Jk!U-?!n!JibDEoKW4NhD%crn3Pe zxV$8yB0vPO<8*?p0X zw)-Ia-DXAxAR@`Nr`g~DBDmBfZAE|xqTHY6hI33#*F!dj^cmR#L=yGY8g2Sg%ia10 zaP86n5xK^*eP?k15nOPRUElHYi+z^3#Uy}8a-b~xovf7oUD*selkEL ziKGy5p>HSoT;B!Un`D4U(u90OWS;?!-JCS8NWM%{B?r(>;w~mb5lI?Hk+0DN*giB{ zoOCikB#EHNrqGaVf0{eaB^e-+WQAIx_DQhGeNH%|WPnH#U2)~NJ{9(vz7x1mxLNs< zL`SV$?Gs`9_Br6xk^v$~RK=BO8j5|I=85x828iIU%bpc={)GFGWNcHw5a${us4@`e zMU9y2vM)q!5^&ZDGzMjo7A~f)l@1V*$A>esGyo!YL^R%uIiqgRypC|U2Z&@AXp!I2 z03wK24nwj45qW%Mf<6Kuf@pN;LjXifY~W-?10bR?la{>)iij}-g%Gy~h}dbP%Z??m z%k2_!5svoSs`y$Wvk}pr4n+jjKxK*}03rxIhYd?8B96=|M72FY#I8mefdGiW4M~j( zKt!!nSOh=>k!*)S14OXgH?(&9O^9ePXrdY# z01=J!6k{_eBG!xq(QEqwBC1m*qVVz+xjYSGit$b;BD)#sq9z&u5sje~f;kkCJ?uD` z(jaLog&$_Zl?LrTBHx~b1&AnzdobZjgZ2dR#-5}G5K#(uXAUEt+QEYWz9$ksBLE_Z zi}vHH01*}6Qwjed03rx!djbX^f}T2-FpU6+AYAMTssIrcE{VAcLn~@oB%Njg&n(Tg_EH0I!_Pi||m-PbSyaoOpw0E(qwegAjKUj-30RR91 literal 0 HcmV?d00001 diff --git a/student-tests/fixtures/sample_progressive.jpg b/student-tests/fixtures/sample_progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..aa39b16d02eab6be1b6e12b07f7e98592b8c0c29 GIT binary patch literal 8520 zcmeHMXH*o~y6tYdfkr|%86@W%MFoK-BOsvx$%r5b5)@D*ND#?6V?c?L1tm((APp)a zC@3gdk|H@tc-4)Jv);PSjC1F{Klc@^UhCAU{e^w@_f^%Y{lWcF5JFu=O$7u7gFs;5 z541l7QUt;9pin3t4ETY;VE6>Ygap7rcH{`07)eG>jwC}ODX1A}C@ATvkVslqT6#uC zW@csz8a56#CJqKBW+to=FaRaMCm%e;x?*j3A#=$Rpk)t_`rm*^QxVFkbM9lZ!&Lfd)Q?* zsn9H=Oe(_4%)Ic_^QnY2F5%8~c5g4R=a7_=+`P@y<{cLwr5g-6$9isNYg0tHN!1MB z6F z_!7)6$qtcy%Pry&kD(=dVQ@7*=1s*TsL|qhRl)GlR~Jt6D;3HG z2~a#Tw^XQwGg^eq*(~l=4qf%TKIBvh{D15St=mu}%@NG2w@smvZk9$~o1h%?=zY|o zdgiG0QLvCq=oULCeKs+)PNBYcW%0bl&FCuPWfG#I=6d{Nx60Ma33Z#L+TT+0*G*m; z@%*x-s$ALi-S!B|=<7Rz!YP#UGF$E)?!hB}kDlt(l>1wPADkNbXTBT8hHgLHc(!X} zPhWopx(}*=^+oChO{kR7F^V|FIzp?~*(W0t?Gr`Fmxwl0dz$@yAyT8_oyYi|;;+)a zChFyT#i{|Fr!mqgZc+WR$E&$hn?7_UJ_X0f(3R-XKR)I#6cjDOXS_0ck8M|NS%!NnA-13KgqKok(?oNZC#KVZ7}rl2Hl$2u&1diq==w7SV>9pKL#Q?wEsU zTp#{B{3eYUE7{W+m3&38uoG69j?q;l<$O;G>|PVSZ3+@_atk_Q?dJU9;oPhB6h-|( zOPI`q@wRu2N_y6O$m?h-)OqE;N}8ynKo!;B)QG7(jWHzwdLjn`1hhE_1cGgTxYma+ z0fHfbArvV>PC>~cehlb;guoma1_{7)r>}T3>zEP@O+n(T<%W3llhmoJmGW}X5?vBD zRYAOxFfcm}MEDdEVQw zm0mjC!)rj^W#A!wzlQ8Y^BU}lPlFCGe-=%A_v*5BD2|{0xPeBwcbR zrh{CRvMwctW3t($vyC)(jYpSVXQW36`l*}Ox8xoc>uqg5`6gq5lHvSA8Jyh*HGW>* z2YvcOGLo2N9c6nApFKnWp--KP>4~MOHP4laXW^;7H)J4TDt)|E3F<1I>Q9n&%bF*r zUU@c<rwRZp@FfbmhM~`7I{epN{L&9-VmwF?7!MB*ifv-wAK$>hNK70lA18WD!3DuW zPQl7%iECuQB`9!+$0t{VmT}s0hrFj$@Fkt($OUw)@3>}`ha}4u@rHE5GIu89#9fT( zkx;AN;>=-v?sf$e@oE+MZ(+xyMkfO%#fO)*J5xj5qis1Ig;*n0Y9d^8(x~8eFQ@1T z7C7FY2#6<(?YW9Uv3aq6l)Rtxq$*gDeon9@vUT8_QRMLyqHc#0GjocaD6V^n*34B2 zb{|Pk%PPwBaO{N5{601$-HJuwi}}6SESBB26qQ9+`DMBVPwq)=rBlzu<11+p?OEoU z5|8C6l=?o0WxXdGRx7_kz`@L#Ivww;w5nKlRg&M*h1f)*Ka0<2Kq}^^)26|NJ2M0G zx_i9&D(Ii*=-j;zDtNr9XJT+Kx2r(+UcTNwsKoC9k=V~2n}&z;7sSlL#meW-iWLPF z7tFp2b^qaM$YlUDd8U?vQnvSb`=e>zNtbXOlZ>)uh@%0wZkF1AK<#TdBawj zOA@_L@#!P_&=bYT>w;%EhC{eplCQ)uIVABfQCL-uF|@-bByqU9-jn5fx*x$LwMpo$ZuKG{_OLHVy`_Eo)cf? zJ_zlHijP<$2%N=G#9`)wGgtdP<^#g#_ZwMC3`>sC! z^0?2^a3^0@@tg2u|G9fHjLs1y_Lf|elCqR#-k~dI`4*umR8Prbre5>kRzfcrR+}}d z3ukBwzlnA2~rKIYAJ8 z5UeQh<_UuVZ=_HtKHh<1!Fc|+poo&vW(ZQ0oK-}gO$_Zq!NM+{k!cwh|NW5-9|a-Q zxIp=ORKesyZ*m83c2i`M;uyYjMCsu^Je=}f~Jy>5S+<`3MM za8GBB%~KDzL;|bGz0Ki{R`<*ZHS}u42HTHSA(L~jSzwJGK!KUxL20m1HY`+BTL_v) za4n4saT}r?ks%nVr{NE`cULjRdKc?oi1>^(dH~hHdOd_@;-D5-DDPBL!stT6WDEJI z+R`=7lJHB;tb$rI-B|A`KuSJXqX*DSSg*f9EwIoCEVNx3$bj*+_Zt%H?Z#x92% zb|vAmq_eVTSONBF6mVpd_b)`^^0vwUK!aOBZ^nq$rh4y4F8`g#`dyUY^kV*|k93`n zPs7AO+pgnb`~YizG4(4JYTbqi4D zujNCF63+4E{_Mw+=Kn2ji{=~cFY%uCWh#HyV(^#5uer3Qz41#?QY4ej?TzCq4Okz* z88^1l0y&!!#%F#~q+6{d-E%(me~l#@L$rmBNULOanM2j9Ky=W&7o-icw^y z=^ht;ZlU_hKYwb$@~}StdTF<1+O9BB5lqZ3`KI-ZhP2O4Pk&Z?_2}2-1r#L`1?}wa z85r1sXtYHpqDo>gQ~MyrZ$Nr8yQ-|9icZ8s?sAt0EFo4!5! z`HN*$QWUWPh6l0u-o!A<}czV_^a5YP~AxvLer~Kp$k5evnx?SB}aT_|x^>H{UbRh^UO9spne;HTa3v~f?HA|U^&=XD>(DHG8TPm=ot0K$>F!(tz;couX&FwFj1OY zl%W%x+$Hk$B;_>OUcvxx3ech=}KW*!ZDbgV_nNX>ffjhr>f_CGw|1E_%uPy@Cl z|JYN(sR1c(S+a;=MQ8ZO^|FlNyE#`6fCG}zvfF3 zcxlVs8=0MHlA3T)yF=py_ZT;xfvTyPx|wmcgn8%ewz8hh-RG0rqKV}idaoTZo0yA} zI~=3-o%0vfG$KxL6;?B%P8T?qN#0LW5larU_tLzJq7xWS3%pNa%;<>WbzWT5z#j^% z4HrEwTq7Qo8pCkRZh^jc4{ulx*ou`GygoQtQ?o7lG~9XM+RxmC44u3#bA?kmTG&k5 zPf$_X%LDy2ILFn8w#$SwfA+Zbj&+%b0L?gaf$G;B>{+Ho$WVfe8+gV9&h47Dl8TP#HFj0weit@w{f7vQ!R8m;zGtw=7 z_g}^rf1CQUrlG5pZ=C+Si&rIOBW(o9`~RV32K2>;zN z0+O8R3)GYHj(x$CWXOH{$()vk@UH7+n%kFT_<37r2Yw&Emj83E+?ai_Trh6Wb zM8i;AjbHYzaZ2j)D5+b!pX|%B?gRzT*5^2@b<-dVQuw=fz`u`^R<>H}KQrEc_dl?z BO#A=< literal 0 HcmV?d00001 diff --git a/student-tests/fixtures/small.png b/student-tests/fixtures/small.png new file mode 100644 index 0000000000000000000000000000000000000000..ff552a5afb3a91bea928d8b46d8f9ef0800f9d00 GIT binary patch literal 93 zcmeAS@N?(olHy`uVBq!ia0vp^8bB<*+qcNQ7{?;LoWnAmdKI;Vst0Ij+d3;+NC literal 0 HcmV?d00001 diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 0000000..bf6951e --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,93 @@ +"""Синтетические изображения для тестов вложений (только стандартная библиотека). + +PNG собирается вручную (zlib + CRC32), поэтому можно получить и корректные +файлы заданного размера, и намеренно испорченные варианты. JPEG — небольшие +fixtures из student-tests/fixtures (синтетические, сгенерированы Pillow). +""" +import os +import struct +import zlib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +PNG_SIG = b"\x89PNG\r\n\x1a\n" +MAX_UPLOAD = 10 * 1024 * 1024 + + +def chunk(ctype: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data) & 0xFFFFFFFF) + + +def ihdr(width, height, depth=8, color=2, interlace=0) -> bytes: + return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)) + + +def png(width=16, height=16, color=2, depth=8, fill=b"\x40", level=6, extra=b"") -> bytes: + channels = {0: 1, 2: 3, 4: 2, 6: 4}[color] + row = b"\x00" + (fill * (width * channels * depth // 8 + 1))[: (width * channels * depth + 7) // 8] + raw = row * height + return PNG_SIG + ihdr(width, height, depth, color) + extra + chunk(b"IDAT", zlib.compress(raw, level)) + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Несжимаемый RGB PNG размером около target_bytes (шум, zlib level 0).""" + width = 1024 + row_bytes = 1 + width * 3 + height = max(1, (target_bytes - 200) // row_bytes) + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + return PNG_SIG + ihdr(width, height) + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"") + + +def png_of_exact_size(size: int) -> bytes: + """Корректный PNG ровно size байт: маленькая картинка + вспомогательный чанк-заполнитель.""" + base = png(8, 8) + pad = size - len(base) - 12 + assert pad >= 0 + filler = chunk(b"zzPd", b"\x00" * pad) # строчная первая буква: вспомогательный, игнорируется + data = base[:8 + 25] + filler + base[8 + 25:] + assert len(data) == size + return data + + +def huge_dimensions_png(width=8000, height=6000) -> bytes: + """48 Мп 1-битная картинка: файл ~50 КБ, распаковка была бы огромной (pixel bomb).""" + row = b"\x00" + b"\x00" * ((width + 7) // 8) + return PNG_SIG + ihdr(width, height, depth=1, color=0) + chunk(b"IDAT", zlib.compress(row * height, 9)) + chunk(b"IEND", b"") + + +def bad_crc_png() -> bytes: + data = bytearray(png(16, 16)) + data[8 + 8 + 13] ^= 0xFF # CRC чанка IHDR + return bytes(data) + + +def truncated_png() -> bytes: + data = png(64, 64, fill=b"\x13\x37\x99") + return data[: len(data) // 2] + + +def wrong_length_png() -> bytes: + """Все CRC верны, но распакованных данных меньше, чем требует IHDR.""" + raw = b"\x00" + b"\x10" * 3 * 4 + return PNG_SIG + ihdr(4, 10) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +def sample_png() -> bytes: + # В compose-сети (ops.py in-network) монтируется только student-tests/, + # поэтому копия публичного fixture лежит и в student-tests/fixtures. + public = ROOT / "tests" / "fixtures" / "sample.png" + return (public if public.is_file() else HERE / "fixtures" / "sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (HERE / "fixtures" / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (HERE / "fixtures" / "sample_progressive.jpg").read_bytes() + + +def truncated_jpeg() -> bytes: + data = sample_jpeg() + return data[: len(data) - 400] diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..c7c540a --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,170 @@ +"""Фоновая нагрузка для сценариев отказа/обновления (только стандартная библиотека). + +Писатели: POST сообщения с Idempotency-Key. Неопределённый результат (обрыв, +timeout, 5xx) повторяется с ТЕМ ЖЕ ключом и телом, как требует контракт, пока +не будет 201. Читатели: GET истории и /users/me. Параллельно опрашивается +страница состояния HAProxy (UP/DOWN реплик), чтобы измерить время обнаружения. + +Каждая попытка записывается как событие (время от старта, тип, успех, статус, +X-Instance-Id, задержка). По событиям считаются RPS/ошибки/перцентили в окнах. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import csv +import io +import math +import threading +import time +import uuid +from urllib.parse import quote +from urllib.request import urlopen + +import common +from common import actor, create_chat, request + + +def percentile(values, p): + if not values: + return None + values = sorted(values) + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) + + +class Load: + def __init__(self, base_url, writers=4, readers=2, timeout=2.0, stats_url=None): + common.BASE_URL = base_url.rstrip("/") + self.writers, self.readers, self.timeout = writers, readers, timeout + self.stats_url = stats_url + self.events = [] # (t, kind, ok, status, instance, latency) + self.lb = [] # (t, tuple(sorted UP servers)) + self.confirmed = [] + self.same_key_retries = 0 + self._lock = threading.Lock() + self._stop = threading.Event() + self._threads = [] + self.alice = actor("lda") + self.bob = actor("ldb") + self.chat = create_chat(self.alice, self.bob) + self.t0 = None + + def now(self): + return time.monotonic() - self.t0 + + def _record(self, kind, ok, status, instance, latency): + with self._lock: + self.events.append((self.now(), kind, ok, status, instance, latency)) + + def _call(self, kind, method, path, token, payload=None, headers=None): + started = time.monotonic() + try: + r = request(method, path, token=token, payload=payload, headers=headers, timeout=self.timeout) + ok = r.status < 500 + self._record(kind, ok, r.status, r.headers.get("X-Instance-Id"), time.monotonic() - started) + return r + except OSError as exc: # обрыв соединения / timeout — результат неизвестен + self._record(kind, False, type(exc).__name__, None, time.monotonic() - started) + return None + + def _writer(self, wid): + n = 0 + path = "/api/v1/chats/%s/messages" % self.chat["id"] + while not self._stop.is_set(): + key = "lg-%d-%d-%s" % (wid, n, uuid.uuid4().hex[:8]) + body = {"text": "load %d/%d %s" % (wid, n, key)} + for attempt in range(200): + r = self._call("w", "POST", path, self.alice["_token"], body, {"Idempotency-Key": key}) + if r is not None and r.status == 201: + with self._lock: + self.confirmed.append(r.json) + break + if r is not None and r.status < 500: + raise AssertionError("unexpected %s on write: %s" % (r.status, r.json)) + with self._lock: + self.same_key_retries += 1 + time.sleep(0.05) # повтор с тем же ключом и телом + n += 1 + + def _reader(self, rid): + i = 0 + while not self._stop.is_set(): + if i % 2: + self._call("r", "GET", "/api/v1/users/me", self.bob["_token"]) + else: + self._call("r", "GET", "/api/v1/chats/%s/messages?limit=20" % self.chat["id"], self.bob["_token"]) + i += 1 + + def _lb_poller(self): + last = None + while not self._stop.is_set(): + try: + data = urlopen(self.stats_url, timeout=1).read().decode("utf-8", "replace") + rows = csv.DictReader(io.StringIO(data.lstrip("# "))) + up = tuple(sorted(r["svname"] for r in rows if r.get("pxname") == "api" + and r.get("svname") not in ("BACKEND", "FRONTEND") + and r.get("status", "").startswith("UP"))) + if up != last: + with self._lock: + self.lb.append((self.now(), up)) + last = up + except Exception: + pass + time.sleep(0.1) + + def start(self): + self.t0 = time.monotonic() + targets = [(self._writer, i) for i in range(self.writers)] + [(self._reader, i) for i in range(self.readers)] + if self.stats_url: + targets.append((self._lb_poller, None)) + for fn, arg in targets: + t = threading.Thread(target=fn, args=(() if arg is None else (arg,)), daemon=True) + t.start() + self._threads.append(t) + return self + + def stop(self): + self._stop.set() + for t in self._threads: + t.join(timeout=30) + + def window(self, a, b): + ev = [e for e in self.events if a <= e[0] < b] + ok = [e for e in ev if e[2]] + dur = max(b - a, 1e-9) + lat = [e[5] for e in ok] + return { + "from_s": round(a, 2), "to_s": round(b, 2), + "requests": len(ev), "ok": len(ok), "errors": len(ev) - len(ok), + "error_ratio": round((len(ev) - len(ok)) / len(ev), 4) if ev else None, + "ok_rps": round(len(ok) / dur, 1), + "write_ok_rps": round(sum(1 for e in ok if e[1] == "w") / dur, 1), + "latency_ms_ok": {"p50": percentile(lat, .5), "p95": percentile(lat, .95), "p99": percentile(lat, .99)}, + "instances": sorted({e[4] for e in ok if e[4]}), + } + + def errors_between(self, a, b): + return [e for e in self.events if a <= e[0] < b and not e[2]] + + def verify(self): + """Все подтверждённые сообщения на месте, дублей (в т.ч. от повторов) нет.""" + items, cursor = [], None + while True: + path = "/api/v1/chats/%s/messages?limit=100" % self.chat["id"] + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = request("GET", path, token=self.bob["_token"], timeout=10).json + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + ids = [m["id"] for m in items] + texts = [m["text"] for m in items] + by_id = {m["id"]: m for m in items} + missing = [m for m in self.confirmed if by_id.get(m["id"]) != m] + return { + "confirmed_writes": len(self.confirmed), + "stored_messages": len(items), + "missing_or_altered": len(missing), + "duplicate_ids": len(ids) - len(set(ids)), + "duplicate_texts": len(texts) - len(set(texts)), + "same_key_retries": self.same_key_retries, + "ok": not missing and len(ids) == len(set(ids)) and len(texts) == len(set(texts)), + } diff --git a/student-tests/race.py b/student-tests/race.py new file mode 100644 index 0000000..ab649f1 --- /dev/null +++ b/student-tests/race.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Конкурентная запись: 10 клиентов x 10 сообщений (перенесено из lab1, Bearer).""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import base64 +import json +import threading +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import urlencode + + +def request(base_url, method, path, token=None, body=None, headers=None): + headers = dict(headers or {}) + headers["Accept"] = "application/json" + + if token: + headers["Authorization"] = "Bearer " + token + + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = "application/json" + + request = urllib.request.Request( + base_url + path, + data=data, + headers=headers, + method=method, + ) + + with urllib.request.urlopen(request, timeout=10) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + +def create_user(base_url, name): + username = "race_%s_%s" % (name, uuid.uuid4().hex[:8]) + password = "Race-password-" + uuid.uuid4().hex + + status, user = request( + base_url, + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": "Race " + name, + "password": password, + }, + ) + assert status == 201 + + credential = base64.b64encode((username + ":" + password).encode()).decode() + status, session = request( + base_url, + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert status == 201 + user["token"] = session["token"] + return user + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--base-url", + default="https://localhost:8443", + ) + args = parser.parse_args() + + base_url = args.base_url.rstrip("/") + + users = [ + create_user(base_url, str(i)) + for i in range(10) + ] + + owner = users[0] + + status, chat = request( + base_url, + "POST", + "/api/v1/chats", + token=owner["token"], + body={ + "title": "Concurrent race test", + "member_ids": [user["id"] for user in users], + }, + ) + + assert status == 201 + + chat_id = chat["id"] + + barrier = threading.Barrier(10) + + def worker(worker_id): + user = users[worker_id] + created = [] + + barrier.wait() + + for message_id in range(10): + text = "race-%d-%d" % (worker_id, message_id) + + status, message = request( + base_url, + "POST", + "/api/v1/chats/%s/messages" % chat_id, + token=user["token"], + body={"text": text}, + headers={"Idempotency-Key": "race-%d-%d-%s" % (worker_id, message_id, uuid.uuid4().hex[:8])}, + ) + + assert status == 201 + created.append(message) + + return created + + with ThreadPoolExecutor(max_workers=10) as executor: + results = list(executor.map(worker, range(10))) + + created = [ + message + for worker_messages in results + for message in worker_messages + ] + + assert len(created) == 100 + + created_ids = {message["id"] for message in created} + created_texts = {message["text"] for message in created} + + assert len(created_ids) == 100 + assert len(created_texts) == 100 + + messages = [] + seen_cursors = set() + cursor = None + page_limit = 13 + + while True: + path = ( + "/api/v1/chats/%s/messages?limit=%d" + % (chat_id, page_limit) + ) + + if cursor is not None: + path += "&" + urlencode({"cursor": cursor}) + + status, page = request( + base_url, + "GET", + path, + token=owner["token"], + ) + + assert status == 200 + + page_messages = page["items"] + messages.extend(page_messages) + + next_cursor = page["next_cursor"] + + if next_cursor is None: + break + + assert next_cursor not in seen_cursors, ( + "Cursor повторился: возможен бесконечный цикл пагинации" + ) + + seen_cursors.add(next_cursor) + cursor = next_cursor + + assert len(messages) == 100 + + history_ids = {message["id"] for message in messages} + history_texts = {message["text"] for message in messages} + + assert len(history_ids) == 100 + assert len(history_texts) == 100 + + assert history_ids == created_ids + assert history_texts == created_texts + + user_ids = {user["id"] for user in users} + + assert all( + message["sender_id"] in user_ids + for message in messages + ) + + print( + "PASS: concurrent write test; " + "10 clients x 10 messages = 100 messages; " + "no lost or duplicate messages" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/revocation_partition.py b/student-tests/revocation_partition.py new file mode 100644 index 0000000..f8dd610 --- /dev/null +++ b/student-tests/revocation_partition.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Минимальное воспроизведение замечания ревью №1 (отзыв токена при разрыве с Redis). + +Прогрев кэша -> разрыв сети API<->Redis без рестарта Redis -> logout -> +восстановление связи -> 8 с опроса старого токена (дольше circuit breaker). +Ожидается только 401. По умолчанию — команды docker compose-стенда: + python student-tests/revocation_partition.py --base-url https://localhost:8443 +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse, shlex, subprocess, sys, time +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common +from common import actor, request + +p = argparse.ArgumentParser() +p.add_argument("--base-url", default="https://localhost:8443") +p.add_argument("--partition", default="docker network disconnect messenger-lab3_backend messenger-lab3-redis-1") +p.add_argument("--heal", default="docker network connect --alias redis messenger-lab3_backend messenger-lab3-redis-1") +args = p.parse_args() +common.BASE_URL = args.base_url.rstrip("/") +ROOT = Path(__file__).resolve().parents[1] +def run(cmd): + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True) +a = actor("repro"); t = a["_token"] +for _ in range(2): request("GET","/api/v1/users/me",token=t) +run(args.partition) +try: + print("logout during partition:", request("DELETE","/api/v1/auth/sessions/current",token=t).status) +finally: + run(args.heal) +seen=set(); end=time.monotonic()+8 +while time.monotonic() healthy/UP -> graceful stop старой, и так N раз). +Проверяется: hook завершился успешно, после обновления обслуживают ТОЛЬКО новые +реплики, подтверждённые сообщения на месте и без дублей; измеряются RPS, +ошибки и задержки до/во время/после. + + python student-tests/rolling_update_load.py --output evidence/rolling_update.json +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import json +import os +import shlex +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +from failover_series import observe_instances # noqa: E402 +from loadgen import Load # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + py = sys.executable or "python" + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) + p.add_argument("--stats-url", default="http://127.0.0.1:%s/stats;csv" % os.environ.get("LB_STATS_PORT", "8404")) + p.add_argument("--update-cmd", default=None, help="default: python scripts/ops.py rolling-update") + p.add_argument("--writers", type=int, default=4) + p.add_argument("--readers", type=int, default=2) + p.add_argument("--pre", type=float, default=5) + p.add_argument("--post", type=float, default=8) + p.add_argument("--output") + args = p.parse_args() + cmd = shlex.split(args.update_cmd) if args.update_cmd else [py, "scripts/ops.py", "rolling-update"] + common.BASE_URL = args.base_url.rstrip("/") + + probe = common.actor("rprobe") + before = observe_instances(probe["_token"]) + print("replicas before: %s" % before, flush=True) + load = Load(args.base_url, writers=args.writers, readers=args.readers, stats_url=args.stats_url).start() + time.sleep(args.pre) + t_upd = load.now() + proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=1200) + t_done = load.now() + print(proc.stdout.strip()[-3000:], flush=True) + time.sleep(args.post) + load.stop() + t_end = load.now() + after = observe_instances(probe["_token"]) + tail_instances = load.window(t_end - 3, t_end)["instances"] + errs = load.errors_between(t_upd, t_done) + report = { + "scenario": "rolling update of all API replicas under continuous load", + "update_command": " ".join(cmd[1:] if cmd[0] == py else cmd), + "update_exit_code": proc.returncode, + "update_seconds": round(t_done - t_upd, 2), + "replicas_before": before, + "replicas_after": after, + "all_replaced": not (set(before) & set(after)) and not (set(before) & set(tail_instances)), + "errors_during_update": len(errs), + "error_kinds": sorted({str(e[3]) for e in errs}), + "windows": { + "before": load.window(0, t_upd), + "during_update": load.window(t_upd, t_done), + "after": load.window(t_done, t_end), + }, + "lb_timeline": [[round(t, 2), list(up)] for t, up in load.lb], + "data": load.verify(), + "load": {"writers": args.writers, "readers": args.readers, "client_timeout_s": 2.0}, + } + report["pass"] = proc.returncode == 0 and report["all_replaced"] and report["data"]["ok"] + w = report["windows"] + print("update %.1fs rc=%d | replicas %s -> %s | errors during update %d (%s) | RPS before %.1f, during %.1f, after %.1f" + " | p95 during %s ms | data ok=%s" + % (report["update_seconds"], proc.returncode, before, after, len(errs), ",".join(report["error_kinds"]) or "-", + w["before"]["ok_rps"], w["during_update"]["ok_rps"], w["after"]["ok_rps"], + w["during_update"]["latency_ms_ok"]["p95"], report["data"]["ok"])) + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + "\n", encoding="utf-8") + print(("PASS" if report["pass"] else "FAIL") + ": rolling update under load") + sys.exit(0 if report["pass"] else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_all.py b/student-tests/run_all.py new file mode 100644 index 0000000..805c8cb --- /dev/null +++ b/student-tests/run_all.py @@ -0,0 +1,51 @@ +"""Запуск собственных тестов, которые идут по HTTP через балансировщик. + + python student-tests/run_all.py --base-url https://localhost:8443 + +Входит: API/auth/конкурентность (lab2), идемпотентность через реплики (lab3), +race.py, вложения lab4 (test_attachments.py, lab5: 202 + ожидание ready), варианты lab5 +(test_pipeline.py). Отказы pipeline (test_pipeline_faults.py) здесь пропускаются — нужен доступ +к БД/брокеру и стенд отказов: python scripts/ops.py pipeline-faults. Тест согласованности БД<->S3 +(test_attachment_consistency.py) здесь пропускается: нужен доступ к БД и S3 — +python scripts/ops.py in-network consistency. Отдельно (меняют состояние стенда или нужен доступ к БД): + python scripts/ops.py in-network storage # тесты хранилища внутри compose-сети + python scripts/ops.py in-network expiry # истечение сессии (api-short-ttl) + python student-tests/failover_series.py # остановка каждой реплики под нагрузкой + python student-tests/rolling_update_load.py # rolling update под нагрузкой + python student-tests/dependency_failure.py # отказ Redis/PostgreSQL (останавливает контейнеры) +""" +import argparse +import os +import subprocess +import sys + +parser = argparse.ArgumentParser() +parser.add_argument("--base-url", default="https://localhost:8443") +parser.add_argument("--expiry-url", default=os.environ.get("EXPIRY_BASE_URL", "")) +parser.add_argument("--with-failures", action="store_true", + help="run student-tests/dependency_failure.py (stops redis/postgres)") +parser.add_argument("--failure-args", default="", + help="extra arguments for dependency_failure.py (custom stop/start commands)") +args = parser.parse_args() + +env = os.environ.copy() +env["BASE_URL"] = args.base_url.rstrip("/") +if args.expiry_url: + env["EXPIRY_BASE_URL"] = args.expiry_url.rstrip("/") + + +def step(title, cmd): + print("\n== " + title, flush=True) + subprocess.run(cmd, env=env, check=True) + + +step("API, auth, concurrency, idempotency tests (DB/expiry tests skip without their env)", + [sys.executable, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_*.py", "-v"]) +step("Concurrent write test", [sys.executable, "student-tests/race.py", "--base-url", args.base_url]) +if args.with_failures: + import shlex + step("Dependency failure scenario", + [sys.executable, "student-tests/dependency_failure.py", "--base-url", args.base_url] + + shlex.split(args.failure_args)) + +print("\nALL STUDENT TESTS PASSED", flush=True) diff --git a/student-tests/s3_outage.py b/student-tests/s3_outage.py new file mode 100644 index 0000000..1bba612 --- /dev/null +++ b/student-tests/s3_outage.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Недоступность S3: деградация только операций с файлами, восстановление без рестарта API. + + python student-tests/s3_outage.py --base-url https://localhost:8443 --output evidence/s3_outage.json + +По умолчанию останавливает/запускает сервис s3 через docker compose и ждёт +прогрева чтения/записи (python scripts/ops.py wait-s3). Для другого стенда — +--stop-cmd / --start-cmd. Тома не удаляются. + +Ожидаемое поведение (S3 НЕ входит в readiness, см. REPORT.md): + * /health/ready = 200, обе реплики остаются в LB; + * регистрация, вход, сообщения и история — работают; + * GET метаданных вложения — 200 (это PostgreSQL); + * POST вложения и GET /content -> 503 unavailable (+ Retry-After) за ограниченное + время (таймауты клиента S3); pending-строку удаляет компенсация + (проверяется в test_attachment_consistency.py через api-fault-s3); + * после возврата S3 тот же токен загружает и скачивает файлы, старый файл цел. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import hashlib +import http.client +import json +import os +import shlex +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import urlparse + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import common # noqa: E402 +import imagegen # noqa: E402 +from common import actor, create_chat, request # noqa: E402 + +ROOT = HERE.parent +MAX_FAIL_SECONDS = 15.0 # connect 2 с x 2 попытки + backoff; ответ клиенту не позже + + +class Check: + def __init__(self): + self.results = [] + + def __call__(self, name, ok, detail=""): + self.results.append({"check": name, "ok": bool(ok), "detail": detail}) + print(("PASS " if ok else "FAIL ") + name + (" — " + detail if detail else ""), flush=True) + + +def run(cmd): + print("$ " + " ".join(cmd), flush=True) + subprocess.run(cmd, cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=180) + + +def upload(token, chat_id, data): + return request("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat_id, token=token, raw=data, + headers={"Content-Type": "image/png"}, timeout=40) + + +def download(token, att_id): + u = urlparse(common.BASE_URL) + conn = (http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=40, context=tlsenv.CONTEXT) if u.scheme == "https" else http.client.HTTPConnection(u.hostname, u.port or 80, timeout=40)) + t = time.monotonic() + conn.request("GET", "/api/v1/attachments/%s/content" % att_id, headers={"Authorization": "Bearer " + token}) + r = conn.getresponse() + body = r.read() + conn.close() + return r.status, body, time.monotonic() - t, r.headers + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) + py = sys.executable or "python3" + p.add_argument("--stop-cmd", default="docker compose --env-file .env stop s3") + p.add_argument("--start-cmd", default="docker compose --env-file .env start s3") + p.add_argument("--wait-cmd", default="%s scripts/ops.py wait-s3" % shlex.quote(py)) + p.add_argument("--output") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + check = Check() + + alice, bob = actor("s3o_a"), actor("s3o_b") + chat = create_chat(alice, bob) + original = imagegen.sample_png() + r = upload(alice["_token"], chat["id"], original) + check("baseline upload 202 (lab5)", r.status == 202, str(r.status)) + att = common.wait_ready(alice["_token"], r.json["id"]) # lab5: обработку делает worker + + run(shlex.split(args.stop_cmd)) + try: + ready = request("GET", "/health/ready") + check("readiness stays 200 while S3 is down (S3 is not a readiness dependency)", ready.status == 200, + str(ready.status)) + m = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "S3 лежит, а чат работает"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + check("text message 201 during S3 outage", m.status == 201, str(m.status)) + h = request("GET", "/api/v1/chats/%s/messages?limit=10" % chat["id"], token=bob["_token"]) + check("history 200 during S3 outage", h.status == 200, str(h.status)) + meta = request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]) + check("attachment metadata 200 (PostgreSQL only)", meta.status == 200, str(meta.status)) + ref = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + check("message referencing existing attachment 201", ref.status == 201, str(ref.status)) + + for i in range(2): + u = upload(alice["_token"], chat["id"], imagegen.png(10 + i, 10)) + code = (u.json or {}).get("error", {}).get("code") + check("upload #%d -> 503 unavailable" % (i + 1), u.status == 503 and code == "unavailable", + "%s %s in %.2fs" % (u.status, code, u.elapsed)) + check("upload #%d failed fast (<= %.0fs)" % (i + 1, MAX_FAIL_SECONDS), u.elapsed <= MAX_FAIL_SECONDS, + "%.2fs" % u.elapsed) + check("upload #%d has Retry-After" % (i + 1), bool(u.headers.get("Retry-After"))) + status, _, elapsed, _ = download(bob["_token"], att["id"]) + check("download -> 503 during outage", status == 503, "%s in %.2fs" % (status, elapsed)) + check("download failed fast", elapsed <= MAX_FAIL_SECONDS, "%.2fs" % elapsed) + me = request("GET", "/api/v1/users/me", token=alice["_token"]) + check("session still valid", me.status == 200, str(me.status)) + finally: + run(shlex.split(args.start_cmd)) + t0 = time.monotonic() + run(shlex.split(args.wait_cmd)) + recovered = round(time.monotonic() - t0, 2) + + check("S3 read/write warm-up after start", True, "%.2fs" % recovered) + status, body, elapsed, headers = download(bob["_token"], att["id"]) + check("old file downloads byte-for-byte after recovery", + status == 200 and hashlib.sha256(body).hexdigest() == att["sha256"], "%s %.2fs" % (status, elapsed)) + u = upload(alice["_token"], chat["id"], imagegen.png(12, 12)) + check("new upload 202 after recovery (same token, no API restart)", u.status == 202, str(u.status)) + + report = {"scenario": "S3 outage and recovery", "max_fail_seconds": MAX_FAIL_SECONDS, + "s3_warmup_seconds": recovered, "checks": check.results, + "pass": all(c["ok"] for c in check.results)} + if args.output: + Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS" if report["pass"] else "FAIL") + sys.exit(0 if report["pass"] else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/security/argon2_bench.py b/student-tests/security/argon2_bench.py new file mode 100644 index 0000000..97d7cd4 --- /dev/null +++ b/student-tests/security/argon2_bench.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""lab7: замер Argon2id на стенде (внутри образа API, с его CPU-лимитом). + + python scripts/ops.py login-budget # = docker compose run --rm api python student-tests/security/argon2_bench.py +Печатает время hash/verify для базовой линии OWASP и более тяжёлых вариантов и оценку +пропускной способности входа на реплику: slots / t_verify, ограниченную CPU-квотой. +""" +import os +import statistics +import time + +from argon2 import PasswordHasher, Type + +N = int(os.environ.get("BENCH_N", "15")) +CPUS = float(os.environ.get("BENCH_CPUS", "0.5")) +SLOTS = int(os.environ.get("PASSWORD_HASH_CONCURRENCY", "4")) +GRID = [(19456, 2, 1), (19456, 3, 1), (47104, 1, 1), (65536, 2, 1), (65536, 3, 1)] +current = (int(os.environ.get("ARGON2_MEMORY_KIB", "19456")), int(os.environ.get("ARGON2_ITERATIONS", "2")), + int(os.environ.get("ARGON2_PARALLELISM", "1"))) +print("Argon2id benchmark: N=%d per row, container CPU quota=%.2f, KDF slots/replica=%d" % (N, CPUS, SLOTS)) +print("%-22s %10s %10s %10s %14s %16s" % ("m(KiB),t,p", "hash p50", "verify p50", "verify p95", "RAM/hash MiB", + "logins/s/replica")) +for m, t, p in GRID: + ph = PasswordHasher(memory_cost=m, time_cost=t, parallelism=p, type=Type.ID) + pw = "benchmark-password-" + str(m) + hashes, verifies = [], [] + h = ph.hash(pw) + for _ in range(N): + t0 = time.perf_counter() + ph.hash(pw) + hashes.append((time.perf_counter() - t0) * 1000) + t0 = time.perf_counter() + ph.verify(h, pw) + verifies.append((time.perf_counter() - t0) * 1000) + v50 = statistics.median(verifies) + v95 = sorted(verifies)[int(0.95 * (len(verifies) - 1))] + # CPU-квота ограничивает сумму по слотам: не больше CPUS ядер на реплику. + rate = min(SLOTS, CPUS) * 1000.0 / v50 + mark = " <- current" if (m, t, p) == current else "" + print("%-22s %9.1fms %9.1fms %9.1fms %14.1f %16.1f%s" % ( + "%d,%d,%d" % (m, t, p), statistics.median(hashes), v50, v95, m / 1024, rate, mark)) diff --git a/student-tests/security/hardening_audit.py b/student-tests/security/hardening_audit.py new file mode 100644 index 0000000..5a65da1 --- /dev/null +++ b/student-tests/security/hardening_audit.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""lab7: аудит hardening запущенного стенда (docker inspect + docker exec), только stdlib. + + python student-tests/security/hardening_audit.py [--project messenger-lab7] + +Для каждого контейнера проекта: privileged, host network/pid/ipc, docker.sock, пользователь +(Config.User и фактический uid процесса), read_only, cap_drop/cap_add, no-new-privileges, +лимиты CPU/RAM/PIDs, опубликованные порты. Отдельно — сегментация сетей и отсутствие egress +у api/worker. Исключения документированы (EXCEPTIONS, REPORT.md) — всё прочее = FAIL. +""" +import argparse +import json +import subprocess +import sys + +ALLOWED_PORTS = {"lb": {("127.0.0.1", "8443")}, "grafana": {("127.0.0.1", "3000")}} +# service -> {проверка: обоснование}; прочие отклонения — ошибка. +EXCEPTIONS = { + "s3": {"root_user": "образ SeaweedFS работает от uid 0; capabilities сброшены (CapDrop ALL), rootfs read-only, " + "запись только в volume /data", + "cap_add": "каталог /data в образе принадлежит не root: возвращены только DAC_OVERRIDE, CHOWN, FOWNER"}, + "grafana": {"read_only": "совместимость Grafana с read-only rootfs не проверена на стенде; caps сброшены, " + "no-new-privileges, uid 472"}, + "tempo-init": {"cap_add": "one-shot chown volume (CAP_CHOWN), завершается до старта tempo", + "root_user": "one-shot chown volume", "limits": "one-shot, завершается за доли секунды", + "read_only": "one-shot chown volume, network_mode none"}, +} +NON_ROOT_REQUIRED = {"api", "worker", "lb", "broker-gate", "otel-gate", "s3-gate", "postgres", "redis", "broker", "openbao", + "bao-agent-api", "bao-agent-worker", "migrate", "s3-init", "prometheus", "loki", "tempo", + "otel-collector", "grafana"} + + +def sh(*cmd, check=True): + res = subprocess.run(list(cmd), capture_output=True, text=True) + if check and res.returncode != 0: + raise SystemExit("%s: %s" % (" ".join(cmd[:3]), res.stderr.strip()[-300:])) + return res + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--project", default="messenger-lab7") + a = p.parse_args() + ids = sh("docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + a.project).stdout.split() + if not ids: + raise SystemExit("no containers for project %s" % a.project) + infos = json.loads(sh("docker", "inspect", *ids).stdout) + fails, rows = [], [] + for c in sorted(infos, key=lambda x: x["Config"]["Labels"].get("com.docker.compose.service", "")): + labels = c["Config"]["Labels"] + svc = labels.get("com.docker.compose.service", "?") + if labels.get("com.docker.compose.oneoff") == "True": + continue + hc = c["HostConfig"] + exc = EXCEPTIONS.get(svc, {}) + problems = [] + running = c["State"]["Status"] == "running" + uid = "-" + if running: + top = sh("docker", "top", c["Id"], "-o", "uid", check=False) + if top.returncode == 0: + uid = ",".join(sorted(set(top.stdout.split()[1:]))) or "-" + else: + top = sh("docker", "top", c["Id"], check=False) + uid = ",".join(sorted({l.split()[0] for l in top.stdout.splitlines()[1:] if l.split()})) or "-" + user = c["Config"].get("User") or "root(image default)" + if hc.get("Privileged"): + problems.append("privileged") + if hc.get("NetworkMode") == "host" or hc.get("PidMode") == "host" or hc.get("IpcMode") == "host": + problems.append("host namespace") + for m in c.get("Mounts", []): + if "docker.sock" in (m.get("Source") or ""): + problems.append("docker.sock mounted") + if running and uid not in ("-",) and any(u in ("0", "root") for u in uid.split(",")) \ + and "root_user" not in exc and svc in NON_ROOT_REQUIRED: + problems.append("process runs as root (%s)" % uid) + if not hc.get("ReadonlyRootfs") and "read_only" not in exc: + problems.append("rootfs writable") + if "ALL" not in (hc.get("CapDrop") or []): + problems.append("cap_drop ALL missing") + if hc.get("CapAdd") and "cap_add" not in exc: + problems.append("cap_add %s" % hc.get("CapAdd")) + if not any("no-new-privileges" in o for o in (hc.get("SecurityOpt") or [])): + problems.append("no-new-privileges missing") + if (not hc.get("Memory") or not hc.get("NanoCpus") or not hc.get("PidsLimit")) and "limits" not in exc: + problems.append("limits mem=%s cpu=%s pids=%s" % (hc.get("Memory"), hc.get("NanoCpus"), hc.get("PidsLimit"))) + published = set() + for port, binds in (hc.get("PortBindings") or {}).items(): + for b in binds or []: + published.add((b.get("HostIp") or "0.0.0.0", b.get("HostPort"))) + if published and published - ALLOWED_PORTS.get(svc, set()): + problems.append("unexpected published ports %s" % sorted(published)) + rows.append((svc, c["Name"].lstrip("/"), user, uid, hc.get("ReadonlyRootfs"), ",".join(hc.get("CapDrop") or []), + ",".join(hc.get("CapAdd") or []) or "-", "%sM" % ((hc.get("Memory") or 0) // 2**20), + sorted(published) or "-", "; ".join(problems) or "ok", + "; ".join("%s: %s" % kv for kv in exc.items()))) + if problems: + fails.append((c["Name"], problems)) + print("%-16s %-34s %-22s %-8s %-6s %-6s %-8s %-6s %-26s %s" % ( + "service", "container", "Config.User", "uid", "ro", "drop", "add", "mem", "published", "result")) + for r in rows: + print("%-16s %-34s %-22s %-8s %-6s %-6s %-8s %-6s %-26s %s" % tuple(str(x)[:34] for x in r[:10])) + if r[10]: + print(" exception: " + r[10]) + # ---- сегментация и egress (изнутри api и worker) + probe = ("import socket,sys\n" + "res={}\n" + "for h,p in [('postgres',5432),('s3',8333),('broker',6380),('otel-collector',4318),('redis',6379)," + "('loki',3100),('tempo',3200),('prometheus',9090),('openbao',8200),('grafana',3000),('api',8443)," + "('1.1.1.1',443)]:\n" + " try:\n" + " socket.create_connection((h,p),timeout=2).close(); res[h]='reachable'\n" + " except socket.gaierror: res[h]='no-dns'\n" + " except OSError as e: res[h]='unreachable'\n" + "print(__import__('json').dumps(res))\n") + expected = { + "api": {"postgres": "reachable", "s3": "reachable", "broker": "reachable", "otel-collector": "reachable", + "redis": "reachable", "loki": "no-dns", "tempo": "no-dns", "prometheus": "no-dns", + "openbao": "no-dns", "grafana": "no-dns", "1.1.1.1": "unreachable"}, + "worker": {"postgres": "reachable", "s3": "reachable", "broker": "reachable", "otel-collector": "reachable", + "loki": "no-dns", "tempo": "no-dns", "prometheus": "no-dns", "openbao": "no-dns", + # api достижим по TCP (общие сети backend/storage/queue), но порт 8443 требует + # mTLS identity lb — сертификат worker отклоняется (tls-audit, lb -> api). + "grafana": "no-dns", "api": "reachable", "1.1.1.1": "unreachable"}, + } + print("\nnetwork segmentation / egress:") + for svc, exp in expected.items(): + cid = sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + a.project, + "--filter", "label=com.docker.compose.service=" + svc).stdout.split() + if not cid: + fails.append((svc, ["not running"])) + continue + got = json.loads(sh("docker", "exec", cid[0], "python", "-c", probe).stdout.strip().splitlines()[-1]) + bad = {h: (got.get(h), v) for h, v in exp.items() if got.get(h) != v + and not (v == "unreachable" and got.get(h) == "no-dns")} + print(" %-7s %s" % (svc, "ok" if not bad else "MISMATCH %s" % bad)) + print(" " + json.dumps(got)) + if bad: + fails.append((svc, ["segmentation %s" % bad])) + print("\n%s: %d container(s) with problems" % ("FAIL" if fails else "PASS", len(fails))) + for name, probs in fails: + print(" %s: %s" % (name, "; ".join(probs))) + return 1 if fails else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/security/leak_check.py b/student-tests/security/leak_check.py new file mode 100644 index 0000000..0a95073 --- /dev/null +++ b/student-tests/security/leak_check.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""lab7: в telemetry нет credentials — пароли, токены сессий, секреты БД/брокера/S3, hash паролей. + +Запуск в контейнере audit (сети telemetry + ingress; клиентский cert grafana для чтения): + python scripts/ops.py leak-check +1) Генерирует «канарейки»: регистрирует пользователя со случайным паролем через LB, делает + неверный вход (другой случайный пароль), верный вход (токен), запросы с токеном, + Basic на бизнес-endpoint и поддельный X-User-Id. +2) Ждёт экспорт OTLP (батчи 5 с, метрики 10 с) и ищет каждую канарейку и каждый рабочий + секрет стенда (пароли ролей БД/Redis/broker, S3 secret keys) в: + Loki (логи всех сервисов, текст и structured metadata), Tempo (полные трассы последних + минут), Prometheus (все имена и значения label'ов). +Код возврата 1 — найдено хоть одно совпадение (значение в отчёт не печатается). +""" +import base64 +import json +import os +import secrets +import ssl +import sys +import time +import urllib.parse +import urllib.request +from pathlib import Path + +SEC = Path(os.environ.get("AUDIT_SECRETS", "/run/secrets")) +CA = str(SEC / "ca_crt") +LB = os.environ.get("AUDIT_LB", "https://lb:8443") + + +def ctx(client=False): + c = ssl.create_default_context(cafile=CA) + if client: + c.load_cert_chain(str(SEC / "grafana_crt"), str(SEC / "grafana_key")) + return c + + +def call(method, path, body=None, headers=None): + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json"} if data else {} + h.update(headers or {}) + req = urllib.request.Request(LB + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, context=ctx(), timeout=10) as r: + return r.status, json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + return e.code, None + + +def get(url): + with urllib.request.urlopen(url, context=ctx(True), timeout=20) as r: + return r.read().decode("utf-8", "replace") + + +def main(): + begin = int(time.time()) + username = "leak_" + secrets.token_hex(4) + password = "canaryPW-" + secrets.token_urlsafe(18) + wrong = "canaryWRONG-" + secrets.token_urlsafe(18) + call("POST", "/api/v1/users", {"username": username, "display_name": "Leak", "password": password}) + call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(("%s:%s" % (username, wrong)).encode()).decode()}) + _, sess = call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(("%s:%s" % (username, password)).encode()).decode()}) + token = (sess or {}).get("token", "") + call("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token, "X-User-Id": secrets.token_hex(8)}) + call("GET", "/api/v1/chats", headers={"Authorization": "Basic " + base64.b64encode( + ("%s:%s" % (username, password)).encode()).decode()}) + call("DELETE", "/api/v1/auth/sessions/current", headers={"Authorization": "Bearer " + token}) + needles = {"user password": password, "wrong password": wrong, "session token": token} + for name in ("db_api_password", "db_worker_password", "cache_api_password", "broker_api_password", + "broker_worker_password", "s3_secret_access_key", "s3_worker_secret_access_key"): + p = SEC / name + if p.exists(): + needles[name] = p.read_text().strip() + patterns = {"argon2 hash": "$argon2id$", "private key": "PRIVATE KEY", "basic header": "Basic ", + "bearer header": "Bearer "} + time.sleep(int(os.environ.get("LEAK_WAIT_SECONDS", "25"))) + end = int(time.time()) + # Только окно этой проверки: положительный контроль доказывает, что найдены СВЕЖИЕ данные. + start = begin - 60 + findings = [] + # Loki + loki_lines = [] + for q in ('{service_name=~".+"}',): + url = "https://loki:3100/loki/api/v1/query_range?" + urllib.parse.urlencode( + {"query": q, "start": str(start * 10**9), "end": str(end * 10**9), "limit": "5000"}) + data = json.loads(get(url)) + for stream in data["data"]["result"]: + for _, line, *meta in stream["values"]: + loki_lines.append(line + json.dumps(meta) + json.dumps(stream.get("stream", {}))) + loki_text = "\n".join(loki_lines) + # Tempo + search = json.loads(get("https://tempo:3200/api/search?" + urllib.parse.urlencode( + {"limit": "100", "start": str(start), "end": str(end)}))) + traces = [] + for t in search.get("traces", [])[:100]: + traces.append(get("https://tempo:3200/api/traces/" + t["traceID"])) + tempo_text = "\n".join(traces) + # Prometheus: все label'ы и их значения + labels = json.loads(get("https://prometheus:9090/api/v1/labels"))["data"] + prom_text = "\n".join(get("https://prometheus:9090/api/v1/label/%s/values" % urllib.parse.quote(l)) + for l in labels) + sources = {"loki": loki_text, "tempo": tempo_text, "prometheus": prom_text} + for src, text in sources.items(): + for name, value in list(needles.items()) + list(patterns.items()): + if value and value in text: + findings.append({"source": src, "what": name}) + # Положительный контроль: поиск не «пустой» — телеметрия этого сценария действительно найдена. + controls = {"loki has messenger-api logs": "messenger-api" in loki_text, + "tempo has POST /api/v1/auth/sessions spans": "/api/v1/auth/sessions" in tempo_text, + "prometheus has http metrics labels": "http_route" in prom_text or "messenger" in prom_text} + report = {"ok": not findings and bool(token) and all(controls.values()), "controls": controls, + "canary_user_created": bool(token), + "searched": {"loki_lines": len(loki_lines), "tempo_traces": len(traces), "prometheus_labels": len(labels)}, + "needles": sorted(list(needles) + list(patterns)), "findings": findings} + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if report["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/security/login_flood.py b/student-tests/security/login_flood.py new file mode 100644 index 0000000..b49e36c --- /dev/null +++ b/student-tests/security/login_flood.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""lab7: бюджет login DoS — поток входов и влияние на остальные запросы. + + python student-tests/security/login_flood.py --base-url https://localhost:8443 --ca-file secrets/pki/ca.crt +Фаза 1 (baseline): латентность GET /users/me с действительным токеном. +Фаза 2 (flood): --concurrency потоков шлют POST /auth/sessions со случайными username +(Argon2id verify «пустышки» на каждый запрос) в течение --duration; параллельно меряется +тот же GET /users/me. Итог: коды ответов потока (401/429/503), RPS попыток входа, +p50/p95 обычного запроса до/во время потока. Ожидание: общий rate limit по IP режет поток +(429) до KDF, семафор KDF ограничивает CPU/RAM, обычные запросы не деградируют кратно. +""" +import argparse +import base64 +import collections +import http.client +import json +import ssl +import statistics +import threading +import time +import uuid +from urllib.parse import urlparse + +p = argparse.ArgumentParser() +p.add_argument("--base-url", default="https://localhost:8443") +p.add_argument("--ca-file", required=True) +p.add_argument("--duration", type=float, default=20) +p.add_argument("--concurrency", type=int, default=16) +a = p.parse_args() +U = urlparse(a.base_url) +CTX = ssl.create_default_context(cafile=a.ca_file) + + +def req(method, path, headers=None, body=None): + c = http.client.HTTPSConnection(U.hostname, U.port, context=CTX, timeout=30) + t0 = time.perf_counter() + try: + c.request(method, path, body=body, headers=headers or {}) + r = c.getresponse() + data = r.read() + return r.status, (time.perf_counter() - t0) * 1000, data + except OSError: + return 0, (time.perf_counter() - t0) * 1000, b"" + finally: + c.close() + + +def pct(xs, q): + xs = sorted(xs) + return xs[int(q * (len(xs) - 1))] if xs else float("nan") + + +user = "flood_" + uuid.uuid4().hex[:8] +pw = "flood-pw-" + uuid.uuid4().hex +req("POST", "/api/v1/users", {"Content-Type": "application/json"}, + json.dumps({"username": user, "display_name": "F", "password": pw})) +s, _, data = req("POST", "/api/v1/auth/sessions", + {"Authorization": "Basic " + base64.b64encode(("%s:%s" % (user, pw)).encode()).decode()}) +token = json.loads(data)["token"] +auth = {"Authorization": "Bearer " + token} +base = [req("GET", "/api/v1/users/me", auth)[1] for _ in range(30)] +stop = threading.Event() +codes = collections.Counter() +lock = threading.Lock() +probe = [] + + +def flooder(): + while not stop.is_set(): + name = "nouser_" + uuid.uuid4().hex[:10] + s, _, _ = req("POST", "/api/v1/auth/sessions", + {"Authorization": "Basic " + base64.b64encode(("%s:wrong-password-1234" % name).encode()).decode()}) + with lock: + codes[s] += 1 + + +def prober(): + while not stop.is_set(): + s, ms, _ = req("GET", "/api/v1/users/me", auth) + probe.append((s, ms)) + time.sleep(0.1) + + +threads = [threading.Thread(target=flooder) for _ in range(a.concurrency)] + [threading.Thread(target=prober)] +t0 = time.monotonic() +for t in threads: + t.start() +time.sleep(a.duration) +stop.set() +for t in threads: + t.join() +elapsed = time.monotonic() - t0 +ok_probe = [ms for s, ms in probe if s == 200] +print(json.dumps({ + "duration_s": round(elapsed, 1), "concurrency": a.concurrency, + "login_attempts": sum(codes.values()), "login_attempts_per_s": round(sum(codes.values()) / elapsed, 1), + "login_status_counts": dict(codes), + "baseline_users_me_ms": {"p50": round(statistics.median(base), 1), "p95": round(pct(base, 0.95), 1)}, + "during_flood_users_me_ms": {"p50": round(statistics.median(ok_probe), 1) if ok_probe else None, + "p95": round(pct(ok_probe, 0.95), 1) if ok_probe else None, + "non_200": sum(1 for s, _ in probe if s != 200), "samples": len(probe)}, +}, indent=2)) diff --git a/student-tests/security/mtls_probe.py b/student-tests/security/mtls_probe.py new file mode 100644 index 0000000..2d2f5df --- /dev/null +++ b/student-tests/security/mtls_probe.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""lab7: одна mTLS-проба реплики API заданным клиентским сертификатом (ротация/отзыв LB). + + python student-tests/security/mtls_probe.py --target api:8443 --server-name api --cert lb.pem --label new +Код возврата 0 — соединение принято и /health/ready = 200; 1 — отклонено (вывод — причина). +""" +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from tls_audit import CA, https # noqa: E402 + +parser = argparse.ArgumentParser() +parser.add_argument("--target", default="api:8443") +parser.add_argument("--server-name", default="api") +parser.add_argument("--cert", required=True) +parser.add_argument("--key") +parser.add_argument("--label", default="") +args = parser.parse_args() +host, port = args.target.rsplit(":", 1) +res = https(host, int(port), args.server_name, "/health/ready", args.cert, args.key, cafile=CA) +print(json.dumps({"label": args.label, "target": args.target, "accepted": res == ("http", 200), "observed": res})) +sys.exit(0 if res == ("http", 200) else 1) diff --git a/student-tests/security/security_http.py b/student-tests/security/security_http.py new file mode 100644 index 0000000..a46567e --- /dev/null +++ b/student-tests/security/security_http.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""lab7: отрицательные проверки на внешнем HTTPS через LB (только стандартная библиотека). + + python student-tests/security/security_http.py --base-url https://localhost:8443 --ca-file secrets/pki/ca.crt + +ACL и вложения: посторонний (eve) не читает чат/историю/metadata/content вложения, не пишет +в чужой чат и не прикрепляет чужой файл к своему чату; X-User-Id не меняет Bearer identity; +Basic на бизнес-endpoint — 401; logout отзывает токен (повтор 401). +Вход: общий для реплик rate limit -> 429 + Retry-After (ответы от >= 2 реплик), даже верный +пароль во время блокировки; ошибки не раскрывают credentials/внутренние детали. +Лимиты: JSON > 64 KiB -> 413 (LB, по Content-Length), chunked > 64 KiB -> 413 (API), +загрузка > 10 MiB -> 413 (API). TLS: неверное имя сервера и пустой trust store -> ошибка +проверки сертификата; plaintext на HTTPS-порт не обслуживается. Grafana — HTTPS с паролем. +Истечение сессии: python scripts/ops.py in-network expiry (экземпляр с TTL 2 с). +""" +import argparse +import base64 +import http.client +import json +import os +import socket +import ssl +import sys +import uuid +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parents[2] +parser = argparse.ArgumentParser() +parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) +parser.add_argument("--ca-file", default=os.environ.get("CA_FILE", str(ROOT / "secrets" / "pki" / "ca.crt"))) +parser.add_argument("--grafana-url", default=os.environ.get("GRAFANA_URL", "https://127.0.0.1:3000")) +args = parser.parse_args() +U = urlparse(args.base_url) +CTX = ssl.create_default_context(cafile=args.ca_file) +RESULTS = [] + + +def check(name, ok, detail=""): + ok = bool(ok) + RESULTS.append(ok) + print("%s %-70s %s" % ("PASS" if ok else "FAIL", name, str(detail)[:90]), flush=True) + + +def call(method, path, body=None, token=None, headers=None, raw=None, ctype="application/json", chunked=False): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) + if data is not None: + h["Content-Type"] = ctype + h.update(headers or {}) + conn = http.client.HTTPSConnection(U.hostname, U.port or 443, context=CTX, timeout=60) + try: + if chunked: + conn.putrequest(method, path) + for k, v in dict(h, **{"Transfer-Encoding": "chunked"}).items(): + conn.putheader(k, v) + conn.endheaders() + conn.send(b"%x\r\n%s\r\n0\r\n\r\n" % (len(data), data)) + else: + conn.request(method, path, body=data, headers=h) + r = conn.getresponse() + payload = r.read() + try: + js = json.loads(payload) if payload else None + except ValueError: + js = None + return r.status, js, {k.lower(): v for k, v in r.getheaders()}, payload + finally: + conn.close() + + +def basic(user, pw): + return {"Authorization": "Basic " + base64.b64encode(("%s:%s" % (user, pw)).encode()).decode()} + + +def actor(prefix): + username = "%s_%s" % (prefix, uuid.uuid4().hex[:10]) + pw = "pw-" + uuid.uuid4().hex + s, user, _, _ = call("POST", "/api/v1/users", {"username": username, "display_name": prefix, "password": pw}) + assert s == 201, s + s, sess, _, _ = call("POST", "/api/v1/auth/sessions", headers=basic(username, pw)) + assert s == 201, s + return {"id": user["id"], "username": username, "pw": pw, "token": sess["token"]} + + +def main(): + alice, bob, eve = actor("alice"), actor("bob"), actor("eve") + s, chat, _, _ = call("POST", "/api/v1/chats", {"title": "secret", "member_ids": [alice["id"], bob["id"]]}, + token=alice["token"]) + png = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + s, att, _, _ = call("POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], raw=png, + token=alice["token"], ctype="image/png") + check("setup: chat + attachment", s == 202 and att and att.get("id"), s) + cid, aid = chat["id"], att["id"] + # ---------------- ACL + for method, path, body in (("GET", "/api/v1/chats/%s" % cid, None), + ("GET", "/api/v1/chats/%s/messages" % cid, None), + ("POST", "/api/v1/chats/%s/messages" % cid, {"text": "intrusion"}), + ("GET", "/api/v1/attachments/%s" % aid, None), + ("GET", "/api/v1/attachments/%s/content" % aid, None)): + s, js, _, _ = call(method, path, body, token=eve["token"], headers={"Idempotency-Key": uuid.uuid4().hex}) + check("outsider %s %s -> 404 (no existence leak)" % (method, path.replace(cid, "{chat}").replace(aid, "{att}")), + s == 404 and js["error"]["code"] == "not_found", s) + s, own, _, _ = call("POST", "/api/v1/chats", {"title": "eve", "member_ids": [eve["id"]]}, token=eve["token"]) + s, js, _, _ = call("POST", "/api/v1/chats/%s/messages" % own["id"], {"text": "", "attachment_ids": [aid]}, + token=eve["token"], headers={"Idempotency-Key": uuid.uuid4().hex}) + check("outsider attaches foreign attachment to own chat -> 404", s == 404, s) + s, me, _, _ = call("GET", "/api/v1/users/me", token=eve["token"], headers={"X-User-Id": alice["id"]}) + check("X-User-Id cannot override Bearer identity", s == 200 and me["id"] == eve["id"], me and me.get("username")) + s, _, h, _ = call("GET", "/api/v1/users/me", headers={"X-User-Id": alice["id"]}) + check("X-User-Id alone -> 401 + WWW-Authenticate Bearer", s == 401 and h.get("www-authenticate", "").startswith( + "Bearer"), h.get("www-authenticate")) + s, _, _, _ = call("GET", "/api/v1/chats", headers=basic(alice["username"], alice["pw"])) + check("Basic on business endpoint -> 401", s == 401, s) + s, _, _, _ = call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex) + check("unknown/forged token -> 401", s == 401, s) + # ---------------- logout + s, _, _, _ = call("DELETE", "/api/v1/auth/sessions/current", token=bob["token"]) + s2, _, _, _ = call("GET", "/api/v1/users/me", token=bob["token"]) + s3, _, _, _ = call("DELETE", "/api/v1/auth/sessions/current", token=bob["token"]) + check("logout 204 -> token rejected 401 -> repeat logout 401", (s, s2, s3) == (204, 401, 401), (s, s2, s3)) + # ---------------- ошибки не раскрывают credentials + s, js, h, raw = call("POST", "/api/v1/auth/sessions", headers=basic(alice["username"], "wrong-password-xyz")) + text = raw.decode("utf-8", "replace") + check("wrong password -> 401, body has no username/password/hash", s == 401 and alice["username"] not in text + and "wrong-password" not in text and "argon" not in text.lower(), text[:80]) + s, js, _, raw = call("POST", "/api/v1/users", raw=b"{not json", ctype="application/json") + text = raw.decode("utf-8", "replace") + check("malformed JSON -> 400 without stack trace/SQL", s == 400 and "Traceback" not in text and "SELECT" not in text + and js["error"]["request_id"], text[:80]) + # ---------------- rate limit входа (общий для реплик) + victim = actor("rl") + statuses, instances, retry = [], set(), None + for _ in range(12): + s, js, h, _ = call("POST", "/api/v1/auth/sessions", headers=basic(victim["username"], "wrong-" + uuid.uuid4().hex)) + statuses.append(s) + instances.add(h.get("x-instance-id")) + if s == 429: + retry = h.get("retry-after") + code = js["error"]["code"] + check("login brute force -> 429 rate_limited + Retry-After", 429 in statuses and retry and retry.isdigit() + and code == "rate_limited", "%s Retry-After=%s" % (statuses, retry)) + check("rate limit is shared: responses from >= 2 replicas", len(instances - {None}) >= 2, sorted(instances - {None})) + s, _, _, _ = call("POST", "/api/v1/auth/sessions", headers=basic(victim["username"], victim["pw"])) + check("correct password while locked out -> 429 (no oracle)", s == 429, s) + # ---------------- лимиты размера + big = json.dumps({"username": "x", "display_name": "y" * 70000, "password": "p" * 20}).encode() + s, js, h, _ = call("POST", "/api/v1/users", raw=big) + check("JSON body 70 KB -> 413 payload_too_large from LB (before any replica)", + s == 413 and js["error"]["code"] == "payload_too_large" and not h.get("x-instance-id"), + "instance=%s" % h.get("x-instance-id")) + s, js, h, _ = call("POST", "/api/v1/users", raw=big, chunked=True) + check("chunked JSON 70 KB -> 413 payload_too_large (API)", s == 413 and js["error"]["code"] == "payload_too_large", + "instance=%s" % h.get("x-instance-id")) + huge = png + b"\0" * (10 * 1024 * 1024 + 1 - len(png)) + s, js, _, _ = call("POST", "/api/v1/chats/%s/attachments?filename=big.png" % cid, raw=huge, token=alice["token"], + ctype="image/png") + check("upload 10 MiB + 1 byte -> 413", s == 413, s) + # ---------------- TLS + try: + raw_sock = socket.create_connection((U.hostname, U.port or 443), timeout=5) + CTX.wrap_socket(raw_sock, server_hostname="evil.example") + check("wrong server name -> certificate verification error", False, "handshake succeeded") + except ssl.SSLCertVerificationError as exc: + check("wrong server name -> certificate verification error", True, exc.verify_message) + empty = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + try: + empty.wrap_socket(socket.create_connection((U.hostname, U.port or 443), timeout=5), server_hostname=U.hostname) + check("empty trust store -> verification error", False, "handshake succeeded") + except ssl.SSLCertVerificationError as exc: + check("empty trust store -> verification error", True, exc.verify_message) + try: + conn = http.client.HTTPConnection(U.hostname, U.port or 443, timeout=5) + conn.request("GET", "/health/ready") + status = conn.getresponse().status + check("plaintext HTTP on HTTPS port is not served", not 200 <= status < 300, status) + except (OSError, http.client.HTTPException) as exc: + check("plaintext HTTP on HTTPS port is not served", True, type(exc).__name__) + # ---------------- Grafana (оператор, локально) + g = urlparse(args.grafana_url) + try: + conn = http.client.HTTPSConnection(g.hostname, g.port, context=CTX, timeout=5) + conn.request("GET", "/api/dashboards/home") + status = conn.getresponse().status + check("grafana over HTTPS requires authentication", status == 401, status) + except OSError as exc: + check("grafana over HTTPS requires authentication (skipped: not reachable)", True, type(exc).__name__) + ok = all(RESULTS) + print("%s: %d/%d checks" % ("PASS" if ok else "FAIL", sum(RESULTS), len(RESULTS))) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/security/tls_audit.py b/student-tests/security/tls_audit.py new file mode 100644 index 0000000..cc606d0 --- /dev/null +++ b/student-tests/security/tls_audit.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""lab7: TLS/mTLS/identity/ACL-аудит ВСЕЙ заявленной матрицы (contracts/trust-matrix.md). + +Запуск (контейнер `audit` подключён к внутренним сетям как посторонний клиент и получает +только тестовые материалы оператора — CA, intruder и сертификаты/пароли ролей): + python scripts/ops.py tls-audit # = docker compose --profile audit run --rm audit student-tests/security/tls_audit.py + +Для каждой связи: положительная проверка (правильный CA/имя/identity -> успех) и +отрицательные: нет клиентского сертификата, чужой CA (с «правильным» CN), сертификат +нашего CA с запрещённой identity (intruder и чужие роли), неверное имя сервера, +plaintext на TLS-порт, ACL-нарушения. Отказ засчитывается только как TLS/соединение/ +авторизационная ошибка — таймаут считается провалом проверки (не доказывает отказ). +Вывод — JSON; код возврата 1, если хоть одна проверка не совпала с ожиданием. +""" +import datetime as dt +import json +import os +import socket +import ssl +import sys +import tempfile +import time +from pathlib import Path + +SEC = Path(os.environ.get("AUDIT_SECRETS", "/run/secrets")) +CA = str(SEC / "ca_crt") +RESULTS = [] +# Порты можно переопределить (AUDIT_PORT_LB=...) — только для запуска вне Docker. +PORT = {k: int(os.environ.get("AUDIT_PORT_" + k, v)) for k, v in + {"LB": 8443, "API": 8443, "CACHE": 6379, "SELF": 8888}.items()} + + +def f(name): + return str(SEC / name) + + +def secret(name): + return (SEC / name).read_text(encoding="utf-8").strip() + + +def record(link, check, expected, ok, detail=""): + RESULTS.append({"link": link, "check": check, "expected": expected, "ok": bool(ok), "observed": str(detail)[:200]}) + + +# ---------------------------------------------------------------- материалы + +def rogue_identity(cn, sans): + """Чужой CA и leaf с «правильным» CN/SAN (например, lb) — должен быть отклонён цепочкой.""" + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + now = dt.datetime.now(dt.timezone.utc) + ca_key = ec.generate_private_key(ec.SECP256R1()) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "Rogue CA")]) + ca = (x509.CertificateBuilder().subject_name(ca_name).issuer_name(ca_name).public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()).not_valid_before(now - dt.timedelta(minutes=5)) + .not_valid_after(now + dt.timedelta(days=1)) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True).sign(ca_key, hashes.SHA256())) + key = ec.generate_private_key(ec.SECP256R1()) + leaf = (x509.CertificateBuilder().subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, cn)])) + .issuer_name(ca_name).public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now - dt.timedelta(minutes=5)).not_valid_after(now + dt.timedelta(days=1)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName(s) for s in sans]), critical=False) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.CLIENT_AUTH, ExtendedKeyUsageOID.SERVER_AUTH]), + critical=False) + .sign(ca_key, hashes.SHA256())) + d = Path(tempfile.mkdtemp(prefix="rogue-")) + (d / "ca.crt").write_bytes(ca.public_bytes(serialization.Encoding.PEM)) + (d / "leaf.crt").write_bytes(leaf.public_bytes(serialization.Encoding.PEM)) + (d / "leaf.key").write_bytes(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, + serialization.NoEncryption())) + return str(d / "leaf.crt"), str(d / "leaf.key"), str(d / "ca.crt") + + +# ---------------------------------------------------------------- транспорт + +def https(host, port, server_name, path="/", cert=None, key=None, cafile=CA, method="GET", body=b"", + ctype=None, timeout=5): + """-> (kind, value): kind in http|verify_error|tls_error|reset|closed|refused|dns_error|timeout.""" + ctx = ssl.create_default_context(cafile=cafile) if cafile else ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if not cafile: + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = True + if cert: + ctx.load_cert_chain(cert, key) + try: + raw = socket.create_connection((host, port), timeout=timeout) + except socket.gaierror as exc: + return "dns_error", exc.strerror + except ConnectionRefusedError: + return "refused", "connection refused" + except socket.timeout: + return "timeout", "connect timeout" + except OSError as exc: + return "refused", exc.strerror + try: + s = ctx.wrap_socket(raw, server_hostname=server_name) + head = "%s %s HTTP/1.1\r\nHost: %s\r\nConnection: close\r\nContent-Length: %d\r\n" % ( + method, path, server_name, len(body)) + if ctype: + head += "Content-Type: %s\r\n" % ctype + s.sendall(head.encode() + b"\r\n" + body) + data = b"" + while True: + chunk = s.recv(65536) + if not chunk: + break + data += chunk + if data.startswith(b"HTTP/"): + return "http", int(data.split(b" ", 2)[1]) + return "closed", "connection closed without HTTP response" + except ssl.SSLCertVerificationError as exc: + return "verify_error", exc.verify_message + except ssl.SSLError as exc: + return "tls_error", exc.reason or str(exc) + except (ConnectionResetError, BrokenPipeError, ConnectionAbortedError) as exc: + return "reset", type(exc).__name__ + except socket.timeout: + return "timeout", "read timeout" + finally: + raw.close() + + +def https_body(host, port, server_name, path, cert, key): + """Текст ответа (причина 503 у Loki/Tempo /ready) — для диагностики в отчёте.""" + import urllib.request + ctx = ssl.create_default_context(cafile=CA) + ctx.load_cert_chain(cert, key) + try: + urllib.request.urlopen("https://%s:%d%s" % (server_name, port, path), context=ctx, timeout=5) + return "" + except Exception as exc: # HTTPError 503 содержит тело с причиной + body = getattr(exc, "read", lambda: b"")() + return body.decode("utf-8", "replace").strip()[:150] + + +def plaintext(host, port, timeout=4): + """HTTP без TLS на TLS-порт: не должно получиться успешного HTTP-ответа 2xx.""" + try: + s = socket.create_connection((host, port), timeout=timeout) + except OSError as exc: + return "refused", str(exc) + try: + s.sendall(b"GET / HTTP/1.0\r\nHost: x\r\n\r\n") + data = s.recv(4096) + if data.startswith(b"HTTP/"): + return "http", int(data.split(b" ", 2)[1]) + return "closed", repr(data[:20]) + except OSError as exc: + return "reset", type(exc).__name__ + finally: + s.close() + + +REJECTED = ("verify_error", "tls_error", "reset", "closed") + + +def expect_rejected(link, check, res): + record(link, check, "rejected (TLS/handshake/close)", res[0] in REJECTED, res) + + +def expect_status(link, check, res, status=200): + record(link, check, "HTTP %d" % status, res == ("http", status), res) + + +def expect_no_success(link, check, res): + ok = not (res[0] == "http" and 200 <= res[1] < 300) + record(link, check, "no successful plaintext response", ok, res) + + +# ---------------------------------------------------------------- связи + +def audit_ingress(): + link = "client -> lb (public TLS)" + expect_status(link, "trusted CA + hostname lb/localhost SAN", https("lb", PORT["LB"], "lb", "/health/ready")) + res = https("lb", PORT["LB"], "lb", "/health/ready", cafile=None) + record(link, "empty trust store", "verify_error", res[0] == "verify_error", res) + res = https("lb", PORT["LB"], "evil.example", "/health/ready") + record(link, "wrong hostname", "verify_error", res[0] == "verify_error", res) + expect_no_success(link, "plaintext HTTP on :8443", plaintext("lb", PORT["LB"])) + res = https("lb", 8404, "lb", "/stats") + record(link, "HAProxy stats not exposed on network", "refused", res[0] in ("refused", "timeout"), res) + + +def audit_lb_api(rogue): + link = "lb -> api (mTLS, allowlist lb + CRL)" + expect_status(link, "identity lb (allowed)", https("api", PORT["API"], "api", "/health/ready", f("lb_client_pem"))) + expect_rejected(link, "no client certificate", https("api", PORT["API"], "api", "/health/ready")) + expect_rejected(link, "rogue CA with CN=lb", https("api", PORT["API"], "api", "/health/ready", rogue[0], rogue[1])) + expect_rejected(link, "same CA, identity intruder", https("api", PORT["API"], "api", "/health/ready", + f("intruder_crt"), f("intruder_key"))) + expect_rejected(link, "same CA, identity worker", https("api", PORT["API"], "api", "/health/ready", + f("worker_crt"), f("worker_key"))) + expect_rejected(link, "same CA, identity api (replica as client)", https("api", PORT["API"], "api", "/health/ready", + f("api_crt"), f("api_key"))) + expect_no_success(link, "plaintext HTTP on :8443", plaintext("api", PORT["API"])) + res = plaintext("api", 8081) + record(link, "health listener 8081 not reachable from network (loopback only)", "refused", + res[0] == "refused", res) + res = https("api", PORT["API"], "evil.example", "/health/ready", f("lb_client_pem")) + record(link, "LB-side check: server name must be api", "verify_error", res[0] == "verify_error", res) + + +def audit_postgres(rogue): + import psycopg + link = "api/worker -> postgres (TLS verify-full, roles)" + api_pw, wrk_pw = secret("db_api_password"), secret("db_worker_password") + + def connect(user, pw, sslmode="verify-full", root=CA, host="postgres"): + return psycopg.connect(host=host, dbname="messenger", user=user, password=pw, sslmode=sslmode, + sslrootcert=root, connect_timeout=5) + + try: + with connect("messenger_api", api_pw) as c: + v = c.execute("SELECT ssl, version FROM pg_stat_ssl WHERE pid = pg_backend_pid()").fetchone() + record(link, "messenger_api verify-full", "TLS session", v and v[0], v) + except Exception as exc: + record(link, "messenger_api verify-full", "TLS session", False, exc) + for name, kwargs, needle in ( + ("plaintext (sslmode=disable)", dict(sslmode="disable"), "no encryption"), + ("rogue CA as trust root", dict(root=rogue[2]), "certificate verify failed"), + ("wrong hostname (IP instead of name)", dict(host=socket.gethostbyname("postgres")), "does not match")): + try: + connect("messenger_api", api_pw, **kwargs).close() + record(link, name, "rejected", False, "connected") + except Exception as exc: + record(link, name, "rejected", needle in str(exc), str(exc).splitlines()[0]) + for user, pw, sql, allowed in ( + ("messenger_worker", wrk_pw, "SELECT count(*) FROM attachments", True), + ("messenger_worker", wrk_pw, "SELECT password_hash FROM users LIMIT 1", False), + ("messenger_worker", wrk_pw, "SELECT token_hash FROM sessions LIMIT 1", False), + ("messenger_api", api_pw, "CREATE TABLE audit_ddl (i int)", False), + ("messenger_api", api_pw, "TRUNCATE users", False)): + try: + with connect(user, pw) as c: + c.execute(sql) + c.rollback() + record(link, "%s: %s" % (user, sql), "allowed" if allowed else "permission denied", allowed, "executed") + except Exception as exc: + record(link, "%s: %s" % (user, sql), "allowed" if allowed else "permission denied", + (not allowed) and "permission denied" in str(exc), str(exc).splitlines()[0]) + + +def redis_client(host, port, cert=None, key=None, user=None, password=None, tls=True, cafile=CA): + import redis + kw = dict(host=host, port=port, socket_timeout=4, socket_connect_timeout=4, username=user, password=password) + if tls: + kw.update(ssl=True, ssl_ca_certs=cafile, ssl_cert_reqs="required", ssl_check_hostname=True) + if cert: + kw.update(ssl_certfile=cert, ssl_keyfile=key) + return redis.Redis(**kw) + + +def redis_try(client, *cmd): + import redis + try: + return "ok", client.execute_command(*cmd) + except redis.exceptions.NoPermissionError as exc: + return "noperm", str(exc) + except redis.exceptions.AuthenticationError as exc: + return "noauth", str(exc) + except redis.exceptions.ResponseError as exc: + text = str(exc) + return ("noauth" if "NOAUTH" in text or "WRONGPASS" in text else + "noperm" if "NOPERM" in text else "error"), text + except (redis.exceptions.ConnectionError, redis.exceptions.TimeoutError, OSError) as exc: + return "conn_error", str(exc)[:120] + + +def audit_cache(rogue): + link = "api -> redis cache (mTLS + ACL)" + pw = secret("cache_api_password") + api = redis_client("redis", PORT["CACHE"], f("api_crt"), f("api_key"), "messenger_api", pw) + res = redis_try(api, "SET", "sess:audit-probe", "1", "PX", "1000") + record(link, "messenger_api SET sess:*", "ok", res[0] == "ok", res) + for cmd in (("SET", "other:key", "1"), ("FLUSHALL",), ("CONFIG", "GET", "*"), ("KEYS", "*")): + res = redis_try(api, *cmd) + record(link, "messenger_api %s" % " ".join(cmd), "NOPERM", res[0] == "noperm", res) + res = redis_try(redis_client("redis", PORT["CACHE"], f("api_crt"), f("api_key")), "GET", "sess:x") + record(link, "cert without AUTH (default user off)", "NOAUTH", res[0] == "noauth", res) + res = redis_try(redis_client("redis", PORT["CACHE"], user="messenger_api", password=pw), "PING") + record(link, "no client certificate", "connection rejected", res[0] == "conn_error", res) + res = redis_try(redis_client("redis", PORT["CACHE"], rogue[0], rogue[1], "messenger_api", pw), "PING") + record(link, "rogue CA client certificate", "connection rejected", res[0] == "conn_error", res) + res = redis_try(redis_client("redis", PORT["CACHE"], user="messenger_api", password=pw, tls=False), "PING") + record(link, "plaintext", "connection rejected", res[0] == "conn_error", res) + + +def audit_broker(rogue): + link = "api/worker -> broker-gate (mTLS allowlist api|worker) -> broker ACL" + ap, wp = secret("broker_api_password"), secret("broker_worker_password") + for name, cert, key in (("no client certificate", None, None), ("rogue CA with CN=api", rogue[0], rogue[1]), + ("same CA, identity intruder", f("intruder_crt"), f("intruder_key")), + ("same CA, identity lb", None, None), ("same CA, identity prometheus", + f("prometheus_crt"), f("prometheus_key"))): + if name.endswith("identity lb"): + cert = key = f("lb_client_pem") + res = redis_try(redis_client("broker", 6380, cert, key, "api_producer", ap), "PING") + record(link, name + " (with valid producer password)", "connection rejected", res[0] == "conn_error", res) + res = redis_try(redis_client("broker", 6380, f("api_crt"), f("api_key")), "PING") + record(link, "identity api without AUTH", "NOAUTH", res[0] == "noauth", res) + producer = redis_client("broker", 6380, f("api_crt"), f("api_key"), "api_producer", ap) + res = redis_try(producer, "PING") + record(link, "api_producer PING", "ok", res[0] == "ok", res) + for cmd in (("XREADGROUP", "GROUP", "attachment-workers", "audit", "COUNT", "1", "STREAMS", "attachments.jobs", ">"), + ("XACK", "attachments.jobs", "attachment-workers", "0-1"), ("XDEL", "attachments.jobs", "0-1"), + ("DEL", "attachments.jobs"), ("FLUSHALL",), ("XADD", "audit.other", "*", "k", "v")): + res = redis_try(producer, *cmd) + record(link, "api_producer %s" % cmd[0] + (" other key" if cmd[1:2] == ("audit.other",) else ""), + "NOPERM", res[0] == "noperm", res) + consumer = redis_client("broker", 6380, f("worker_crt"), f("worker_key"), "worker_consumer", wp) + res = redis_try(consumer, "XLEN", "attachments.jobs") + record(link, "worker_consumer XLEN", "ok", res[0] == "ok", res) + for cmd in (("SET", "x", "1"), ("FLUSHALL",), ("CONFIG", "GET", "*"), ("DEL", "attachments.jobs")): + res = redis_try(consumer, *cmd) + record(link, "worker_consumer %s" % cmd[0], "NOPERM", res[0] == "noperm", res) + res = redis_try(redis_client("broker", 6379, user="api_producer", password=ap, tls=False), "PING") + record(link, "Redis port 6379 not reachable from network (loopback only)", "connection refused", + res[0] == "conn_error", res) + res = redis_try(redis_client("broker", 6380, user="api_producer", password=ap, tls=False), "PING") + record(link, "plaintext to gate", "connection rejected", res[0] == "conn_error", res) + + +def audit_s3(): + import boto3 + from botocore.config import Config + from botocore.exceptions import ClientError + link = "api/worker -> s3 (TLS, scoped identities)" + expect_status(link, "TLS verify-full GET /healthz", https("s3", 8333, "s3", "/healthz")) + res = https("s3", 8333, "evil.example", "/healthz") + record(link, "wrong hostname", "verify_error", res[0] == "verify_error", res) + expect_no_success(link, "plaintext on :8333", plaintext("s3", 8333)) + for port, what in ((8334, "S3 API behind gate"), (18334, "S3 gateway gRPC (unauthenticated)"), + (18333, "S3 gRPC default port"), (9333, "master"), (8888, "filer"), (18888, "filer gRPC")): + res = plaintext("s3", port) + record(link, "%s :%d not reachable from network" % (what, port), "refused", res[0] == "refused", res) + + def client(prefix): + return boto3.client("s3", endpoint_url="https://s3:8333", region_name="us-east-1", verify=CA, + aws_access_key_id=secret(prefix + "access_key_id"), + aws_secret_access_key=secret(prefix + "secret_access_key"), + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, + retries={"max_attempts": 1})) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + api, wrk = client("s3_"), client("s3_worker_") + cases = ( + ("api PUT staging/", api, lambda c: c.put_object(Bucket=bucket, Key="staging/_audit-probe", Body=b"a"), True), + ("api DELETE staging/", api, lambda c: c.delete_object(Bucket=bucket, Key="staging/_audit-probe"), True), + ("api PUT final/ (worker prefix)", api, + lambda c: c.put_object(Bucket=bucket, Key="final/_audit-forged", Body=b"a"), False), + ("api LIST bucket", api, lambda c: c.list_objects_v2(Bucket=bucket, MaxKeys=1), False), + ("api CREATE bucket", api, lambda c: c.create_bucket(Bucket="audit-forbidden"), False), + ("worker LIST bucket", wrk, lambda c: c.list_objects_v2(Bucket=bucket, MaxKeys=1), True), + ("worker PUT outside its prefixes", wrk, lambda c: c.put_object(Bucket=bucket, Key="_other/_audit", Body=b"a"), + False), + ("worker DELETE bucket", wrk, lambda c: c.delete_bucket(Bucket=bucket), False), + ) + for label, c, op, allowed in cases: + try: + op(c) + record(link, label, "allowed" if allowed else "AccessDenied", allowed, "allowed") + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + record(link, label, "allowed" if allowed else "AccessDenied", + (not allowed) and code in ("AccessDenied", "Forbidden"), code) + + +def audit_collector(rogue): + link = "api/worker -> otel-gate (mTLS allowlist api|worker) -> collector" + body, ct = b"{}", "application/json" + for name, cert, key in (("identity api", f("api_crt"), f("api_key")), + ("identity worker", f("worker_crt"), f("worker_key"))): + expect_status(link, name + " POST /v1/logs", https("otel-collector", 4318, "otel-collector", "/v1/logs", + cert, key, method="POST", body=body, ctype=ct)) + for name, cert, key in (("no client certificate", None, None), ("rogue CA with CN=api", rogue[0], rogue[1]), + ("same CA, identity intruder", f("intruder_crt"), f("intruder_key")), + ("same CA, identity prometheus", f("prometheus_crt"), f("prometheus_key")), + ("same CA, identity lb", f("lb_client_pem"), None)): + expect_rejected(link, name + " POST /v1/logs", https("otel-collector", 4318, "otel-collector", "/v1/logs", + cert, key, method="POST", body=body, ctype=ct)) + expect_no_success(link, "plaintext on :4318", plaintext("otel-collector", 4318)) + link2 = "prometheus -> otel-gate metrics (allowlist prometheus)" + expect_status(link2, "identity prometheus /metrics :8889", + https("otel-collector", 8889, "otel-collector", "/metrics", f("prometheus_crt"), f("prometheus_key"))) + expect_status(link2, "identity prometheus /metrics :8888 (self)", + https("otel-collector", PORT["SELF"], "otel-collector", "/metrics", f("prometheus_crt"), f("prometheus_key"))) + expect_rejected(link2, "identity api /metrics :8889", + https("otel-collector", 8889, "otel-collector", "/metrics", f("api_crt"), f("api_key"))) + for port in (14318, 18889, 18888, 13133): + res = plaintext("otel-collector", port) + record(link2, "collector loopback port %d not reachable" % port, "refused", res[0] == "refused", res) + + +def audit_backends(rogue): + for host, port, path, link in ( + ("loki", 3100, "/ready", "collector/grafana -> loki (mTLS)"), + ("tempo", 3200, "/ready", "grafana -> tempo (mTLS)"), + ("prometheus", 9090, "/-/ready", "grafana -> prometheus (mTLS)")): + # Loki/Tempo после старта несколько десятков секунд отвечают 503 (кольцо ingester'а + # прогревается). Это не про TLS: ждём готовности до 90 с, иначе показываем причину. + res = https(host, port, host, path, f("grafana_crt"), f("grafana_key")) + deadline = time.monotonic() + 90 + while res == ("http", 503) and time.monotonic() < deadline: + time.sleep(5) + res = https(host, port, host, path, f("grafana_crt"), f("grafana_key")) + if res == ("http", 503): + res = ("http", 503, https_body(host, port, host, path, f("grafana_crt"), f("grafana_key"))) + expect_status(link, "client cert grafana (TLS ok, service ready)", res) + expect_rejected(link, "no client certificate", https(host, port, host, path)) + expect_rejected(link, "rogue CA client certificate", https(host, port, host, path, rogue[0], rogue[1])) + expect_no_success(link, "plaintext", plaintext(host, port)) + link = "collector -> tempo OTLP gRPC (mTLS)" + expect_rejected(link, "no client certificate", https("tempo", 4317, "tempo", "/")) + + +def audit_openbao(): + link = "agents -> openbao (TLS + AppRole)" + res = https("openbao", 8200, "openbao", "/v1/sys/health") + record(link, "TLS verify-full /v1/sys/health", "HTTP 200 (unsealed)", res == ("http", 200), res) + res = https("openbao", 8200, "evil.example", "/v1/sys/health") + record(link, "wrong hostname", "verify_error", res[0] == "verify_error", res) + expect_no_success(link, "plaintext on :8200", plaintext("openbao", 8200)) + res = https("openbao", 8200, "openbao", "/v1/kv/data/messenger/api") + record(link, "read KV without token", "HTTP 403", res == ("http", 403), res) + + +def main(): + rogue = rogue_identity("lb", ["lb", "api"]) + for fn in (audit_ingress, lambda: audit_lb_api(rogue), lambda: audit_postgres(rogue), lambda: audit_cache(rogue), + lambda: audit_broker(rogue), audit_s3, lambda: audit_collector(rogue), lambda: audit_backends(rogue), + audit_openbao): + try: + fn() + except Exception as exc: # одна сломанная связь не прячет остальные + record("audit", getattr(fn, "__name__", "section"), "no exception", False, repr(exc)) + failed = [r for r in RESULTS if not r["ok"]] + print(json.dumps({"ok": not failed, "total": len(RESULTS), "failed": len(failed), "checks": RESULTS}, + ensure_ascii=False, indent=2)) + for r in RESULTS: + print("%s %-62s %-52s %s" % ("PASS" if r["ok"] else "FAIL", r["link"][:62], r["check"][:52], r["observed"][:60]), + file=sys.stderr) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..98a246d --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,139 @@ +"""Поведенческие тесты API, перенесённые из lab1 и переведённые на Bearer.""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import base64 +import json +import unittest +import uuid +from urllib.parse import urlencode + +from common import actor, create_chat, request + + +class ApiBehaviorTests(unittest.TestCase): + + def test_malformed_json_returns_400(self): + r = request("POST", "/api/v1/users", raw=b'{"username": ') + self.assertEqual(r.status, 400, r.json) + self.assertIn("request_id", r.json["error"]) + self.assertEqual(r.json["error"]["request_id"], r.headers.get("X-Request-Id")) + + def test_registration_without_password_returns_400(self): + r = request("POST", "/api/v1/users", payload={ + "username": "nopass_" + uuid.uuid4().hex[:10], + "display_name": "No password", + }) + self.assertEqual(r.status, 400, r.json) + + def test_short_password_returns_400(self): + r = request("POST", "/api/v1/users", payload={ + "username": "short_" + uuid.uuid4().hex[:10], + "display_name": "Short", + "password": "elevenchars", + }) + self.assertEqual(r.status, 400, r.json) + + def test_password_never_returned(self): + r = request("POST", "/api/v1/users", payload={ + "username": "secret_" + uuid.uuid4().hex[:10], + "display_name": "Secret", + "password": "Very-secret-password-42", + }) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(set(r.json), {"id", "username", "display_name", "created_at"}) + + def test_duplicate_username_returns_409(self): + payload = { + "username": "duplicate_" + uuid.uuid4().hex[:10], + "display_name": "First User", + "password": "Duplicate-pass-123", + } + self.assertEqual(request("POST", "/api/v1/users", payload=payload).status, 201) + self.assertEqual(request("POST", "/api/v1/users", payload=payload).status, 409) + + def test_empty_message_returns_400(self): + owner = actor() + chat = create_chat(owner) + r = request("POST", "/api/v1/chats/%s/messages" % chat["id"], + token=owner["_token"], payload={"text": " "}) + self.assertEqual(r.status, 400, r.json) + + def test_sender_id_in_body_rejected(self): + owner = actor() + chat = create_chat(owner) + r = request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=owner["_token"], + payload={"text": "hi", "sender_id": owner["id"]}) + self.assertEqual(r.status, 400, r.json) + + def test_outsider_cannot_access_chat(self): + owner, member, outsider = actor(), actor(), actor() + chat = create_chat(owner, member) + chat_path = "/api/v1/chats/%s" % chat["id"] + self.assertEqual(request("GET", chat_path, token=outsider["_token"]).status, 404) + self.assertEqual(request("GET", chat_path + "/messages", token=outsider["_token"]).status, 404) + r = request("POST", chat_path + "/messages", token=outsider["_token"], + payload={"text": "Unauthorized message"}) + self.assertEqual(r.status, 404) + self.assertEqual(request("GET", chat_path, token=member["_token"]).status, 200) + + def test_naive_timestamp_cursor_returns_400(self): + owner = actor() + chat = create_chat(owner) + payload = { + "resource_id": chat["id"], + "created_at": "2026-10-02T12:00:00", + "item_id": str(uuid.uuid4()), + } + encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + r = request("GET", "/api/v1/chats/%s/messages?%s" % (chat["id"], urlencode({"cursor": encoded})), + token=owner["_token"]) + self.assertEqual(r.status, 400, r.json) + + def test_cursor_from_other_chat_returns_400(self): + owner = actor() + chat_a, chat_b = create_chat(owner), create_chat(owner) + for i in range(3): + request("POST", "/api/v1/chats/%s/messages" % chat_a["id"], token=owner["_token"], + payload={"text": "m%d" % i}) + page = request("GET", "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], token=owner["_token"]).json + self.assertTrue(page["next_cursor"]) + r = request("GET", "/api/v1/chats/%s/messages?%s" % (chat_b["id"], urlencode({"cursor": page["next_cursor"]})), + token=owner["_token"]) + self.assertEqual(r.status, 400, r.json) + + def test_pagination_returns_all_messages_without_duplicates(self): + owner = actor() + chat = create_chat(owner) + path = "/api/v1/chats/%s/messages" % chat["id"] + expected = [] + for index in range(8): + r = request("POST", path, token=owner["_token"], payload={"text": "page-%d" % index}) + self.assertEqual(r.status, 201, r.json) + expected.append(r.json["id"]) + received, cursor = [], None + while True: + query = {"limit": 3} + if cursor: + query["cursor"] = cursor + page = request("GET", path + "?" + urlencode(query), token=owner["_token"]).json + received.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(received, expected) + + def test_chat_list_pagination(self): + owner = actor() + created = [create_chat(owner)["id"] for _ in range(5)] + got, cursor = [], None + while True: + q = "?limit=2" + ("&" + urlencode({"cursor": cursor}) if cursor else "") + page = request("GET", "/api/v1/chats" + q, token=owner["_token"]).json + got.extend(c["id"] for c in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(got, created) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_attachment_consistency.py b/student-tests/test_attachment_consistency.py new file mode 100644 index 0000000..b3153b6 --- /dev/null +++ b/student-tests/test_attachment_consistency.py @@ -0,0 +1,311 @@ +"""Согласованность PostgreSQL <-> S3 для вложений (нужен прямой доступ к БД и S3). + +lab5: API пишет байты в staging/ (а не originals/), строка pending ссылается на +staging_key; успешная загрузка отвечает 202. Логика сборщика та же. + +Запуск внутри compose-сети (БД и S3 наружу не публикуются): + python scripts/ops.py in-network consistency +Без DATABASE_URL(_FILE) и S3-настроек тесты пропускаются. + +Что доказывается: + * сбой между S3 PUT и COMMIT (pending + объект) обнаруживается dry-run'ом, + исправляется --apply, повторный запуск ничего не меняет (идемпотентность); + * свежие pending (моложе grace) и ready/используемые вложения не удаляются; + * объект-сирота (нет строки) удаляется только после grace; + * ready без объекта — только отчёт (missing_objects), без авто-удаления; + * обрыв загрузки клиентом не оставляет ни строки, ни объекта; + * внедрённые отказы в API (если заданы FAULT_*_URL — экземпляры api-fault-*): + ошибка S3 PUT -> 503 и компенсация; ошибка БД после PUT -> 503, объект удалён; + падение процесса после PUT -> остаётся pending+объект, их убирает сборщик. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import hashlib +import json +import os +import socket +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import urlparse +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +sys.path.insert(0, str(HERE)) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) +HAS_S3 = bool(os.environ.get("S3_ENDPOINT")) +GRACE = 900 + + +@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured") +class AttachmentConsistencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool + from src.objectstore import ObjectStore + from src.security import hash_password + from src.storage import Storage + from src import reconcile + + cls.settings = load_settings() + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + cls.store = ObjectStore(cls.settings) + cls.rec = reconcile + user = cls.storage.create_user("cons_" + uuid.uuid4().hex[:16], "Consistency", hash_password("Consistency-pw-1")) + cls.user = user + cls.chat = cls.storage.create_chat("consistency", [user.id]) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + # helpers + + def pending_with_object(self, age_seconds=0, put=True): + att_id = uuid.uuid4() + key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex) + data = b"\x89PNG fake bytes " + uuid.uuid4().bytes + ok = self.storage.create_pending_attachment(att_id, self.chat.id, self.user.id, "p.png", "image/png", + len(data), hashlib.sha256(data).hexdigest(), 1, 1, key) + self.assertTrue(ok) + if put: + self.store.put(key, data, "image/png", hashlib.md5(data).digest(), hashlib.sha256(data).hexdigest()) + if age_seconds: + self.backdate(att_id, age_seconds) + return att_id, key + + def backdate(self, att_id, seconds): + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET created_at = created_at - make_interval(secs => %s) WHERE id = %s", + (seconds, att_id)) + + def row(self, att_id): + with self.pool.connection() as conn: + return conn.execute("SELECT state FROM attachments WHERE id = %s", (att_id,)).fetchone() + + def run_gc(self, apply, grace=GRACE, sweep=True): + report = None + for _ in range(50): # фоновый GC реплики мог держать lock + report = self.rec.run_once(self.pool, self.store, grace, apply, orphan_sweep=sweep) + if report is not None: + return report + time.sleep(0.2) + self.fail("reconciler lock is busy") + + # tests + + def test_crash_between_put_and_commit_dry_run_then_apply_then_noop(self): + att_id, key = self.pending_with_object(age_seconds=GRACE + 60) + dry = self.run_gc(apply=False, sweep=False) + self.assertIn(str(att_id), dry["stale_pending_ids"]) + self.assertIsNotNone(self.row(att_id), "dry-run must not change anything") + self.assertIsNotNone(self.store.head(key)) + applied = self.run_gc(apply=True, sweep=False) + self.assertGreaterEqual(applied["stale_pending_rows_deleted"], 1) + self.assertIsNone(self.row(att_id)) + self.assertIsNone(self.store.head(key)) + again = self.run_gc(apply=True, sweep=False) + self.assertNotIn(str(att_id), again["stale_pending_ids"]) + + def test_fresh_pending_is_kept(self): + att_id, key = self.pending_with_object(age_seconds=0) + self.run_gc(apply=True) + self.assertEqual(self.row(att_id), ("pending",)) + self.assertIsNotNone(self.store.head(key)) + self.storage.delete_pending_attachment(att_id) + self.store.delete(key) + + def test_ready_and_referenced_are_never_collected(self): + att_id, key = self.pending_with_object() + self.assertIsNotNone(self.storage.mark_attachment_ready(att_id)) + msg, _ = self.storage.create_message(self.chat.id, self.user.id, "", uuid.uuid4().hex, os.urandom(32), + 86400, [att_id]) + self.assertEqual(msg.attachment_ids, [att_id]) + self.backdate(att_id, 10 * GRACE) + self.run_gc(apply=True, grace=GRACE) + self.assertEqual(self.row(att_id), ("ready",)) + self.assertIsNotNone(self.store.head(key)) + # Даже прямое удаление строки запрещено БД, пока на неё ссылается сообщение. + from psycopg import errors + with self.assertRaises(errors.ForeignKeyViolation): + with self.pool.connection() as conn: + conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,)) + + def test_orphan_object_deleted_only_after_grace(self): + key = "%s%s" % (self.rec.OBJECT_PREFIX, uuid.uuid4().hex) + self.store.put(key, b"orphan", "image/png", hashlib.md5(b"orphan").digest(), "0" * 64) + report = self.run_gc(apply=True, grace=GRACE) + self.assertGreaterEqual(report["orphan_objects_within_grace"], 1) + self.assertIsNotNone(self.store.head(key), "young orphan must survive (grace period)") + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, False, report, verify_ready=False) + self.assertIn(key, report["orphan_objects_sample"]) + self.assertIsNotNone(self.store.head(key), "dry-run must not delete") + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False) + self.assertIsNone(self.store.head(key)) + report = {} + self.rec.sweep_orphans(self.pool, self.store, 0, True, report, verify_ready=False) + self.assertNotIn(key, report["orphan_objects_sample"]) + + def test_missing_object_is_reported_not_deleted(self): + att_id, key = self.pending_with_object() + self.storage.mark_attachment_ready(att_id) + self.store.delete(key) # внедрённая потеря байтов + report = self.run_gc(apply=True) + self.assertIn(str(att_id), report["missing_object_ids"]) + self.assertEqual(self.row(att_id), ("ready",)) + with self.pool.connection() as conn: # убрать искусственную запись, на неё нет ссылок + conn.execute("DELETE FROM attachments WHERE id = %s", (att_id,)) + + def test_pending_cannot_be_attached(self): + att_id, key = self.pending_with_object() + from src.storage import AttachmentNotFound + with self.assertRaises(AttachmentNotFound): + self.storage.create_message(self.chat.id, self.user.id, "x", uuid.uuid4().hex, os.urandom(32), + 86400, [att_id]) + self.storage.delete_pending_attachment(att_id) + self.store.delete(key) + + +# ------------------------------------------------------------------ HTTP + DB + +def _http(base, method, path, token=None, body=None, headers=None, timeout=20): + h = dict(headers or {}) + if token: + h["Authorization"] = "Bearer " + token + data = body + if isinstance(body, dict): + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + req = Request(base.rstrip("/") + path, data=data, headers=h, method=method) + try: + with urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read() or b"null") + except HTTPError as e: + return e.code, json.loads(e.read() or b"null") + + +def _actor(base): + import base64 + name = "flt_" + uuid.uuid4().hex[:16] + pw = "Fault-" + uuid.uuid4().hex + s, user = _http(base, "POST", "/api/v1/users", body={"username": name, "display_name": "F", "password": pw}) + assert s == 201, (s, user) + cred = base64.b64encode((name + ":" + pw).encode()).decode() + s, sess = _http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert s == 201, (s, sess) + s, chat = _http(base, "POST", "/api/v1/chats", sess["token"], {"title": "faults", "member_ids": [user["id"]]}) + assert s == 201, (s, chat) + return user, sess["token"], chat + + +@unittest.skipUnless(HAS_DB and HAS_S3, "DATABASE_URL and S3 settings are not configured") +class UploadFailureTests(unittest.TestCase): + """Отказы на пути загрузки, наблюдаемые со стороны БД и бакета.""" + + @classmethod + def setUpClass(cls): + AttachmentConsistencyTests.setUpClass.__func__(cls) + import imagegen + cls.png = imagegen.sample_png() + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def rows_of(self, owner_id): + with self.pool.connection() as conn: + return conn.execute("SELECT id, state, COALESCE(staging_key, object_key) FROM attachments " + "WHERE owner_id = %s", + (owner_id,)).fetchall() + + def object_count(self): + total, token = 0, None + while True: + items, token = self.store.list_page("staging/", token) + total += len(items) + if not token: + return total + + def upload(self, base, token, chat): + return _http(base, "POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], token, + self.png, {"Content-Type": "image/png"}) + + @unittest.skipUnless(os.environ.get("BASE_URL"), "BASE_URL is not set") + def test_aborted_upload_leaves_no_row_and_no_object(self): + base = os.environ["BASE_URL"] + user, token, chat = _actor(base) + before = self.object_count() + u = urlparse(base) + sock = socket.create_connection((u.hostname, u.port or 80), timeout=5) + head = ("POST /api/v1/chats/%s/attachments?filename=a.png HTTP/1.1\r\nHost: %s\r\n" + "Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n" + % (chat["id"], u.hostname, token, 5 * 1024 * 1024)).encode() + sock.sendall(head + self.png + b"\x00" * 200000) # ~0.2 MiB из заявленных 5 MiB + time.sleep(0.5) + sock.close() # обрыв + time.sleep(2) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertEqual(self.object_count(), before) + s, att = self.upload(base, token, chat) # сервис продолжает работать + self.assertEqual(s, 202, att) + + @unittest.skipUnless(os.environ.get("FAULT_S3_URL"), "FAULT_S3_URL (api-fault-s3) is not set") + def test_s3_put_failure_returns_503_and_compensates(self): + base = os.environ["FAULT_S3_URL"] + user, token, chat = _actor(base) + s, body = self.upload(base, token, chat) + self.assertEqual((s, body["error"]["code"]), (503, "unavailable")) + self.assertEqual(self.rows_of(user["id"]), [], "pending row must be removed by compensation") + + @unittest.skipUnless(os.environ.get("FAULT_DB_URL"), "FAULT_DB_URL (api-fault-db) is not set") + def test_db_failure_after_put_returns_503_and_deletes_object(self): + base = os.environ["FAULT_DB_URL"] + user, token, chat = _actor(base) + before = self.object_count() + s, body = self.upload(base, token, chat) + self.assertEqual((s, body["error"]["code"]), (503, "unavailable")) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertEqual(self.object_count(), before, "object must be deleted by compensation") + + @unittest.skipUnless(os.environ.get("FAULT_CRASH_URL"), "FAULT_CRASH_URL (api-fault-crash) is not set") + def test_process_crash_after_put_is_collected_by_gc(self): + base = os.environ["FAULT_CRASH_URL"] + user, token, chat = _actor(base) + try: + self.upload(base, token, chat) + self.fail("instance was expected to crash") + except (URLError, OSError, ConnectionError): + pass # процесс умер между S3 PUT и COMMIT + rows = self.rows_of(user["id"]) + self.assertEqual(len(rows), 1) + att_id, state, key = rows[0] + self.assertEqual(state, "pending") + self.assertIsNotNone(self.store.head(key), "S3 PUT happened before the crash") + # Через grace сборщик убирает и строку, и объект (ускоряем, сдвигая created_at). + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET created_at = created_at - interval '1 hour' WHERE id = %s", + (att_id,)) + from src import reconcile + report = None + for _ in range(50): + report = reconcile.run_once(self.pool, self.store, GRACE, True, orphan_sweep=False) + if report: + break + time.sleep(0.2) + self.assertIn(str(att_id), report["stale_pending_ids"]) + self.assertEqual(self.rows_of(user["id"]), []) + self.assertIsNone(self.store.head(key)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..2fc1ea4 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,310 @@ +"""Вложения lab4 через балансировщик (HTTP, только стандартная библиотека). + +lab5: загрузка возвращает 202 queued; там, где нужны байты, тест ждёт ready +(common.wait_ready). Варианты проверяет student-tests/test_pipeline.py. + + BASE_URL=https://localhost:8443 python -m unittest discover -s student-tests -p test_attachments.py -v + +Покрывает уровень 4: слишком большой файл (с Content-Length и chunked без него), +повреждённые изображения, подмену MIME, pixel bomb, опасные filename, чужое +attachment_id, привязку файла другого чата, неизвестный вариант. Во всех +негативных случаях дополнительно проверяется: ответ < 500, общий формат ошибки. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import hashlib +import http.client +import sys +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote, urlparse + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import common # noqa: E402 +import imagegen as img # noqa: E402 +from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402 + +MAX = 10 * 1024 * 1024 + + +def upload(token, chat_id, data, content_type="image/png", filename="photo.png", headers=None): + path = "/api/v1/chats/%s/attachments" % chat_id + if filename is not None: + path += "?filename=" + quote(filename, safe="") + h = {"Content-Type": content_type} + h.update(headers or {}) + return request("POST", path, token=token, raw=data, headers=h, timeout=60) + + +def message(token, chat_id, text, attachment_ids=None, key=None): + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, payload=body, + headers={"Idempotency-Key": key or uuid.uuid4().hex}) + + +def raw_get(path, token): + """GET, возвращающий байты (common.request декодирует только JSON).""" + u = urlparse(BASE_URL) + conn = (http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=30, context=tlsenv.CONTEXT) if u.scheme == "https" else http.client.HTTPConnection(u.hostname, u.port or 80, timeout=30)) + conn.request("GET", path, headers={"Authorization": "Bearer " + token}) + resp = conn.getresponse() + data = resp.read() + conn.close() + return resp.status, resp.headers, data + + +def chunked_upload(token, chat_id, total, piece=256 * 1024): + """Transfer-Encoding: chunked без Content-Length: лимит должен сработать по факту чтения.""" + u = urlparse(BASE_URL) + conn = (http.client.HTTPSConnection(u.hostname, u.port or 443, timeout=60, context=tlsenv.CONTEXT) if u.scheme == "https" else http.client.HTTPConnection(u.hostname, u.port or 80, timeout=60)) + path = "/api/v1/chats/%s/attachments?filename=big.png" % chat_id + conn.putrequest("POST", path) + conn.putheader("Authorization", "Bearer " + token) + conn.putheader("Content-Type", "image/png") + conn.putheader("Transfer-Encoding", "chunked") + conn.endheaders() + first = img.PNG_SIG + b"\x00" * (piece - 8) + sent = 0 + try: + while sent < total: + part = first if sent == 0 else b"\x00" * min(piece, total - sent) + conn.send(b"%x\r\n" % len(part) + part + b"\r\n") + sent += len(part) + conn.send(b"0\r\n\r\n") + except (BrokenPipeError, ConnectionResetError): + pass # сервер мог закрыть приём раньше; ответ всё равно читаем + resp = conn.getresponse() + status = resp.status + resp.read() + conn.close() + return status + + +@unittest.skipUnless(reachable(BASE_URL), "API is not reachable at %s" % BASE_URL) +class AttachmentTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("att_a") + cls.bob = actor("att_b") + cls.eve = actor("att_e") + cls.chat = create_chat(cls.alice, cls.bob) + cls.eve_chat = create_chat(cls.eve) + cls.other_chat = create_chat(cls.alice) # чат Алисы без Боба + + def assertError(self, r, status, code=None): + self.assertEqual(r.status, status, (r.status, r.json)) + self.assertLess(r.status, 500) + self.assertIsInstance(r.json, dict) + self.assertIn("error", r.json) + if code: + self.assertEqual(r.json["error"]["code"], code) + self.assertEqual(r.json["error"]["request_id"], r.headers.get("X-Request-Id")) + + def upload_ok(self, data, content_type="image/png", filename="photo.png", who=None, chat=None, ready=False): + who = who or self.alice + chat = chat or self.chat + r = upload(who["_token"], chat["id"], data, content_type, filename) + self.assertEqual(r.status, 202, (r.status, r.json)) + att = r.json + self.assertEqual(att["status"], "queued") + self.assertEqual(att["variants"], []) + self.assertIsNone(att["error_code"]) + self.assertEqual(att["size_bytes"], len(data)) + self.assertEqual(att["sha256"], hashlib.sha256(data).hexdigest()) + self.assertEqual(att["content_type"], content_type) + self.assertEqual(att["chat_id"], chat["id"]) + self.assertEqual(att["owner_id"], who["id"]) + self.assertNotIn("object_key", att) + if ready: + done = wait_ready(who["_token"], att["id"]) + for k in ("id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "created_at"): + self.assertEqual(done[k], att[k], k) + return done + return att + + # ------------------------------------------------------------- happy path + + def test_png_roundtrip_member_reads_outsider_404(self): + data = img.sample_png() + att = self.upload_ok(data, filename="Отпуск 2026 (1).png", ready=True) + self.assertEqual(att["filename"], "Отпуск 2026 (1).png") + meta = request("GET", "/api/v1/attachments/" + att["id"], token=self.bob["_token"]) + self.assertEqual(meta.status, 200) + self.assertEqual(meta.json, att) + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual(status, 200) + self.assertEqual(body, data) + self.assertEqual(headers.get_content_type(), "image/png") + self.assertEqual(int(headers["Content-Length"]), len(data)) + self.assertEqual(headers.get("X-Content-Type-Options"), "nosniff") + self.assertIn("no-store", headers.get("Cache-Control", "")) + for suffix in ("", "/content"): + self.assertError(request("GET", "/api/v1/attachments/" + att["id"] + suffix, token=self.eve["_token"]), + 404, "not_found") + self.assertError(request("GET", "/api/v1/attachments/" + att["id"] + suffix), 401, "unauthorized") + + def test_jpeg_baseline_and_progressive(self): + for data in (img.sample_jpeg(), img.progressive_jpeg()): + att = self.upload_ok(data, "image/jpeg", "photo.jpg", ready=True) + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.alice["_token"]) + self.assertEqual((status, body, headers.get_content_type()), (200, data, "image/jpeg")) + + def test_content_type_with_parameters_is_accepted(self): + self.upload_ok(img.png(8, 8), "image/png", "x.png") + r = upload(self.alice["_token"], self.chat["id"], img.png(8, 8), "image/png; charset=binary", "x.png") + self.assertEqual(r.status, 202, r.json) + + def test_exactly_max_size_is_accepted(self): + data = img.png_of_exact_size(MAX) + att = self.upload_ok(data, filename="max.png", ready=True) + status, _, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual(status, 200) + self.assertEqual(hashlib.sha256(body).hexdigest(), att["sha256"]) + + def test_original_variant_and_unknown_variants(self): + att = self.upload_ok(img.png(4, 4), ready=True) + base = "/api/v1/attachments/%s/content" % att["id"] + status, _, _ = raw_get(base + "?variant=original", self.bob["_token"]) + self.assertEqual(status, 200) + # lab5: у ready есть thumbnail (вариант создаёт worker). + status, headers, _ = raw_get(base + "?variant=thumbnail", self.bob["_token"]) + self.assertEqual((status, headers.get_content_type()), (200, "image/jpeg")) + self.assertError(request("GET", base + "?variant=../../etc", token=self.bob["_token"]), 400, "invalid_request") + + # ------------------------------------------------------------- limits + + def test_too_large_with_content_length(self): + r = upload(self.alice["_token"], self.chat["id"], img.png_of_exact_size(MAX + 1), filename="big.png") + self.assertError(r, 413, "payload_too_large") + + def test_too_large_chunked_without_content_length(self): + status = chunked_upload(self.alice["_token"], self.chat["id"], MAX + 512 * 1024) + self.assertEqual(status, 413) + + def test_too_many_pixels(self): + data = img.huge_dimensions_png(8000, 6000) # 48 Мп при размере файла ~50 КБ + self.assertLess(len(data), MAX) + self.assertError(upload(self.alice["_token"], self.chat["id"], data), 413, "payload_too_large") + + # ------------------------------------------------------------- content checks + + def test_corrupted_images_are_415(self): + cases = { + "truncated png": img.truncated_png(), + "bad crc png": img.bad_crc_png(), + "png header + random": img.PNG_SIG + uuid.uuid4().bytes * 64, + "png with wrong data length": img.wrong_length_png(), + "truncated jpeg": img.truncated_jpeg(), + "empty body": b"", + "plain text": "не картинка".encode(), + "gif": b"GIF89a" + b"\x00" * 64, + } + for name, data in cases.items(): + ctype = "image/jpeg" if "jpeg" in name else "image/png" + with self.subTest(name): + self.assertError(upload(self.alice["_token"], self.chat["id"], data, ctype), 415, + "unsupported_media_type") + + def test_mime_spoofing_is_415(self): + cases = [ + (img.sample_png(), "image/jpeg"), + (img.sample_jpeg(), "image/png"), + (img.sample_png(), "image/gif"), + (img.sample_png(), "application/octet-stream"), + (img.sample_png(), "multipart/form-data; boundary=x"), + (img.sample_png(), "text/html"), + ] + for data, ctype in cases: + with self.subTest(ctype): + self.assertError(upload(self.alice["_token"], self.chat["id"], data, ctype), 415, + "unsupported_media_type") + + def test_dangerous_filenames_are_400(self): + for name in ("../etc/passwd", "a/b.png", "a\\b.png", "nul\x00.png", "bell\x07.png", "new\nline.png", + "", " ", ".", "..", "x" * 256, "rtl‮gnp.exe"): + with self.subTest(repr(name)): + self.assertError(upload(self.alice["_token"], self.chat["id"], img.png(2, 2), filename=name), + 400, "invalid_request") + self.assertError(upload(self.alice["_token"], self.chat["id"], img.png(2, 2), filename=None), + 400, "invalid_request") + self.upload_ok(img.png(2, 2), filename="x" * 255) + + # ------------------------------------------------------------- access + + def test_outsider_and_unknown_chat_upload_404(self): + self.assertError(upload(self.eve["_token"], self.chat["id"], img.png(2, 2)), 404, "not_found") + self.assertError(upload(self.alice["_token"], str(uuid.uuid4()), img.png(2, 2)), 404, "not_found") + self.assertError(upload(self.alice["_token"], "not-a-uuid", img.png(2, 2)), 400, "invalid_request") + self.assertError(upload(None, self.chat["id"], img.png(2, 2)), 401, "unauthorized") + + def test_invalid_and_unknown_attachment_ids(self): + self.assertError(request("GET", "/api/v1/attachments/not-a-uuid", token=self.bob["_token"]), 400) + self.assertError(request("GET", "/api/v1/attachments/%s" % uuid.uuid4(), token=self.bob["_token"]), 404) + self.assertError(request("GET", "/api/v1/attachments/%s/content" % uuid.uuid4(), token=self.bob["_token"]), 404) + + # ------------------------------------------------------------- messages + + def test_message_with_attachments_visible_to_members(self): + a1 = self.upload_ok(img.png(3, 3)) + a2 = self.upload_ok(img.sample_jpeg(), "image/jpeg", "b.jpg") + r = message(self.alice["_token"], self.chat["id"], "", [a2["id"], a1["id"]]) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(r.json["text"], "") + self.assertEqual(r.json["attachment_ids"], [a2["id"], a1["id"]]) # порядок сохранён + # Любой участник может сослаться на вложение чата повторно. + r2 = message(self.bob["_token"], self.chat["id"], "и снова", [a1["id"]]) + self.assertEqual(r2.status, 201, r2.json) + page = request("GET", "/api/v1/chats/%s/messages?limit=100" % self.chat["id"], token=self.bob["_token"]) + by_id = {m["id"]: m for m in page.json["items"]} + self.assertEqual(by_id[r.json["id"]]["attachment_ids"], [a2["id"], a1["id"]]) + self.assertEqual(by_id[r2.json["id"]]["attachment_ids"], [a1["id"]]) + text_only = message(self.alice["_token"], self.chat["id"], "без вложений") + self.assertEqual(text_only.json["attachment_ids"], []) + + def test_attachment_from_other_chat_is_404_and_nothing_saved(self): + foreign = self.upload_ok(img.png(2, 2), chat=self.other_chat) # чат Алисы без Боба + key = uuid.uuid4().hex + r = message(self.alice["_token"], self.chat["id"], "чужой файл", [foreign["id"]], key=key) + self.assertError(r, 404, "not_found") + # Отказ не «сжёг» ключ: тот же ключ с корректным телом создаёт сообщение. + ok = message(self.alice["_token"], self.chat["id"], "чужой файл", [], key=key) + self.assertEqual(ok.status, 201, ok.json) + # Посторонняя Ева не может прикрепить файл Алисы в своём чате. + a = self.upload_ok(img.png(2, 2)) + self.assertError(message(self.eve["_token"], self.eve_chat["id"], "x", [a["id"]]), 404, "not_found") + self.assertError(message(self.alice["_token"], self.chat["id"], "x", [str(uuid.uuid4())]), 404) + + def test_message_attachment_validation(self): + a = self.upload_ok(img.png(2, 2)) + cid, tok = self.chat["id"], self.alice["_token"] + self.assertError(message(tok, cid, ""), 400) + self.assertError(message(tok, cid, " "), 400) + self.assertError(message(tok, cid, "dup", [a["id"], a["id"]]), 400) + self.assertError(message(tok, cid, "many", [str(uuid.uuid4()) for _ in range(11)]), 400) + self.assertError(message(tok, cid, "bad", ["not-a-uuid"]), 400) + self.assertError(message(tok, cid, "bad", "not-a-list"), 400) + r = message(tok, cid, " ", [a["id"]]) # пробельный текст допустим с вложением + self.assertEqual(r.status, 201, r.json) + + def test_idempotency_includes_attachment_ids(self): + a1 = self.upload_ok(img.png(2, 2)) + a2 = self.upload_ok(img.png(3, 3)) + key = uuid.uuid4().hex + first = message(self.alice["_token"], self.chat["id"], "пара", [a1["id"], a2["id"]], key=key) + self.assertEqual(first.status, 201) + replay = message(self.alice["_token"], self.chat["id"], "пара", [a1["id"], a2["id"]], key=key) + self.assertEqual((replay.status, replay.json), (201, first.json)) + self.assertError(message(self.alice["_token"], self.chat["id"], "пара", [a2["id"], a1["id"]], key=key), 409) + self.assertError(message(self.alice["_token"], self.chat["id"], "пара", [a1["id"]], key=key), 409) + # Отсутствующее поле == [] для идемпотентности. + k2 = uuid.uuid4().hex + m1 = message(self.alice["_token"], self.chat["id"], "без поля", None, key=k2) + m2 = message(self.alice["_token"], self.chat["id"], "без поля", [], key=k2) + self.assertEqual((m2.status, m2.json), (201, m1.json)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 0000000..eea12c5 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,110 @@ +"""Аутентификация: Basic только для входа, Bearer для остального, logout.""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import base64 +import unittest +import uuid + +from common import actor, basic, login, register, request + + +def assert_challenge(test, r, scheme): + test.assertEqual(r.status, 401, r.json) + test.assertTrue(r.headers.get("WWW-Authenticate", "").startswith(scheme), r.headers.get("WWW-Authenticate")) + test.assertEqual(r.json["error"]["code"], "unauthorized") + + +class AuthTests(unittest.TestCase): + + def test_wrong_password_and_unknown_user_look_the_same(self): + user = register("wp") + wrong = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(user["username"], "Wrong-password-000")}) + unknown = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic("nouser_" + uuid.uuid4().hex[:10], "Wrong-password-000")}) + for r in (wrong, unknown): + assert_challenge(self, r, 'Basic realm="messenger"') + self.assertIn('charset="UTF-8"', r.headers.get("WWW-Authenticate")) + self.assertEqual(wrong.json["error"]["message"], unknown.json["error"]["message"]) + + def test_malformed_basic_headers(self): + for header in (None, "Basic", "Basic !!!notbase64", "Bearer abc", + "Basic " + base64.b64encode(b"no-colon-here").decode()): + headers = {"Authorization": header} if header else {} + r = request("POST", "/api/v1/auth/sessions", headers=headers) + assert_challenge(self, r, "Basic") + + def test_session_shape_and_token_entropy(self): + user = register("shape") + r = request("POST", "/api/v1/auth/sessions", + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201, r.json) + self.assertEqual(set(r.json), {"token", "token_type", "expires_at", "user"}) + self.assertEqual(r.json["token_type"], "Bearer") + token = r.json["token"] + # 32 случайных байта в base64url = 43 символа (256 бит) + self.assertGreaterEqual(len(token), 43) + self.assertNotIn(user["username"], token) + self.assertNotIn(user["id"], token) + self.assertTrue(r.json["expires_at"].endswith("Z")) + self.assertNotEqual(token, login(user), "each login must issue a new token") + + def test_password_with_colon_and_unicode(self): + password = "пароль:с:двоеточием-🙂" + user = register("colon", password=password) + token = login(user) + self.assertEqual(request("GET", "/api/v1/users/me", token=token).json["id"], user["id"]) + + def test_login_with_body_rejected(self): + """Контракт: вход — только заголовок Basic, без тела. Лишний ввод -> 400, сессии нет.""" + user = register("body") + auth = {"Authorization": basic(user["username"], user["_password"])} + cases = [ + dict(payload={"unexpected": True}), + dict(payload={}), + dict(raw=b"not json at all"), + dict(raw=b"username=x&password=y", headers={"Content-Type": "application/x-www-form-urlencoded"}), + ] + for case in cases: + headers = dict(auth) + headers.update(case.pop("headers", {})) + r = request("POST", "/api/v1/auth/sessions", headers=headers, **case) + self.assertEqual(r.status, 400, (case, r.json)) + self.assertEqual(r.json["error"]["code"], "invalid_request") + self.assertNotIn("token", r.json) + # Без тела тот же заголовок работает. + self.assertEqual(request("POST", "/api/v1/auth/sessions", headers=auth).status, 201) + + def test_basic_on_business_endpoint_rejected(self): + user = register("basic") + r = request("GET", "/api/v1/users/me", + headers={"Authorization": basic(user["username"], user["_password"])}) + assert_challenge(self, r, 'Bearer realm="messenger"') + + def test_x_user_id_gives_no_access(self): + user = actor("xuid") + other = actor("xuid") + assert_challenge(self, request("GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}), "Bearer") + r = request("GET", "/api/v1/users/me", token=user["_token"], headers={"X-User-Id": other["id"]}) + self.assertEqual(r.json["id"], user["id"]) + + def test_unknown_and_garbage_tokens(self): + for token in ("x" * 43, "", "a b", "z" * 1000): + r = request("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert_challenge(self, r, "Bearer") + + def test_logout_revokes_only_current_session(self): + user = register("logout") + t1, t2 = login(user), login(user) + # Прогреваем кэш Redis для t1, чтобы проверить инвалидацию кэша. + self.assertEqual(request("GET", "/api/v1/users/me", token=t1).status, 200) + self.assertEqual(request("GET", "/api/v1/users/me", token=t1).status, 200) + r = request("DELETE", "/api/v1/auth/sessions/current", token=t1) + self.assertEqual(r.status, 204) + self.assertIsNone(r.json) + assert_challenge(self, request("GET", "/api/v1/users/me", token=t1), "Bearer") + assert_challenge(self, request("DELETE", "/api/v1/auth/sessions/current", token=t1), "Bearer") + self.assertEqual(request("GET", "/api/v1/users/me", token=t2).status, 200) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 0000000..defb99f --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,81 @@ +"""Гонка регистрации одинакового username и rollback составной операции.""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import actor, basic, request + +PARALLEL = 16 + + +class RegistrationRaceTests(unittest.TestCase): + + def test_same_username_parallel_registration(self): + username = "race_" + uuid.uuid4().hex[:12] + barrier = threading.Barrier(PARALLEL) + + def attempt(i): + password = "Race-password-%02d-%s" % (i, uuid.uuid4().hex[:6]) + barrier.wait() + r = request("POST", "/api/v1/users", timeout=30, payload={ + "username": username, "display_name": "Racer %d" % i, "password": password, + }) + return r.status, r.json, password + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(attempt, range(PARALLEL))) + + statuses = sorted(s for s, _, _ in results) + self.assertEqual(statuses.count(201), 1, statuses) + self.assertEqual(statuses.count(409), PARALLEL - 1, statuses) + for status, body, _ in results: + if status == 409: + self.assertEqual(body["error"]["code"], "conflict") + + # Победитель входит своим паролем; пароли проигравших не подходят. + winner = next(r for r in results if r[0] == 201) + loser = next(r for r in results if r[0] == 409) + ok = request("POST", "/api/v1/auth/sessions", headers={"Authorization": basic(username, winner[2])}) + self.assertEqual(ok.status, 201) + self.assertEqual(ok.json["user"]["id"], winner[1]["id"]) + bad = request("POST", "/api/v1/auth/sessions", headers={"Authorization": basic(username, loser[2])}) + self.assertEqual(bad.status, 401) + + +class ChatRollbackTests(unittest.TestCase): + + def _chat_ids(self, owner): + r = request("GET", "/api/v1/chats?limit=100", token=owner["_token"]) + self.assertEqual(r.status, 200) + return [c["id"] for c in r.json["items"]] + + def test_unknown_member_rolls_back_whole_chat(self): + """Чат вставляется первым, затем участники по порядку (сначала сам владелец). + + Неизвестный участник вызывает нарушение внешнего ключа на 2-й строке + chat_members. Без транзакции у владельца остался бы чат с одним + участником; с транзакцией не остаётся ничего. + """ + owner, member = actor("rb"), actor("rb") + before = self._chat_ids(owner) + r = request("POST", "/api/v1/chats", token=owner["_token"], payload={ + "title": "Must roll back", + "member_ids": [owner["id"], member["id"], str(uuid.uuid4())], + }) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(self._chat_ids(owner), before, "partial chat is visible after failed creation") + self.assertEqual(self._chat_ids(member), [], "partial membership is visible to other member") + + def test_duplicate_and_missing_caller_rejected(self): + owner, member = actor("rb"), actor("rb") + for ids in ([owner["id"], owner["id"]], [member["id"]]): + r = request("POST", "/api/v1/chats", token=owner["_token"], + payload={"title": "bad", "member_ids": ids}) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(self._chat_ids(owner), []) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..da6766a --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,50 @@ +"""Истечение сессии при коротком SESSION_TTL_SECONDS. + +Нужен отдельный экземпляр API с коротким TTL (compose-профиль ttl, порт 8081): + docker compose --profile ttl up -d api-short-ttl + EXPIRY_BASE_URL=http://127.0.0.1:8081 python -m unittest discover -s student-tests -p test_expiry.py +Без EXPIRY_BASE_URL тест пропускается (основной стенд работает с TTL >= 300). +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import datetime as dt +import os +import time +import unittest + +from common import basic, register, request + +EXPIRY_BASE_URL = os.environ.get("EXPIRY_BASE_URL", "").rstrip("/") + + +@unittest.skipUnless(EXPIRY_BASE_URL, "set EXPIRY_BASE_URL to the short-TTL API instance") +class ExpiryTests(unittest.TestCase): + + def test_session_expires_even_when_cached(self): + user = register("exp", base_url=EXPIRY_BASE_URL) + r = request("POST", "/api/v1/auth/sessions", base_url=EXPIRY_BASE_URL, + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201, r.json) + token = r.json["token"] + expires_at = dt.datetime.fromisoformat(r.json["expires_at"].replace("Z", "+00:00")) + ttl = (expires_at - dt.datetime.now(dt.timezone.utc)).total_seconds() + self.assertLess(ttl, 30, "EXPIRY_BASE_URL must point to a short-TTL instance") + + # Два запроса подряд: первый кладёт сессию в Redis, второй читает из кэша. + for _ in range(2): + self.assertEqual(request("GET", "/api/v1/users/me", token=token, base_url=EXPIRY_BASE_URL).status, 200) + + time.sleep(max(ttl, 0) + 1.5) + r = request("GET", "/api/v1/users/me", token=token, base_url=EXPIRY_BASE_URL) + self.assertEqual(r.status, 401, r.json) + self.assertTrue(r.headers.get("WWW-Authenticate", "").startswith("Bearer")) + r = request("DELETE", "/api/v1/auth/sessions/current", token=token, base_url=EXPIRY_BASE_URL) + self.assertEqual(r.status, 401, "logout with expired token must be 401") + + # Новый вход после истечения работает. + r = request("POST", "/api/v1/auth/sessions", base_url=EXPIRY_BASE_URL, + headers={"Authorization": basic(user["username"], user["_password"])}) + self.assertEqual(r.status, 201) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..01534ab --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,173 @@ +"""Idempotency-Key через балансировщик (lab3). Только стандартная библиотека. + + BASE_URL=https://localhost:8443 python -m unittest discover -s student-tests -p test_idempotency.py -v + +Главная проверка — одновременные повторы с одним ключом, которые LB +раскладывает по РАЗНЫМ репликам (это проверяется по X-Instance-Id), +создают ровно одно сообщение. Остальное — семантика из contracts/README.md. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import json +import threading +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +from common import actor, create_chat, login, request + +PARALLEL = 24 + + +def history(token, chat_id): + items, cursor = [], None + for _ in range(100): + path = "/api/v1/chats/%s/messages?limit=100" % chat_id + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = request("GET", path, token=token).json + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + raise AssertionError("pagination did not finish") + + +def post(token, chat_id, body, key, raw=None): + # None -> заголовок не отправляется (см. common.request). + headers = {"Idempotency-Key": key} + if raw is not None: + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, raw=raw, headers=headers) + return request("POST", "/api/v1/chats/%s/messages" % chat_id, token=token, payload=body, headers=headers) + + +def parallel(n, fn): + barrier = threading.Barrier(n) + + def run(i): + barrier.wait() + return fn(i) + + with ThreadPoolExecutor(max_workers=n) as pool: + return list(pool.map(run, range(n))) + + +class IdempotencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("ida") + cls.bob = actor("idb") + cls.eve = actor("ide") + cls.chat = create_chat(cls.alice, cls.bob) + cls.chat2 = create_chat(cls.alice, cls.bob) + + def test_concurrent_same_key_across_replicas_creates_one_message(self): + key = "par-" + uuid.uuid4().hex + text = "одновременно " + key + results = parallel(PARALLEL, lambda i: post(self.alice["_token"], self.chat["id"], {"text": text}, key)) + statuses = [r.status for r in results] + self.assertEqual(statuses, [201] * PARALLEL, statuses) + ids = {r.json["id"] for r in results} + self.assertEqual(len(ids), 1, "same key produced different messages") + bodies = {json.dumps(r.json, sort_keys=True) for r in results} + self.assertEqual(len(bodies), 1, "replays must return the identical Message") + instances = {r.headers.get("X-Instance-Id") for r in results} + self.assertGreaterEqual(len(instances), 2, "requests did not reach >= 2 replicas: %s" % instances) + stored = [m for m in history(self.bob["_token"], self.chat["id"]) if m["text"] == text] + self.assertEqual(len(stored), 1, "history must contain exactly one message") + replayed = sum(1 for r in results if r.headers.get("Idempotency-Replayed") == "true") + self.assertEqual(replayed, PARALLEL - 1) + print("\n %d parallel POSTs, one key -> 1 message; replicas: %s" % (PARALLEL, sorted(instances))) + + def test_concurrent_same_key_different_bodies_one_winner(self): + key = "race-" + uuid.uuid4().hex + results = parallel(12, lambda i: post(self.alice["_token"], self.chat["id"], {"text": "вариант %d" % i}, key)) + winners = [r for r in results if r.status == 201] + losers = [r for r in results if r.status == 409] + self.assertEqual(len(winners) + len(losers), 12, [r.status for r in results]) + self.assertGreaterEqual(len(winners), 1) + self.assertEqual(len({r.json["id"] for r in winners}), 1, "two bodies won with one key") + for r in losers: + self.assertEqual(r.json["error"]["code"], "conflict") + texts = [m["text"] for m in history(self.alice["_token"], self.chat["id"]) if m["text"].startswith("вариант ")] + self.assertEqual(len([t for t in texts if t == winners[0].json["text"]]), 1) + + def test_sequential_replay_and_conflict(self): + key = "seq-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], {"text": "раз"}, key) + self.assertEqual(first.status, 201) + again = post(self.alice["_token"], self.chat["id"], {"text": "раз"}, key) + self.assertEqual((again.status, again.json), (201, first.json)) + other = post(self.alice["_token"], self.chat["id"], {"text": "два"}, key) + self.assertEqual(other.status, 409) + self.assertEqual(other.json["error"]["code"], "conflict") + + def test_json_formatting_and_key_order_do_not_matter(self): + key = "fmt-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], None, key, raw=b'{"text":"format"}') + spaced = post(self.alice["_token"], self.chat["id"], None, key, raw=b'{ \n "text" : "format"\n}') + self.assertEqual(first.status, 201) + self.assertEqual((spaced.status, spaced.json), (201, first.json)) + + def test_missing_or_invalid_key_is_400(self): + token, chat_id = self.alice["_token"], self.chat["id"] + # Не-ASCII ключ передаём сырыми UTF-8 байтами (urllib пропускает только latin-1). + non_ascii = "ключ".encode("utf-8").decode("latin-1") + bad_keys = [None, "", " ", "a" * 129, non_ascii, "has space", "semi;colon", "slash/key"] + for key in bad_keys: + with self.subTest(key=key): + r = post(token, chat_id, {"text": "x"}, key) + self.assertEqual(r.status, 400, r.json) + self.assertEqual(r.json["error"]["code"], "invalid_request") + ok = post(token, chat_id, {"text": "x"}, "A" * 128) + self.assertEqual(ok.status, 201) + ok2 = post(token, chat_id, {"text": "y"}, "a.b_c:d-1") + self.assertEqual(ok2.status, 201) + + def test_auth_checked_before_key(self): + r = post(None, self.chat["id"], {"text": "x"}, None) + self.assertEqual(r.status, 401) + + def test_scope_is_user_and_chat(self): + key = "scope-" + uuid.uuid4().hex + a1 = post(self.alice["_token"], self.chat["id"], {"text": "scope"}, key) + b1 = post(self.bob["_token"], self.chat["id"], {"text": "scope"}, key) + a2 = post(self.alice["_token"], self.chat2["id"], {"text": "scope"}, key) + self.assertEqual([a1.status, b1.status, a2.status], [201, 201, 201]) + self.assertEqual(len({a1.json["id"], b1.json["id"], a2.json["id"]}), 3) + # Другое тело другим пользователем с тем же ключом — не конфликт. + b2 = post(self.bob["_token"], self.chat2["id"], {"text": "другое"}, key) + self.assertEqual(b2.status, 201) + + def test_key_survives_new_session_of_same_user(self): + key = "sess-" + uuid.uuid4().hex + first = post(self.alice["_token"], self.chat["id"], {"text": "до перелогина"}, key) + token2 = login(self.alice) + replay = post(token2, self.chat["id"], {"text": "до перелогина"}, key) + self.assertEqual((replay.status, replay.json), (201, first.json)) + + def test_failed_access_does_not_burn_key(self): + key = "burn-" + uuid.uuid4().hex + denied = post(self.eve["_token"], self.chat["id"], {"text": "чужое"}, key) + self.assertEqual(denied.status, 404) + # Тот же ключ у участника и даже у того же постороннего позже — не занят. + mine = post(self.alice["_token"], self.chat["id"], {"text": "своё"}, key) + self.assertEqual(mine.status, 201) + invalid = post(self.alice["_token"], self.chat2["id"], {"text": ""}, key) + self.assertEqual(invalid.status, 400) + valid = post(self.alice["_token"], self.chat2["id"], {"text": "после 400"}, key) + self.assertEqual(valid.status, 201) + + def test_replay_rechecks_session(self): + key = "revoked-" + uuid.uuid4().hex + token = login(self.bob) + first = post(token, self.chat["id"], {"text": "до logout"}, key) + self.assertEqual(first.status, 201) + self.assertEqual(request("DELETE", "/api/v1/auth/sessions/current", token=token).status, 204) + replay = post(token, self.chat["id"], {"text": "до logout"}, key) + self.assertEqual(replay.status, 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_pipeline.py b/student-tests/test_pipeline.py new file mode 100644 index 0000000..956e98f --- /dev/null +++ b/student-tests/test_pipeline.py @@ -0,0 +1,179 @@ +"""lab5: асинхронный pipeline вложений через балансировщик (HTTP, только stdlib). + + BASE_URL=https://localhost:8443 python -m unittest discover -s student-tests -p test_pipeline.py -v + +Проверяет контракт вариантов по ФАКТИЧЕСКИМ байтам (а не только по метаданным): + * 202 queued -> ready; варианты thumbnail (JPEG, вписан в 256), crop (JPEG 128x128), + optimized (WebP, вписан в 1280, без увеличения); size/sha256 совпадают с байтами; + * EXIF orientation применяется (400x100 с orientation=6 -> 64x256); + * маленький исходник не увеличивается в thumbnail/optimized, crop увеличивается; + * прозрачность компонуется на белый (JPEG thumbnail прозрачного PNG — белый); + * в вариантах нет EXIF (APP1) — метаданные удалены; + * original после обработки неизменен (SHA-256), variant=original совпадает; + * повторный GET стабилен (повтор не создаёт новых вариантов); + * сообщение может ссылаться на queued-вложение; посторонний не видит вариантов. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import hashlib +import struct +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent / "tests")) +from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402 +from image_probe import jpeg_size # noqa: E402 +from test_attachments import raw_get, upload # noqa: E402 + +FIX = HERE / "fixtures" + + +def webp_size(data): + """Размеры WebP по заголовку RIFF (VP8 / VP8L / VP8X).""" + if data[:4] != b"RIFF" or data[8:12] != b"WEBP": + raise AssertionError("not a WebP file") + chunk = data[12:16] + if chunk == b"VP8 ": + w, h = struct.unpack("> 14) & 0x3FFF) + 1 + if chunk == b"VP8X": + w = int.from_bytes(data[24:27], "little") + 1 + h = int.from_bytes(data[27:30], "little") + 1 + return w, h + raise AssertionError("unknown WebP chunk %r" % chunk) + + +def jpeg_has_exif(data): + i = 2 + while i + 4 <= len(data) and data[i] == 0xFF: + marker = data[i + 1] + if marker == 0xDA: + return False + length = int.from_bytes(data[i + 2:i + 4], "big") + if marker == 0xE1 and data[i + 4:i + 10] == b"Exif\x00\x00": + return True + i += 2 + length + return False + + +def jpeg_center_is_white(data): + """Грубая проверка без декодера: JPEG целиком белого кадра очень мал и его + размеры совпадают; точную проверку цвета делает test_pipeline_faults (Pillow).""" + return len(data) < 2000 + + +@unittest.skipUnless(reachable(BASE_URL), "API is not reachable at %s" % BASE_URL) +class PipelineTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.alice = actor("pipe_a") + cls.bob = actor("pipe_b") + cls.eve = actor("pipe_e") + cls.chat = create_chat(cls.alice, cls.bob) + + def accept(self, data, ctype, name): + r = upload(self.alice["_token"], self.chat["id"], data, ctype, name) + self.assertEqual(r.status, 202, r.json) + self.assertEqual(r.json["status"], "queued") + self.assertEqual(r.json["variants"], []) + self.assertIsNone(r.json["error_code"]) + return r.json + + def variants_of(self, att): + return {v["name"]: v for v in att["variants"]} + + def fetch_variant(self, att, name): + status, headers, body = raw_get("/api/v1/attachments/%s/content?variant=%s" % (att["id"], name), + self.bob["_token"]) + self.assertEqual(status, 200, name) + v = self.variants_of(att)[name] + self.assertEqual(headers.get_content_type(), v["content_type"]) + self.assertEqual(len(body), v["size_bytes"]) + self.assertEqual(hashlib.sha256(body).hexdigest(), v["sha256"]) + return body + + def check_all(self, data, ctype, name, thumb, opt): + att = self.accept(data, ctype, name) + ready = wait_ready(self.bob["_token"], att["id"]) + self.assertIsNone(ready["error_code"]) + v = self.variants_of(ready) + self.assertEqual(sorted(v), ["crop", "optimized", "thumbnail"]) + self.assertEqual((v["thumbnail"]["content_type"], v["crop"]["content_type"], v["optimized"]["content_type"]), + ("image/jpeg", "image/jpeg", "image/webp")) + self.assertEqual((v["thumbnail"]["width"], v["thumbnail"]["height"]), thumb) + self.assertEqual((v["crop"]["width"], v["crop"]["height"]), (128, 128)) + self.assertEqual((v["optimized"]["width"], v["optimized"]["height"]), opt) + t = self.fetch_variant(ready, "thumbnail") + c = self.fetch_variant(ready, "crop") + o = self.fetch_variant(ready, "optimized") + self.assertEqual(jpeg_size(t), thumb) + self.assertEqual(jpeg_size(c), (128, 128)) + self.assertEqual(webp_size(o), opt) + self.assertFalse(jpeg_has_exif(t) or jpeg_has_exif(c), "EXIF must be stripped from variants") + self.assertNotIn(b"Exif\x00\x00", o) + # Оригинал неизменен. + status, headers, body = raw_get("/api/v1/attachments/%s/content" % att["id"], self.bob["_token"]) + self.assertEqual((status, body, headers.get_content_type()), (200, data, ctype)) + status, _, body2 = raw_get("/api/v1/attachments/%s/content?variant=original" % att["id"], self.bob["_token"]) + self.assertEqual((status, body2), (200, data)) + return ready, t + + def test_fixture_320x200(self): + data = (HERE.parent / "tests/fixtures/sample.png").read_bytes() + self.check_all(data, "image/png", "sample.png", (256, 160), (320, 200)) + + def test_exif_orientation_applied_and_stripped(self): + data = (FIX / "orient6.jpg").read_bytes() + self.assertIn(b"Exif", data) + self.check_all(data, "image/jpeg", "orient6.jpg", (64, 256), (100, 400)) + + def test_small_source_not_upscaled_except_crop(self): + data = (FIX / "small.png").read_bytes() + self.check_all(data, "image/png", "small.png", (40, 20), (40, 20)) + + def test_transparency_on_white(self): + data = (FIX / "transparent.png").read_bytes() + _, thumb = self.check_all(data, "image/png", "transparent.png", (256, 128), (600, 300)) + self.assertTrue(jpeg_center_is_white(thumb)) + + def test_progressive_jpeg(self): + data = (FIX / "sample_progressive.jpg").read_bytes() + self.check_all(data, "image/jpeg", "p.jpg", (256, 160), (320, 200)) + + def test_result_is_stable_and_private(self): + data = (FIX / "sample.jpg").read_bytes() + att = self.accept(data, "image/jpeg", "stable.jpg") + # queued/processing-вложение уже можно прикрепить к сообщению. + m = request("POST", "/api/v1/chats/%s/messages" % self.chat["id"], token=self.alice["_token"], + payload={"text": "", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + self.assertEqual(m.status, 201, m.json) + ready = wait_ready(self.alice["_token"], att["id"]) + time.sleep(1.0) + again = request("GET", "/api/v1/attachments/" + att["id"], token=self.bob["_token"]).json + self.assertEqual(again, ready) # повторные чтения не меняют результат + for suffix in ("", "/content?variant=thumbnail", "/content?variant=crop"): + r = request("GET", "/api/v1/attachments/" + att["id"] + suffix, token=self.eve["_token"]) + self.assertEqual(r.status, 404) + + def test_unknown_variant_is_400(self): + data = (FIX / "small.png").read_bytes() + att = self.accept(data, "image/png", "v.png") + wait_ready(self.bob["_token"], att["id"]) + r = request("GET", "/api/v1/attachments/%s/content?variant=%s" % (att["id"], quote("big")), + token=self.bob["_token"]) + self.assertEqual(r.status, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..fd06449 --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,311 @@ +"""lab5, уровень 4: внедрённые отказы pipeline. Запуск — внутри compose-сети: + + python scripts/ops.py pipeline-faults + +(пересоздаёт worker с compose.faults.yaml, поднимает api-fault-publish/api-fault-dup, +запускает этот файл в one-off контейнере образа API, затем возвращает обычный worker). + +Нужен прямой доступ к PostgreSQL (журнал job_events — доказательство того, ЧТО +произошло, а не только итогового статуса), к S3 и брокеру — берутся из тех же +secrets, что у API. Без окружения тесты пропускаются. + +Сценарии и ожидаемые инварианты: + A. crash между COMMIT и publish (api-fault-publish) -> задачу публикует sweeper + другого процесса, ready без ручного создания нового задания; + B. crash после XADD до подтверждения в outbox (api-fault-dup) -> дубль сообщения, + worker делает no-op, ровно один succeeded и один набор вариантов; + C. worker умер после записи объектов в S3 до COMMIT -> после рестарта тот же + consumer забирает свой PEL, перезаписывает ТЕ ЖЕ байты, ready; + D. worker умер после COMMIT до XACK -> повторная доставка -> no-op already_succeeded; + E. временная ошибка -> retry с backoff+jitter (интервалы в заданных границах), ready на 3-й попытке; + F. постоянная ошибка -> failed сразу (без повторов), DLQ, content 409, прикрепить нельзя (409); + G. попытки исчерпаны -> failed retry_exhausted -> явный replay -> ready; повторный + replay успешного задания -> no-op (не исполняется дубль); + H. зависший worker (обработка дольше аренды) -> lease_expired, новая попытка у + другого consumer, ready; проснувшийся старый результат НЕ публикует (fencing, lease_lost); + I. staging удалён после безопасной фиксации; + J. мусорное сообщение в брокере (не по схеме) -> ack без падения worker. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import hashlib +import os +import re +import sys +import time +import unittest +import uuid +from pathlib import Path +from urllib.parse import quote + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parent)) +from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402 + +FAULT_PUBLISH_URL = os.environ.get("FAULT_PUBLISH_URL", "").rstrip("/") +FAULT_DUP_URL = os.environ.get("FAULT_DUP_URL", "").rstrip("/") +FIX = HERE / "fixtures" + + +def _db_available(): + try: + from src.config import load_settings + load_settings() + return True + except Exception: + return False + + +DB_OK = _db_available() + + +def upload(token, chat_id, data, filename, base_url=None, ctype="image/png"): + return request("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, quote(filename)), + token=token, raw=data, headers={"Content-Type": ctype}, base_url=base_url, timeout=60) + + +@unittest.skipUnless(reachable(BASE_URL) and DB_OK, "needs API at %s and DB/S3/broker env" % BASE_URL) +class PipelineFaultTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool + from src.jobs import JobStore + from src.objectstore import ObjectStore + cls.settings = load_settings() + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=15) + cls.jobs = JobStore(cls.pool, cls.settings) + cls.store = ObjectStore(cls.settings) + cls.alice = actor("flt_a") + cls.bob = actor("flt_b") + cls.chat = create_chat(cls.alice, cls.bob) + cls.png = (HERE.parent / "tests/fixtures/sample.png").read_bytes() + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + # ------------------------------------------------------------ helpers + + def accept(self, filename, base_url=None, data=None): + r = upload(self.alice["_token"], self.chat["id"], data or self.png, filename, base_url) + self.assertEqual(r.status, 202, (r.status, r.json)) + self.assertEqual(r.json["status"], "queued") + return r.json + + def job(self, att_id): + with self.pool.connection() as conn: + row = conn.execute("SELECT job_id, status, attempt, max_attempts, replay_count, last_error_code " + "FROM attachment_jobs WHERE attachment_id = %s", (att_id,)).fetchone() + return dict(zip(("job_id", "status", "attempt", "max_attempts", "replay_count", "last_error_code"), row)) + + def events(self, att_id): + with self.pool.connection() as conn: + return conn.execute( + "SELECT event, attempt, COALESCE(detail, ''), COALESCE(actor, ''), " + "EXTRACT(EPOCH FROM at)::float FROM job_events WHERE job_id IN " + "(SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id", (att_id,)).fetchall() + + def names(self, att_id): + return [e[0] for e in self.events(att_id)] + + def wait_event(self, att_id, event, timeout=60.0, detail=None): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for e in self.events(att_id): + if e[0] == event and (detail is None or detail in e[2]): + return e + time.sleep(0.3) + self.fail("event %s (%s) not seen for %s; events: %s" % (event, detail, att_id, self.names(att_id))) + + def assert_single_result(self, att_id): + ready = request("GET", "/api/v1/attachments/" + att_id, token=self.bob["_token"]).json + self.assertEqual(ready["status"], "ready") + self.assertEqual(sorted(v["name"] for v in ready["variants"]), ["crop", "optimized", "thumbnail"]) + with self.pool.connection() as conn: + n = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id = %s", + (att_id,)).fetchone()[0] + self.assertEqual(n, 3, "exactly one set of variants") + self.assertEqual(self.names(att_id).count("succeeded"), 1, "exactly one published result") + return ready + + def reference_variants(self): + """Эталон: те же байты, загруженные без отказов. Детерминизм вывода.""" + if not hasattr(self.__class__, "_ref"): + att = self.accept("reference.png") + self.__class__._ref = {v["name"]: v["sha256"] + for v in wait_ready(self.bob["_token"], att["id"])["variants"]} + return self.__class__._ref + + # ------------------------------------------------------------ A, B: outbox + + @unittest.skipUnless(FAULT_PUBLISH_URL and reachable(FAULT_PUBLISH_URL), "api-fault-publish is not running") + def test_a_crash_between_commit_and_publish(self): + att = self.accept("a.png", base_url=FAULT_PUBLISH_URL) # 202 получен ДО падения + ready = wait_ready(self.bob["_token"], att["id"], timeout=60) + ev = self.events(att["id"]) + fault = [e for e in ev if e[0] == "fault_injected"] + self.assertTrue(fault, "fault instance did not crash before publish") + published = [e for e in ev if e[0] == "published"] + self.assertEqual(len(published), 1) + self.assertNotEqual(published[0][3], fault[0][3], "row must be published by ANOTHER process") + self.assertFalse(reachable(FAULT_PUBLISH_URL), "fault instance should be dead") + self.assertEqual(self.job(att["id"])["attempt"], 1, "same job, no new job created manually") + self.assert_single_result(att["id"]) + self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants()) + + @unittest.skipUnless(FAULT_DUP_URL and reachable(FAULT_DUP_URL), "api-fault-dup is not running") + def test_b_crash_after_publish_duplicate_is_noop(self): + att = self.accept("b.png", base_url=FAULT_DUP_URL) + wait_ready(self.bob["_token"], att["id"], timeout=60) + self.wait_event(att["id"], "redelivery_noop", timeout=30) + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ C, D: worker crash + + def test_c_worker_crash_after_side_effects_before_commit(self): + att = self.accept("fault-crash-after-effects.png") + ready = wait_ready(self.bob["_token"], att["id"], timeout=120) + names = self.names(att["id"]) + self.assertIn("fault_injected", names) + self.assertIn("lease_takeover", names, "restarted consumer must take over its own stale lease") + self.assert_single_result(att["id"]) + self.assertEqual(self.job(att["id"])["attempt"], 1, "redelivery does not create a new attempt") + self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants(), + "repeated side effect writes identical bytes") + + def test_d_worker_crash_after_commit_before_ack(self): + att = self.accept("fault-crash-before-ack.png") + wait_ready(self.bob["_token"], att["id"], timeout=60) + self.wait_event(att["id"], "fault_injected", timeout=30) + self.wait_event(att["id"], "redelivery_noop", timeout=120, detail="already_succeeded") + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ E, F, G: retry / DLQ / replay + + def test_e_transient_errors_retry_with_backoff(self): + att = self.accept("fault-transient-until-3.png") + wait_ready(self.bob["_token"], att["id"], timeout=60) + ev = self.events(att["id"]) + retries = [e for e in ev if e[0] == "retry_scheduled"] + self.assertEqual([e[1] for e in retries], [1, 2]) + claims = [e for e in ev if e[0] == "claimed"] + self.assertEqual([e[1] for e in claims], [1, 2, 3]) + # Значения worker'а из compose.faults.yaml (у one-off контейнера свои, обычные). + base, cap = self.settings_value("FAULTS_RETRY_BASE_SECONDS", 1.0), self.settings_value("FAULTS_RETRY_MAX_SECONDS", 4.0) + for (r, nxt) in zip(retries, claims[1:]): + delay = float(re.search(r"in ([0-9.]+)s", r[2]).group(1)) + d = min(cap, base * 2 ** (r[1] - 1)) + self.assertGreaterEqual(delay, d / 2 - 1e-6) + self.assertLessEqual(delay, d + 1e-6) + self.assertGreaterEqual(nxt[4] - r[4], delay - 0.05, "next attempt must not start before backoff") + self.assertEqual(self.job(att["id"])["attempt"], 3) + self.assert_single_result(att["id"]) + + def settings_value(self, name, default): + return float(os.environ.get(name) or default) + + def test_f_permanent_error_fails_without_retry(self): + att = self.accept("fault-permanent.png") + done = wait_ready(self.bob["_token"], att["id"], timeout=60, allow_failed=True) + self.assertEqual((done["status"], done["error_code"], done["variants"]), ("failed", "invalid_image", [])) + self.assertNotIn("retry_scheduled", self.names(att["id"])) + self.assertEqual(self.job(att["id"])["attempt"], 1) + with self.pool.connection() as conn: + dl = conn.execute("SELECT error_code FROM dead_letters WHERE attachment_id = %s", (att["id"],)).fetchone() + self.assertEqual(dl[0], "invalid_image") + r = request("GET", "/api/v1/attachments/%s/content" % att["id"], token=self.bob["_token"]) + self.assertEqual((r.status, r.json["error"]["code"]), (409, "conflict")) + m = request("POST", "/api/v1/chats/%s/messages" % self.chat["id"], token=self.alice["_token"], + payload={"text": "x", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + self.assertEqual(m.status, 409) + + def test_g_retry_exhausted_dlq_then_explicit_replay(self): + att = self.accept("fault-transient-until-replay.png") + done = wait_ready(self.bob["_token"], att["id"], timeout=90, allow_failed=True) + self.assertEqual((done["status"], done["error_code"]), ("failed", "retry_exhausted")) + job = self.job(att["id"]) + self.assertEqual((job["status"], job["attempt"]), ("failed", job["max_attempts"])) + self.assertEqual(self.names(att["id"]).count("retry_scheduled"), job["max_attempts"] - 1) + outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay") + self.assertEqual(outcome, "replayed") + wait_ready(self.bob["_token"], att["id"], timeout=60) + job2 = self.job(att["id"]) + self.assertEqual((job2["status"], job2["attempt"], job2["replay_count"]), + ("succeeded", job["attempt"] + 1, 1)) + # Повторный replay успешного задания — no-op: дубль не исполняется. + outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay") + self.assertEqual(outcome, "noop") + self.assertEqual(self.job(att["id"])["attempt"], job2["attempt"]) + with self.pool.connection() as conn: + dl = conn.execute("SELECT replayed_at IS NOT NULL, replay_count FROM dead_letters WHERE attachment_id = %s", + (att["id"],)).fetchone() + self.assertEqual(dl, (True, 1)) + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ H: hung worker + + def test_h_hung_worker_lease_expires_and_is_fenced(self): + att = self.accept("fault-slow.png") + wait_ready(self.bob["_token"], att["id"], timeout=90) + self.wait_event(att["id"], "lease_expired", timeout=5) + self.wait_event(att["id"], "lease_lost", timeout=80) + job = self.job(att["id"]) + self.assertEqual(job["attempt"], 2) + self.assert_single_result(att["id"]) + + # ------------------------------------------------------------ I, J + + def test_i_staging_removed_after_safe_commit(self): + att = self.accept("plain.png") + wait_ready(self.bob["_token"], att["id"], timeout=60) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + with self.pool.connection() as conn: + key = conn.execute("SELECT staging_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0] + if key is None: + break + time.sleep(0.3) + self.assertIsNone(key) + with self.pool.connection() as conn: + final = conn.execute("SELECT object_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0] + self.assertTrue(final.startswith("final/")) + data = self.store.read_all(final, 20 * 1024 * 1024) + self.assertEqual(hashlib.sha256(data).hexdigest(), att["sha256"]) + with self.pool.connection() as conn: + leftovers = conn.execute( + "SELECT count(*) FROM job_events WHERE event = 'accepted' AND attachment_id = %s", (att["id"],) + ).fetchone()[0] + self.assertEqual(leftovers, 1) + + def test_j_garbage_broker_message_is_acked(self): + from src.broker import Broker + b = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group) + mid = b.client.xadd(self.settings.broker_stream, {"job": '{"not":"a job"}'}).decode() + mid2 = b.publish({"schema_version": 1, "job_id": str(uuid.uuid4()), "type": "attachment.process.v1", + "attachment_id": str(uuid.uuid4()), "pipeline_version": 1, "attempt": 1, + "created_at": "2026-01-01T00:00:00Z"}) + deadline = time.monotonic() + 20 + pending = None + while time.monotonic() < deadline: + pending = [p for p in b.client.xpending_range(self.settings.broker_stream, self.settings.broker_group, + "-", "+", 100) + if p["message_id"].decode() in (mid, mid2)] + info = b.client.xinfo_groups(self.settings.broker_stream)[0] + delivered = info["last-delivered-id"].decode() + if not pending and _id(delivered) >= _id(mid2): + break + time.sleep(0.3) + self.assertEqual(pending, [], "poison/unknown messages must be acknowledged") + b.close() + + +def _id(x): + a, _, b = x.partition("-") + return int(a), int(b or 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_session_cache.py b/student-tests/test_session_cache.py new file mode 100644 index 0000000..e097270 --- /dev/null +++ b/student-tests/test_session_cache.py @@ -0,0 +1,149 @@ +"""Отзыв сессии при сбое инвалидации кэша (регрессия замечания ревью №1). + +Сценарий ревью: прогретый кэш -> связь API с Redis разорвана -> logout (204) +-> связь восстановлена с ТЕМ ЖЕ Redis, без его рестарта, положительная запись +отозванного токена ещё жива. Ожидание: токен отклоняется сразу, а не через +SESSION_CACHE_TTL_SECONDS. + +Redis здесь заменён управляемой заглушкой с тем же интерфейсом, что и +SessionCache (get/put/revoke): «разрыв» = операции падают, но данные заглушки +сохраняются — ровно как у живого Redis за сетевым разрывом. PostgreSQL — +настоящая (нужен DATABASE_URL/DATABASE_URL_FILE, иначе тест пропускается): + make test-storage +""" +import os +import sys +import time +import unittest +import uuid +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) + + +class PartitionableCache: + """Кэш, который «помнит» данные во время разрыва, но недоступен для API.""" + + def __init__(self): + self.data = {} + self.partitioned = False + + def get(self, digest): + from src.sessions import REVOKED + if self.partitioned: + return None + value = self.data.get(digest) + if value is None: + return None + if value == REVOKED: + return REVOKED + user, expires_at = value + return (user, expires_at) if expires_at > datetime.now(timezone.utc) else None + + def put(self, digest, user, expires_at): + if not self.partitioned: + self.data.setdefault(digest, (user, expires_at)) # SET NX + + def revoke(self, digest): + from src.sessions import REVOKED + if self.partitioned: + return False + self.data[digest] = REVOKED + return True + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class SessionRevocationCacheTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + cls.settings = load_settings() + wait_and_migrate(cls.settings) + cls.pool = create_pool(cls.settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + cls.user = cls.storage.create_user("rvk_" + uuid.uuid4().hex[:12], "Revoke", + hash_password("Revoke-test-password")) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def service(self, cache): + from src.sessions import SessionService + return SessionService(self.storage, cache, self.settings) + + def pending(self, digest): + with self.pool.connection() as conn: + return conn.execute("SELECT count(*) FROM session_cache_invalidations WHERE token_hash = %s", + (digest,)).fetchone()[0] + + def warm(self, svc, cache, token): + from src.security import token_digest + self.assertEqual(svc.resolve(token).id, self.user.id) + self.assertIsInstance(cache.data.get(token_digest(token)), tuple, "cache was not warmed") + + def test_revoked_token_rejected_after_partition_heals(self): + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + digest = token_digest(token) + self.warm(svc, cache, token) + + cache.partitioned = True + self.assertTrue(svc.revoke(token), "logout must succeed (PostgreSQL is the source of truth)") + self.assertIsInstance(cache.data[digest], tuple, "precondition: stale positive entry survived") + self.assertEqual(self.pending(digest), 1, "invalidation debt must be committed with revoked_at") + self.assertIsNone(svc.resolve(token), "during partition") + + cache.partitioned = False # тот же «Redis», данные не потеряны + self.assertIsNone(svc.resolve(token), "immediately after partition heals") + time.sleep(svc.RECHECK_SECONDS + 0.1) + self.assertIsNone(svc.resolve(token), "after reconciliation") + self.assertEqual(cache.data[digest], "revoked", "reconciliation must write the tombstone") + self.assertEqual(self.pending(digest), 0, "debt must be cleared after the tombstone") + + # После погашения долга кэшу снова доверяют: другой токен обслуживается из кэша. + other, _ = svc.create(self.user) + self.warm(svc, cache, other) + + def test_debt_survives_api_restart(self): + """Процесс, выполнивший logout, умер до восстановления связи — новый процесс не доверяет кэшу.""" + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + self.warm(svc, cache, token) + cache.partitioned = True + self.assertTrue(svc.revoke(token)) + del svc # «рестарт API»: состояние процесса потеряно + + cache.partitioned = False + restarted = self.service(cache) + self.assertIsNone(restarted.resolve(token)) + self.assertEqual(cache.data[token_digest(token)], "revoked") + + def test_normal_logout_leaves_no_debt(self): + from src.security import token_digest + cache = PartitionableCache() + svc = self.service(cache) + token, _ = svc.create(self.user) + self.warm(svc, cache, token) + self.assertTrue(svc.revoke(token)) + self.assertEqual(cache.data[token_digest(token)], "revoked") + self.assertEqual(self.pending(token_digest(token)), 0) + self.assertIsNone(svc.resolve(token)) + self.assertFalse(svc.revoke(token), "second logout must fail") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/test_storage_idempotency.py b/student-tests/test_storage_idempotency.py new file mode 100644 index 0000000..b433420 --- /dev/null +++ b/student-tests/test_storage_idempotency.py @@ -0,0 +1,144 @@ +"""Идемпотентность на уровне хранилища (нужен прямой доступ к PostgreSQL). + +* сбой ПОСЛЕ вставки ключа и сообщения внутри транзакции -> не остаётся ни ключа, + ни сообщения, и повтор с тем же ключом создаёт сообщение (5xx не фиксируется); +* 16 потоков с одним ключом -> ровно одна вставка, остальные — повтор; +* истёкший ключ можно занять заново; очистка удаляет только истёкшие строки и + выполняется одной репликой за раз (advisory lock). + +Запуск внутри compose-сети (БД наружу не публикуется): + python scripts/ops.py in-network storage +Без DATABASE_URL/DATABASE_URL_FILE тест пропускается. +""" +import hashlib +import os +import sys +import threading +import time +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) +DAY = 86400 + + +class InjectedFailure(RuntimeError): + pass + + +def fp(text): + return hashlib.sha256(text.encode()).digest() + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class StorageIdempotencyTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + settings = load_settings() + wait_and_migrate(settings) + cls.pool = create_pool(settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + h = hash_password("Idempotency-test-password") + cls.user = cls.storage.create_user("idm_" + uuid.uuid4().hex[:12], "Idem", h) + cls.chat = cls.storage.create_chat("idem-" + uuid.uuid4().hex[:8], [cls.user.id]) + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def count(self, sql, params): + with self.pool.connection() as conn: + return conn.execute(sql, params).fetchone()[0] + + def keys(self, key): + return self.count("SELECT count(*) FROM idempotency_keys WHERE user_id = %s AND idem_key = %s", + (self.user.id, key)) + + def test_failure_inside_transaction_stores_nothing(self): + key = "fail-" + uuid.uuid4().hex + text = "сбой " + key + original = self.storage._insert_message + seen = {} + + def insert_then_fail(conn, message_id, chat_id, sender_id, body): + original(conn, message_id, chat_id, sender_id, body) + seen["keys"] = conn.execute("SELECT count(*) FROM idempotency_keys WHERE idem_key = %s", (key,)).fetchone()[0] + seen["messages"] = conn.execute("SELECT count(*) FROM messages WHERE id = %s", (message_id,)).fetchone()[0] + raise InjectedFailure("crash after both inserts, before COMMIT") + + self.storage._insert_message = insert_then_fail + try: + with self.assertRaises(InjectedFailure): + self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + finally: + self.storage._insert_message = original + self.assertEqual((seen["keys"], seen["messages"]), (1, 1), "inserts were visible inside the transaction") + self.assertEqual(self.keys(key), 0) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 0) + # Клиент повторяет неподтверждённый запрос с тем же ключом — теперь успешно. + msg, replayed = self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + self.assertFalse(replayed) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 1) + + def test_threads_same_key_single_insert(self): + key = "thr-" + uuid.uuid4().hex + text = "потоки " + key + barrier = threading.Barrier(16) + + def call(_): + barrier.wait() + return self.storage.create_message(self.chat.id, self.user.id, text, key, fp(text), DAY) + + with ThreadPoolExecutor(max_workers=16) as pool: + results = list(pool.map(call, range(16))) + self.assertEqual(sum(1 for _, replayed in results if not replayed), 1) + self.assertEqual(len({m.id for m, _ in results}), 1) + self.assertEqual(self.count("SELECT count(*) FROM messages WHERE text = %s", (text,)), 1) + + def test_conflict_on_different_body(self): + from src.storage import IdempotencyConflict + key = "cnf-" + uuid.uuid4().hex + self.storage.create_message(self.chat.id, self.user.id, "a", key, fp("a"), DAY) + with self.assertRaises(IdempotencyConflict): + self.storage.create_message(self.chat.id, self.user.id, "b", key, fp("b"), DAY) + + def test_expired_key_can_be_reused_and_is_cleaned(self): + key = "exp-" + uuid.uuid4().hex + first, _ = self.storage.create_message(self.chat.id, self.user.id, "старый", key, fp("старый"), 1) + time.sleep(1.2) + # Истёк: другое тело уже не конфликт, создаётся новое сообщение. + second, replayed = self.storage.create_message(self.chat.id, self.user.id, "новый", key, fp("новый"), DAY) + self.assertFalse(replayed) + self.assertNotEqual(first.id, second.id) + self.assertEqual(self.keys(key), 1) + # Очистка удаляет только истёкшие ключи. + k2 = "exp2-" + uuid.uuid4().hex + self.storage.create_message(self.chat.id, self.user.id, "x", k2, fp("x"), 1) + time.sleep(1.2) + while self.storage.delete_expired_idempotency_keys(): + pass + self.assertEqual(self.keys(k2), 0) + self.assertEqual(self.keys(key), 1) + + def test_cleanup_runs_on_one_replica_at_a_time(self): + from src.storage import IDEMPOTENCY_CLEANUP_LOCK_ID + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (IDEMPOTENCY_CLEANUP_LOCK_ID,)) + self.assertIsNone(self.storage.delete_expired_idempotency_keys()) + self.assertIsNotNone(self.storage.delete_expired_idempotency_keys()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_storage_rollback.py b/student-tests/test_storage_rollback.py new file mode 100644 index 0000000..305f0ec --- /dev/null +++ b/student-tests/test_storage_rollback.py @@ -0,0 +1,101 @@ +"""Rollback на уровне хранилища с внедрённым сбоем (нужен прямой доступ к БД). + +Сбой вызывается ПОСЛЕ того, как строки chats и chat_members уже вставлены +внутри транзакции. Проверяем, что в БД не осталось ни чата, ни участников. + +Запуск внутри compose-сети (БД наружу не публикуется): + docker compose run --rm --no-deps -v "$PWD/student-tests:/app/student-tests:ro" \ + api python -m unittest discover -s student-tests -p test_storage_rollback.py -v +Без DATABASE_URL/DATABASE_URL_FILE тест пропускается. +""" +import os +import sys +import unittest +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +HAS_DB = bool(os.environ.get("DATABASE_URL") or os.environ.get("DATABASE_URL_FILE")) + + +class InjectedFailure(RuntimeError): + pass + + +@unittest.skipUnless(HAS_DB, "DATABASE_URL is not configured") +class StorageRollbackTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + from src.config import load_settings + from src.db import create_pool, wait_and_migrate + from src.security import hash_password + from src.storage import Storage + + settings = load_settings() + wait_and_migrate(settings) + cls.pool = create_pool(settings) + cls.pool.wait(timeout=10) + cls.storage = Storage(cls.pool) + h = hash_password("Rollback-test-password") + cls.users = [ + cls.storage.create_user("rbk_" + uuid.uuid4().hex[:12], "Rollback", h) + for _ in range(3) + ] + + @classmethod + def tearDownClass(cls): + cls.pool.close() + + def count(self, sql, params): + with self.pool.connection() as conn: + return conn.execute(sql, params).fetchone()[0] + + def test_failure_after_inserts_leaves_nothing(self): + title = "rollback-" + uuid.uuid4().hex + original = self.storage._insert_members + seen = {} + + def insert_then_fail(conn, chat_id, member_ids): + original(conn, chat_id, member_ids) + # Внутри транзакции строки видны — значит вставка действительно была. + seen["chats"] = conn.execute("SELECT count(*) FROM chats WHERE title = %s", (title,)).fetchone()[0] + seen["members"] = conn.execute("SELECT count(*) FROM chat_members WHERE chat_id = %s", (chat_id,)).fetchone()[0] + seen["chat_id"] = chat_id + raise InjectedFailure("simulated crash between statements") + + self.storage._insert_members = insert_then_fail + try: + with self.assertRaises(InjectedFailure): + self.storage.create_chat(title, [u.id for u in self.users]) + finally: + self.storage._insert_members = original + + self.assertEqual(seen["chats"], 1) + self.assertEqual(seen["members"], 3) + self.assertEqual(self.count("SELECT count(*) FROM chats WHERE title = %s", (title,)), 0) + self.assertEqual(self.count("SELECT count(*) FROM chat_members WHERE chat_id = %s", (seen["chat_id"],)), 0) + + def test_no_orphan_chats_in_database(self): + orphans = self.count( + "SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)", ()) + self.assertEqual(orphans, 0) + + def test_migrations_are_idempotent(self): + from src.config import load_settings + from src.db import apply_migrations + self.assertEqual(apply_migrations(load_settings()), []) + self.assertEqual(self.count("SELECT count(*) FROM schema_migrations WHERE version = %s", ("001_init",)), 1) + + def test_passwords_stored_as_argon2id(self): + with self.pool.connection() as conn: + stored = conn.execute("SELECT password_hash FROM users WHERE id = %s", + (self.users[0].id,)).fetchone()[0] + self.assertTrue(stored.startswith("$argon2id$"), stored[:10]) + self.assertNotIn("Rollback-test-password", stored) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/tlsenv.py b/student-tests/tlsenv.py new file mode 100644 index 0000000..eecd8c0 --- /dev/null +++ b/student-tests/tlsenv.py @@ -0,0 +1,35 @@ +"""lab7: общий TLS-контекст собственных тестов (только стандартная библиотека). + +Импорт модуля устанавливает глобальный urllib-opener с проверкой сертификата и hostname: + CA_FILE CA стенда (по умолчанию secrets/pki/ca.crt рядом с репозиторием) + CLIENT_CERT_FILE PEM с клиентским cert(+key) — только для прямых вызовов реплики + CLIENT_KEY_FILE (in-network тесты обращаются к api-short-ttl, минуя LB, с identity lb) +Отключить проверку сертификата нельзя: такого параметра нет намеренно. +""" +import os +import ssl +import urllib.request +from pathlib import Path + +DEFAULT_BASE_URL = "https://localhost:8443" +_ROOT = Path(__file__).resolve().parents[1] + + +def ca_file(): + value = os.environ.get("CA_FILE", "").strip() + if value: + return value + local = _ROOT / "secrets" / "pki" / "ca.crt" + return str(local) if local.exists() else None + + +def context(): + ctx = ssl.create_default_context(cafile=ca_file()) + pem = os.environ.get("CLIENT_CERT_FILE", "").strip() + if pem: + ctx.load_cert_chain(pem, os.environ.get("CLIENT_KEY_FILE", "").strip() or None) + return ctx + + +CONTEXT = context() +urllib.request.install_opener(urllib.request.build_opener(urllib.request.HTTPSHandler(context=CONTEXT))) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..370b744 --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Эксперимент: N одновременных загрузок ~9.5 MiB и пик памяти API-реплик. + +Гипотеза: память реплики ограничена сверху + baseline + MAX_CONCURRENT_UPLOADS x (~2 x размер файла) + разбор изображения, +и не растёт с числом одновременных клиентов: лишние ждут слот +(UPLOAD_QUEUE_TIMEOUT_SECONDS), а не читают тело в память. + +Источники памяти: + всегда VmHWM процесса API (пик RSS, считает ядро), обнулённый перед + экспериментом через /proc/PID/clear_refs — главный показатель; + --docker + docker stats (опрос ~1 с) по контейнерам сервиса api + + cgroup memory.peak (ядро >= 5.19) до и после; + --pids 123,456 + /proc//status: VmRSS каждые 20 мс (запуск без Docker). + + python student-tests/upload_memory.py --base-url https://localhost:8443 \ + --concurrency 10 --docker --output evidence/upload_memory.json + +Изображения синтетические (шум, PNG без сжатия). Токены не сохраняются. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import hashlib +import json +import os +import platform +import re +import statistics +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import common # noqa: E402 +import imagegen # noqa: E402 + +MiB = 1024 * 1024 +UNITS = {"b": 1, "kib": 1024, "mib": MiB, "gib": 1024 * MiB, "kb": 1000, "mb": 1000 ** 2, "gb": 1000 ** 3} + + +def parse_mem(text): + m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", text) + return float(m.group(1)) * UNITS.get(m.group(2).lower(), 1) if m else 0.0 + + +class Sampler(threading.Thread): + def __init__(self, args): + super().__init__(daemon=True) + self.args = args + self.stop = threading.Event() + self.samples = {} # name -> [bytes] + + def add(self, name, value): + self.samples.setdefault(name, []).append(value) + + def run(self): + while not self.stop.is_set(): + if self.args.pids: + for pid in self.args.pids.split(","): + try: + status = Path("/proc/%s/status" % pid).read_text() + rss = int(re.search(r"VmRSS:\s+(\d+) kB", status).group(1)) * 1024 + self.add("pid-" + pid, rss) + except (OSError, AttributeError): + pass + time.sleep(0.02) + else: + out = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"] + docker_api_ids(self.args), + capture_output=True, text=True).stdout + for line in out.splitlines(): + name, usage = line.split("\t", 1) + self.add(name, parse_mem(usage.split("/")[0])) + + +def docker_api_ids(args): + out = subprocess.run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + args.project, + "--filter", "label=com.docker.compose.service=api"], capture_output=True, text=True).stdout + return out.split() + + +def proc_hwm(args, reset=False): + """Пик RSS процесса (VmHWM, считает ядро — без пропусков между опросами). + + --docker: PID 1 контейнера api (python, exec-форма CMD); --pids: заданные PID. + reset=True обнуляет пик (echo 5 > /proc/PID/clear_refs) перед экспериментом. + """ + result = {} + if args.docker: + for cid in docker_api_ids(args): + name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, + text=True).stdout.strip().lstrip("/") + if reset: + subprocess.run(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"], + capture_output=True) + out = subprocess.run(["docker", "exec", cid, "cat", "/proc/1/status"], capture_output=True, + text=True).stdout + m = re.search(r"VmHWM:\s+(\d+) kB", out) + if m: + result[name] = int(m.group(1)) * 1024 + else: + for pid in args.pids.split(","): + try: + if reset: + Path("/proc/%s/clear_refs" % pid).write_text("5") + m = re.search(r"VmHWM:\s+(\d+) kB", Path("/proc/%s/status" % pid).read_text()) + result["pid-" + pid] = int(m.group(1)) * 1024 + except (OSError, AttributeError): + pass + return result + + +def cgroup_peaks(args): + """cgroup v2 memory.peak (или v1 max_usage_in_bytes) каждого контейнера api.""" + peaks = {} + for cid in docker_api_ids(args): + for path in ("/sys/fs/cgroup/memory.peak", "/sys/fs/cgroup/memory/memory.max_usage_in_bytes"): + r = subprocess.run(["docker", "exec", cid, "cat", path], capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip().isdigit(): + name = subprocess.run(["docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, + text=True).stdout.strip().lstrip("/") + peaks[name] = int(r.stdout.strip()) + break + return peaks + + +def one_upload(base, token, chat_id, data): + req = Request(base + "/api/v1/chats/%s/attachments?filename=mem.png" % chat_id, data=data, method="POST", + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}) + t0 = time.monotonic() + try: + with urlopen(req, timeout=120) as r: + body = json.loads(r.read()) + ok = body["sha256"] == hashlib.sha256(data).hexdigest() + return r.status, time.monotonic() - t0, r.headers.get("X-Instance-Id"), ok + except HTTPError as e: + return e.code, time.monotonic() - t0, e.headers.get("X-Instance-Id"), False + except (URLError, OSError): + return 0, time.monotonic() - t0, None, False + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default=os.environ.get("BASE_URL", "https://localhost:8443")) + p.add_argument("--concurrency", type=int, default=10) + p.add_argument("--rounds", type=int, default=2, help="повторы волны одновременных загрузок") + p.add_argument("--size-mib", type=float, default=9.5) + p.add_argument("--docker", action="store_true") + p.add_argument("--project", default=os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab4")) + p.add_argument("--pids", default="") + p.add_argument("--label", default="") + p.add_argument("--output", default="") + args = p.parse_args() + if not args.docker and not args.pids: + p.error("choose --docker or --pids") + base = args.base_url.rstrip("/") + common.BASE_URL = base + + owner = common.actor("mem") + chat = common.create_chat(owner) + files = [imagegen.noise_png(int(args.size_mib * MiB)) for _ in range(args.concurrency)] + assert all(len(f) <= 10 * MiB for f in files) + + before = cgroup_peaks(args) if args.docker else {} + proc_hwm(args, reset=True) + sampler = Sampler(args) + sampler.start() + time.sleep(1.5) # baseline + baseline = {k: max(v) for k, v in sampler.samples.items()} + results = [] + started = time.monotonic() + for _ in range(args.rounds): + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results += list(pool.map(lambda d: one_upload(base, owner["_token"], chat["id"], d), files)) + duration = time.monotonic() - started + time.sleep(1.0) + sampler.stop.set() + sampler.join(timeout=5) + after = cgroup_peaks(args) if args.docker else {} + hwm = proc_hwm(args) + + statuses = {} + for status, _, _, _ in results: + statuses[str(status)] = statuses.get(str(status), 0) + 1 + lat = sorted(r[1] for r in results if r[0] in (201, 202)) + + def pct(q): + return round(lat[min(len(lat) - 1, int(q * len(lat)))], 3) if lat else None + + memory = {} + for name, values in sampler.samples.items(): + memory[name] = { + "baseline_mib": round(baseline.get(name, values[0]) / MiB, 1), + "peak_mib": round(max(values) / MiB, 1), + "growth_mib": round((max(values) - baseline.get(name, values[0])) / MiB, 1), + "samples": len(values), + } + report = { + "label": args.label, + "date_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "host": "%s %s, %d CPU" % (platform.system(), platform.machine(), os.cpu_count() or 0), + "source": "docker stats + cgroup peak" if args.docker else "/proc VmRSS (20 ms)", + "concurrency": args.concurrency, + "rounds": args.rounds, + "file_mib": round(len(files[0]) / MiB, 2), + "uploads": len(results), + "statuses": statuses, + "sha256_ok": sum(1 for r in results if r[3]), + "instances": sorted({r[2] for r in results if r[2]}), + "duration_s": round(duration, 2), + "latency_201_s": {"p50": pct(0.5), "p95": pct(0.95), "max": lat[-1] if lat else None, + "mean": round(statistics.mean(lat), 3) if lat else None}, + "memory": memory, + } + report["process_peak_rss_mib"] = {k: round(v / MiB, 1) for k, v in hwm.items()} + if args.docker: + report["cgroup_peak_mib_before"] = {k: round(v / MiB, 1) for k, v in before.items()} + report["cgroup_peak_mib_after"] = {k: round(v / MiB, 1) for k, v in after.items()} + text = json.dumps(report, ensure_ascii=False, indent=2) + print(text) + if args.output: + Path(args.output).write_text(text + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/student-tests/wait_url.py b/student-tests/wait_url.py new file mode 100644 index 0000000..73f5fed --- /dev/null +++ b/student-tests/wait_url.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Ждать HTTP 200 по URL (до 90 с). Используется внутри compose-сети: ops.py in-network.""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import sys +import time +from urllib.request import urlopen + +url = sys.argv[1] +deadline = time.monotonic() + float(sys.argv[2] if len(sys.argv) > 2 else 90) +while time.monotonic() < deadline: + try: + if urlopen(url, timeout=2).status == 200: + print("ready: " + url) + sys.exit(0) + except OSError: + pass + time.sleep(0.5) +print("not ready: " + url) +sys.exit(1) diff --git a/student-tests/worker_outage.py b/student-tests/worker_outage.py new file mode 100644 index 0000000..df5ff27 --- /dev/null +++ b/student-tests/worker_outage.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""lab5, уровень 3: сценарий защиты «worker остановлен -> upload 202 -> рестарт +API и broker -> worker запущен -> ready». Запускается с хоста (управляет Docker +через scripts/ops.py), печатает только ID и статусы — без токенов. + + python student-tests/worker_outage.py --output evidence/worker_outage.json + +Шаги: + 1. остановить worker (docker stop); + 2. загрузить файл -> 202 queued; GET -> queued; content -> 409; сообщение с этим + вложением и обычное сообщение -> 201 (остальной API работает); + 3. брокер: сообщение лежит в потоке (lag >= 1) — `ops.py jobs stats`; + 4. перезапустить реплики API, затем broker (volumes сохраняются); + 5. GET -> всё ещё queued (никто синхронно не обработал), сообщение в брокере + пережило рестарт (lag >= 1); + 6. запустить worker -> ready, thumbnail 256x160 — настоящий JPEG, SHA-256 совпадает. + +Команды управления переопределяются (--stop-worker и т.д.) для другого стенда. +""" +import tlsenv # noqa: F401 lab7: HTTPS с проверкой CA стенда +import argparse +import hashlib +import json +import shlex +import subprocess +import sys +import time +import uuid +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(ROOT / "tests")) +import common # noqa: E402 +from image_probe import jpeg_size # noqa: E402 +from test_attachments import raw_get, upload # noqa: E402 + +PY = sys.executable + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--base-url", default="https://localhost:8443") + p.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY)) + p.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY)) + p.add_argument("--restart-api", default="%s scripts/ops.py api-restart" % shlex.quote(PY)) + p.add_argument("--restart-broker", default="%s scripts/ops.py broker-restart" % shlex.quote(PY)) + p.add_argument("--broker-stats", default="%s scripts/ops.py jobs stats" % shlex.quote(PY)) + p.add_argument("--output", default="") + args = p.parse_args() + common.BASE_URL = args.base_url.rstrip("/") + timeline = [] + t0 = time.monotonic() + + def note(step, **kw): + kw.update(step=step, t=round(time.monotonic() - t0, 2)) + timeline.append(kw) + print(json.dumps(kw, ensure_ascii=False), flush=True) + + def sh(cmd): + started = time.monotonic() + out = subprocess.run(shlex.split(cmd), cwd=ROOT, capture_output=True, text=True, timeout=600) + if out.returncode != 0: + raise SystemExit("command failed: %s\n%s" % (cmd, out.stderr[-800:])) + return out.stdout, round(time.monotonic() - started, 2) + + def broker_lag(): + out, _ = sh(args.broker_stats) + try: + data = json.loads(out[out.index("{"):]) + g = data["broker"]["groups"][0] + return {"length": data["broker"]["length"], "lag": g.get("lag"), "pending": g["pending"], + "jobs": data["jobs"], "outbox_rows": data["outbox"]["rows"]} + except (ValueError, KeyError, IndexError, TypeError): + return {"raw": out[-300:]} + + alice = common.actor("out_a") + bob = common.actor("out_b") + chat = common.create_chat(alice, bob) + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + + _, dt = sh(args.stop_worker) + note("worker stopped", seconds=dt) + + r = upload(alice["_token"], chat["id"], data, "image/png", "outage.png") + assert r.status == 202 and r.json["status"] == "queued", (r.status, r.json) + att = r.json + note("upload accepted", status=r.status, attachment_id=att["id"], attachment_status=att["status"]) + + time.sleep(3) # быстрый путь relay + sweeper успевают опубликовать + meta = common.request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]).json + content = common.request("GET", "/api/v1/attachments/%s/content" % att["id"], token=bob["_token"]) + assert meta["status"] == "queued", meta + assert content.status == 409, content.status + m1 = common.request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=alice["_token"], + payload={"text": "файл ещё в очереди", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + m2 = common.request("POST", "/api/v1/chats/%s/messages" % chat["id"], token=bob["_token"], + payload={"text": "обычное сообщение"}, headers={"Idempotency-Key": uuid.uuid4().hex}) + assert m1.status == 201 and m2.status == 201, (m1.status, m2.status) + note("while worker is down", metadata_status=meta["status"], content_status=content.status, + message_with_attachment=m1.status, plain_message=m2.status, message_id=m1.json["id"]) + note("broker before restarts", **broker_lag()) + + _, dt = sh(args.restart_api) + note("api replicas restarted", seconds=dt) + _, dt = sh(args.restart_broker) + note("broker restarted (volume kept)", seconds=dt) + time.sleep(2) + meta = common.request("GET", "/api/v1/attachments/" + att["id"], token=bob["_token"]).json + assert meta["status"] == "queued", meta + stats = broker_lag() + note("after restarts", metadata_status=meta["status"], **stats) + + _, dt = sh(args.start_worker) + note("worker started", seconds=dt) + ready = common.wait_ready(bob["_token"], att["id"], timeout=90) + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + status, headers, body = raw_get("/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], bob["_token"]) + assert status == 200 and headers.get_content_type() == "image/jpeg" + assert hashlib.sha256(body).hexdigest() == thumb["sha256"] and jpeg_size(body) == (256, 160) + status, _, original = raw_get("/api/v1/attachments/%s/content" % att["id"], bob["_token"]) + assert status == 200 and original == data + note("ready", attachment_id=att["id"], variants=[v["name"] for v in ready["variants"]], + thumbnail_jpeg_size=list(jpeg_size(body)), original_sha256_ok=True) + print("PASS: accepted job survived worker stop + API restart + broker restart and became ready") + if args.output: + Path(args.output).write_text(json.dumps(timeline, ensure_ascii=False, indent=2), encoding="utf-8") + + +if __name__ == "__main__": + main()