From c52b2157e088e16e5bf768b46682c33ad5dd7981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Thu, 1 Oct 2026 21:17:29 +0300 Subject: [PATCH 01/13] chore: keep lab plans and notes local --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b5a7189..f101a4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ # Local/private state +/docs/plans/ +/LAB_NOTES.md .env .env.* !.env.example -- 2.54.0 From 22e1764d84835a4ee5e2983444a63d240998348e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:16:47 +0300 Subject: [PATCH 02/13] feat: Add pinned telemetry stack and Grafana provisioning (U1) Wire OTLP metrics, logs, and traces to Prometheus, Loki, and Tempo with bounded Collector queues. Provision Grafana sources and a dashboard based on the stores' and Collector's documented metrics. Runtime integration awaits the application Compose stack; static syntax and wiring checks pass. --- compose.telemetry.yaml | 59 +++++++++++++++ infra/grafana/dashboards/telemetry.json | 46 ++++++++++++ .../provisioning/dashboards/dashboards.yaml | 7 ++ .../provisioning/datasources/datasources.yaml | 18 +++++ infra/loki/config.yaml | 32 +++++++++ infra/otel/collector.yaml | 72 +++++++++++++++++++ infra/prometheus/prometheus.yml | 16 +++++ infra/tempo/config.yaml | 17 +++++ 8 files changed, 267 insertions(+) create mode 100644 compose.telemetry.yaml create mode 100644 infra/grafana/dashboards/telemetry.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/config.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/tempo/config.yaml diff --git a/compose.telemetry.yaml b/compose.telemetry.yaml new file mode 100644 index 0000000..8201791 --- /dev/null +++ b/compose.telemetry.yaml @@ -0,0 +1,59 @@ +# Add with: docker compose -f compose.yaml -f compose.telemetry.yaml up -d +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:0.128.0 + command: ["--config=/etc/otelcol-contrib/collector.yaml"] + mem_limit: 384m + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro + depends_on: + - prometheus + - loki + - tempo + + prometheus: + image: prom/prometheus:v3.5.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --web.enable-remote-write-receiver + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus_data:/prometheus + + loki: + image: grafana/loki:3.5.0 + command: ["-config.file=/etc/loki/config.yaml"] + volumes: + - ./infra/loki/config.yaml:/etc/loki/config.yaml:ro + - loki_data:/loki + + tempo: + image: grafana/tempo:2.9.5 + command: ["-config.file=/etc/tempo/config.yaml"] + volumes: + - ./infra/tempo/config.yaml:/etc/tempo/config.yaml:ro + - tempo_data:/var/tempo + + grafana: + image: grafana/grafana:12.0.2 + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in local .env} + volumes: + - grafana_data:/var/lib/grafana + - ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro + - ./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + depends_on: + - prometheus + - loki + - tempo + +volumes: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/infra/grafana/dashboards/telemetry.json b/infra/grafana/dashboards/telemetry.json new file mode 100644 index 0000000..4811866 --- /dev/null +++ b/infra/grafana/dashboards/telemetry.json @@ -0,0 +1,46 @@ +{ + "uid": "lab6-telemetry", + "title": "Lab 6 · Telemetry stack", + "tags": ["lab6"], + "timezone": "browser", + "schemaVersion": 40, + "version": 1, + "refresh": "10s", + "time": { "from": "now-1h", "to": "now" }, + "panels": [ + { + "id": 1, + "type": "timeseries", + "title": "Telemetry stores and collector reachable", + "description": "Prometheus self-scrape and internal /metrics endpoints. API and worker panels follow their actual exported metric names.", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "refId": "A", + "expr": "up{job=~\"prometheus|otel-collector|loki|tempo\"}", + "legendFormat": "{{job}}" + } + ], + "fieldConfig": { "defaults": { "min": 0, "max": 1 }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 } + }, + { + "id": 2, + "type": "timeseries", + "title": "Collector exporter queue size", + "description": "Bounded exporter queues, sourced from the Collector's own Prometheus endpoint.", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "refId": "A", + "expr": "otelcol_exporter_queue_size{job=\"otel-collector\"}", + "legendFormat": "{{exporter}}" + } + ], + "fieldConfig": { "defaults": { "min": 0 }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 } + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..e172240 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: Lab 6 + type: file + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..f03287e --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/loki/config.yaml b/infra/loki/config.yaml new file mode 100644 index 0000000..508633a --- /dev/null +++ b/infra/loki/config.yaml @@ -0,0 +1,32 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..013d283 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,72 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + timeout: 2s + send_batch_size: 512 + +exporters: + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + remote_write_queue: + enabled: true + queue_size: 1000 + num_consumers: 1 + retry_on_failure: + enabled: true + max_elapsed_time: 30s + otlphttp/loki: + endpoint: http://loki:3100/otlp + sending_queue: + enabled: true + queue_size: 64 + retry_on_failure: + enabled: true + max_elapsed_time: 30s + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true + sending_queue: + enabled: true + queue_size: 64 + retry_on_failure: + enabled: true + max_elapsed_time: 30s + +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + without_type_suffix: true + without_units: true + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..b56d200 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: loki + static_configs: + - targets: ["loki:3100"] + - job_name: tempo + static_configs: + - targets: ["tempo:3200"] diff --git a/infra/tempo/config.yaml b/infra/tempo/config.yaml new file mode 100644 index 0000000..9f813a2 --- /dev/null +++ b/infra/tempo/config.yaml @@ -0,0 +1,17 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks -- 2.54.0 From c60b3e652be77bf3c8f9da66a9e5ab5a95eb305c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:43:04 +0300 Subject: [PATCH 03/13] feat: Carry runnable lab5 business stack into lab6 (U1) Import committed lab5 baseline 03899d7 via git archive: API replicas, PostgreSQL, private S3, RabbitMQ, publisher, worker, migrations, and hooks. Preserve lab6 contracts and the separate telemetry stack. Static checks pass; Docker runtime integration remains pending. --- Dockerfile | 12 ++ compose.yaml | 225 +++++++++++++++++++ infra/bootstrap-s3.sh | 16 ++ infra/haproxy.cfg | 26 +++ infra/minio-api-policy.json | 15 ++ infra/minio-maintenance-policy.json | 15 ++ migrations/001_initial.sql | 36 ++++ migrations/002_sessions.sql | 7 + migrations/003_message_replays.sql | 11 + migrations/004_attachments.sql | 32 +++ migrations/005_attachment_owner.sql | 2 + migrations/006_pipeline.sql | 74 +++++++ requirements.txt | 10 + scripts/bootstrap_broker.py | 44 ++++ scripts/migrate.py | 30 +++ scripts/restart.sh | 5 + scripts/setup-local-secrets.py | 42 ++++ scripts/stop-one.sh | 13 ++ src/api.py | 117 ++++++++++ src/attachment_read.py | 72 +++++++ src/attachments.py | 171 +++++++++++++++ src/auth.py | 43 ++++ src/db.py | 38 ++++ src/image_variants.py | 83 +++++++ src/main.py | 67 ++++++ src/models.py | 68 ++++++ src/pipeline_jobs.py | 278 ++++++++++++++++++++++++ src/publisher.py | 70 ++++++ src/server.py | 14 ++ src/staging_cleanup.py | 104 +++++++++ src/store.py | 297 ++++++++++++++++++++++++++ src/worker.py | 216 +++++++++++++++++++ student-tests/test_image_pipeline.py | 74 +++++++ student-tests/test_outbox_payload.py | 35 +++ student-tests/test_staging_cleanup.py | 46 ++++ student-tests/test_upload_handoff.py | 61 ++++++ student-tests/test_upload_queued.py | 26 +++ student-tests/test_variants_api.py | 47 ++++ 38 files changed, 2542 insertions(+) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/bootstrap-s3.sh create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio-api-policy.json create mode 100644 infra/minio-maintenance-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_sessions.sql create mode 100644 migrations/003_message_replays.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_attachment_owner.sql create mode 100644 migrations/006_pipeline.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap_broker.py create mode 100644 scripts/migrate.py create mode 100755 scripts/restart.sh create mode 100644 scripts/setup-local-secrets.py create mode 100755 scripts/stop-one.sh create mode 100644 src/api.py create mode 100644 src/attachment_read.py create mode 100644 src/attachments.py create mode 100644 src/auth.py create mode 100644 src/db.py create mode 100644 src/image_variants.py create mode 100644 src/main.py create mode 100644 src/models.py create mode 100644 src/pipeline_jobs.py create mode 100644 src/publisher.py create mode 100644 src/server.py create mode 100644 src/staging_cleanup.py create mode 100644 src/store.py create mode 100644 src/worker.py create mode 100644 student-tests/test_image_pipeline.py create mode 100644 student-tests/test_outbox_payload.py create mode 100644 student-tests/test_staging_cleanup.py create mode 100644 student-tests/test_upload_handoff.py create mode 100644 student-tests/test_upload_queued.py create mode 100644 student-tests/test_variants_api.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1ecad6d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app +COPY --chown=app:app src/ src/ +COPY --chown=app:app migrations/ migrations/ +COPY --chown=app:app scripts/migrate.py scripts/migrate.py +USER app +EXPOSE 8080 +CMD ["python", "-m", "src.server"] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..8bdc432 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,225 @@ +name: mtusi2026lab5 + +services: + postgres: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + + migrate: + build: . + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + restart: "no" + + s3-volume-init: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + user: "0:0" + entrypoint: ["/bin/sh", "-c"] + command: ["chown -R 100:100 /data"] + volumes: [s3_data:/data] + restart: "no" + + s3: + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + secrets: [minio_root_user, minio_root_password] + volumes: [s3_data:/data] + depends_on: + s3-volume-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9000/minio/health/ready"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 384m + + s3-bootstrap: + image: bitnamilegacy/minio-client:2025.7.21-debian-12-r3@sha256:73bd39f7899a0cef12b8dd5df13aa93a3ed1aaa44236542442e9ac76819ac158 + user: "0:0" + entrypoint: ["/bin/sh", "/bootstrap.sh"] + command: [] + volumes: + - ./infra/bootstrap-s3.sh:/bootstrap.sh:ro + - ./infra/minio-api-policy.json:/policies/minio-api-policy.json:ro + - ./infra/minio-maintenance-policy.json:/policies/minio-maintenance-policy.json:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + - s3_maintenance_key_id + - s3_maintenance_secret_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + broker: + image: rabbitmq:4.1.8-management + hostname: broker + restart: unless-stopped + volumes: + - broker_data:/var/lib/rabbitmq + - ./secrets/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 24 + + broker-init: + image: python:3.12.14-alpine3.24 + restart: "no" + depends_on: + broker: + condition: service_healthy + environment: + BROKER_URL_FILE: /run/secrets/broker_url + volumes: + - ./scripts/bootstrap_broker.py:/app/bootstrap_broker.py:ro + secrets: [broker_url] + command: ["python", "/app/bootstrap_broker.py"] + + api: + build: . + scale: 2 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"] + interval: 2s + timeout: 3s + retries: 20 + start_period: 5s + stop_grace_period: 10s + cpus: 0.5 + mem_limit: 256m + + publisher: + build: . + command: ["python", "-m", "src.publisher"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: attachments.process + secrets: [database_url, broker_url] + depends_on: + migrate: + condition: service_completed_successfully + broker-init: + condition: service_completed_successfully + restart: unless-stopped + + staging-cleanup: + build: . + command: ["python", "-m", "src.staging_cleanup"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: s3:9000 + S3_MAINTENANCE_KEY_ID_FILE: /run/secrets/s3_maintenance_key_id + S3_MAINTENANCE_SECRET_KEY_FILE: /run/secrets/s3_maintenance_secret_key + STAGING_GRACE_SECONDS: ${STAGING_GRACE_SECONDS:-3600} + STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60} + secrets: [database_url, s3_maintenance_key_id, s3_maintenance_secret_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + restart: unless-stopped + + worker: + build: . + command: ["python", "-m", "src.worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: attachments.process + S3_ENDPOINT: s3:9000 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + depends_on: + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + broker-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 0.5 + mem_limit: 768m + + lb: + image: haproxy:3.0.11-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: 0.5 + mem_limit: 128m + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_maintenance_key_id: + file: ./secrets/s3_maintenance_key_id + s3_maintenance_secret_key: + file: ./secrets/s3_maintenance_secret_key + broker_url: + file: ./secrets/broker_url + +volumes: + postgres_data: + s3_data: + broker_data: diff --git a/infra/bootstrap-s3.sh b/infra/bootstrap-s3.sh new file mode 100644 index 0000000..becaf21 --- /dev/null +++ b/infra/bootstrap-s3.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +read_secret() { + tr -d '\n' < "/run/secrets/$1" +} + +mc alias set local http://s3:9000 "$(read_secret minio_root_user)" "$(read_secret minio_root_password)" >/dev/null +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null +mc admin policy create local messenger-api /policies/minio-api-policy.json >/dev/null +mc admin policy create local messenger-maintenance /policies/minio-maintenance-policy.json >/dev/null +mc admin user add local "$(read_secret s3_access_key_id)" "$(read_secret s3_secret_access_key)" >/dev/null +mc admin policy attach local messenger-api --user "$(read_secret s3_access_key_id)" >/dev/null +mc admin user add local "$(read_secret s3_maintenance_key_id)" "$(read_secret s3_maintenance_secret_key)" >/dev/null +mc admin policy attach local messenger-maintenance --user "$(read_secret s3_maintenance_key_id)" >/dev/null diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..40952ab --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,26 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 2s + timeout client 15s + timeout server 15s + +resolvers docker + nameserver docker 127.0.0.11:53 + resolve_retries 2 + timeout retry 1s + hold valid 1s + +frontend public + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 0000000..54be94d --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/infra/minio-maintenance-policy.json b/infra/minio-maintenance-policy.json new file mode 100644 index 0000000..6beb6bd --- /dev/null +++ b/infra/minio-maintenance-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + } + ] +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..dbfb006 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX chats_history_idx ON chats (created_at, id); + +CREATE TABLE memberships ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); + +CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_sessions.sql b/migrations/002_sessions.sql new file mode 100644 index 0000000..5d2f364 --- /dev/null +++ b/migrations/002_sessions.sql @@ -0,0 +1,7 @@ +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_user_idx ON sessions (user_id); diff --git a/migrations/003_message_replays.sql b/migrations/003_message_replays.sql new file mode 100644 index 0000000..e5a8627 --- /dev/null +++ b/migrations/003_message_replays.sql @@ -0,0 +1,11 @@ +CREATE TABLE message_replays ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + request_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, request_key) +); + +CREATE INDEX message_replays_expiry_idx ON message_replays (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..576f30b --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,32 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (id, chat_id) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/005_attachment_owner.sql b/migrations/005_attachment_owner.sql new file mode 100644 index 0000000..6db4290 --- /dev/null +++ b/migrations/005_attachment_owner.sql @@ -0,0 +1,2 @@ +ALTER TABLE attachments ADD COLUMN owner_id uuid NOT NULL REFERENCES users(id); +ALTER TABLE attachments ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(); diff --git a/migrations/006_pipeline.sql b/migrations/006_pipeline.sql new file mode 100644 index 0000000..f332a6f --- /dev/null +++ b/migrations/006_pipeline.sql @@ -0,0 +1,74 @@ +-- Lab5: PostgreSQL is the source of truth; RabbitMQ carries only job IDs. +ALTER TABLE attachments + ADD COLUMN staging_key text UNIQUE, + ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version integer NOT NULL CHECK (pipeline_version > 0), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt > 0), + status text NOT NULL DEFAULT 'queued' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_token uuid, + lease_expires_at timestamptz, + error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (attachment_id, pipeline_version), + CHECK ((status = 'processing') = + (lease_token IS NOT NULL AND lease_expires_at IS NOT NULL)) +); + +CREATE INDEX attachment_jobs_due_idx + ON attachment_jobs (next_attempt_at, id) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_expired_idx + ON attachment_jobs (lease_expires_at, id) WHERE status = 'processing'; + +-- One logical publication per managed attempt. A crash after broker confirm +-- but before published_at may republish; consumers must tolerate that duplicate. +CREATE TABLE attachment_outbox ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + attempt integer NOT NULL CHECK (attempt > 0), + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + claim_token uuid, + claim_expires_at timestamptz, + published_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, attempt), + CHECK ((claim_token IS NULL) = (claim_expires_at IS NULL)) +); + +CREATE INDEX attachment_outbox_pending_idx + ON attachment_outbox (available_at, id) WHERE published_at IS NULL; + +-- Internal 'original' is stored alongside public thumbnail/crop/optimized +-- so every physical effect has the same versioned identity and checksum. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + object_key text NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name), + FOREIGN KEY (attachment_id, pipeline_version) + REFERENCES attachment_jobs (attachment_id, pipeline_version) ON DELETE CASCADE +); + +-- An API upload reserves its staging key before S3 PUT. Cleanup only removes +-- objects without a committed job and without an unexpired upload marker. +CREATE TABLE attachment_uploads ( + id uuid PRIMARY KEY, + staging_key text NOT NULL UNIQUE, + lease_expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX attachment_uploads_expired_idx ON attachment_uploads (lease_expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a9a509e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.118.3 +pydantic==2.12.4 +uvicorn==0.38.0 +psycopg[binary,pool]==3.2.12 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +minio==7.2.20 +Pillow==11.3.0 +urllib3==2.8.0 +pika==1.4.4 diff --git a/scripts/bootstrap_broker.py b/scripts/bootstrap_broker.py new file mode 100644 index 0000000..25d2780 --- /dev/null +++ b/scripts/bootstrap_broker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Idempotently declare the durable processing queue via RabbitMQ management API.""" + +import base64 +import json +import os +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote, unquote, urlsplit +from urllib.request import Request, urlopen + + +broker_url = open(os.environ.get("BROKER_URL_FILE", "/run/secrets/broker_url"), encoding="utf-8").read().strip() +parsed = urlsplit(broker_url) +if parsed.scheme != "amqp" or not parsed.username or not parsed.password: + raise SystemExit("Invalid broker URL file") +credentials = f"{unquote(parsed.username)}:{unquote(parsed.password)}".encode() +headers = { + "Authorization": "Basic " + base64.b64encode(credentials).decode("ascii"), + "Content-Type": "application/json", +} +url = "http://broker:15672/api/queues/%2F/" + quote("attachments.process", safe="") +body = json.dumps( + {"durable": True, "auto_delete": False, "arguments": {"x-queue-type": "quorum"}} +).encode() + +for attempt in range(30): + try: + with urlopen(Request(url, data=body, headers=headers, method="PUT"), timeout=3): + pass + with urlopen(Request(url, headers=headers), timeout=3) as response: + queue = json.load(response) + if queue.get("type") != "quorum" or not queue.get("durable"): + raise SystemExit("Processing queue is not durable quorum") + print("Durable quorum queue attachments.process is ready") + break + except HTTPError as exc: + if exc.code in (400, 401, 403, 406): + raise SystemExit(f"RabbitMQ queue declaration failed (HTTP {exc.code})") from None + except URLError: + pass + if attempt == 29: + raise SystemExit("RabbitMQ management API did not become ready") + time.sleep(1) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 0000000..e346d4f --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,30 @@ +"""Apply numbered SQL migrations once, atomically and under a DB lock.""" + +from pathlib import Path +import sys + +import psycopg + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import database_url + + +def main() -> None: + migration_dir = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(2026, 2)") + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())" + ) + for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")): + version = int(path.name.split("_", 1)[0]) + if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone(): + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..d8a434a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api lb publisher worker staging-cleanup broker postgres s3 +docker compose --env-file .env up -d --wait --scale api=2 diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py new file mode 100644 index 0000000..168168d --- /dev/null +++ b/scripts/setup-local-secrets.py @@ -0,0 +1,42 @@ +"""Create local Compose secrets without printing or overwriting existing values.""" + +import os +import secrets +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] / "secrets" + + +def write(name: str, value: str) -> None: + fd = os.open(ROOT / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as out: + out.write(value) + + +def main() -> None: + ROOT.mkdir(mode=0o700, exist_ok=True) + db_names = {"postgres_password", "database_url"} + s3_names = { + "minio_root_user", "minio_root_password", "s3_access_key_id", "s3_secret_access_key", + "s3_maintenance_key_id", "s3_maintenance_secret_key", + } + for names in (db_names, s3_names): + existing = {name for name in names if (ROOT / name).exists()} + if existing and existing != names: + raise SystemExit(f"Incomplete secret group: {', '.join(sorted(names - existing))}") + if not (ROOT / "postgres_password").exists(): + password = secrets.token_urlsafe(32) + write("postgres_password", password) + write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") + if not (ROOT / "minio_root_user").exists(): + write("minio_root_user", "lab5admin") + write("minio_root_password", secrets.token_urlsafe(32)) + write("s3_access_key_id", secrets.token_hex(10)) + write("s3_secret_access_key", secrets.token_urlsafe(32)) + write("s3_maintenance_key_id", secrets.token_hex(10)) + write("s3_maintenance_secret_key", secrets.token_urlsafe(32)) + + +if __name__ == "__main__": + main() diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..99c65a2 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +for container in $(docker compose --env-file .env ps -q api); do + instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$container" >/dev/null + exit 0 + fi +done +echo "target API instance not found" >&2 +exit 1 diff --git a/src/api.py b/src/api.py new file mode 100644 index 0000000..69bc46a --- /dev/null +++ b/src/api.py @@ -0,0 +1,117 @@ +"""Messenger HTTP routes.""" + +import re +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from .auth import basic_credentials, bearer_token +from .attachments import storage_ready, upload +from .attachment_read import download, get_view +from .models import CreateChat, CreateMessage, CreateUser +from .store import store + +router = APIRouter() + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + result = store.ready() + storage_ready() + return result + + +@router.post("/api/v1/users", status_code=201) +def register_user(body: CreateUser): + return store.register(body.username, body.display_name, body.password) + + +@router.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request, response: Response): + if await request.body(): + raise HTTPException(400) + username, password = basic_credentials(request.headers.get("authorization")) + response.headers["Cache-Control"] = "no-store" + return store.create_session(username, password) + + +def current_user(request: Request) -> dict: + return store.authenticated(bearer_token(request.headers.get("authorization"))) + + +def parse_limit(value: str) -> int: + try: + number = int(value) if value.isdecimal() else 0 + except ValueError: + raise HTTPException(400) from None + if not 1 <= number <= 100: + raise HTTPException(400) + return number + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + store.revoke_session(bearer_token(request.headers.get("authorization"))) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +def user_me(actor: dict = Depends(current_user)): + return actor + + +@router.post("/api/v1/chats", status_code=201) +def create_chat(body: CreateChat, actor: dict = Depends(current_user)): + return store.create_chat(actor["id"], body.title, body.member_ids) + + +@router.get("/api/v1/chats") +def list_chats( + limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user) +): + return store.list_chats(actor["id"], parse_limit(limit), cursor) + + +@router.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: UUID, actor: dict = Depends(current_user)): + return store.chat(actor["id"], str(chat_id)) + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: UUID, request: Request, filename: str = Query(...), actor: dict = Depends(current_user)): + return await upload(request, str(chat_id), actor["id"], filename) + + +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment(attachment_id: UUID, actor: dict = Depends(current_user)): + return get_view(actor["id"], attachment_id) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def get_attachment_content( + attachment_id: UUID, variant: str = Query("original"), actor: dict = Depends(current_user) +): + return download(actor["id"], attachment_id, variant) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)): + key = request.headers.get("Idempotency-Key") + if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400) + return store.create_message(actor["id"], str(chat_id), body.text, key, body.attachment_ids) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: UUID, + limit: str = Query("50"), + cursor: str | None = Query(None, min_length=1), + actor: dict = Depends(current_user), +): + return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor) diff --git a/src/attachment_read.py b/src/attachment_read.py new file mode 100644 index 0000000..34f91a6 --- /dev/null +++ b/src/attachment_read.py @@ -0,0 +1,72 @@ +"""Authorize metadata and stream versioned variants from the private bucket.""" + +import logging +from uuid import UUID + +from fastapi import HTTPException +from fastapi.responses import StreamingResponse + +from .attachments import BUCKET, attachment_view, storage +from .store import connection + + +logger = logging.getLogger(__name__) + + +def metadata(actor_id: str, attachment_id: UUID) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a " + "JOIN memberships m ON m.chat_id=a.chat_id " + "WHERE a.id=%s AND m.user_id=%s", + (attachment_id, UUID(actor_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return row + + +def download(actor_id: str, attachment_id: UUID, variant: str) -> StreamingResponse: + row = metadata(actor_id, attachment_id) + if row["status"] != "ready": + raise HTTPException(409) + with connection() as conn: + selected = conn.execute( + "SELECT object_key,content_type,size_bytes FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + if selected is None: + raise HTTPException(404) + try: + obj = storage().get_object(BUCKET, selected["object_key"]) + except Exception as exc: + logger.warning("S3 read failed for attachment %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + + def chunks(): + try: + yield from obj.stream(64 * 1024) + finally: + obj.close() + obj.release_conn() + + return StreamingResponse( + chunks(), media_type=selected["content_type"], + headers={"Content-Length": str(selected["size_bytes"])} + ) + + +def get_view(actor_id: str, attachment_id: UUID) -> dict: + row = metadata(actor_id, attachment_id) + view = attachment_view(row) + if row["status"] == "ready": + with connection() as conn: + variants = conn.execute( + "SELECT name,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s " + "AND name <> 'original' ORDER BY name", + (attachment_id, row["pipeline_version"]), + ).fetchall() + view["variants"] = [dict(variant) for variant in variants] + return view diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..818c5a5 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,171 @@ +"""Private S3 access shared by attachment routes and maintenance jobs.""" + +import asyncio +import hashlib +import logging +import os +import unicodedata +from functools import lru_cache +from pathlib import Path +from tempfile import SpooledTemporaryFile +from uuid import UUID, uuid4 + +from fastapi import HTTPException, Request +from minio import Minio +from minio.error import S3Error +from PIL import Image, UnidentifiedImageError +from urllib3 import PoolManager, Retry, Timeout + +from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload +from .store import connection, store, timestamp + + +BUCKET = "messenger-attachments" +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +logger = logging.getLogger(__name__) + + +def secret(name: str) -> str: + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + +@lru_cache(maxsize=1) +def storage() -> Minio: + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_ACCESS_KEY_ID"), + secret_key=secret("S3_SECRET_ACCESS_KEY"), + secure=False, + region="us-east-1", + http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)), + ) + + +def storage_ready() -> None: + try: + storage().stat_object(BUCKET, "health/ready-probe") + except S3Error as exc: + if exc.code != "NoSuchKey": + raise HTTPException(503) from None + except Exception: + raise HTTPException(503) from None + + +def attachment_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), + "filename": row["filename"], "content_type": row["content_type"], + "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], + "variants": [], "error_code": row["error_code"], + "created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]), + } + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename in {".", ".."}: + raise HTTPException(400) + if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename): + raise HTTPException(400) + + +def validate_image(stream, declared_type: str) -> str: + try: + with Image.open(stream) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type is None or actual_type != declared_type: + raise HTTPException(415) + if image.width * image.height > MAX_PIXELS: + raise HTTPException(413) + image.verify() + stream.seek(0) + with Image.open(stream) as image: + image.load() + return actual_type + except Image.DecompressionBombError: + raise HTTPException(413) from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise HTTPException(415) from None + + +def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None: + """Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum.""" + info = client.stat_object(BUCKET, key) + if info.size != size or info.content_type.split(";", 1)[0] != content_type: + raise ValueError("staging metadata mismatch") + obj = client.get_object(BUCKET, key) + try: + digest = hashlib.sha256() + received = 0 + for chunk in obj.stream(64 * 1024): + received += len(chunk) + if received > size: + raise ValueError("staging size mismatch") + digest.update(chunk) + if received != size or digest.hexdigest() != sha256: + raise ValueError("staging checksum mismatch") + finally: + obj.close() + obj.release_conn() + + +def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict: + attachment_id = uuid4() + staging_key = f"staging/{attachment_id}" + object_key = f"originals/{attachment_id}" + with connection() as conn: + reserve_upload(conn, attachment_id, staging_key) + + try: + stream.seek(0) + client = storage() + client.put_object(BUCKET, staging_key, stream, size, content_type=content_type) + with connection() as conn: + if not renew_upload(conn, attachment_id): + raise HTTPException(503) + verify_staging(client, staging_key, size, sha256, content_type) + with connection() as conn: + row, _job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id), + filename=filename, staging_key=staging_key, object_key=object_key, + content_type=content_type, size_bytes=size, sha256=sha256, + ) + except HTTPException: + raise + except Exception as exc: + # A lost COMMIT acknowledgement may leave a valid queued job. Keep the + # staging object; cleanup must check PostgreSQL before deleting it. + logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__) + raise HTTPException(503) from None + return attachment_view(row) + + +async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict: + await asyncio.to_thread(store.chat, actor_id, chat_id) + validate_filename(filename) + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in {"image/png", "image/jpeg"}: + raise HTTPException(415) + length = request.headers.get("content-length") + if length is not None: + try: + if int(length) > MAX_BYTES: + raise HTTPException(413) + except ValueError: + raise HTTPException(400) from None + + with SpooledTemporaryFile(max_size=1024 * 1024) as stream: + digest = hashlib.sha256() + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise HTTPException(413) + stream.write(chunk) + digest.update(chunk) + stream.seek(0) + validate_image(stream, declared_type) + return await asyncio.to_thread( + persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream + ) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..5714b17 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,43 @@ +"""Parse HTTP credentials without exposing them in errors or logs.""" + +import base64 +import binascii +import hashlib +import re + +from fastapi import HTTPException + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z") + + +def unauthorized(challenge: str) -> HTTPException: + return HTTPException(401, headers={"WWW-Authenticate": challenge}) + + +def basic_credentials(header: str | None) -> tuple[str, str]: + if not header or not header.startswith("Basic "): + raise unauthorized(BASIC_CHALLENGE) + try: + raw = base64.b64decode(header[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeError, ValueError): + raise unauthorized(BASIC_CHALLENGE) from None + if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128: + raise unauthorized(BASIC_CHALLENGE) + return username, password + + +def bearer_token(header: str | None) -> str: + if not header or not header.startswith("Bearer "): + raise unauthorized(BEARER_CHALLENGE) + token = header[7:] + if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token): + raise unauthorized(BEARER_CHALLENGE) + return token + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..832ff07 --- /dev/null +++ b/src/db.py @@ -0,0 +1,38 @@ +"""Bounded PostgreSQL pool shared by API storage and health checks.""" + +import os +import atexit +from functools import lru_cache +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from psycopg_pool import PoolTimeout + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return os.environ.get("DATABASE_URL", "postgresql:///messenger") + + +@lru_cache(maxsize=1) +def pool() -> ConnectionPool: + result = ConnectionPool( + database_url(), min_size=0, max_size=8, timeout=5, + kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row}, + open=True, + ) + atexit.register(result.close) + return result + + +def ready() -> bool: + try: + with pool().connection() as conn: + conn.execute("SELECT 1 FROM users LIMIT 0") + return True + except (psycopg.Error, PoolTimeout, OSError): + return False diff --git a/src/image_variants.py b/src/image_variants.py new file mode 100644 index 0000000..72a8cd3 --- /dev/null +++ b/src/image_variants.py @@ -0,0 +1,83 @@ +"""Deterministic, bounded image variants for one accepted PNG or JPEG.""" + +import hashlib +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class InvalidImage(ValueError): + pass + + +def _fit(image: Image.Image, bound: int) -> Image.Image: + factor = min(1, bound / max(image.size)) + size = (max(1, int(image.width * factor)), max(1, int(image.height * factor))) + return image.resize(size, Image.Resampling.LANCZOS) if size != image.size else image + + +def _jpeg(image: Image.Image) -> bytes: + output = BytesIO() + image.save(output, format="JPEG", quality=85, optimize=True) + return output.getvalue() + + +def _variant(data: bytes, content_type: str, image: Image.Image) -> dict: + return { + "data": data, + "content_type": content_type, + "size_bytes": len(data), + "width": image.width, + "height": image.height, + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def render(data: bytes, content_type: str) -> dict[str, dict]: + """Preserve original bytes; orient, flatten alpha, strip metadata on derivatives.""" + if not 0 < len(data) <= MAX_BYTES or content_type not in {"image/png", "image/jpeg"}: + raise InvalidImage("invalid original") + try: + with Image.open(BytesIO(data)) as source: + if {"PNG": "image/png", "JPEG": "image/jpeg"}.get(source.format) != content_type: + raise InvalidImage("wrong image type") + if source.width * source.height > MAX_PIXELS: + raise InvalidImage("too many pixels") + source.load() + original_size = source.size + oriented = ImageOps.exif_transpose(source) + if oriented.mode in {"RGBA", "LA", "P"}: + rgba = oriented.convert("RGBA") + image = Image.new("RGB", rgba.size, "white") + image.paste(rgba, mask=rgba.getchannel("A")) + else: + image = oriented.convert("RGB") + + thumbnail = _fit(image, 256) + side = min(image.size) + left = (image.width - side) // 2 + top = (image.height - side) // 2 + crop = image.resize( + (128, 128), Image.Resampling.LANCZOS, + box=(left, top, left + side, top + side), + ) + optimized = _fit(image, 1280) + webp = BytesIO() + optimized.save(webp, format="WEBP", quality=82, method=4) + return { + "original": { + "data": data, "content_type": content_type, + "size_bytes": len(data), "width": original_size[0], "height": original_size[1], + "sha256": hashlib.sha256(data).hexdigest(), + }, + "thumbnail": _variant(_jpeg(thumbnail), "image/jpeg", thumbnail), + "crop": _variant(_jpeg(crop), "image/jpeg", crop), + "optimized": _variant(webp.getvalue(), "image/webp", optimized), + } + except (UnidentifiedImageError, OSError, SyntaxError, Image.DecompressionBombError) as exc: + raise InvalidImage("image decoding failed") from exc diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..26dea4b --- /dev/null +++ b/src/main.py @@ -0,0 +1,67 @@ +"""FastAPI entry point and contract-wide response handling.""" + +from uuid import uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from .api import router + +app = FastAPI() +app.include_router(router) +INSTANCE_ID = uuid4().hex + +ERRORS = { + 400: ("invalid_request", "Неверный запрос"), + 401: ("unauthorized", "Требуется действительный пользователь"), + 404: ("not_found", "Объект не найден"), + 409: ("conflict", "Конфликт данных"), + 413: ("payload_too_large", "Слишком большой запрос"), + 415: ("unsupported_media_type", "Неподдерживаемый тип данных"), + 429: ("rate_limited", "Слишком много запросов"), + 503: ("unavailable", "Сервис недоступен"), +} + + +def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse: + code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка")) + request_id = request.state.request_id + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})}, + ) + + +@app.middleware("http") +async def request_contract(request: Request, call_next): + request.state.request_id = str(uuid4()) + is_upload = request.url.path.startswith("/api/v1/chats/") and request.url.path.endswith("/attachments") + if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions" and not is_upload: + media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") + if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"): + return error_response(request, 415) + try: + (await request.body()).decode("utf-8") + except UnicodeDecodeError: + return error_response(request, 400) + response = await call_next(request) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(RequestValidationError) +async def validation_error(request: Request, exc: RequestValidationError): + return error_response(request, 400) + + +@app.exception_handler(HTTPException) +async def http_error(request: Request, exc: HTTPException): + return error_response(request, exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def unexpected_error(request: Request, exc: Exception): + return error_response(request, 500) diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..06b16be --- /dev/null +++ b/src/models.py @@ -0,0 +1,68 @@ +"""Request bodies from the lab 1 contract.""" + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + + +class StrictBody(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUser(StrictBody): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def nonblank_name(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank display name") + return value + + @field_validator("password") + @classmethod + def nonblank_password(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank password") + return value + + +class CreateChat(StrictBody): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def nonblank_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + @classmethod + def valid_members(cls, value: list[str]) -> list[str]: + members = [str(UUID(member_id)) for member_id in value] + if len(set(members)) != len(members): + raise ValueError("duplicate member") + return members + + +class CreateMessage(StrictBody): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def valid_attachments(cls, value: list[str]) -> list[str]: + ids = [str(UUID(item)) for item in value] + if len(set(ids)) != len(ids): + raise ValueError("duplicate attachment") + return ids + + @model_validator(mode="after") + def text_or_attachment(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("message needs text or attachment") + return self diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py new file mode 100644 index 0000000..5fa477f --- /dev/null +++ b/src/pipeline_jobs.py @@ -0,0 +1,278 @@ +"""Atomic PostgreSQL state changes for the attachment processing pipeline. + +Callers own the connection and must commit before replying 202 or acking AMQP. +S3 writes and RabbitMQ publication happen outside these transactions. +""" + +from datetime import timezone +from uuid import uuid4 + + +REQUIRED_VARIANTS = {"original", "thumbnail", "crop", "optimized"} + + +def job_payload(row): + """Broker message per jobs.schema.json: identifiers only, never file bytes.""" + return { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "created_at": row["created_at"].astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), + } + + +def reserve_upload(conn, attachment_id, staging_key, lease_seconds=300): + """Record an active staging PUT before it starts.""" + if lease_seconds < 1: + raise ValueError("invalid upload lease") + with conn.transaction(): + conn.execute( + "INSERT INTO attachment_uploads (id,staging_key,lease_expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s))", + (attachment_id, staging_key, lease_seconds), + ) + + +def renew_upload(conn, attachment_id, lease_seconds=300): + """Keep a slow, active upload protected from orphan cleanup.""" + if lease_seconds < 1: + raise ValueError("invalid upload lease") + with conn.transaction(): + return conn.execute( + "UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()+make_interval(secs => %s) " + "WHERE id=%s AND lease_expires_at>clock_timestamp() RETURNING id", + (lease_seconds, attachment_id), + ).fetchone() is not None + + +def queue_attachment(conn, *, attachment_id, chat_id, owner_id, filename, + staging_key, object_key, content_type, size_bytes, sha256, + pipeline_version=1): + """Commit attachment, job, and outbox after the full S3 staging PUT.""" + job_id = uuid4() + with conn.transaction(): + marker = conn.execute( + "SELECT id FROM attachment_uploads WHERE id=%s AND staging_key=%s " + "AND lease_expires_at>clock_timestamp() FOR UPDATE", + (attachment_id, staging_key), + ).fetchone() + if marker is None: + raise ValueError("upload reservation is missing") + row = conn.execute( + "INSERT INTO attachments " + "(id,chat_id,owner_id,filename,object_key,staging_key,content_type,size_bytes,sha256,status,pipeline_version) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s) " + "RETURNING id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,error_code,created_at,updated_at", + (attachment_id, chat_id, owner_id, filename, object_key, staging_key, + content_type, size_bytes, sha256, pipeline_version), + ).fetchone() + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) VALUES (%s,%s,%s)", + (job_id, attachment_id, pipeline_version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (attachment_id,)) + return row, job_id + + +def claim_outbox(conn, limit=10, lease_seconds=30): + """Lease pending publications; publisher confirms must call confirm_outbox.""" + if not 1 <= limit <= 100 or lease_seconds < 1: + raise ValueError("invalid outbox lease") + token = uuid4() + with conn.transaction(): + rows = conn.execute( + "WITH due AS (" + " SELECT id FROM attachment_outbox" + " WHERE published_at IS NULL AND available_at<=clock_timestamp()" + " AND (claim_expires_at IS NULL OR claim_expires_at %s)" + " FROM due,attachment_jobs j" + " WHERE o.id=due.id AND j.id=o.job_id" + " RETURNING o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at", + (limit, token, lease_seconds), + ).fetchall() + return token, rows + + +def confirm_outbox(conn, outbox_id, token): + """Mark published only after RabbitMQ publisher confirm.""" + with conn.transaction(): + return conn.execute( + "UPDATE attachment_outbox SET published_at=clock_timestamp()," + "claim_token=NULL,claim_expires_at=NULL " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL RETURNING id", + (outbox_id, token), + ).fetchone() is not None + + +def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): + """A stale/redelivered message cannot steal an active or finished job.""" + if lease_seconds < 1: + raise ValueError("invalid job lease") + token = uuid4() + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.staging_key,a.object_key,a.sha256,a.content_type,a.size_bytes," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "queued" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attempt"] != delivery_attempt or + job["next_attempt_at"] > conn.execute("SELECT clock_timestamp() AS now").fetchone()["now"]): + return None + conn.execute( + "UPDATE attachment_jobs SET status='processing',lease_token=%s," + "lease_expires_at=clock_timestamp()+make_interval(secs => %s)," + "updated_at=clock_timestamp() WHERE id=%s", + (token, lease_seconds, job_id), + ) + conn.execute( + "UPDATE attachments SET status='processing',updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s AND status='queued'", + (job["attachment_id"], job["pipeline_version"]), + ) + return {**job, "lease_token": token} + + +def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): + if max_attempts < 1 or delay_seconds < 0: + raise ValueError("invalid retry configuration") + failed = permanent or job["attempt"] >= max_attempts + if failed: + code = error_code if permanent else "retry_exhausted" + conn.execute( + "UPDATE attachment_jobs SET status='failed',lease_token=NULL,lease_expires_at=NULL," + "error_code=%s,updated_at=clock_timestamp() WHERE id=%s", + (code, job["id"]), + ) + conn.execute( + "UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (code, job["attachment_id"], job["pipeline_version"]), + ) + else: + attempt = job["attempt"] + 1 + due = conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,lease_token=NULL,lease_expires_at=NULL," + "next_attempt_at=clock_timestamp()+make_interval(secs => %s)," + "error_code=%s,updated_at=clock_timestamp() WHERE id=%s RETURNING next_attempt_at", + (attempt, delay_seconds, error_code, job["id"]), + ).fetchone()["next_attempt_at"] + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt,available_at) VALUES (%s,%s,%s)", + (job["id"], attempt, due), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + return "failed" if failed else "queued" + + +def finish_attempt(conn, job_id, token, *, max_attempts, delay_seconds=0, + error_code="processing_failed", permanent=False): + """Persist retry or terminal failure before the worker acks delivery.""" + if error_code not in {"invalid_image", "processing_failed", "storage_unavailable", "worker_timeout"}: + raise ValueError("unsafe error code") + with conn.transaction(): + job = conn.execute( + "SELECT * FROM attachment_jobs WHERE id=%s FOR UPDATE", (job_id,) + ).fetchone() + if job is None or job["status"] != "processing" or job["lease_token"] != token: + raise ValueError("job lease is no longer owned") + return _reschedule_locked( + conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds, + error_code=error_code, permanent=permanent, + ) + + +def recover_expired(conn, *, max_attempts, delay_seconds=0, limit=20): + """Requeue dead workers; old delivery attempts become harmless duplicates.""" + if not 1 <= limit <= 100: + raise ValueError("invalid recovery batch") + with conn.transaction(): + jobs = conn.execute( + "SELECT * FROM attachment_jobs WHERE status='processing' " + "AND lease_expires_at<=clock_timestamp() " + "ORDER BY lease_expires_at,id LIMIT %s FOR UPDATE SKIP LOCKED", + (limit,), + ).fetchall() + return [ + (job["id"], _reschedule_locked( + conn, job, max_attempts=max_attempts, delay_seconds=delay_seconds, + error_code="worker_timeout", permanent=False, + )) + for job in jobs + ] + + +def complete_job(conn, job_id, token, variants): + """Record every versioned effect, then expose ready atomically. + + Returns True when this version is still current. A duplicate ready job is + a harmless no-op; an old version never changes the current attachment. + """ + if set(variants) != REQUIRED_VARIANTS: + raise ValueError("incomplete variant set") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.object_key,a.sha256,a.content_type,a.size_bytes," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j", + (job_id,), + ).fetchone() + if job is None: + raise ValueError("unknown job") + if job["status"] == "ready": + return job["pipeline_version"] == job["current_pipeline_version"] + if job["status"] != "processing" or job["lease_token"] != token: + raise ValueError("job lease is no longer owned") + original = variants["original"] + if (original["object_key"] != job["object_key"] or + original["sha256"] != job["sha256"] or + original["content_type"] != job["content_type"] or + original["size_bytes"] != job["size_bytes"]): + raise ValueError("original does not match accepted upload") + for name, value in variants.items(): + fields = (value["object_key"], value["content_type"], value["size_bytes"], + value["width"], value["height"], value["sha256"]) + conn.execute( + "INSERT INTO attachment_variants " + "(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,width,height,sha256) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + (job["attachment_id"], job["pipeline_version"], name, *fields), + ) + stored = conn.execute( + "SELECT object_key,content_type,size_bytes,width,height,sha256 " + "FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (job["attachment_id"], job["pipeline_version"], name), + ).fetchone() + if stored is None or tuple(stored[key] for key in + ("object_key", "content_type", "size_bytes", "width", "height", "sha256")) != fields: + raise ValueError("versioned variant conflicts with a previous effect") + conn.execute( + "UPDATE attachment_jobs SET status='ready',lease_token=NULL,lease_expires_at=NULL," + "error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (job_id,), + ) + current = conn.execute( + "UPDATE attachments SET status='ready',error_code=NULL," + "updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s RETURNING id", + (job["attachment_id"], job["pipeline_version"]), + ).fetchone() + return current is not None diff --git a/src/publisher.py b/src/publisher.py new file mode 100644 index 0000000..53fdfb8 --- /dev/null +++ b/src/publisher.py @@ -0,0 +1,70 @@ +"""Publish PostgreSQL outbox jobs with RabbitMQ confirms.""" + +import json +import logging +import os +import time +from pathlib import Path + +import pika +import psycopg +from psycopg_pool import PoolTimeout + +from .db import pool +from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload + + +logger = logging.getLogger(__name__) +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") + + +def publish_batch(channel): + with pool().connection() as conn: + token, rows = claim_outbox(conn, limit=10, lease_seconds=30) + for row in rows: + body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8") + channel.basic_publish( + exchange="", routing_key=QUEUE, body=body, mandatory=True, + properties=pika.BasicProperties( + content_type="application/json", delivery_mode=2, + message_id=str(row["job_id"]), type="attachment.process.v1", + ), + ) + # basic_publish returns only after a broker confirm in confirm mode. + with pool().connection() as conn: + if not confirm_outbox(conn, row["id"], token): + logger.warning("Outbox claim expired before confirm for row %s", row["id"]) + return len(rows) + + +def main(): + logging.basicConfig(level=logging.INFO) + logging.getLogger("pika").setLevel(logging.ERROR) + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = 30 + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + while broker.is_open: + if not publish_batch(channel): + broker.process_data_events(time_limit=1) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Publisher reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + try: + broker.close() + except (pika.exceptions.AMQPError, OSError): + pass + + +if __name__ == "__main__": + main() diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..20fb9c9 --- /dev/null +++ b/src/server.py @@ -0,0 +1,14 @@ +"""Run the ASGI worker as the container's main process.""" + +import os + +import uvicorn + + +if __name__ == "__main__": + uvicorn.run( + "src.main:app", + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=int(os.environ.get("APP_PORT", "8080")), + workers=1, + ) diff --git a/src/staging_cleanup.py b/src/staging_cleanup.py new file mode 100644 index 0000000..72d70b3 --- /dev/null +++ b/src/staging_cleanup.py @@ -0,0 +1,104 @@ +"""Remove abandoned staging objects after a grace period or a committed result.""" + +import logging +import os +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +from minio import Minio + +from .db import pool + + +logger = logging.getLogger(__name__) +BUCKET = "messenger-attachments" + + +def storage(): + def secret(name): + return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() + + return Minio( + os.environ.get("S3_ENDPOINT", "s3:9000"), + access_key=secret("S3_MAINTENANCE_KEY_ID"), + secret_key=secret("S3_MAINTENANCE_SECRET_KEY"), + secure=False, + region="us-east-1", + ) + + +def clean_once(client, grace_seconds=3600): + """Return removed count; lock DB state through each S3 delete.""" + if grace_seconds < 1: + raise ValueError("invalid staging grace period") + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + removed = 0 + for obj in client.list_objects(BUCKET, prefix="staging/", recursive=True): + key = obj.object_name + suffix = key.removeprefix("staging/") + try: + if key != f"staging/{UUID(suffix)}": + continue + except ValueError: + continue + with pool().connection() as conn, conn.transaction(): + attachment = conn.execute( + "SELECT a.status,EXISTS(" + "SELECT 1 FROM attachment_variants v WHERE v.attachment_id=a.id " + "AND v.pipeline_version=a.pipeline_version AND v.name='original' " + "AND v.object_key=a.object_key) AS original_ready " + "FROM attachments a WHERE a.staging_key=%s FOR UPDATE OF a", + (key,), + ).fetchone() + if attachment: + if attachment["status"] != "ready" or not attachment["original_ready"]: + continue + else: + if obj.last_modified is None or obj.last_modified > cutoff: + continue + marker = conn.execute( + "SELECT lease_expires_at>clock_timestamp() AS active " + "FROM attachment_uploads WHERE staging_key=%s FOR UPDATE", + (key,), + ).fetchone() + if marker and marker["active"]: + continue + client.remove_object(BUCKET, key) + if not attachment: + conn.execute( + "DELETE FROM attachment_uploads WHERE staging_key=%s " + "AND lease_expires_at<=clock_timestamp()", + (key,), + ) + removed += 1 + with pool().connection() as conn, conn.transaction(): + conn.execute( + "DELETE FROM attachment_uploads u WHERE u.lease_expires_at<" + "clock_timestamp()-make_interval(secs => %s) " + "AND NOT EXISTS (SELECT 1 FROM attachments a WHERE a.staging_key=u.staging_key)", + (grace_seconds,), + ) + return removed + + +def main(): + logging.basicConfig(level=logging.INFO) + client = storage() + grace = int(os.environ.get("STAGING_GRACE_SECONDS", "3600")) + interval = int(os.environ.get("STAGING_CLEANUP_INTERVAL_SECONDS", "60")) + if interval < 1: + raise ValueError("invalid cleanup interval") + while True: + try: + removed = clean_once(client, grace) + if removed: + logger.info("Removed %s staging objects", removed) + except Exception as exc: + logger.warning("Staging cleanup retrying after %s", type(exc).__name__) + time.sleep(interval) + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..298109a --- /dev/null +++ b/src/store.py @@ -0,0 +1,297 @@ +"""PostgreSQL-backed users, chats and message history.""" + +import base64 +import binascii +import hashlib +import json +import os +import secrets +from contextlib import contextmanager +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +import psycopg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException +from psycopg.errors import UniqueViolation +from psycopg_pool import PoolTimeout + +from . import db +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized + +password_hasher = PasswordHasher() + + +def timestamp(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def encode_cursor(resource: str, item: dict) -> str: + value = [resource, item["created_at"], item["id"]] + return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=") + + +def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True) + value = json.loads(raw) + if not isinstance(value, list) or len(value) != 3 or value[0] != resource: + raise ValueError + date_text, item_id = value[1:] + if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str): + raise ValueError + date = datetime.fromisoformat(date_text[:-1] + "+00:00") + if date.utcoffset().total_seconds() != 0: + raise ValueError + return date, UUID(item_id) + except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError): + raise HTTPException(400) from None + + +@contextmanager +def connection(): + try: + with db.pool().connection() as conn: + yield conn + except (psycopg.Error, PoolTimeout, OSError): + raise HTTPException(503) from None + + +def user_view(row: dict) -> dict: + return { + "id": str(row["id"]), "username": row["username"], + "display_name": row["display_name"], "created_at": timestamp(row["created_at"]), + } + + +def chat_view(row: dict) -> dict: + return { + "id": str(row["id"]), "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": timestamp(row["created_at"]), + } + + +def message_view(row: dict) -> dict: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), "text": row["text"], + "created_at": timestamp(row["created_at"]), + "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], + } + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids""" + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id=m.id ORDER BY a.position) AS attachment_ids""" + + +class Store: + def ready(self) -> dict: + if not db.ready(): + raise HTTPException(503) + return {"status": "ok"} + + def register(self, username: str, display_name: str, password: str) -> dict: + password_hash = password_hasher.hash(password) + with connection() as conn: + try: + row = conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) " + "RETURNING id,username,display_name,created_at", + (uuid4(), username, display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise HTTPException(409) from None + return user_view(row) + + def user(self, user_id: str | None) -> dict: + if not user_id: + raise HTTPException(401) + try: + user_uuid = UUID(user_id) + except (ValueError, TypeError): + raise HTTPException(401) from None + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,) + ).fetchone() + if row is None: + raise HTTPException(401) + return user_view(row) + + def create_session(self, username: str, password: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s", + (username,), + ).fetchone() + try: + valid = row is not None and password_hasher.verify(row["password_hash"], password) + except (VerificationError, InvalidHashError): + valid = False + if not valid: + raise unauthorized(BASIC_CHALLENGE) + token = secrets.token_urlsafe(32) + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + expiry = conn.execute( + "INSERT INTO sessions (token_hash,user_id,expires_at) " + "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at", + (token_hash(token), row["id"], ttl), + ).fetchone()["expires_at"] + return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)} + + def authenticated(self, token: str) -> dict: + with connection() as conn: + row = conn.execute( + "SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s " + "JOIN users u ON u.id=s.user_id " + "WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + return user_view(row) + + def revoke_session(self, token: str) -> None: + with connection() as conn: + row = conn.execute( + "DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash", + (token_hash(token),), + ).fetchone() + if row is None: + raise unauthorized(BEARER_CHALLENGE) + + def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict: + if actor_id not in member_ids: + raise HTTPException(400) + members = [UUID(member_id) for member_id in member_ids] + with connection() as conn: + count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"] + if count != len(members): + raise HTTPException(400) + row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone() + for position, member_id in enumerate(members): + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)", + (row["id"], member_id, position), + ) + return chat_view({**row, "member_ids": members}) + + def chat(self, actor_id: str, chat_id: str) -> dict: + with connection() as conn: + row = conn.execute( + f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id " + "WHERE access.user_id=%s AND c.id=%s", + (UUID(actor_id), UUID(chat_id)), + ).fetchone() + if row is None: + raise HTTPException(404) + return chat_view(row) + + def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict: + after = decode_cursor(cursor, f"chats:{actor_id}") + query = (f"SELECT {CHAT_COLUMNS} FROM chats c " + "JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s") + params = [UUID(actor_id)] + if after: + query += " AND (c.created_at,c.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY c.created_at,c.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + rows = conn.execute(query, params).fetchall() + items = [chat_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None} + + def create_message(self, actor_id: str, chat_id: str, text: str, key: str, attachment_ids: list[str] | None = None) -> dict: + ids = [UUID(item) for item in attachment_ids or []] + if len(ids) > 10 or len(set(ids)) != len(ids) or (not text.strip() and not ids): + raise HTTPException(400) + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + request_hash = hashlib.sha256(json.dumps( + {"text": text, "attachment_ids": [str(item) for item in ids]}, ensure_ascii=False, sort_keys=True, + ).encode()).digest() + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + conn.execute( + "DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s " + "AND created_at dict: + after = decode_cursor(cursor, f"messages:{chat_id}") + chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id) + query = f"SELECT {MESSAGE_COLUMNS} FROM messages m WHERE m.chat_id=%s" + params = [chat_uuid] + if after: + query += " AND (m.created_at,m.id)>(%s,%s)" + params.extend(after) + query += " ORDER BY m.created_at,m.id LIMIT %s" + params.append(limit + 1) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid) + ).fetchone() + if member is None: + raise HTTPException(404) + rows = conn.execute(query, params).fetchall() + items = [message_view(row) for row in rows[:limit]] + return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None} + + +store = Store() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..b0c2f7f --- /dev/null +++ b/src/worker.py @@ -0,0 +1,216 @@ +"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects.""" + +import hashlib +import json +import logging +import os +import random +import re +import signal +import time +from contextlib import contextmanager +from datetime import datetime +from io import BytesIO +from pathlib import Path +from uuid import UUID + +import pika +import psycopg +from minio.error import S3Error +from psycopg_pool import PoolTimeout + +from .attachments import BUCKET, storage +from .db import pool +from .image_variants import InvalidImage, MAX_BYTES, render +from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired + + +logger = logging.getLogger(__name__) +QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) +LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) + + +def parse_message(body: bytes) -> dict: + if len(body) > 4096: + raise ValueError("oversized job message") + item = json.loads(body) + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if (not isinstance(item, dict) or not required <= item.keys() or + item.keys() - required - {"traceparent", "tracestate"} or + type(item["schema_version"]) is not int or item["schema_version"] != 1 or + item["type"] != "attachment.process.v1" or + type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or + type(item["attempt"]) is not int or item["attempt"] < 1 or + not isinstance(item["job_id"], str) or + not isinstance(item["attachment_id"], str) or + not isinstance(item["created_at"], str) or + ("traceparent" in item and + (not isinstance(item["traceparent"], str) or + re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or + ("tracestate" in item and + (not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))): + raise ValueError("invalid job message") + UUID(item["job_id"]) + UUID(item["attachment_id"]) + created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00")) + if created_at.utcoffset() is None: + raise ValueError("job timestamp lacks timezone") + return item + + +def _source_bytes(job) -> bytes: + if not 0 < job["size_bytes"] <= MAX_BYTES: + raise InvalidImage("invalid accepted size") + # Reprocessing uses the byte-identical final original, so ready staging + # can be reclaimed without making a future pipeline version impossible. + key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"] + response = storage().get_object(BUCKET, key) + try: + chunks = [] + size = 0 + digest = hashlib.sha256() + for chunk in response.stream(64 * 1024): + size += len(chunk) + if size > job["size_bytes"]: + raise InvalidImage("source exceeds accepted size") + chunks.append(chunk) + digest.update(chunk) + if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]: + raise InvalidImage("source checksum mismatch") + return b"".join(chunks) + finally: + response.close() + response.release_conn() + + +def _write_variants(job, variants: dict) -> dict: + extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + stored = {} + for name, variant in variants.items(): + key = (job["object_key"] if name == "original" else + f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}") + data = variant["data"] + storage().put_object( + BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"] + ) + stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"} + stored[name]["object_key"] = key + return stored + + +def _delay(attempt: int) -> float: + base = min(30, 2 ** min(attempt - 1, 5)) + return base + random.uniform(0, base / 2) + + +@contextmanager +def _deadline(seconds: int): + def expired(_signal, _frame): + raise TimeoutError("image job exceeded worker deadline") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def process_message(item: dict) -> str: + with pool().connection() as conn: + job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS) + if job is None: + return "duplicate_or_stale" + token = job["lease_token"] + try: + with _deadline(LEASE_SECONDS): + data = _source_bytes(job) + variants = render(data, job["content_type"]) + stored = _write_variants(job, variants) + with pool().connection() as conn: + current = complete_job(conn, job["id"], token, stored) + if current and job["staging_key"]: + try: + storage().remove_object(BUCKET, job["staging_key"]) + except Exception as exc: + # Periodic cleanup recovers this post-COMMIT gap. + logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__) + return "ready" + except psycopg.Error: + raise + except Exception as exc: + if isinstance(exc, InvalidImage): + code, permanent = "invalid_image", True + elif isinstance(exc, TimeoutError): + code, permanent = "worker_timeout", False + elif isinstance(exc, S3Error) or isinstance(exc, OSError): + code, permanent = "storage_unavailable", False + elif isinstance(exc, ValueError): + code, permanent = "processing_failed", True + else: + code, permanent = "processing_failed", False + logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code) + with pool().connection() as conn: + try: + return finish_attempt( + conn, job["id"], token, max_attempts=MAX_ATTEMPTS, + delay_seconds=_delay(job["attempt"]), error_code=code, + permanent=permanent, + ) + except ValueError: + # A lease recovery already owns the next attempt. + return "stale_lease" + + +def main() -> None: + logging.basicConfig(level=logging.INFO) + if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: + raise SystemExit("invalid worker retry settings") + url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() + params = pika.URLParameters(url) + params.heartbeat = max(120, LEASE_SECONDS * 2) + params.socket_timeout = 5 + params.blocked_connection_timeout = 10 + while True: + broker = None + try: + broker = pika.BlockingConnection(params) + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.basic_qos(prefetch_count=1) + last_recovery = 0.0 + while broker.is_open: + if time.monotonic() - last_recovery >= 5: + with pool().connection() as conn: + recover_expired(conn, max_attempts=MAX_ATTEMPTS) + last_recovery = time.monotonic() + method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False) + if method is None: + broker.process_data_events(time_limit=1) + continue + try: + item = parse_message(body) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + logger.warning("Rejecting invalid job message") + channel.basic_reject(method.delivery_tag, requeue=False) + continue + try: + result = process_message(item) + channel.basic_ack(method.delivery_tag) + logger.info("Job %s: %s", item["job_id"], result) + except (psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__) + channel.basic_nack(method.delivery_tag, requeue=True) + time.sleep(2) + except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc: + logger.warning("Worker reconnecting after %s", type(exc).__name__) + time.sleep(2) + finally: + if broker is not None and broker.is_open: + broker.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py new file mode 100644 index 0000000..404b863 --- /dev/null +++ b/student-tests/test_image_pipeline.py @@ -0,0 +1,74 @@ +"""Small checks for the worker's deterministic image and payload boundary.""" + +import json +import unittest +from io import BytesIO + +from PIL import Image + +from src.image_variants import InvalidImage, render +from src.worker import parse_message + + +def encoded(image, format="PNG", **options): + stream = BytesIO() + image.save(stream, format=format, **options) + return stream.getvalue() + + +class ImagePipelineTest(unittest.TestCase): + def test_contract_sizes_original_and_content_types(self): + original = encoded(Image.new("RGB", (320, 200), "red")) + result = render(original, "image/png") + self.assertEqual(result["original"]["data"], original) + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (256, 160)) + self.assertEqual((result["crop"]["width"], result["crop"]["height"]), (128, 128)) + self.assertEqual((result["optimized"]["width"], result["optimized"]["height"]), (320, 200)) + self.assertEqual([result[name]["content_type"] for name in ("thumbnail", "crop", "optimized")], + ["image/jpeg", "image/jpeg", "image/webp"]) + + def test_alpha_is_white_and_small_image_is_not_upscaled(self): + original = encoded(Image.new("RGBA", (24, 12), (255, 0, 0, 0))) + result = render(original, "image/png") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (24, 12)) + with Image.open(BytesIO(result["thumbnail"]["data"])) as thumb: + self.assertTrue(all(channel >= 245 for channel in thumb.getpixel((0, 0)))) + + def test_exif_orientation_and_center_crop(self): + image = Image.new("RGB", (30, 10), "green") + image.paste("red", (0, 0, 10, 10)) + image.paste("blue", (20, 0, 30, 10)) + exif = Image.Exif() + exif[274] = 6 + original = encoded(image, "JPEG", exif=exif) + result = render(original, "image/jpeg") + self.assertEqual((result["thumbnail"]["width"], result["thumbnail"]["height"]), (10, 30)) + with Image.open(BytesIO(result["crop"]["data"])) as crop: + red, green, blue = crop.getpixel((64, 64)) + self.assertGreater(green, red) + self.assertGreater(green, blue) + + def test_rejects_wrong_type_and_corrupt_file(self): + original = encoded(Image.new("RGB", (2, 2), "black")) + with self.assertRaises(InvalidImage): + render(original, "image/jpeg") + with self.assertRaises(InvalidImage): + render(b"broken", "image/png") + + def test_payload_contains_only_versioned_job_identifiers(self): + body = { + "schema_version": 1, + "job_id": "11111111-1111-4111-8111-111111111111", + "type": "attachment.process.v1", + "attachment_id": "22222222-2222-4222-8222-222222222222", + "pipeline_version": 1, + "attempt": 1, + "created_at": "2026-09-01T12:00:00Z", + } + self.assertEqual(parse_message(json.dumps(body).encode()), body) + with self.assertRaises(ValueError): + parse_message(json.dumps({**body, "staging_key": "private"}).encode()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_outbox_payload.py b/student-tests/test_outbox_payload.py new file mode 100644 index 0000000..0fe6b3f --- /dev/null +++ b/student-tests/test_outbox_payload.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Keep broker messages within the public job schema and free of file data.""" + +import json +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "tests")) +from client import validate # noqa: E402 +from src.pipeline_jobs import job_payload # noqa: E402 + + +class OutboxPayloadTest(unittest.TestCase): + def test_job_message_has_only_contract_fields(self): + schema = json.loads((ROOT / "contracts/jobs.schema.json").read_text(encoding="utf-8")) + row = { + "job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 2, + "attempt": 3, "created_at": datetime.now(timezone.utc), + "object_key": "staging/must-not-leak", "bytes": b"must-not-leak", + } + message = job_payload(row) + validate(message, schema) + self.assertEqual(set(message), set(schema["required"])) + self.assertNotIn("staging", json.dumps(message)) + self.assertNotIn("bytes", json.dumps(message)) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_staging_cleanup.py b/student-tests/test_staging_cleanup.py new file mode 100644 index 0000000..4dddb28 --- /dev/null +++ b/student-tests/test_staging_cleanup.py @@ -0,0 +1,46 @@ +"""Run inside the cleanup image against an isolated pair of staging keys.""" + +import time +import sys +from io import BytesIO +from pathlib import Path +from uuid import uuid4 + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from src.db import pool +from src.pipeline_jobs import reserve_upload +from src.staging_cleanup import BUCKET, clean_once, storage + + +client = storage() +orphan_id, active_id = uuid4(), uuid4() +orphan_key, active_key = f"staging/{orphan_id}", f"staging/{active_id}" + + +def exists(key): + return any(obj.object_name == key for obj in client.list_objects(BUCKET, prefix=key)) + + +try: + for key in (orphan_key, active_key): + client.put_object(BUCKET, key, BytesIO(b"test"), 4, content_type="image/png") + with pool().connection() as conn: + reserve_upload(conn, active_id, active_key, lease_seconds=60) + time.sleep(2) + clean_once(client, grace_seconds=1) + assert not exists(orphan_key), "old orphan was not removed" + assert exists(active_key), "active upload was removed" + + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_uploads SET lease_expires_at=clock_timestamp()-interval '1 second' WHERE id=%s", + (active_id,), + ) + clean_once(client, grace_seconds=1) + assert not exists(active_key), "expired orphan was not removed" + print("PASS: cleanup removes old orphans and preserves active uploads") +finally: + for key in (orphan_key, active_key): + client.remove_object(BUCKET, key) + with pool().connection() as conn: + conn.execute("DELETE FROM attachment_uploads WHERE id=%s", (active_id,)) diff --git a/student-tests/test_upload_handoff.py b/student-tests/test_upload_handoff.py new file mode 100644 index 0000000..8a26182 --- /dev/null +++ b/student-tests/test_upload_handoff.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Exercise 202 handoff while RabbitMQ and worker are unavailable.""" + +import subprocess +import sys +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run # noqa: E402 + + +def compose(*args): + subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, + ) + + +def ready_after_restart(client): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not recover after restart") + + +def main(): + args = arguments("Lab5 durable upload handoff with stopped broker and worker") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher", "broker") + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked") + + compose("up", "-d", "broker") + compose("restart", "api") + ready_after_restart(client) + check( + client.call("GET", f"/api/v1/attachments/{attachment['id']}", actor=bob, schema="Attachment")["json"]["status"] == "queued", + "accepted job disappeared before publisher restart", + ) + compose("up", "-d", "publisher", "worker") + client.wait_ready(bob, attachment) + result = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(result["bytes"] == original, "accepted original changed after recovery") + finally: + compose("up", "-d", "broker", "publisher", "worker") + print("PASS: accepted upload survived broker/API outage and reached ready") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_upload_queued.py b/student-tests/test_upload_queued.py new file mode 100644 index 0000000..ec6521d --- /dev/null +++ b/student-tests/test_upload_queued.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Public U2 check while publisher and worker are stopped.""" + +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 durable queued upload") + client = Client(args.base_url, args.ca_file) + alice, bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload did not enter queued state") + client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", 409, actor=bob) + message = client.message(alice, chat, "", attachments=[attachment["id"]])["json"] + check(message["attachment_ids"] == [attachment["id"]], "queued attachment cannot be linked") + print("PASS: 202 queued upload is readable and linkable; content is pending") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_variants_api.py b/student-tests/test_variants_api.py new file mode 100644 index 0000000..fd4aee8 --- /dev/null +++ b/student-tests/test_variants_api.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Check every public variant against committed metadata and original bytes.""" + +import hashlib +import sys +from pathlib import Path + + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def main(): + args = arguments("Lab5 ready variants") + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + attachment, original = client.attachment(alice, chat) + ready = client.wait_ready(bob, attachment) + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "incomplete variant set") + expected = { + "thumbnail": ("image/jpeg", 256, 160), + "crop": ("image/jpeg", 128, 128), + "optimized": ("image/webp", 320, 200), + } + for name, (content_type, width, height) in expected.items(): + variant = variants[name] + check((variant["content_type"], variant["width"], variant["height"]) == + (content_type, width, height), f"wrong {name} metadata") + response = client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", actor=bob + ) + check(response["headers"].get_content_type() == content_type, f"wrong {name} MIME") + check(len(response["bytes"]) == variant["size_bytes"] and + hashlib.sha256(response["bytes"]).hexdigest() == variant["sha256"], + f"wrong {name} bytes") + client.call( + "GET", f"/api/v1/attachments/{attachment['id']}/content?variant={name}", + 404, actor=eve, + ) + downloaded = client.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob) + check(downloaded["bytes"] == original, "original bytes changed") + print("PASS: all three variants match metadata; original preserved; outsider denied") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From 26f091c9b4ec43b44468e84db69d410c65cfeccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 14:54:25 +0300 Subject: [PATCH 04/13] fix: Include broker bootstrap helper in lab6 baseline (U1) --- scripts/init_local_broker.py | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 scripts/init_local_broker.py diff --git a/scripts/init_local_broker.py b/scripts/init_local_broker.py new file mode 100644 index 0000000..e24bf49 --- /dev/null +++ b/scripts/init_local_broker.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Create local RabbitMQ credentials without writing them to tracked files.""" + +from pathlib import Path +from secrets import token_hex +from urllib.parse import quote + + +root = Path(__file__).resolve().parents[1] +secret_dir = root / "secrets" +secret_dir.mkdir(mode=0o700, exist_ok=True) +secret_dir.chmod(0o700) +config = secret_dir / "rabbitmq.conf" +url_file = secret_dir / "broker_url" +username = "lab5_app" + +if config.exists(): + values = dict( + line.split(" = ", 1) + for line in config.read_text(encoding="utf-8").splitlines() + if " = " in line + ) + if values.get("default_user") != username or not values.get("default_pass"): + raise SystemExit("Existing RabbitMQ config cannot be reused") + password = values["default_pass"] +else: + password = token_hex(24) + config.write_text( + f"default_user = {username}\ndefault_pass = {password}\n", + encoding="utf-8", + ) +config.chmod(0o600) +url_file.write_text( + f"amqp://{username}:{quote(password, safe='')}@broker:5672/%2F\n", + encoding="utf-8", +) +url_file.chmod(0o600) +print("Local broker credentials are ready in ignored secrets/") -- 2.54.0 From 8eab283f6f4d711c53f052894a6406c95a4afad0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 15:09:55 +0300 Subject: [PATCH 05/13] feat: Instrument durable API-to-worker telemetry (U2) --- .gitignore | 1 + compose.telemetry.yaml | 7 ++- compose.yaml | 8 ++- migrations/008_trace_context.sql | 7 +++ requirements.txt | 4 ++ scripts/setup-local-secrets.py | 2 + src/attachments.py | 25 +++++--- src/main.py | 40 +++++++++++- src/pipeline_jobs.py | 16 +++-- src/publisher.py | 42 +++++++++---- src/telemetry.py | 86 +++++++++++++++++++++++++ src/worker.py | 53 +++++++++++++--- student-tests/test_outbox_payload.py | 6 +- student-tests/test_telemetry.py | 94 ++++++++++++++++++++++++++++ 14 files changed, 355 insertions(+), 36 deletions(-) create mode 100644 migrations/008_trace_context.sql create mode 100644 src/telemetry.py create mode 100644 student-tests/test_telemetry.py diff --git a/.gitignore b/.gitignore index f101a4f..860fedd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .env .env.* !.env.example +compose.local.yaml secrets/ certs/private/ *.key diff --git a/compose.telemetry.yaml b/compose.telemetry.yaml index 8201791..02f35c4 100644 --- a/compose.telemetry.yaml +++ b/compose.telemetry.yaml @@ -41,7 +41,8 @@ services: ports: - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" environment: - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in local .env} + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password + secrets: [grafana_admin_password] volumes: - grafana_data:/var/lib/grafana - ./infra/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro @@ -57,3 +58,7 @@ volumes: loki_data: tempo_data: grafana_data: + +secrets: + grafana_admin_password: + file: ./secrets/grafana_admin_password diff --git a/compose.yaml b/compose.yaml index 8bdc432..345da2d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,4 @@ -name: mtusi2026lab5 +name: mtusi2026lab6 services: postgres: @@ -112,6 +112,8 @@ services: S3_BUCKET: messenger-attachments S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_VERSION: lab6 secrets: [database_url, s3_access_key_id, s3_secret_access_key] depends_on: migrate: @@ -135,6 +137,8 @@ services: DATABASE_URL_FILE: /run/secrets/database_url BROKER_URL_FILE: /run/secrets/broker_url BROKER_QUEUE: attachments.process + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_VERSION: lab6 secrets: [database_url, broker_url] depends_on: migrate: @@ -175,6 +179,8 @@ services: WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + APP_VERSION: lab6 secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] depends_on: migrate: diff --git a/migrations/008_trace_context.sql b/migrations/008_trace_context.sql new file mode 100644 index 0000000..f992f9a --- /dev/null +++ b/migrations/008_trace_context.sql @@ -0,0 +1,7 @@ +-- Preserve the upload span context across the durable outbox and every retry. +-- Version 007 is reserved for the lab5 replay/audit migration. +ALTER TABLE attachment_jobs + ADD COLUMN traceparent text + CHECK (traceparent IS NULL OR traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'), + ADD COLUMN tracestate text + CHECK (tracestate IS NULL OR length(tracestate) <= 512); diff --git a/requirements.txt b/requirements.txt index a9a509e..2655fc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,7 @@ minio==7.2.20 Pillow==11.3.0 urllib3==2.8.0 pika==1.4.4 +opentelemetry-api==1.45.0 +opentelemetry-sdk==1.45.0 +opentelemetry-semantic-conventions==0.66b0 +opentelemetry-exporter-otlp-proto-http==1.45.0 diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py index 168168d..6fbac6d 100644 --- a/scripts/setup-local-secrets.py +++ b/scripts/setup-local-secrets.py @@ -36,6 +36,8 @@ def main() -> None: write("s3_secret_access_key", secrets.token_urlsafe(32)) write("s3_maintenance_key_id", secrets.token_hex(10)) write("s3_maintenance_secret_key", secrets.token_urlsafe(32)) + if not (ROOT / "grafana_admin_password").exists(): + write("grafana_admin_password", secrets.token_urlsafe(32)) if __name__ == "__main__": diff --git a/src/attachments.py b/src/attachments.py index 818c5a5..13cb080 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -13,6 +13,8 @@ from uuid import UUID, uuid4 from fastapi import HTTPException, Request from minio import Minio from minio.error import S3Error +from opentelemetry import trace +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from PIL import Image, UnidentifiedImageError from urllib3 import PoolManager, Retry, Timeout @@ -25,6 +27,8 @@ MAX_BYTES = 10 * 1024 * 1024 MAX_PIXELS = 40_000_000 Image.MAX_IMAGE_PIXELS = MAX_PIXELS logger = logging.getLogger(__name__) +tracer = trace.get_tracer(__name__) +trace_context = TraceContextTextMapPropagator() def secret(name: str) -> str: @@ -120,17 +124,24 @@ def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: try: stream.seek(0) client = storage() - client.put_object(BUCKET, staging_key, stream, size, content_type=content_type) + with tracer.start_as_current_span("s3.stage_upload", record_exception=False, set_status_on_exception=False): + client.put_object(BUCKET, staging_key, stream, size, content_type=content_type) with connection() as conn: if not renew_upload(conn, attachment_id): raise HTTPException(503) - verify_staging(client, staging_key, size, sha256, content_type) + with tracer.start_as_current_span("s3.verify_staging", record_exception=False, set_status_on_exception=False): + verify_staging(client, staging_key, size, sha256, content_type) + carrier = {} + trace_context.inject(carrier) with connection() as conn: - row, _job_id = queue_attachment( - conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id), - filename=filename, staging_key=staging_key, object_key=object_key, - content_type=content_type, size_bytes=size, sha256=sha256, - ) + with tracer.start_as_current_span("db.queue_attachment", record_exception=False, + set_status_on_exception=False): + row, _job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id), + filename=filename, staging_key=staging_key, object_key=object_key, + content_type=content_type, size_bytes=size, sha256=sha256, + traceparent=carrier.get("traceparent"), tracestate=carrier.get("tracestate"), + ) except HTTPException: raise except Exception as exc: diff --git a/src/main.py b/src/main.py index 26dea4b..d8a500c 100644 --- a/src/main.py +++ b/src/main.py @@ -1,16 +1,27 @@ -"""FastAPI entry point and contract-wide response handling.""" +"""FastAPI entry point, contract responses, and safe HTTP telemetry.""" +import logging +from time import monotonic from uuid import uuid4 from fastapi import FastAPI, HTTPException, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from .api import router +from . import telemetry app = FastAPI() app.include_router(router) INSTANCE_ID = uuid4().hex +telemetry.configure("messenger-api") +HTTP_REQUESTS, HTTP_DURATION = telemetry.http_instruments() +TRACER = trace.get_tracer(__name__) +TRACE_CONTEXT = TraceContextTextMapPropagator() +logger = logging.getLogger(__name__) ERRORS = { 400: ("invalid_request", "Неверный запрос"), @@ -37,6 +48,33 @@ def error_response(request: Request, status: int, extra_headers: dict | None = N @app.middleware("http") async def request_contract(request: Request, call_next): request.state.request_id = str(uuid4()) + started = monotonic() + status = 500 + context = TRACE_CONTEXT.extract(request.headers) + with TRACER.start_as_current_span("HTTP request", context=context, kind=SpanKind.SERVER, + record_exception=False, set_status_on_exception=False) as span: + try: + response = await _contract_response(request, call_next) + status = response.status_code + return response + finally: + route = getattr(request.scope.get("route"), "path", "unmatched") + status_class = f"{status // 100}xx" + attributes = {"method": request.method, "route": route, "status_class": status_class} + HTTP_REQUESTS.add(1, attributes) + HTTP_DURATION.record(monotonic() - started, attributes) + span.update_name(f"{request.method} {route}") + span.set_attribute("http.request.method", request.method) + span.set_attribute("http.route", route) + span.set_attribute("http.response.status_code", status) + span.set_attribute("request.id", request.state.request_id) + if status >= 500: + span.set_status(Status(StatusCode.ERROR)) + logger.info("http_request", extra={"event": "http_request", "request_id": request.state.request_id, + "method": request.method, "route": route, "status_class": status_class}) + + +async def _contract_response(request: Request, call_next): is_upload = request.url.path.startswith("/api/v1/chats/") and request.url.path.endswith("/attachments") if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions" and not is_upload: media_type, _, charset = request.headers.get("content-type", "").lower().partition(";") diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py index 5fa477f..47b2342 100644 --- a/src/pipeline_jobs.py +++ b/src/pipeline_jobs.py @@ -13,7 +13,7 @@ REQUIRED_VARIANTS = {"original", "thumbnail", "crop", "optimized"} def job_payload(row): """Broker message per jobs.schema.json: identifiers only, never file bytes.""" - return { + payload = { "schema_version": 1, "job_id": str(row["job_id"]), "type": "attachment.process.v1", @@ -22,6 +22,10 @@ def job_payload(row): "attempt": row["attempt"], "created_at": row["created_at"].astimezone(timezone.utc).isoformat().replace("+00:00", "Z"), } + for key in ("traceparent", "tracestate"): + if row.get(key): + payload[key] = row[key] + return payload def reserve_upload(conn, attachment_id, staging_key, lease_seconds=300): @@ -50,7 +54,7 @@ def renew_upload(conn, attachment_id, lease_seconds=300): def queue_attachment(conn, *, attachment_id, chat_id, owner_id, filename, staging_key, object_key, content_type, size_bytes, sha256, - pipeline_version=1): + pipeline_version=1, traceparent=None, tracestate=None): """Commit attachment, job, and outbox after the full S3 staging PUT.""" job_id = uuid4() with conn.transaction(): @@ -70,8 +74,9 @@ def queue_attachment(conn, *, attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, pipeline_version), ).fetchone() conn.execute( - "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) VALUES (%s,%s,%s)", - (job_id, attachment_id, pipeline_version), + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version,traceparent,tracestate) " + "VALUES (%s,%s,%s,%s,%s)", + (job_id, attachment_id, pipeline_version, traceparent, tracestate), ) conn.execute( "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", @@ -97,7 +102,8 @@ def claim_outbox(conn, limit=10, lease_seconds=30): " SET claim_token=%s,claim_expires_at=clock_timestamp()+make_interval(secs => %s)" " FROM due,attachment_jobs j" " WHERE o.id=due.id AND j.id=o.job_id" - " RETURNING o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at", + " RETURNING o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at," + "j.traceparent,j.tracestate", (limit, token, lease_seconds), ).fetchall() return token, rows diff --git a/src/publisher.py b/src/publisher.py index 53fdfb8..209d24b 100644 --- a/src/publisher.py +++ b/src/publisher.py @@ -8,36 +8,52 @@ from pathlib import Path import pika import psycopg +from opentelemetry import trace +from opentelemetry.trace import SpanKind +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from psycopg_pool import PoolTimeout from .db import pool from .pipeline_jobs import claim_outbox, confirm_outbox, job_payload +from .telemetry import configure -logger = logging.getLogger(__name__) +logger = logging.getLogger("src.publisher") QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +tracer = trace.get_tracer(__name__) +trace_context = TraceContextTextMapPropagator() def publish_batch(channel): with pool().connection() as conn: token, rows = claim_outbox(conn, limit=10, lease_seconds=30) for row in rows: - body = json.dumps(job_payload(row), separators=(",", ":")).encode("utf-8") - channel.basic_publish( - exchange="", routing_key=QUEUE, body=body, mandatory=True, - properties=pika.BasicProperties( - content_type="application/json", delivery_mode=2, - message_id=str(row["job_id"]), type="attachment.process.v1", - ), - ) - # basic_publish returns only after a broker confirm in confirm mode. - with pool().connection() as conn: - if not confirm_outbox(conn, row["id"], token): - logger.warning("Outbox claim expired before confirm for row %s", row["id"]) + parent = trace_context.extract({key: row[key] for key in ("traceparent", "tracestate") if row.get(key)}) + with tracer.start_as_current_span("attachments.publish", context=parent, kind=SpanKind.PRODUCER, + record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("messaging.destination.name", QUEUE) + span.set_attribute("messaging.message.attempt", row["attempt"]) + carrier = {} + trace_context.inject(carrier) + body = json.dumps({**job_payload(row), **carrier}, separators=(",", ":")).encode("utf-8") + channel.basic_publish( + exchange="", routing_key=QUEUE, body=body, mandatory=True, + properties=pika.BasicProperties( + content_type="application/json", delivery_mode=2, + message_id=str(row["job_id"]), type="attachment.process.v1", + ), + ) + # basic_publish returns only after a broker confirm in confirm mode. + with pool().connection() as conn: + if not confirm_outbox(conn, row["id"], token): + logger.warning("Outbox claim expired before confirm for row %s", row["id"]) + logger.info("job_published", extra={"event": "job_published", "job_type": "attachment.process.v1", + "attempt": row["attempt"]}) return len(rows) def main(): + configure("messenger-publisher") logging.basicConfig(level=logging.INFO) logging.getLogger("pika").setLevel(logging.ERROR) url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..9d6408c --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,86 @@ +"""Bounded, process-local OTLP setup and low-cardinality instruments.""" + +import logging +import os +from functools import lru_cache +from uuid import uuid4 + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + + +HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10) +JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120) + + +@lru_cache(maxsize=1) +def configure(service_name: str) -> bool: + """Export asynchronously when Compose supplies a Collector endpoint.""" + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").rstrip("/") + if not endpoint: + return False + resource = Resource.create({ + "service.name": service_name, + "service.version": os.environ.get("APP_VERSION", "lab6"), + "service.instance.id": uuid4().hex, + "deployment.environment.name": "lab", + }) + + traces = TracerProvider(resource=resource) + traces.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces", timeout=2), + max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000, + )) + trace.set_tracer_provider(traces) + + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{endpoint}/v1/metrics", timeout=2), + export_interval_millis=5000, export_timeout_millis=2000, + ) + metrics.set_meter_provider(MeterProvider( + resource=resource, metric_readers=[reader], + views=[ + View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), + View(instrument_name="messenger.jobs.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + ], + )) + + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=f"{endpoint}/v1/logs", timeout=2), + max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=1000, + )) + set_logger_provider(logs) + app_logger = logging.getLogger("src") + app_logger.setLevel(logging.INFO) + app_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + return True + + +@lru_cache(maxsize=1) +def http_instruments(): + meter = metrics.get_meter("messenger.http") + return ( + meter.create_counter("messenger.http.requests", unit="1"), + meter.create_histogram("messenger.http.duration", unit="s"), + ) + + +@lru_cache(maxsize=1) +def job_instruments(): + meter = metrics.get_meter("messenger.jobs") + return ( + meter.create_counter("messenger.jobs.processed", unit="1"), + meter.create_histogram("messenger.jobs.duration", unit="s"), + ) diff --git a/src/worker.py b/src/worker.py index b0c2f7f..364b58b 100644 --- a/src/worker.py +++ b/src/worker.py @@ -17,18 +17,24 @@ from uuid import UUID import pika import psycopg from minio.error import S3Error +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from psycopg_pool import PoolTimeout from .attachments import BUCKET, storage from .db import pool from .image_variants import InvalidImage, MAX_BYTES, render from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired +from .telemetry import configure, job_instruments -logger = logging.getLogger(__name__) +logger = logging.getLogger("src.worker") QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) +tracer = trace.get_tracer(__name__) +trace_context = TraceContextTextMapPropagator() def parse_message(body: bytes) -> dict: @@ -126,11 +132,15 @@ def process_message(item: dict) -> str: token = job["lease_token"] try: with _deadline(LEASE_SECONDS): - data = _source_bytes(job) - variants = render(data, job["content_type"]) - stored = _write_variants(job, variants) + with tracer.start_as_current_span("s3.read_source", record_exception=False, set_status_on_exception=False): + data = _source_bytes(job) + with tracer.start_as_current_span("image.render", record_exception=False, set_status_on_exception=False): + variants = render(data, job["content_type"]) + with tracer.start_as_current_span("s3.write_variants", record_exception=False, set_status_on_exception=False): + stored = _write_variants(job, variants) with pool().connection() as conn: - current = complete_job(conn, job["id"], token, stored) + with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False): + current = complete_job(conn, job["id"], token, stored) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) @@ -152,6 +162,7 @@ def process_message(item: dict) -> str: else: code, permanent = "processing_failed", False logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code) + trace.get_current_span().add_event("job_retry_or_failure", {"error.code": code}) with pool().connection() as conn: try: return finish_attempt( @@ -164,7 +175,36 @@ def process_message(item: dict) -> str: return "stale_lease" +def traced_process_message(item: dict) -> str: + parent = trace_context.extract({key: item[key] for key in ("traceparent", "tracestate") if key in item}) + with tracer.start_as_current_span("attachments.process", context=parent, kind=SpanKind.CONSUMER, + record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("messaging.destination.name", QUEUE) + span.set_attribute("messaging.message.attempt", item["attempt"]) + span.set_attribute("job.id", item["job_id"]) + span.set_attribute("attachment.id", item["attachment_id"]) + started = time.monotonic() + outcome = "error" + try: + outcome = process_message(item) + return outcome + except (psycopg.Error, PoolTimeout, OSError): + outcome = "database_unavailable" + raise + finally: + attributes = {"job_type": item["type"], "outcome": outcome} + processed, duration = job_instruments() + processed.add(1, attributes) + duration.record(time.monotonic() - started, attributes) + span.set_attribute("job.outcome", outcome) + if outcome in {"failed", "database_unavailable", "error"}: + span.set_status(Status(StatusCode.ERROR)) + logger.info("job_processed", extra={"event": "job_processed", "job_type": item["type"], + "outcome": outcome, "attempt": item["attempt"]}) + + def main() -> None: + configure("messenger-worker") logging.basicConfig(level=logging.INFO) if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: raise SystemExit("invalid worker retry settings") @@ -197,9 +237,8 @@ def main() -> None: channel.basic_reject(method.delivery_tag, requeue=False) continue try: - result = process_message(item) + traced_process_message(item) channel.basic_ack(method.delivery_tag) - logger.info("Job %s: %s", item["job_id"], result) except (psycopg.Error, PoolTimeout, OSError) as exc: logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__) channel.basic_nack(method.delivery_tag, requeue=True) diff --git a/student-tests/test_outbox_payload.py b/student-tests/test_outbox_payload.py index 0fe6b3f..b1c37de 100644 --- a/student-tests/test_outbox_payload.py +++ b/student-tests/test_outbox_payload.py @@ -23,10 +23,14 @@ class OutboxPayloadTest(unittest.TestCase): "job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 2, "attempt": 3, "created_at": datetime.now(timezone.utc), "object_key": "staging/must-not-leak", "bytes": b"must-not-leak", + "traceparent": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + "tracestate": "lab=upload", } message = job_payload(row) validate(message, schema) - self.assertEqual(set(message), set(schema["required"])) + self.assertEqual(set(message), set(schema["required"]) | {"traceparent", "tracestate"}) + self.assertEqual(message["traceparent"], row["traceparent"]) + self.assertEqual(message["tracestate"], row["tracestate"]) self.assertNotIn("staging", json.dumps(message)) self.assertNotIn("bytes", json.dumps(message)) diff --git a/student-tests/test_telemetry.py b/student-tests/test_telemetry.py new file mode 100644 index 0000000..d5b8837 --- /dev/null +++ b/student-tests/test_telemetry.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""The durable job carrier preserves one trace across broker boundaries.""" + +import json +import sys +import unittest +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import Mock, patch +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from opentelemetry import trace # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.trace import SpanKind # noqa: E402 +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator # noqa: E402 + + +exporter = InMemorySpanExporter() +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(exporter)) +trace.set_tracer_provider(provider) + +from src.pipeline_jobs import job_payload # noqa: E402 +from src import worker # noqa: E402 + + +class TelemetryTest(unittest.TestCase): + def setUp(self): + exporter.clear() + + def test_failed_job_has_low_cardinality_metric_and_error_span(self): + item = { + "job_id": str(uuid4()), "attachment_id": str(uuid4()), + "type": "attachment.process.v1", "attempt": 1, + } + counter, duration = Mock(), Mock() + with patch.object(worker, "process_message", return_value="failed"), \ + patch.object(worker, "job_instruments", return_value=(counter, duration)): + self.assertEqual(worker.traced_process_message(item), "failed") + labels = {"job_type": "attachment.process.v1", "outcome": "failed"} + counter.add.assert_called_once_with(1, labels) + self.assertEqual(duration.record.call_args.args[1], labels) + self.assertEqual(exporter.get_finished_spans()[-1].status.status_code.name, "ERROR") + + def test_upload_publish_consume_and_retry_share_causal_trace(self): + propagator = TraceContextTextMapPropagator() + tracer = trace.get_tracer(__name__) + row = { + "job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 1, + "attempt": 1, "created_at": datetime.now(timezone.utc), + } + + with tracer.start_as_current_span("upload") as upload: + upload_carrier = {} + propagator.inject(upload_carrier) + row.update(upload_carrier) + + producer_parent = propagator.extract(upload_carrier) + with tracer.start_as_current_span("publish", context=producer_parent, kind=SpanKind.PRODUCER) as producer: + producer_carrier = {} + propagator.inject(producer_carrier) + message = {**job_payload(row), **producer_carrier} + + with patch.object(worker, "process_message", return_value="queued"): + self.assertEqual(worker.traced_process_message(worker.parse_message(json.dumps(message).encode())), "queued") + # A managed retry uses the same durable upload parent but a new + # publisher attempt; both worker attempts remain in one trace. + row["attempt"] = 2 + retry_parent = propagator.extract(upload_carrier) + with tracer.start_as_current_span("retry_publish", context=retry_parent, + kind=SpanKind.PRODUCER) as retry_producer: + retry_carrier = {} + propagator.inject(retry_carrier) + retry_message = {**job_payload(row), **retry_carrier} + self.assertEqual(worker.traced_process_message(worker.parse_message(json.dumps(retry_message).encode())), "queued") + + spans = exporter.get_finished_spans() + consumers = [span for span in spans if span.name == "attachments.process"] + self.assertEqual(len(consumers), 2) + self.assertEqual(consumers[0].parent.span_id, producer.get_span_context().span_id) + self.assertEqual(consumers[1].parent.span_id, retry_producer.get_span_context().span_id) + self.assertEqual(producer.parent.span_id, upload.get_span_context().span_id) + self.assertEqual(retry_producer.parent.span_id, upload.get_span_context().span_id) + self.assertEqual({span.context.trace_id for span in spans}, {upload.get_span_context().trace_id}) + + +if __name__ == "__main__": + unittest.main() -- 2.54.0 From e5d8796dc6ad6749a1057f9eec3662801046e6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 15:12:27 +0300 Subject: [PATCH 06/13] feat: Carry audited pipeline replay and fault probes into lab6 (U4) --- migrations/007_replay_audit.sql | 16 ++ scripts/measure-image-pipeline.py | 107 ++++++++++++ src/pipeline_jobs.py | 2 +- src/replay.py | 64 ++++++++ src/reprocess.py | 57 +++++++ src/worker.py | 10 ++ student-tests/publish_gap_inner.py | 59 +++++++ student-tests/test_pipeline_faults.py | 188 ++++++++++++++++++++++ student-tests/test_publish_confirm_gap.py | 45 ++++++ 9 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 migrations/007_replay_audit.sql create mode 100644 scripts/measure-image-pipeline.py create mode 100644 src/replay.py create mode 100644 src/reprocess.py create mode 100644 student-tests/publish_gap_inner.py create mode 100644 student-tests/test_pipeline_faults.py create mode 100644 student-tests/test_publish_confirm_gap.py diff --git a/migrations/007_replay_audit.sql b/migrations/007_replay_audit.sql new file mode 100644 index 0000000..cbe4c37 --- /dev/null +++ b/migrations/007_replay_audit.sql @@ -0,0 +1,16 @@ +-- A replay starts a new bounded run without deleting earlier outbox history. +ALTER TABLE attachment_jobs + ADD COLUMN replay_start_attempt integer NOT NULL DEFAULT 1 + CHECK (replay_start_attempt > 0 AND replay_start_attempt <= attempt); + +CREATE TABLE attachment_job_replays ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + previous_attempt integer NOT NULL CHECK (previous_attempt > 0), + new_attempt integer NOT NULL CHECK (new_attempt > previous_attempt), + previous_error_code text NOT NULL, + operator_name text NOT NULL CHECK (length(operator_name) BETWEEN 1 AND 80), + reason text NOT NULL CHECK (length(reason) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (job_id, new_attempt) +); diff --git a/scripts/measure-image-pipeline.py b/scripts/measure-image-pipeline.py new file mode 100644 index 0000000..6b27caa --- /dev/null +++ b/scripts/measure-image-pipeline.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Small synthetic image workload; reports upload and time-to-ready separately.""" + +import argparse +import hashlib +import json +import math +import os +import struct +import sys +import time +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png_noise(width, height): + raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")) + + +def percentile(values, percentile_value): + values = sorted(values) + return round(values[max(0, math.ceil(len(values) * percentile_value) - 1)], 3) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--concurrency", type=int, default=4) + parser.add_argument("--per-size", type=int, default=4) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100: + parser.error("invalid workload size") + + setup = Client(args.base_url) + alice, _bob, _eve, chat = setup.bootstrap() + images = { + "small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(), + "noise_800x600": png_noise(800, 600), + } + items = [(name, data) for name, data in images.items() for _ in range(args.per_size)] + + def work(item): + name, data = item + client = Client(args.base_url, timeout=20) + start = time.monotonic() + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}.png", + 202, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + accepted = time.monotonic() + attachment = response["json"] + if attachment["sha256"] != hashlib.sha256(data).hexdigest(): + raise Failure("accepted checksum mismatch") + client.wait_ready(alice, attachment, wait_seconds=180) + ready = time.monotonic() + return {"profile": name, "size_bytes": len(data), + "upload_ms": round((accepted - start) * 1000, 3), + "ready_ms": round((ready - start) * 1000, 3)} + + started = time.monotonic() + completed, failures = [], [] + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + for future in as_completed([executor.submit(work, item) for item in items]): + try: + completed.append(future.result()) + except (Failure, OSError, ValueError) as exc: + failures.append(type(exc).__name__) + elapsed = time.monotonic() - started + result = { + "lab": 6, "workload": "synthetic PNG upload and poll until ready", + "concurrency": args.concurrency, "per_size": args.per_size, + "elapsed_seconds": round(elapsed, 3), "accepted_ready": len(completed), + "failed": len(failures), "failure_types": sorted(set(failures)), + "profiles": {}, + } + for name, data in images.items(): + rows = [item for item in completed if item["profile"] == name] + result["profiles"][name] = { + "size_bytes": len(data), "successful": len(rows), + "upload_ms": {f"p{p}": percentile([r["upload_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + "ready_ms": {f"p{p}": percentile([r["ready_ms"] for r in rows], p / 100) + for p in (50, 95, 99)} if rows else {}, + } + Path(args.output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + if failures or len(completed) != len(items): + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/src/pipeline_jobs.py b/src/pipeline_jobs.py index 47b2342..4be6206 100644 --- a/src/pipeline_jobs.py +++ b/src/pipeline_jobs.py @@ -155,7 +155,7 @@ def claim_job(conn, job_id, delivery_attempt, lease_seconds=60): def _reschedule_locked(conn, job, *, max_attempts, delay_seconds, error_code, permanent): if max_attempts < 1 or delay_seconds < 0: raise ValueError("invalid retry configuration") - failed = permanent or job["attempt"] >= max_attempts + failed = permanent or job["attempt"] - job["replay_start_attempt"] + 1 >= max_attempts if failed: code = error_code if permanent else "retry_exhausted" conn.execute( diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..96e7596 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,64 @@ +"""Explicit, audited replay of one failed current-version job.""" + +import argparse +from uuid import UUID + +from .db import pool + + +def replay_failed(conn, job_id: UUID, operator: str, reason: str) -> int: + if not 1 <= len(operator) <= 80 or not 1 <= len(reason) <= 200: + raise ValueError("operator and reason are required") + with conn.transaction(): + job = conn.execute( + "SELECT j.*,a.status AS attachment_status," + "a.pipeline_version AS current_pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id " + "WHERE j.id=%s FOR UPDATE OF j,a", + (job_id,), + ).fetchone() + if (job is None or job["status"] != "failed" or + job["pipeline_version"] != job["current_pipeline_version"] or + job["attachment_status"] != "failed"): + raise ValueError("only a failed current-version job can be replayed") + attempt = job["attempt"] + 1 + conn.execute( + "UPDATE attachment_jobs SET status='queued',attempt=%s,replay_start_attempt=%s," + "next_attempt_at=clock_timestamp(),error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s", + (attempt, attempt, job_id), + ) + conn.execute( + "UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() " + "WHERE id=%s AND pipeline_version=%s", + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,%s)", + (job_id, attempt), + ) + conn.execute( + "INSERT INTO attachment_job_replays " + "(job_id,previous_attempt,new_attempt,previous_error_code,operator_name,reason) " + "VALUES (%s,%s,%s,%s,%s,%s)", + (job_id, job["attempt"], attempt, job["error_code"], operator, reason), + ) + return attempt + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("job_id", type=UUID) + parser.add_argument("--operator", required=True) + parser.add_argument("--reason", required=True) + args = parser.parse_args() + try: + with pool().connection() as conn: + attempt = replay_failed(conn, args.job_id, args.operator, args.reason) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued job {args.job_id} attempt {attempt}") + + +if __name__ == "__main__": + main() diff --git a/src/reprocess.py b/src/reprocess.py new file mode 100644 index 0000000..f7f4354 --- /dev/null +++ b/src/reprocess.py @@ -0,0 +1,57 @@ +"""Start a new pipeline version from a verified, already ready original.""" + +import argparse +from uuid import UUID, uuid4 + +from .db import pool + + +def start_new_version(conn, attachment_id: UUID) -> tuple[int, UUID]: + with conn.transaction(): + row = conn.execute( + "SELECT * FROM attachments WHERE id=%s FOR UPDATE", (attachment_id,) + ).fetchone() + if row is None or row["status"] != "ready": + raise ValueError("only a ready attachment can start a new version") + original = conn.execute( + "SELECT object_key,size_bytes,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s AND name='original'", + (attachment_id, row["pipeline_version"]), + ).fetchone() + if (original is None or original["object_key"] != row["object_key"] or + original["size_bytes"] != row["size_bytes"] or + original["sha256"] != row["sha256"]): + raise ValueError("verified original is missing") + version = row["pipeline_version"] + 1 + job_id = uuid4() + conn.execute( + "UPDATE attachments SET pipeline_version=%s,status='queued',error_code=NULL," + "updated_at=clock_timestamp() WHERE id=%s", + (version, attachment_id), + ) + conn.execute( + "INSERT INTO attachment_jobs (id,attachment_id,pipeline_version) " + "VALUES (%s,%s,%s)", + (job_id, attachment_id, version), + ) + conn.execute( + "INSERT INTO attachment_outbox (job_id,attempt) VALUES (%s,1)", + (job_id,), + ) + return version, job_id + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", type=UUID) + args = parser.parse_args() + try: + with pool().connection() as conn: + version, job_id = start_new_version(conn, args.attachment_id) + except ValueError as exc: + parser.error(str(exc)) + print(f"Queued attachment {args.attachment_id} version {version} job {job_id}") + + +if __name__ == "__main__": + main() diff --git a/src/worker.py b/src/worker.py index 364b58b..ddcede4 100644 --- a/src/worker.py +++ b/src/worker.py @@ -110,6 +110,13 @@ def _delay(attempt: int) -> float: return base + random.uniform(0, base / 2) +def _fault(point: str, job_id) -> None: + """Test-only process death at a named durable boundary for one job.""" + if (os.environ.get("LAB5_FAULT_POINT") == point and + os.environ.get("LAB5_FAULT_JOB_ID") == str(job_id)): + os._exit(72) + + @contextmanager def _deadline(seconds: int): def expired(_signal, _frame): @@ -130,6 +137,7 @@ def process_message(item: dict) -> str: if job is None: return "duplicate_or_stale" token = job["lease_token"] + _fault("after_claim_before_s3", job["id"]) try: with _deadline(LEASE_SECONDS): with tracer.start_as_current_span("s3.read_source", record_exception=False, set_status_on_exception=False): @@ -138,9 +146,11 @@ def process_message(item: dict) -> str: variants = render(data, job["content_type"]) with tracer.start_as_current_span("s3.write_variants", record_exception=False, set_status_on_exception=False): stored = _write_variants(job, variants) + _fault("after_s3_before_result_commit", job["id"]) with pool().connection() as conn: with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False): current = complete_job(conn, job["id"], token, stored) + _fault("after_result_commit_before_ack", job["id"]) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) diff --git a/student-tests/publish_gap_inner.py b/student-tests/publish_gap_inner.py new file mode 100644 index 0000000..e1e0559 --- /dev/null +++ b/student-tests/publish_gap_inner.py @@ -0,0 +1,59 @@ +"""Run in a one-shot publisher container to inject publish-before-confirm gap.""" + +import json +import os +from pathlib import Path +from uuid import UUID, uuid4 + +import pika + +from src.db import pool +from src.publisher import QUEUE, job_payload, publish_batch + + +attachment_id = UUID(os.environ["FAULT_ATTACHMENT_ID"]) +url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() +broker = pika.BlockingConnection(pika.URLParameters(url)) +try: + channel = broker.channel() + channel.queue_declare(queue=QUEUE, passive=True) + channel.confirm_delivery() + token = uuid4() + with pool().connection() as conn: + row = conn.execute( + "SELECT o.id,o.job_id,o.attempt,j.attachment_id,j.pipeline_version,j.created_at " + "FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id " + "WHERE j.attachment_id=%s AND j.pipeline_version=1 AND o.attempt=1", + (attachment_id,), + ).fetchone() + assert row is not None + claimed = conn.execute( + "UPDATE attachment_outbox SET claim_token=%s," + "claim_expires_at=clock_timestamp()+interval '30 seconds' " + "WHERE id=%s AND claim_token IS NULL AND published_at IS NULL RETURNING id", + (token, row["id"]), + ).fetchone() + assert claimed is not None + body = json.dumps(job_payload(row), separators=(",", ":")).encode() + channel.basic_publish( + exchange="", routing_key=QUEUE, mandatory=True, body=body, + properties=pika.BasicProperties(content_type="application/json", delivery_mode=2), + ) + # Simulate process death after broker confirm, before PostgreSQL confirm. + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_outbox SET claim_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND claim_token=%s AND published_at IS NULL", + (row["id"], token), + ) + published = publish_batch(channel) + with pool().connection() as conn: + outbox = conn.execute( + "SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox " + "WHERE id=%s", (row["id"],), + ).fetchone() + count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count + assert published >= 1 and outbox["confirmed"] and count >= 2 + print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count})) +finally: + broker.close() diff --git a/student-tests/test_pipeline_faults.py b/student-tests/test_pipeline_faults.py new file mode 100644 index 0000000..63a8e8e --- /dev/null +++ b/student-tests/test_pipeline_faults.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Deterministic fault windows for the versioned attachment pipeline. + +Run in a one-shot worker image while the normal worker and publisher are stopped. +All rows and objects are synthetic and stay in the educational volumes. +""" + +import hashlib +import json +import os +import subprocess +import sys +import time +from io import BytesIO +from uuid import uuid4 + +from PIL import Image + +from src.attachments import BUCKET, storage +from src.db import pool +from src.pipeline_jobs import ( + claim_job, finish_attempt, queue_attachment, recover_expired, reserve_upload, +) +from src.replay import replay_failed +from src.reprocess import start_new_version +from src.worker import process_message + + +image = BytesIO() +Image.new("RGB", (320, 200), "#2765a8").save(image, format="PNG") +DATA = image.getvalue() +SHA = hashlib.sha256(DATA).hexdigest() + + +def create_case(): + user_id, chat_id, attachment_id = uuid4(), uuid4(), uuid4() + staging_key = f"staging/{attachment_id}" + with pool().connection() as conn: + with conn.transaction(): + conn.execute( + "INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,%s,%s,%s)", + (user_id, f"fault_{user_id.hex[:20]}", "Pipeline fault fixture", "synthetic"), + ) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s)", (chat_id, "Pipeline fault fixture")) + conn.execute( + "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", + (chat_id, user_id), + ) + reserve_upload(conn, attachment_id, staging_key) + storage().put_object(BUCKET, staging_key, BytesIO(DATA), len(DATA), content_type="image/png") + with pool().connection() as conn: + _, job_id = queue_attachment( + conn, attachment_id=attachment_id, chat_id=chat_id, owner_id=user_id, + filename="fixture.png", staging_key=staging_key, + object_key=f"originals/{attachment_id}", content_type="image/png", + size_bytes=len(DATA), sha256=SHA, + ) + return attachment_id, job_id + + +def item(job_id, attempt=1): + return {"job_id": str(job_id), "attempt": attempt} + + +def state(attachment_id, version): + with pool().connection() as conn: + attachment = conn.execute( + "SELECT status,pipeline_version,sha256 FROM attachments WHERE id=%s", (attachment_id,) + ).fetchone() + rows = conn.execute( + "SELECT name,sha256 FROM attachment_variants " + "WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name", + (attachment_id, version), + ).fetchall() + return attachment, {row["name"]: row["sha256"] for row in rows} + + +def assert_ready(attachment_id, version): + attachment, variants = state(attachment_id, version) + assert attachment["status"] == "ready" and attachment["pipeline_version"] == version + assert attachment["sha256"] == SHA + assert set(variants) == {"original", "thumbnail", "crop", "optimized"} + assert variants["original"] == SHA + return variants + + +def expire_and_recover(job_id): + with pool().connection() as conn: + conn.execute( + "UPDATE attachment_jobs SET lease_expires_at=clock_timestamp()-interval '1 second' " + "WHERE id=%s AND status='processing'", + (job_id,), + ) + recovered = recover_expired(conn, max_attempts=5) + assert (job_id, "queued") in recovered + + +def crash(job_id, point): + child = subprocess.run( + [sys.executable, "-c", "from src.worker import process_message; " + f"process_message({{'job_id': '{job_id}', 'attempt': 1}})"], + env={**os.environ, "LAB5_FAULT_POINT": point, "LAB5_FAULT_JOB_ID": str(job_id)}, + capture_output=True, text=True, + ) + assert child.returncode == 72, (point, child.returncode, child.stderr) + + +def main(): + results = {} + + attachment, job = create_case() + crash(job, "after_result_commit_before_ack") + first = assert_ready(attachment, 1) + assert process_message(item(job)) == "duplicate_or_stale" + assert assert_ready(attachment, 1) == first + results["after_result_commit_before_ack_duplicate"] = "pass" + + attachment, job = create_case() + crash(job, "after_claim_before_s3") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["before_side_effect_lease_recovery"] = "pass" + + attachment, job = create_case() + crash(job, "after_s3_before_result_commit") + expire_and_recover(job) + assert process_message(item(job, 2)) == "ready" + assert_ready(attachment, 1) + results["after_s3_before_result_commit"] = "pass" + + attachment, old_job = create_case() + assert process_message(item(old_job)) == "ready" + with pool().connection() as conn: + version, new_job = start_new_version(conn, attachment) + assert version == 2 + assert process_message(item(old_job)) == "duplicate_or_stale" + assert process_message(item(new_job)) == "ready" + newer = assert_ready(attachment, 2) + assert process_message(item(old_job)) == "duplicate_or_stale" + assert assert_ready(attachment, 2) == newer + assert len(state(attachment, 1)[1]) == 4 + results["duplicate_reorder_new_version"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=5, + error_code="invalid_image", permanent=True, + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "verified synthetic replay") + assert attempt == 2 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + with pool().connection() as conn: + count = conn.execute( + "SELECT count(*) AS n FROM attachment_job_replays WHERE job_id=%s", (job,) + ).fetchone()["n"] + assert count == 1 + results["failed_registry_operator_replay"] = "pass" + + attachment, job = create_case() + with pool().connection() as conn: + claimed = claim_job(conn, job, 1) + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + delay_seconds=0.01, error_code="storage_unavailable", + ) == "queued" + time.sleep(0.03) + with pool().connection() as conn: + claimed = claim_job(conn, job, 2) + assert claimed is not None + assert finish_attempt( + conn, job, claimed["lease_token"], max_attempts=2, + error_code="storage_unavailable", + ) == "failed" + attempt = replay_failed(conn, job, "lab5-test", "retry budget reset after repair") + assert attempt == 3 + assert process_message(item(job, attempt)) == "ready" + assert_ready(attachment, 1) + results["bounded_transient_retry_then_replay"] = "pass" + + print(json.dumps(results, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_publish_confirm_gap.py b/student-tests/test_publish_confirm_gap.py new file mode 100644 index 0000000..1006997 --- /dev/null +++ b/student-tests/test_publish_confirm_gap.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Show that broker confirm without outbox confirmation safely republishes.""" + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run # noqa: E402 + + +def compose(*args, input_data=None): + return subprocess.run( + ["docker", "compose", *args], cwd=ROOT, + input=input_data, check=True, + ) + + +def main(): + args = arguments("Lab6 publish/confirm gap") + client = Client(args.base_url, args.ca_file) + compose("stop", "worker", "publisher") + attachment = None + try: + alice, bob, _eve, chat = client.bootstrap() + attachment, _original = client.attachment(alice, chat) + check(attachment["status"] == "queued", "upload was not queued") + source = (ROOT / "student-tests/publish_gap_inner.py").read_bytes() + compose( + "run", "--rm", "--no-deps", "-T", + "-e", f"FAULT_ATTACHMENT_ID={attachment['id']}", "publisher", "python", "-", + input_data=source, + ) + finally: + compose("up", "-d", "publisher", "worker") + ready = client.wait_ready(bob, attachment) + check(ready["status"] == "ready", "republished job did not finish") + check(len(ready["variants"]) == 3, "duplicate delivery changed variant set") + print("PASS: confirmed broker copy republished after lost DB confirm; worker reached ready once") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From 9fac497d1a370dd5af723baaf53671bdc374244d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 15:25:13 +0300 Subject: [PATCH 07/13] test: Bound broker queue observation in confirm-gap probe --- student-tests/publish_gap_inner.py | 8 +++++++- student-tests/test_publish_confirm_gap.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/student-tests/publish_gap_inner.py b/student-tests/publish_gap_inner.py index e1e0559..b27d6a5 100644 --- a/student-tests/publish_gap_inner.py +++ b/student-tests/publish_gap_inner.py @@ -2,6 +2,7 @@ import json import os +import time from pathlib import Path from uuid import UUID, uuid4 @@ -52,7 +53,12 @@ try: "SELECT published_at IS NOT NULL AS confirmed FROM attachment_outbox " "WHERE id=%s", (row["id"],), ).fetchone() - count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count + deadline = time.monotonic() + 3 + while True: + count = channel.queue_declare(queue=QUEUE, passive=True).method.message_count + if count >= 2 or time.monotonic() >= deadline: + break + time.sleep(0.05) assert published >= 1 and outbox["confirmed"] and count >= 2 print(json.dumps({"confirmed_after_republish": True, "queued_copies_before_worker": count})) finally: diff --git a/student-tests/test_publish_confirm_gap.py b/student-tests/test_publish_confirm_gap.py index 1006997..7524a49 100644 --- a/student-tests/test_publish_confirm_gap.py +++ b/student-tests/test_publish_confirm_gap.py @@ -13,7 +13,7 @@ from client import Client, arguments, check, run # noqa: E402 def compose(*args, input_data=None): return subprocess.run( - ["docker", "compose", *args], cwd=ROOT, + ["docker", "compose", "-f", "compose.yaml", "-f", "compose.telemetry.yaml", *args], cwd=ROOT, input=input_data, check=True, ) -- 2.54.0 From 5deecdf545f66a94cd11585bc7a39a8ae2986736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 15:32:51 +0300 Subject: [PATCH 08/13] feat: Add live dashboard and freeze SLO alerts (U3/U4 setup) --- .env.example | 2 + compose.telemetry.yaml | 3 + .../lab6-http-baseline-2api-2026-10-02.json | 17 + .../lab6-image-baseline-2api-2026-10-02.json | 40 ++ evidence/slo-thresholds-2026-10-02.md | 48 ++ infra/grafana/dashboards/telemetry.json | 580 +++++++++++++++++- infra/prometheus/prometheus.yml | 6 +- infra/prometheus/rules.yml | 91 +++ infra/runbooks/incident.md | 54 ++ scripts/setup-local-secrets.py | 2 +- scripts/slo_watch.py | 69 +++ src/db.py | 2 + src/publisher.py | 30 +- src/telemetry.py | 42 ++ src/worker.py | 12 +- 15 files changed, 976 insertions(+), 22 deletions(-) create mode 100644 evidence/lab6-http-baseline-2api-2026-10-02.json create mode 100644 evidence/lab6-image-baseline-2api-2026-10-02.json create mode 100644 evidence/slo-thresholds-2026-10-02.md create mode 100644 infra/prometheus/rules.yml create mode 100644 infra/runbooks/incident.md create mode 100644 scripts/slo_watch.py diff --git a/.env.example b/.env.example index 2b32655..fbd7637 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,5 @@ OTEL_SERVICE_NAME=messenger-api OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +GRAFANA_PORT=3000 +PROMETHEUS_PORT=9090 diff --git a/compose.telemetry.yaml b/compose.telemetry.yaml index 02f35c4..89358ff 100644 --- a/compose.telemetry.yaml +++ b/compose.telemetry.yaml @@ -13,6 +13,8 @@ services: prometheus: image: prom/prometheus:v3.5.0 + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" command: - --config.file=/etc/prometheus/prometheus.yml - --storage.tsdb.path=/prometheus @@ -20,6 +22,7 @@ services: - --web.enable-remote-write-receiver volumes: - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/rules.yml:/etc/prometheus/rules.yml:ro - prometheus_data:/prometheus loki: diff --git a/evidence/lab6-http-baseline-2api-2026-10-02.json b/evidence/lab6-http-baseline-2api-2026-10-02.json new file mode 100644 index 0000000..0751892 --- /dev/null +++ b/evidence/lab6-http-baseline-2api-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.01, + "concurrency": 4, + "successful_requests": 18260, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 608.462, + "successful_latency_ms": { + "p50": 4.289, + "p95": 14.966, + "p99": 59.664 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/lab6-image-baseline-2api-2026-10-02.json b/evidence/lab6-image-baseline-2api-2026-10-02.json new file mode 100644 index 0000000..408b1c7 --- /dev/null +++ b/evidence/lab6-image-baseline-2api-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 13.029, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 1309.41, + "p95": 1618.28, + "p99": 1645.672 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 34.166, + "p95": 52.97, + "p99": 84.31 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 988.508, + "p95": 1304.924, + "p99": 1332.116 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 24.969, + "p95": 31.176, + "p99": 33.817 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/slo-thresholds-2026-10-02.md b/evidence/slo-thresholds-2026-10-02.md new file mode 100644 index 0000000..4c39d7d --- /dev/null +++ b/evidence/slo-thresholds-2026-10-02.md @@ -0,0 +1,48 @@ +# Зафиксированные SLO до инъекций отказов — 2026-10-02 + +Стенд: Colima/arm64, 2 CPU, 2 GiB RAM, две API-реплики за HAProxy, +serial worker, PostgreSQL/MinIO/RabbitMQ, Collector/Prometheus/Loki/Tempo/ +Grafana. На момент фиксации fault matrix из перенесённой lab5 выполнена, но +**SLO-инциденты ЛР6 ещё не запускались**. Правила ниже лежат в +`infra/prometheus/rules.yml`; `promtool check rules` обнаружил 6 правил. + +## Baseline + +- 30,010 с, четыре closed-loop клиента, 18 260 успешных POST сообщений, + 0 ошибок, 608,462 RPS, latency успехов p50/p95/p99 + 4,289/14,966/59,664 мс: `lab6-http-baseline-2api-2026-10-02.json`. +- 20 PNG 320×200 (111 817 байт) + 20 PNG 800×600 (1 441 103 байта), + четыре клиента: 40/40 `ready`, 0 ошибок, 13,029 с. p95 от начала upload + до ready 1,305/1,618 с по профилям: `lab6-image-baseline-2api-2026-10-02.json`. + Poll раз в 0,3 с; SDK histogram измеряет от durable job creation до result + COMMIT и не включает upload до принятия. + +## Цели и формулы + +- **API:** 99% бизнес HTTP-запросов без 5xx за скользящие 10 минут. + Numerator — 5xx, denominator — все non-health ответы, включая 4xx; + HAProxy-generated 503 не видны SDK (ограничение), поэтому эксперимент + обращается к живому API-процессу напрямую при DB outage. При нулевом + traffic SLI неизвестен, а не равен 100%. +- **Ready:** 95% завершённых image jobs достигают `ready` не позднее 5 секунд + после durable acceptance за скользящие 10 минут. Числитель плохих jobs: + `rate(ready_duration_count) - rate(ready_duration_bucket{le="5"})`, + знаменатель — `rate(ready_duration_count)`. Застрявшие jobs ещё не + завершены и не попадают в histogram; queue oldest age и отдельный alert + закрывают слепую зону. + +Локальные burn-rate окна уменьшены пропорционально времени демонстрации: +short требует одновременно 15 и 30 секунд, threshold 14,4 бюджета и `for:10s`; +long требует одновременно 60 и 120 секунд, threshold 6 и `for:10s`. +Для API error budget 1% это error ratio >14,4% / >6%; для ready budget 5% +slow ratio >72% / >30%. Оценка правил каждые 5 секунд. Operational alerts: +HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при +`for:10s` и sample freshness <15 с. Порог выбран до сценариев; +эксперимент не меняет его после наблюдений. + +Профиль опыта: одинаковые API/worker/DB лимиты и синтетические данные. +Короткий сбой ожидается увидеть short-правилом без long; длинный — обоими. +Записываются UTC timestamps воздействия, pending/firing/inactive из +Prometheus rule API каждые 5 с, detection/recovery, false positives на +спокойном baseline и выбранные SLI/queue values. Эти окна учебные и не +являются production SLO. diff --git a/infra/grafana/dashboards/telemetry.json b/infra/grafana/dashboards/telemetry.json index 4811866..adfc8da 100644 --- a/infra/grafana/dashboards/telemetry.json +++ b/infra/grafana/dashboards/telemetry.json @@ -1,19 +1,26 @@ { "uid": "lab6-telemetry", - "title": "Lab 6 · Telemetry stack", - "tags": ["lab6"], + "title": "Lab 6 · Messenger signals", + "tags": [ + "lab6" + ], "timezone": "browser", "schemaVersion": 40, - "version": 1, + "version": 2, "refresh": "10s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "panels": [ { "id": 1, "type": "timeseries", - "title": "Telemetry stores and collector reachable", - "description": "Prometheus self-scrape and internal /metrics endpoints. API and worker panels follow their actual exported metric names.", - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "title": "Stores", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { "refId": "A", @@ -21,26 +28,565 @@ "legendFormat": "{{job}}" } ], - "fieldConfig": { "defaults": { "min": 0, "max": 1 }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 } + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "timeseries", - "title": "Collector exporter queue size", - "description": "Bounded exporter queues, sourced from the Collector's own Prometheus endpoint.", - "datasource": { "type": "prometheus", "uid": "prometheus" }, + "title": "HTTP RPS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m]))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + } + }, + { + "id": 3, + "type": "timeseries", + "title": "HTTP 5xx ratio", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + } + }, + { + "id": 4, + "type": "timeseries", + "title": "HTTP p50/p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_http_duration_seconds_bucket{route!~\"/health/.*\"}[5m])))", + "legendFormat": "series 3" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Job outcomes", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(outcome)(rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Job p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_jobs_duration_seconds_bucket[5m])))", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Queue depth", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_queue_depth{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_depth{queue_name=\"attachments.process\"})<15))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Oldest queued job", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "max(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"} and (time()-timestamp(messenger_queue_oldest_age_seconds{queue_name=\"attachments.process\"})<15))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + } + }, + { + "id": 9, + "type": "timeseries", + "title": "DB pool used / waiting", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(messenger_db_pool_used)", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum by(service_name)(messenger_db_pool_waiting)", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Worker busy / capacity", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(messenger_worker_busy)", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum(messenger_worker_capacity)", + "legendFormat": "series 2" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Time to ready p50/p95/p99", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.50,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99,sum by(le)(rate(messenger_attachments_ready_duration_seconds_bucket[5m])))", + "legendFormat": "series 3" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Process CPU", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(rate(messenger_process_cpu_time_seconds_total[1m]))", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Process RSS", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "sum by(service_name)(messenger_process_rss_bytes)", + "legendFormat": "series 1" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Collector queue / failures", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "targets": [ { "refId": "A", "expr": "otelcol_exporter_queue_size{job=\"otel-collector\"}", - "legendFormat": "{{exporter}}" + "legendFormat": "series 1" + }, + { + "refId": "B", + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total[1m]))", + "legendFormat": "series 2" + }, + { + "refId": "C", + "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))", + "legendFormat": "series 3" + }, + { + "refId": "D", + "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "legendFormat": "series 4" } ], - "fieldConfig": { "defaults": { "min": 0 }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom" } }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 } + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Alerts firing", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\",alertname=~\"Lab6.*\"}", + "legendFormat": "{{alertname}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + } } ] } diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml index b56d200..01012e9 100644 --- a/infra/prometheus/prometheus.yml +++ b/infra/prometheus/prometheus.yml @@ -1,5 +1,9 @@ global: - scrape_interval: 15s + scrape_interval: 5s + evaluation_interval: 5s + +rule_files: + - /etc/prometheus/rules.yml scrape_configs: - job_name: prometheus diff --git a/infra/prometheus/rules.yml b/infra/prometheus/rules.yml new file mode 100644 index 0000000..3cd1414 --- /dev/null +++ b/infra/prometheus/rules.yml @@ -0,0 +1,91 @@ +groups: + - name: lab6-operational + interval: 5s + rules: + - alert: Lab6HighHttpErrorRate + expr: | + sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])), 0.001) > 0.10 + for: 10s + labels: + severity: warning + annotations: + summary: More than 10% of business HTTP requests return 5xx + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6QueueLag + expr: | + (messenger_queue_oldest_age_seconds{queue_name="attachments.process"} + and (time() - timestamp(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) < 15)) > 10 + for: 10s + labels: + severity: warning + annotations: + summary: Oldest queued image job is over 10 seconds old + runbook: See infra/runbooks/incident.md#queue-lag + - name: lab6-slo-burn + interval: 5s + rules: + - alert: Lab6ApiBurnShort + expr: | + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[15s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[15s])),0.001) > 0.144) + and + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001) > 0.144) + for: 10s + labels: + severity: page + slo: api + window: short + annotations: + summary: API error budget burning over 14.4x in both 15s and 30s windows + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6ApiBurnLong + expr: | + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[60s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[60s])),0.001) > 0.06) + and + ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[120s])) or vector(0)) + / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[120s])),0.001) > 0.06) + for: 10s + labels: + severity: warning + slo: api + window: long + annotations: + summary: API error budget burning over 6x in both 60s and 120s windows + runbook: See infra/runbooks/incident.md#http-errors + - alert: Lab6ReadyBurnShort + expr: | + ((sum(rate(messenger_attachments_ready_duration_seconds_count[15s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[15s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[15s])),0.001) > 0.72) + and + ((sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001) > 0.72) + for: 10s + labels: + severity: page + slo: ready + window: short + annotations: + summary: Ready-time error budget burning over 14.4x in both 15s and 30s windows + runbook: See infra/runbooks/incident.md#queue-lag + - alert: Lab6ReadyBurnLong + expr: | + ((sum(rate(messenger_attachments_ready_duration_seconds_count[60s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[60s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[60s])),0.001) > 0.30) + and + ((sum(rate(messenger_attachments_ready_duration_seconds_count[120s])) + - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[120s]))) + / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[120s])),0.001) > 0.30) + for: 10s + labels: + severity: warning + slo: ready + window: long + annotations: + summary: Ready-time error budget burning over 6x in both 60s and 120s windows + runbook: See infra/runbooks/incident.md#queue-lag diff --git a/infra/runbooks/incident.md b/infra/runbooks/incident.md new file mode 100644 index 0000000..5de27a4 --- /dev/null +++ b/infra/runbooks/incident.md @@ -0,0 +1,54 @@ +# Lab 6: local incident runbook + +Use Grafana at `http://localhost:3000` (dashboard **Lab 6 · Messenger signals**) or +Prometheus rules at `/alerts` inside the Compose network. All times below are +UTC. The dashboard uses stored Prometheus data; Tempo and Loki are separate +datasources. Keep the last hour visible before diagnosing a recovered incident. + +## HTTP errors + +`Lab6HighHttpErrorRate` fires when 5xx / all non-health HTTP requests exceeds +10% over 30 seconds for another 10 seconds. Client 4xx are in the denominator +and have a separate panel; they are not server failures. Check HTTP RPS first: +an empty or tiny denominator makes an error ratio unreliable. Compare 5xx +ratio with DB pool used/waiting and `up` for the telemetry stores. In Tempo, +find an `HTTP` server span in the time range and inspect its `db.*` child span +duration/status. Query Loki `{service_name="messenger-api"}` for the same +trace ID. A growing pool wait and failing DB span identify a database incident; +a long server span with fast DB children points to the application path. +Restart PostgreSQL only after confirming its container state and persistent +volume. Confirm `up`, 5xx ratio, and `/health/ready` recover; record first +`pending`, `firing`, and `inactive` timestamps from the rule API. + +## Queue lag + +`Lab6QueueLag` fires when the oldest durable queued job exceeds 10 seconds for +another 10 seconds. `messenger_queue_depth` counts rows with `status='queued'` +in PostgreSQL, including an outbox row awaiting publication. It is sampled by +the publisher. First compare queue depth, oldest age, `messenger_worker_busy` +and `messenger_worker_capacity`. If busy disappears and capacity is absent, +inspect the worker container; if capacity is present but processing outcome +rates stop, inspect broker connectivity and `job_published` events in Loki. +If jobs fail repeatedly, inspect `messenger_jobs_processed_total` grouped by +outcome and the corresponding consumer trace; do not replay until the cause +is corrected. Restart the worker, then verify queued count/age fall, ready +durations appear, and the alert returns to inactive. + +## Follow an upload + +Take the `trace_id` from the API `attachment_uploaded` log in Loki. Tempo +should contain the API upload span, publisher `attachments.publish` producer +span, worker `attachments.process` consumer span and its S3/render/DB children. +The worker `job_processed` log has the same trace ID. Job identifiers are +synthetic test identifiers; metric labels contain only route templates, +status classes, job type/outcome and queue name. Never copy request bodies, +tokens, signed object URLs or raw image bytes into evidence. + +## Missing telemetry + +If a signal is absent, check Collector `otelcol_exporter_queue_size`, +`otelcol_exporter_send_failed_*` and store `up` before concluding the business +service recovered. The SDK uses asynchronous bounded queues and 2-second +export timeouts; Collector queues are finite. Export loss during a backend +outage is expected and must be reported. A missing sample is not zero errors. +Telemetry stores are not part of API readiness. diff --git a/scripts/setup-local-secrets.py b/scripts/setup-local-secrets.py index 6fbac6d..8cb5829 100644 --- a/scripts/setup-local-secrets.py +++ b/scripts/setup-local-secrets.py @@ -30,7 +30,7 @@ def main() -> None: write("postgres_password", password) write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") if not (ROOT / "minio_root_user").exists(): - write("minio_root_user", "lab5admin") + write("minio_root_user", "lab6admin") write("minio_root_password", secrets.token_urlsafe(32)) write("s3_access_key_id", secrets.token_hex(10)) write("s3_secret_access_key", secrets.token_urlsafe(32)) diff --git a/scripts/slo_watch.py b/scripts/slo_watch.py new file mode 100644 index 0000000..2c0b0eb --- /dev/null +++ b/scripts/slo_watch.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Capture local Prometheus alert states and SLI values without credentials.""" + +import argparse +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen + + +QUERIES = { + "http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)', + "queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})', + "ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)', + "collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)', +} + + +def fetch(base, path, params=None): + url = base + path + ("?" + urlencode(params) if params else "") + with urlopen(url, timeout=4) as response: + result = json.load(response) + if result["status"] != "success": + raise RuntimeError(result) + return result["data"] + + +def snapshot(base): + groups = fetch(base, "/api/v1/rules", {"type": "alert"})["groups"] + alerts = [rule for group in groups for rule in group["rules"]] + row = { + "timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "alerts": {item["name"]: item["state"] + for item in alerts if item["name"].startswith("Lab6")}, + "values": {}, + } + for name, query in QUERIES.items(): + values = fetch(base, "/api/v1/query", {"query": query})["result"] + row["values"][name] = float(values[0]["value"][1]) if values else None + return row + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--prometheus", default="http://127.0.0.1:9090") + parser.add_argument("--duration", type=int, default=60) + parser.add_argument("--interval", type=int, default=5) + parser.add_argument("--output", required=True) + args = parser.parse_args() + if not 5 <= args.duration <= 900 or not 1 <= args.interval <= 30: + parser.error("invalid duration or interval") + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + end = time.monotonic() + args.duration + with path.open("w", encoding="utf-8") as out: + while True: + row = snapshot(args.prometheus) + out.write(json.dumps(row, sort_keys=True) + "\n") + out.flush() + if time.monotonic() + args.interval > end: + break + time.sleep(args.interval) + print(f"saved {path}") + + +if __name__ == "__main__": + main() diff --git a/src/db.py b/src/db.py index 832ff07..48947b0 100644 --- a/src/db.py +++ b/src/db.py @@ -26,6 +26,8 @@ def pool() -> ConnectionPool: open=True, ) atexit.register(result.close) + from .telemetry import register_pool_metrics + register_pool_metrics(result) return result diff --git a/src/publisher.py b/src/publisher.py index 209d24b..02d2590 100644 --- a/src/publisher.py +++ b/src/publisher.py @@ -8,7 +8,8 @@ from pathlib import Path import pika import psycopg -from opentelemetry import trace +from opentelemetry import metrics, trace +from opentelemetry.metrics import Observation from opentelemetry.trace import SpanKind from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from psycopg_pool import PoolTimeout @@ -24,6 +25,32 @@ tracer = trace.get_tracer(__name__) trace_context = TraceContextTextMapPropagator() +def register_queue_metrics(): + """Sample the durable backlog; broker delivery does not erase queued jobs.""" + meter = metrics.get_meter("messenger.queue") + + def backlog(_): + try: + with pool().connection() as conn: + row = conn.execute( + "SELECT count(*) AS depth," + "coalesce(extract(epoch FROM clock_timestamp()-min(created_at)),0) AS oldest_age " + "FROM attachment_jobs WHERE status='queued'" + ).fetchone() + return row + except (psycopg.Error, PoolTimeout, OSError): + return None + + def observe(field): + def callback(options): + row = backlog(options) + return [Observation(row[field], {"queue_name": QUEUE})] if row is not None else [] + return callback + + meter.create_observable_gauge("messenger.queue.depth", callbacks=[observe("depth")]) + meter.create_observable_gauge("messenger.queue.oldest_age", callbacks=[observe("oldest_age")], unit="s") + + def publish_batch(channel): with pool().connection() as conn: token, rows = claim_outbox(conn, limit=10, lease_seconds=30) @@ -54,6 +81,7 @@ def publish_batch(channel): def main(): configure("messenger-publisher") + register_queue_metrics() logging.basicConfig(level=logging.INFO) logging.getLogger("pika").setLevel(logging.ERROR) url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip() diff --git a/src/telemetry.py b/src/telemetry.py index 9d6408c..8d68d31 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -2,10 +2,12 @@ import logging import os +import time from functools import lru_cache from uuid import uuid4 from opentelemetry import metrics, trace +from opentelemetry.metrics import Observation from opentelemetry._logs import set_logger_provider from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter @@ -22,6 +24,7 @@ from opentelemetry.sdk.trace.export import BatchSpanProcessor HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10) JOB_BUCKETS = (0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120) +READY_BUCKETS = (0.5, 1, 2, 5, 10, 30, 60, 120, 300) @lru_cache(maxsize=1) @@ -53,6 +56,7 @@ def configure(service_name: str) -> bool: views=[ View(instrument_name="messenger.http.duration", aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS)), View(instrument_name="messenger.jobs.duration", aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS)), + View(instrument_name="messenger.attachments.ready_duration", aggregation=ExplicitBucketHistogramAggregation(READY_BUCKETS)), ], )) @@ -65,6 +69,15 @@ def configure(service_name: str) -> bool: app_logger = logging.getLogger("src") app_logger.setLevel(logging.INFO) app_logger.addHandler(LoggingHandler(level=logging.INFO, logger_provider=logs)) + meter = metrics.get_meter("messenger.runtime") + meter.create_observable_counter("messenger.process.cpu.time", callbacks=[ + lambda _: [Observation(time.process_time())], + ], unit="s") + def rss(_): + with open("/proc/self/statm", encoding="ascii") as statm: + return [Observation(int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE"))] + + meter.create_observable_gauge("messenger.process.rss", callbacks=[rss], unit="By") return True @@ -84,3 +97,32 @@ def job_instruments(): meter.create_counter("messenger.jobs.processed", unit="1"), meter.create_histogram("messenger.jobs.duration", unit="s"), ) + + +@lru_cache(maxsize=1) +def pipeline_instruments(): + meter = metrics.get_meter("messenger.pipeline") + return ( + meter.create_histogram("messenger.attachments.ready_duration", unit="s"), + meter.create_up_down_counter("messenger.worker.busy"), + ) + + +def register_worker_capacity(): + metrics.get_meter("messenger.pipeline").create_observable_gauge( + "messenger.worker.capacity", callbacks=[lambda _: [Observation(1)]], + ) + + +def register_pool_metrics(pool): + """Read local pool counters without querying the database.""" + meter = metrics.get_meter("messenger.pool") + + def observe(_): + stats = pool.get_stats() + return [Observation(stats["pool_size"] - stats["pool_available"])] + + meter.create_observable_gauge("messenger.db.pool.used", callbacks=[observe]) + meter.create_observable_gauge("messenger.db.pool.waiting", callbacks=[ + lambda _: [Observation(pool.get_stats()["requests_waiting"])], + ]) diff --git a/src/worker.py b/src/worker.py index ddcede4..b51e8a0 100644 --- a/src/worker.py +++ b/src/worker.py @@ -9,7 +9,7 @@ import re import signal import time from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timezone from io import BytesIO from pathlib import Path from uuid import UUID @@ -26,7 +26,7 @@ from .attachments import BUCKET, storage from .db import pool from .image_variants import InvalidImage, MAX_BYTES, render from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired -from .telemetry import configure, job_instruments +from .telemetry import configure, job_instruments, pipeline_instruments, register_worker_capacity logger = logging.getLogger("src.worker") @@ -151,6 +151,10 @@ def process_message(item: dict) -> str: with tracer.start_as_current_span("db.complete_job", record_exception=False, set_status_on_exception=False): current = complete_job(conn, job["id"], token, stored) _fault("after_result_commit_before_ack", job["id"]) + if current: + ready_duration, _ = pipeline_instruments() + ready_duration.record(max(0, (datetime.now(timezone.utc) - job["created_at"]).total_seconds()), + {"job_type": "attachment.process.v1"}) if current and job["staging_key"]: try: storage().remove_object(BUCKET, job["staging_key"]) @@ -194,6 +198,8 @@ def traced_process_message(item: dict) -> str: span.set_attribute("job.id", item["job_id"]) span.set_attribute("attachment.id", item["attachment_id"]) started = time.monotonic() + _, busy = pipeline_instruments() + busy.add(1) outcome = "error" try: outcome = process_message(item) @@ -202,6 +208,7 @@ def traced_process_message(item: dict) -> str: outcome = "database_unavailable" raise finally: + busy.add(-1) attributes = {"job_type": item["type"], "outcome": outcome} processed, duration = job_instruments() processed.add(1, attributes) @@ -215,6 +222,7 @@ def traced_process_message(item: dict) -> str: def main() -> None: configure("messenger-worker") + register_worker_capacity() logging.basicConfig(level=logging.INFO) if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1: raise SystemExit("invalid worker retry settings") -- 2.54.0 From 43bcde73de6fa2228d841c6add3e78875faaaf99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 15:35:32 +0300 Subject: [PATCH 09/13] fix: Hold long SLO burn alert through brief spikes (U4 setup) --- evidence/slo-thresholds-2026-10-02.md | 5 +++-- infra/prometheus/rules.yml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/evidence/slo-thresholds-2026-10-02.md b/evidence/slo-thresholds-2026-10-02.md index 4c39d7d..1ecb8be 100644 --- a/evidence/slo-thresholds-2026-10-02.md +++ b/evidence/slo-thresholds-2026-10-02.md @@ -33,7 +33,7 @@ Grafana. На момент фиксации fault matrix из перенесён Локальные burn-rate окна уменьшены пропорционально времени демонстрации: short требует одновременно 15 и 30 секунд, threshold 14,4 бюджета и `for:10s`; -long требует одновременно 60 и 120 секунд, threshold 6 и `for:10s`. +long требует одновременно 60 и 120 секунд, threshold 6 и `for:90s`. Для API error budget 1% это error ratio >14,4% / >6%; для ready budget 5% slow ratio >72% / >30%. Оценка правил каждые 5 секунд. Operational alerts: HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при @@ -41,7 +41,8 @@ HTTP 5xx >10% за 30 с при `for:10s`, oldest queued age >10 с при эксперимент не меняет его после наблюдений. Профиль опыта: одинаковые API/worker/DB лимиты и синтетические данные. -Короткий сбой ожидается увидеть short-правилом без long; длинный — обоими. +Короткий сбой ожидается увидеть short-правилом без long; длительное `for` +подавляет long на кратких всплесках. Длинный сбой должен вызвать оба. Записываются UTC timestamps воздействия, pending/firing/inactive из Prometheus rule API каждые 5 с, detection/recovery, false positives на спокойном baseline и выбранные SLI/queue values. Эти окна учебные и не diff --git a/infra/prometheus/rules.yml b/infra/prometheus/rules.yml index 3cd1414..1b0729d 100644 --- a/infra/prometheus/rules.yml +++ b/infra/prometheus/rules.yml @@ -47,7 +47,7 @@ groups: and ((sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[120s])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[120s])),0.001) > 0.06) - for: 10s + for: 90s labels: severity: warning slo: api @@ -81,7 +81,7 @@ groups: ((sum(rate(messenger_attachments_ready_duration_seconds_count[120s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[120s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[120s])),0.001) > 0.30) - for: 10s + for: 90s labels: severity: warning slo: ready -- 2.54.0 From f1963e5f199c40310d486f7a1257f439367620d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 17:06:13 +0300 Subject: [PATCH 10/13] Bound Tempo blocks and complete telemetry diagnostics --- compose.telemetry.yaml | 1 + infra/grafana/dashboards/telemetry.json | 22 ++++++++++++++++++---- infra/tempo/config.yaml | 5 +++++ scripts/slo_watch.py | 2 +- src/store.py | 16 +++++++++++----- 5 files changed, 36 insertions(+), 10 deletions(-) diff --git a/compose.telemetry.yaml b/compose.telemetry.yaml index 89358ff..2513324 100644 --- a/compose.telemetry.yaml +++ b/compose.telemetry.yaml @@ -34,6 +34,7 @@ services: tempo: image: grafana/tempo:2.9.5 + mem_limit: 768m command: ["-config.file=/etc/tempo/config.yaml"] volumes: - ./infra/tempo/config.yaml:/etc/tempo/config.yaml:ro diff --git a/infra/grafana/dashboards/telemetry.json b/infra/grafana/dashboards/telemetry.json index adfc8da..d4d3559 100644 --- a/infra/grafana/dashboards/telemetry.json +++ b/infra/grafana/dashboards/telemetry.json @@ -94,7 +94,7 @@ "targets": [ { "refId": "A", - "expr": "sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", + "expr": "(sum(rate(messenger_http_requests_total{status_class=\"5xx\",route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{route!~\"/health/.*\"}[1m])),0.001)", "legendFormat": "series 1" } ], @@ -519,17 +519,17 @@ }, { "refId": "B", - "expr": "sum(rate(otelcol_exporter_send_failed_metric_points_total[1m]))", + "expr": "sum(rate(otelcol_exporter_send_failed_metric_points[1m]))", "legendFormat": "series 2" }, { "refId": "C", - "expr": "sum(rate(otelcol_exporter_send_failed_log_records_total[1m]))", + "expr": "sum(rate(otelcol_exporter_send_failed_log_records[1m]))", "legendFormat": "series 3" }, { "refId": "D", - "expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))", + "expr": "sum(rate(otelcol_exporter_send_failed_spans[1m]))", "legendFormat": "series 4" } ], @@ -587,6 +587,20 @@ "x": 0, "y": 56 } + }, + { + "id": 16, + "type": "timeseries", + "title": "HTTP 4xx RPS (client responses)", + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "targets": [{ + "refId": "A", + "expr": "sum(rate(messenger_http_requests_total{status_class=\"4xx\",route!~\"/health/.*\"}[1m])) or vector(0)", + "legendFormat": "4xx" + }], + "fieldConfig": {"defaults": {"unit": "reqps", "min": 0}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}}, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 56} } ] } diff --git a/infra/tempo/config.yaml b/infra/tempo/config.yaml index 9f813a2..14f76b5 100644 --- a/infra/tempo/config.yaml +++ b/infra/tempo/config.yaml @@ -8,6 +8,11 @@ distributor: grpc: endpoint: 0.0.0.0:4317 +ingester: + max_block_bytes: 5000000 + max_block_duration: 1m + concurrent_flushes: 1 + storage: trace: backend: local diff --git a/scripts/slo_watch.py b/scripts/slo_watch.py index 2c0b0eb..1f4b98a 100644 --- a/scripts/slo_watch.py +++ b/scripts/slo_watch.py @@ -14,7 +14,7 @@ QUERIES = { "http_5xx_ratio_30s": 'sum(rate(messenger_http_requests_total{status_class="5xx",route!~"/health/.*"}[30s])) / clamp_min(sum(rate(messenger_http_requests_total{route!~"/health/.*"}[30s])),0.001)', "queue_oldest_seconds": 'max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"})', "ready_slow_ratio_30s": '(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])) - sum(rate(messenger_attachments_ready_duration_seconds_bucket{le="5"}[30s]))) / clamp_min(sum(rate(messenger_attachments_ready_duration_seconds_count[30s])),0.001)', - "collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans_total)', + "collector_failed_spans": 'sum(otelcol_exporter_send_failed_spans)', } diff --git a/src/store.py b/src/store.py index 298109a..0a701b0 100644 --- a/src/store.py +++ b/src/store.py @@ -14,6 +14,8 @@ import psycopg from argon2 import PasswordHasher from argon2.exceptions import InvalidHashError, VerificationError from fastapi import HTTPException +from opentelemetry import trace +from opentelemetry.trace import Status, StatusCode from psycopg.errors import UniqueViolation from psycopg_pool import PoolTimeout @@ -21,6 +23,7 @@ from . import db from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized password_hasher = PasswordHasher() +tracer = trace.get_tracer(__name__) def timestamp(value: datetime) -> str: @@ -53,11 +56,14 @@ def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | @contextmanager def connection(): - try: - with db.pool().connection() as conn: - yield conn - except (psycopg.Error, PoolTimeout, OSError): - raise HTTPException(503) from None + with tracer.start_as_current_span("db.store", record_exception=False, set_status_on_exception=False) as span: + span.set_attribute("db.system.name", "postgresql") + try: + with db.pool().connection() as conn: + yield conn + except (psycopg.Error, PoolTimeout, OSError): + span.set_status(Status(StatusCode.ERROR)) + raise HTTPException(503) from None def user_view(row: dict) -> dict: -- 2.54.0 From f61d0705749a7ca385a6eba4b28fc2c20a0cb925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 17:06:45 +0300 Subject: [PATCH 11/13] Record SLO and observability fault experiments --- evidence/queue-worker-stop-2026-10-02.json | 324 +++ evidence/ready-throttle-load-2026-10-02.json | 40 + ...ready-throttle-strong-load-2026-10-02.json | 26 + ...ady-throttle-strong-watch-2026-10-02.jsonl | 32 + .../ready-throttle-watch-2026-10-02.jsonl | 30 + evidence/slo-long-db-2026-10-02.json | 2262 +++++++++++++++++ evidence/slo-quiet-baseline-2026-10-02.jsonl | 6 + evidence/slo-short-db-2026-10-02.json | 1584 ++++++++++++ .../telemetry-backend-outage-2026-10-02.json | 150 ++ ...telemetry-outage-http-load-2026-10-02.json | 17 + evidence/tempo-recovery-load-2026-10-02.json | 17 + scripts/measure-image-pipeline.py | 3 + student-tests/test_observability_failure.py | 118 + student-tests/test_queue_incident.py | 89 + student-tests/test_slo_alerts.py | 138 + 15 files changed, 4836 insertions(+) create mode 100644 evidence/queue-worker-stop-2026-10-02.json create mode 100644 evidence/ready-throttle-load-2026-10-02.json create mode 100644 evidence/ready-throttle-strong-load-2026-10-02.json create mode 100644 evidence/ready-throttle-strong-watch-2026-10-02.jsonl create mode 100644 evidence/ready-throttle-watch-2026-10-02.jsonl create mode 100644 evidence/slo-long-db-2026-10-02.json create mode 100644 evidence/slo-quiet-baseline-2026-10-02.jsonl create mode 100644 evidence/slo-short-db-2026-10-02.json create mode 100644 evidence/telemetry-backend-outage-2026-10-02.json create mode 100644 evidence/telemetry-outage-http-load-2026-10-02.json create mode 100644 evidence/tempo-recovery-load-2026-10-02.json create mode 100644 student-tests/test_observability_failure.py create mode 100644 student-tests/test_queue_incident.py create mode 100644 student-tests/test_slo_alerts.py diff --git a/evidence/queue-worker-stop-2026-10-02.json b/evidence/queue-worker-stop-2026-10-02.json new file mode 100644 index 0000000..0abbac5 --- /dev/null +++ b/evidence/queue-worker-stop-2026-10-02.json @@ -0,0 +1,324 @@ +{ + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:23:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:04+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 4.44206, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:09+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 9.447189, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:14+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 14.45216, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "pending", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:19+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 19.454665, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "pending", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:24+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 24.459164, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:29+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 29.464961, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:34+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 34.474193, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "firing", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:39+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": null + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:24:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:25:04+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:25:09+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "queue_alert_states": [ + "firing", + "inactive", + "pending" + ], + "queued_uploads": 8, + "ready_uploads": 8, + "times": { + "all_ready": "2026-10-02T13:24:33+00:00", + "started": "2026-10-02T13:23:44+00:00", + "uploads_accepted": "2026-10-02T13:23:54+00:00", + "worker_started": "2026-10-02T13:24:32+00:00", + "worker_stopped": "2026-10-02T13:23:54+00:00" + } +} diff --git a/evidence/ready-throttle-load-2026-10-02.json b/evidence/ready-throttle-load-2026-10-02.json new file mode 100644 index 0000000..c63deac --- /dev/null +++ b/evidence/ready-throttle-load-2026-10-02.json @@ -0,0 +1,40 @@ +{ + "accepted_ready": 40, + "concurrency": 4, + "elapsed_seconds": 29.794, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 20, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 4105.489, + "p95": 4756.876, + "p99": 5080.984 + }, + "size_bytes": 1441103, + "successful": 20, + "upload_ms": { + "p50": 29.131, + "p95": 52.165, + "p99": 53.056 + } + }, + "small_320x200": { + "ready_ms": { + "p50": 1573.785, + "p95": 1916.581, + "p99": 1919.551 + }, + "size_bytes": 111817, + "successful": 20, + "upload_ms": { + "p50": 17.66, + "p95": 27.092, + "p99": 66.386 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/ready-throttle-strong-load-2026-10-02.json b/evidence/ready-throttle-strong-load-2026-10-02.json new file mode 100644 index 0000000..6cba4bd --- /dev/null +++ b/evidence/ready-throttle-strong-load-2026-10-02.json @@ -0,0 +1,26 @@ +{ + "accepted_ready": 30, + "concurrency": 4, + "elapsed_seconds": 72.988, + "failed": 0, + "failure_types": [], + "lab": 6, + "per_size": 30, + "profiles": { + "noise_800x600": { + "ready_ms": { + "p50": 9476.253, + "p95": 10359.763, + "p99": 10437.545 + }, + "size_bytes": 1441103, + "successful": 30, + "upload_ms": { + "p50": 27.409, + "p95": 68.138, + "p99": 83.338 + } + } + }, + "workload": "synthetic PNG upload and poll until ready" +} diff --git a/evidence/ready-throttle-strong-watch-2026-10-02.jsonl b/evidence/ready-throttle-strong-watch-2026-10-02.jsonl new file mode 100644 index 0000000..23f8fa9 --- /dev/null +++ b/evidence/ready-throttle-strong-watch-2026-10-02.jsonl @@ -0,0 +1,32 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:31+00:00", "values": {"collector_failed_spans": 20872.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:36+00:00", "values": {"collector_failed_spans": 20878.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:41+00:00", "values": {"collector_failed_spans": 20900.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:46+00:00", "values": {"collector_failed_spans": 20912.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:51+00:00", "values": {"collector_failed_spans": 20918.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:56+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:01+00:00", "values": {"collector_failed_spans": 20930.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:06+00:00", "values": {"collector_failed_spans": 21312.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.699382, "ready_slow_ratio_30s": 0.6666666666666666}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:36:11+00:00", "values": {"collector_failed_spans": 21914.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.8333333333333333}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:16+00:00", "values": {"collector_failed_spans": 22138.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.303315, "ready_slow_ratio_30s": 0.875}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "pending"}, "timestamp": "2026-10-02T13:36:21+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 0.9}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:26+00:00", "values": {"collector_failed_spans": 22264.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.819518, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:31+00:00", "values": {"collector_failed_spans": 22752.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:36+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.800324, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:41+00:00", "values": {"collector_failed_spans": 22807.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:46+00:00", "values": {"collector_failed_spans": 22817.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 6.705292, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:51+00:00", "values": {"collector_failed_spans": 22846.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:36:56+00:00", "values": {"collector_failed_spans": 22852.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.390955, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:01+00:00", "values": {"collector_failed_spans": 22870.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:06+00:00", "values": {"collector_failed_spans": 22876.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 5.015189, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:11+00:00", "values": {"collector_failed_spans": 22888.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "firing"}, "timestamp": "2026-10-02T13:37:16+00:00", "values": {"collector_failed_spans": 22899.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:21+00:00", "values": {"collector_failed_spans": 22917.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:26+00:00", "values": {"collector_failed_spans": 22929.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "pending", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:31+00:00", "values": {"collector_failed_spans": 22939.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 1.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:36+00:00", "values": {"collector_failed_spans": 22951.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:41+00:00", "values": {"collector_failed_spans": 22963.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:46+00:00", "values": {"collector_failed_spans": 22975.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:51+00:00", "values": {"collector_failed_spans": 22987.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:37:56+00:00", "values": {"collector_failed_spans": 22993.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "firing", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:01+00:00", "values": {"collector_failed_spans": 23016.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:38:06+00:00", "values": {"collector_failed_spans": 23022.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/ready-throttle-watch-2026-10-02.jsonl b/evidence/ready-throttle-watch-2026-10-02.jsonl new file mode 100644 index 0000000..9ac20cb --- /dev/null +++ b/evidence/ready-throttle-watch-2026-10-02.jsonl @@ -0,0 +1,30 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:53+00:00", "values": {"collector_failed_spans": 20481.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:32:58+00:00", "values": {"collector_failed_spans": 20491.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:03+00:00", "values": {"collector_failed_spans": 20509.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:08+00:00", "values": {"collector_failed_spans": 20527.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:13+00:00", "values": {"collector_failed_spans": 20531.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:18+00:00", "values": {"collector_failed_spans": 20537.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:23+00:00", "values": {"collector_failed_spans": 20549.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:28+00:00", "values": {"collector_failed_spans": 20561.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:33+00:00", "values": {"collector_failed_spans": 20585.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.155266, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:38+00:00", "values": {"collector_failed_spans": 20591.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.998314, "ready_slow_ratio_30s": 0.05458974358974353}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:43+00:00", "values": {"collector_failed_spans": 20609.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.081209, "ready_slow_ratio_30s": 0.09646666666666656}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:48+00:00", "values": {"collector_failed_spans": 20627.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.683843, "ready_slow_ratio_30s": 0.1055743589743589}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:53+00:00", "values": {"collector_failed_spans": 20637.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.220981, "ready_slow_ratio_30s": 0.11358709677419333}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:33:58+00:00", "values": {"collector_failed_spans": 20643.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 2.589964, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:03+00:00", "values": {"collector_failed_spans": 20655.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:08+00:00", "values": {"collector_failed_spans": 20661.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:14+00:00", "values": {"collector_failed_spans": 20679.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:19+00:00", "values": {"collector_failed_spans": 20701.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:24+00:00", "values": {"collector_failed_spans": 20707.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:29+00:00", "values": {"collector_failed_spans": 20719.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:34+00:00", "values": {"collector_failed_spans": 20731.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:39+00:00", "values": {"collector_failed_spans": 20743.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:44+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:49+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:54+00:00", "values": {"collector_failed_spans": 20773.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:34:59+00:00", "values": {"collector_failed_spans": 20797.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:04+00:00", "values": {"collector_failed_spans": 20803.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:09+00:00", "values": {"collector_failed_spans": 20821.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:14+00:00", "values": {"collector_failed_spans": 20838.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T13:35:19+00:00", "values": {"collector_failed_spans": 20848.0, "http_5xx_ratio_30s": 0.0, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/slo-long-db-2026-10-02.json b/evidence/slo-long-db-2026-10-02.json new file mode 100644 index 0000000..cc27859 --- /dev/null +++ b/evidence/slo-long-db-2026-10-02.json @@ -0,0 +1,2262 @@ +{ + "case": "long", + "events": [ + [ + "2026-10-02T13:16:15+00:00", + 401 + ], + [ + "2026-10-02T13:16:15+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:16+00:00", + 401 + ], + [ + "2026-10-02T13:16:17+00:00", + 401 + ], + [ + "2026-10-02T13:16:17+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:18+00:00", + 401 + ], + [ + "2026-10-02T13:16:19+00:00", + 401 + ], + [ + "2026-10-02T13:16:19+00:00", + 401 + ], + [ + "2026-10-02T13:16:20+00:00", + 401 + ], + [ + "2026-10-02T13:16:20+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:21+00:00", + 401 + ], + [ + "2026-10-02T13:16:22+00:00", + 401 + ], + [ + "2026-10-02T13:16:22+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:23+00:00", + 401 + ], + [ + "2026-10-02T13:16:24+00:00", + 401 + ], + [ + "2026-10-02T13:16:24+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:25+00:00", + 401 + ], + [ + "2026-10-02T13:16:26+00:00", + 401 + ], + [ + "2026-10-02T13:16:26+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:27+00:00", + 401 + ], + [ + "2026-10-02T13:16:28+00:00", + 401 + ], + [ + "2026-10-02T13:16:28+00:00", + 401 + ], + [ + "2026-10-02T13:16:29+00:00", + 401 + ], + [ + "2026-10-02T13:16:29+00:00", + 401 + ], + [ + "2026-10-02T13:16:30+00:00", + 503 + ], + [ + "2026-10-02T13:16:30+00:00", + 503 + ], + [ + "2026-10-02T13:16:35+00:00", + 503 + ], + [ + "2026-10-02T13:16:41+00:00", + 503 + ], + [ + "2026-10-02T13:16:46+00:00", + 503 + ], + [ + "2026-10-02T13:16:52+00:00", + 503 + ], + [ + "2026-10-02T13:16:57+00:00", + 503 + ], + [ + "2026-10-02T13:17:02+00:00", + 503 + ], + [ + "2026-10-02T13:17:08+00:00", + 503 + ], + [ + "2026-10-02T13:17:13+00:00", + 503 + ], + [ + "2026-10-02T13:17:19+00:00", + 503 + ], + [ + "2026-10-02T13:17:24+00:00", + 503 + ], + [ + "2026-10-02T13:17:29+00:00", + 503 + ], + [ + "2026-10-02T13:17:35+00:00", + 503 + ], + [ + "2026-10-02T13:17:40+00:00", + 503 + ], + [ + "2026-10-02T13:17:46+00:00", + 503 + ], + [ + "2026-10-02T13:17:51+00:00", + 503 + ], + [ + "2026-10-02T13:17:57+00:00", + 503 + ], + [ + "2026-10-02T13:18:02+00:00", + 503 + ], + [ + "2026-10-02T13:18:07+00:00", + 503 + ], + [ + "2026-10-02T13:18:13+00:00", + 503 + ], + [ + "2026-10-02T13:18:18+00:00", + 503 + ], + [ + "2026-10-02T13:18:24+00:00", + 503 + ], + [ + "2026-10-02T13:18:29+00:00", + 503 + ], + [ + "2026-10-02T13:18:34+00:00", + 503 + ], + [ + "2026-10-02T13:18:40+00:00", + 503 + ], + [ + "2026-10-02T13:18:45+00:00", + 503 + ], + [ + "2026-10-02T13:18:51+00:00", + 503 + ], + [ + "2026-10-02T13:18:56+00:00", + 503 + ], + [ + "2026-10-02T13:19:01+00:00", + 503 + ], + [ + "2026-10-02T13:19:07+00:00", + 503 + ], + [ + "2026-10-02T13:19:12+00:00", + 503 + ], + [ + "2026-10-02T13:19:18+00:00", + 503 + ], + [ + "2026-10-02T13:19:23+00:00", + 503 + ], + [ + "2026-10-02T13:19:28+00:00", + 503 + ], + [ + "2026-10-02T13:19:34+00:00", + 503 + ], + [ + "2026-10-02T13:19:39+00:00", + 503 + ], + [ + "2026-10-02T13:19:45+00:00", + 503 + ], + [ + "2026-10-02T13:19:50+00:00", + 503 + ], + [ + "2026-10-02T13:19:55+00:00", + 503 + ], + [ + "2026-10-02T13:20:01+00:00", + 503 + ], + [ + "2026-10-02T13:20:06+00:00", + 503 + ], + [ + "2026-10-02T13:20:12+00:00", + 503 + ], + [ + "2026-10-02T13:20:17+00:00", + 503 + ], + [ + "2026-10-02T13:20:23+00:00", + 503 + ], + [ + "2026-10-02T13:20:28+00:00", + 503 + ], + [ + "2026-10-02T13:20:29+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:30+00:00", + 401 + ], + [ + "2026-10-02T13:20:31+00:00", + 401 + ], + [ + "2026-10-02T13:20:31+00:00", + 401 + ], + [ + "2026-10-02T13:20:32+00:00", + 401 + ], + [ + "2026-10-02T13:20:32+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:33+00:00", + 401 + ], + [ + "2026-10-02T13:20:34+00:00", + 401 + ], + [ + "2026-10-02T13:20:34+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:35+00:00", + 401 + ], + [ + "2026-10-02T13:20:36+00:00", + 401 + ], + [ + "2026-10-02T13:20:36+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:37+00:00", + 401 + ], + [ + "2026-10-02T13:20:38+00:00", + 401 + ], + [ + "2026-10-02T13:20:38+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:39+00:00", + 401 + ], + [ + "2026-10-02T13:20:40+00:00", + 401 + ], + [ + "2026-10-02T13:20:40+00:00", + 401 + ], + [ + "2026-10-02T13:20:41+00:00", + 401 + ], + [ + "2026-10-02T13:20:41+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:42+00:00", + 401 + ], + [ + "2026-10-02T13:20:43+00:00", + 401 + ], + [ + "2026-10-02T13:20:43+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:44+00:00", + 401 + ], + [ + "2026-10-02T13:20:45+00:00", + 401 + ], + [ + "2026-10-02T13:20:45+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:46+00:00", + 401 + ], + [ + "2026-10-02T13:20:47+00:00", + 401 + ], + [ + "2026-10-02T13:20:47+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:48+00:00", + 401 + ], + [ + "2026-10-02T13:20:49+00:00", + 401 + ], + [ + "2026-10-02T13:20:49+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:50+00:00", + 401 + ], + [ + "2026-10-02T13:20:51+00:00", + 401 + ], + [ + "2026-10-02T13:20:51+00:00", + 401 + ], + [ + "2026-10-02T13:20:52+00:00", + 401 + ], + [ + "2026-10-02T13:20:52+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:53+00:00", + 401 + ], + [ + "2026-10-02T13:20:54+00:00", + 401 + ], + [ + "2026-10-02T13:20:54+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:55+00:00", + 401 + ], + [ + "2026-10-02T13:20:56+00:00", + 401 + ], + [ + "2026-10-02T13:20:56+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:57+00:00", + 401 + ], + [ + "2026-10-02T13:20:58+00:00", + 401 + ], + [ + "2026-10-02T13:20:58+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:20:59+00:00", + 401 + ], + [ + "2026-10-02T13:21:00+00:00", + 401 + ], + [ + "2026-10-02T13:21:00+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:01+00:00", + 401 + ], + [ + "2026-10-02T13:21:02+00:00", + 401 + ], + [ + "2026-10-02T13:21:02+00:00", + 401 + ], + [ + "2026-10-02T13:21:03+00:00", + 401 + ], + [ + "2026-10-02T13:21:03+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:04+00:00", + 401 + ], + [ + "2026-10-02T13:21:05+00:00", + 401 + ], + [ + "2026-10-02T13:21:05+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:06+00:00", + 401 + ], + [ + "2026-10-02T13:21:07+00:00", + 401 + ], + [ + "2026-10-02T13:21:07+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:08+00:00", + 401 + ], + [ + "2026-10-02T13:21:09+00:00", + 401 + ], + [ + "2026-10-02T13:21:09+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:10+00:00", + 401 + ], + [ + "2026-10-02T13:21:11+00:00", + 401 + ], + [ + "2026-10-02T13:21:11+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:12+00:00", + 401 + ], + [ + "2026-10-02T13:21:13+00:00", + 401 + ], + [ + "2026-10-02T13:21:13+00:00", + 401 + ], + [ + "2026-10-02T13:21:14+00:00", + 401 + ], + [ + "2026-10-02T13:21:14+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:15+00:00", + 401 + ], + [ + "2026-10-02T13:21:16+00:00", + 401 + ], + [ + "2026-10-02T13:21:16+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:17+00:00", + 401 + ], + [ + "2026-10-02T13:21:18+00:00", + 401 + ], + [ + "2026-10-02T13:21:18+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:19+00:00", + 401 + ], + [ + "2026-10-02T13:21:20+00:00", + 401 + ], + [ + "2026-10-02T13:21:20+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:21+00:00", + 401 + ], + [ + "2026-10-02T13:21:22+00:00", + 401 + ], + [ + "2026-10-02T13:21:22+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:23+00:00", + 401 + ], + [ + "2026-10-02T13:21:24+00:00", + 401 + ], + [ + "2026-10-02T13:21:24+00:00", + 401 + ], + [ + "2026-10-02T13:21:25+00:00", + 401 + ], + [ + "2026-10-02T13:21:25+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:26+00:00", + 401 + ], + [ + "2026-10-02T13:21:27+00:00", + 401 + ], + [ + "2026-10-02T13:21:27+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:28+00:00", + 401 + ], + [ + "2026-10-02T13:21:29+00:00", + 401 + ], + [ + "2026-10-02T13:21:29+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:30+00:00", + 401 + ], + [ + "2026-10-02T13:21:31+00:00", + 401 + ], + [ + "2026-10-02T13:21:31+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:32+00:00", + 401 + ], + [ + "2026-10-02T13:21:33+00:00", + 401 + ], + [ + "2026-10-02T13:21:33+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:34+00:00", + 401 + ], + [ + "2026-10-02T13:21:35+00:00", + 401 + ], + [ + "2026-10-02T13:21:35+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:36+00:00", + 401 + ], + [ + "2026-10-02T13:21:37+00:00", + 401 + ], + [ + "2026-10-02T13:21:37+00:00", + 401 + ], + [ + "2026-10-02T13:21:38+00:00", + 401 + ], + [ + "2026-10-02T13:21:38+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:39+00:00", + 401 + ], + [ + "2026-10-02T13:21:40+00:00", + 401 + ], + [ + "2026-10-02T13:21:40+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:41+00:00", + 401 + ], + [ + "2026-10-02T13:21:42+00:00", + 401 + ], + [ + "2026-10-02T13:21:42+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:43+00:00", + 401 + ], + [ + "2026-10-02T13:21:44+00:00", + 401 + ], + [ + "2026-10-02T13:21:44+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:45+00:00", + 401 + ], + [ + "2026-10-02T13:21:46+00:00", + 401 + ], + [ + "2026-10-02T13:21:46+00:00", + 401 + ], + [ + "2026-10-02T13:21:47+00:00", + 401 + ], + [ + "2026-10-02T13:21:47+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:48+00:00", + 401 + ], + [ + "2026-10-02T13:21:49+00:00", + 401 + ], + [ + "2026-10-02T13:21:49+00:00", + 401 + ], + [ + "2026-10-02T13:21:50+00:00", + 401 + ] + ], + "planned_seconds": { + "outage": 180, + "post": 130, + "pre": 15 + }, + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:14+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:19+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:24+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:29+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:34+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:39+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05263157894736842, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:44+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10714285714285715, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:49+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.23529411764705882, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:54+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:16:59+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:17:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:18:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:19:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.8333333333333333, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.8, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10714285714285715, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:45+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.07407407407407408, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:50+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0196078431372549, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:20:55+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:00+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "firing", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:05+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:10+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:15+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:20+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:25+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:30+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:35+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:21:40+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "times": { + "db_healthy": "2026-10-02T13:19:32+00:00", + "db_stopped": "2026-10-02T13:16:30+00:00", + "restart_requested": "2026-10-02T13:19:30+00:00", + "started": "2026-10-02T13:16:14+00:00", + "stop_requested": "2026-10-02T13:16:29+00:00" + }, + "watch_errors": [] +} diff --git a/evidence/slo-quiet-baseline-2026-10-02.jsonl b/evidence/slo-quiet-baseline-2026-10-02.jsonl new file mode 100644 index 0000000..e6029c0 --- /dev/null +++ b/evidence/slo-quiet-baseline-2026-10-02.jsonl @@ -0,0 +1,6 @@ +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:28+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:33+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:38+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:43+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:48+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} +{"alerts": {"Lab6ApiBurnLong": "inactive", "Lab6ApiBurnShort": "inactive", "Lab6HighHttpErrorRate": "inactive", "Lab6QueueLag": "inactive", "Lab6ReadyBurnLong": "inactive", "Lab6ReadyBurnShort": "inactive"}, "timestamp": "2026-10-02T12:33:53+00:00", "values": {"collector_failed_spans": null, "http_5xx_ratio_30s": null, "queue_oldest_seconds": 0.0, "ready_slow_ratio_30s": 0.0}} diff --git a/evidence/slo-short-db-2026-10-02.json b/evidence/slo-short-db-2026-10-02.json new file mode 100644 index 0000000..8f6090e --- /dev/null +++ b/evidence/slo-short-db-2026-10-02.json @@ -0,0 +1,1584 @@ +{ + "case": "short", + "events": [ + [ + "2026-10-02T13:13:13+00:00", + 401 + ], + [ + "2026-10-02T13:13:13+00:00", + 401 + ], + [ + "2026-10-02T13:13:14+00:00", + 401 + ], + [ + "2026-10-02T13:13:14+00:00", + 401 + ], + [ + "2026-10-02T13:13:15+00:00", + 401 + ], + [ + "2026-10-02T13:13:15+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:16+00:00", + 401 + ], + [ + "2026-10-02T13:13:17+00:00", + 401 + ], + [ + "2026-10-02T13:13:17+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:18+00:00", + 401 + ], + [ + "2026-10-02T13:13:19+00:00", + 401 + ], + [ + "2026-10-02T13:13:19+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:20+00:00", + 401 + ], + [ + "2026-10-02T13:13:21+00:00", + 401 + ], + [ + "2026-10-02T13:13:21+00:00", + 401 + ], + [ + "2026-10-02T13:13:22+00:00", + 401 + ], + [ + "2026-10-02T13:13:22+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:23+00:00", + 401 + ], + [ + "2026-10-02T13:13:24+00:00", + 401 + ], + [ + "2026-10-02T13:13:24+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:25+00:00", + 401 + ], + [ + "2026-10-02T13:13:26+00:00", + 401 + ], + [ + "2026-10-02T13:13:26+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:27+00:00", + 401 + ], + [ + "2026-10-02T13:13:28+00:00", + 401 + ], + [ + "2026-10-02T13:13:33+00:00", + 503 + ], + [ + "2026-10-02T13:13:39+00:00", + 503 + ], + [ + "2026-10-02T13:13:44+00:00", + 503 + ], + [ + "2026-10-02T13:13:49+00:00", + 503 + ], + [ + "2026-10-02T13:13:55+00:00", + 503 + ], + [ + "2026-10-02T13:14:00+00:00", + 503 + ], + [ + "2026-10-02T13:14:06+00:00", + 503 + ], + [ + "2026-10-02T13:14:11+00:00", + 503 + ], + [ + "2026-10-02T13:14:16+00:00", + 503 + ], + [ + "2026-10-02T13:14:22+00:00", + 503 + ], + [ + "2026-10-02T13:14:27+00:00", + 503 + ], + [ + "2026-10-02T13:14:33+00:00", + 503 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:35+00:00", + 401 + ], + [ + "2026-10-02T13:14:36+00:00", + 401 + ], + [ + "2026-10-02T13:14:36+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:37+00:00", + 401 + ], + [ + "2026-10-02T13:14:38+00:00", + 401 + ], + [ + "2026-10-02T13:14:38+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:39+00:00", + 401 + ], + [ + "2026-10-02T13:14:40+00:00", + 401 + ], + [ + "2026-10-02T13:14:40+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:41+00:00", + 401 + ], + [ + "2026-10-02T13:14:42+00:00", + 401 + ], + [ + "2026-10-02T13:14:42+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:43+00:00", + 401 + ], + [ + "2026-10-02T13:14:44+00:00", + 401 + ], + [ + "2026-10-02T13:14:44+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:45+00:00", + 401 + ], + [ + "2026-10-02T13:14:46+00:00", + 401 + ], + [ + "2026-10-02T13:14:46+00:00", + 401 + ], + [ + "2026-10-02T13:14:47+00:00", + 401 + ], + [ + "2026-10-02T13:14:47+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:48+00:00", + 401 + ], + [ + "2026-10-02T13:14:49+00:00", + 401 + ], + [ + "2026-10-02T13:14:49+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:50+00:00", + 401 + ], + [ + "2026-10-02T13:14:51+00:00", + 401 + ], + [ + "2026-10-02T13:14:51+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:52+00:00", + 401 + ], + [ + "2026-10-02T13:14:53+00:00", + 401 + ], + [ + "2026-10-02T13:14:53+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:54+00:00", + 401 + ], + [ + "2026-10-02T13:14:55+00:00", + 401 + ], + [ + "2026-10-02T13:14:55+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:56+00:00", + 401 + ], + [ + "2026-10-02T13:14:57+00:00", + 401 + ], + [ + "2026-10-02T13:14:57+00:00", + 401 + ], + [ + "2026-10-02T13:14:58+00:00", + 401 + ], + [ + "2026-10-02T13:14:58+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:14:59+00:00", + 401 + ], + [ + "2026-10-02T13:15:00+00:00", + 401 + ], + [ + "2026-10-02T13:15:00+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:01+00:00", + 401 + ], + [ + "2026-10-02T13:15:02+00:00", + 401 + ], + [ + "2026-10-02T13:15:02+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:03+00:00", + 401 + ], + [ + "2026-10-02T13:15:04+00:00", + 401 + ], + [ + "2026-10-02T13:15:04+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:05+00:00", + 401 + ], + [ + "2026-10-02T13:15:06+00:00", + 401 + ], + [ + "2026-10-02T13:15:06+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:07+00:00", + 401 + ], + [ + "2026-10-02T13:15:08+00:00", + 401 + ], + [ + "2026-10-02T13:15:08+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:09+00:00", + 401 + ], + [ + "2026-10-02T13:15:10+00:00", + 401 + ], + [ + "2026-10-02T13:15:10+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:11+00:00", + 401 + ], + [ + "2026-10-02T13:15:12+00:00", + 401 + ], + [ + "2026-10-02T13:15:12+00:00", + 401 + ], + [ + "2026-10-02T13:15:13+00:00", + 401 + ], + [ + "2026-10-02T13:15:13+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:14+00:00", + 401 + ], + [ + "2026-10-02T13:15:15+00:00", + 401 + ], + [ + "2026-10-02T13:15:15+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:16+00:00", + 401 + ], + [ + "2026-10-02T13:15:17+00:00", + 401 + ], + [ + "2026-10-02T13:15:17+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:18+00:00", + 401 + ], + [ + "2026-10-02T13:15:19+00:00", + 401 + ], + [ + "2026-10-02T13:15:19+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:20+00:00", + 401 + ], + [ + "2026-10-02T13:15:21+00:00", + 401 + ], + [ + "2026-10-02T13:15:21+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:22+00:00", + 401 + ], + [ + "2026-10-02T13:15:23+00:00", + 401 + ], + [ + "2026-10-02T13:15:23+00:00", + 401 + ], + [ + "2026-10-02T13:15:24+00:00", + 401 + ], + [ + "2026-10-02T13:15:24+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:25+00:00", + 401 + ], + [ + "2026-10-02T13:15:26+00:00", + 401 + ], + [ + "2026-10-02T13:15:26+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:27+00:00", + 401 + ], + [ + "2026-10-02T13:15:28+00:00", + 401 + ], + [ + "2026-10-02T13:15:28+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:29+00:00", + 401 + ], + [ + "2026-10-02T13:15:30+00:00", + 401 + ], + [ + "2026-10-02T13:15:30+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:31+00:00", + 401 + ], + [ + "2026-10-02T13:15:32+00:00", + 401 + ], + [ + "2026-10-02T13:15:32+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:33+00:00", + 401 + ], + [ + "2026-10-02T13:15:34+00:00", + 401 + ], + [ + "2026-10-02T13:15:34+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:35+00:00", + 401 + ], + [ + "2026-10-02T13:15:36+00:00", + 401 + ], + [ + "2026-10-02T13:15:36+00:00", + 401 + ], + [ + "2026-10-02T13:15:37+00:00", + 401 + ], + [ + "2026-10-02T13:15:37+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:38+00:00", + 401 + ], + [ + "2026-10-02T13:15:39+00:00", + 401 + ], + [ + "2026-10-02T13:15:39+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:40+00:00", + 401 + ], + [ + "2026-10-02T13:15:41+00:00", + 401 + ], + [ + "2026-10-02T13:15:41+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:42+00:00", + 401 + ], + [ + "2026-10-02T13:15:43+00:00", + 401 + ], + [ + "2026-10-02T13:15:43+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:44+00:00", + 401 + ], + [ + "2026-10-02T13:15:45+00:00", + 401 + ], + [ + "2026-10-02T13:15:45+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:46+00:00", + 401 + ], + [ + "2026-10-02T13:15:47+00:00", + 401 + ], + [ + "2026-10-02T13:15:47+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:48+00:00", + 401 + ], + [ + "2026-10-02T13:15:49+00:00", + 401 + ], + [ + "2026-10-02T13:15:49+00:00", + 401 + ], + [ + "2026-10-02T13:15:50+00:00", + 401 + ], + [ + "2026-10-02T13:15:50+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:51+00:00", + 401 + ], + [ + "2026-10-02T13:15:52+00:00", + 401 + ], + [ + "2026-10-02T13:15:52+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:53+00:00", + 401 + ], + [ + "2026-10-02T13:15:54+00:00", + 401 + ], + [ + "2026-10-02T13:15:54+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:55+00:00", + 401 + ], + [ + "2026-10-02T13:15:56+00:00", + 401 + ], + [ + "2026-10-02T13:15:56+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:57+00:00", + 401 + ], + [ + "2026-10-02T13:15:58+00:00", + 401 + ] + ], + "planned_seconds": { + "outage": 50, + "post": 90, + "pre": 15 + }, + "prometheus_samples": [ + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": null, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05239864559019707, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.10986596449397801, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:53+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.1914814703057944, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "pending", + "Lab6HighHttpErrorRate": "pending", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:13:58+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:03+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:08+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "firing", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 1.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.25, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.1111111111111111, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "firing", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:53+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.05263157894736842, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:14:58+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.02, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:03+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "pending", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:08+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:13+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:18+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:23+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:28+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:33+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:38+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:43+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + }, + { + "alerts": { + "Lab6ApiBurnLong": "inactive", + "Lab6ApiBurnShort": "inactive", + "Lab6HighHttpErrorRate": "inactive", + "Lab6QueueLag": "inactive", + "Lab6ReadyBurnLong": "inactive", + "Lab6ReadyBurnShort": "inactive" + }, + "timestamp": "2026-10-02T13:15:48+00:00", + "values": { + "collector_failed_spans": null, + "http_5xx_ratio_30s": 0.0, + "queue_oldest_seconds": 0.0, + "ready_slow_ratio_30s": 0.0 + } + } + ], + "times": { + "db_healthy": "2026-10-02T13:14:21+00:00", + "db_stopped": "2026-10-02T13:13:28+00:00", + "restart_requested": "2026-10-02T13:14:18+00:00", + "started": "2026-10-02T13:13:13+00:00", + "stop_requested": "2026-10-02T13:13:28+00:00" + }, + "watch_errors": [] +} diff --git a/evidence/telemetry-backend-outage-2026-10-02.json b/evidence/telemetry-backend-outage-2026-10-02.json new file mode 100644 index 0000000..3ead1ad --- /dev/null +++ b/evidence/telemetry-backend-outage-2026-10-02.json @@ -0,0 +1,150 @@ +{ + "collector_counter_delta": { + "otelcol_exporter_enqueue_failed_spans": 19486.0, + "otelcol_exporter_send_failed_log_records": 11063.0, + "otelcol_exporter_send_failed_metric_points": 484.0, + "otelcol_exporter_send_failed_spans": 16547.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "collector_counters_after": { + "otelcol_exporter_enqueue_failed_spans": 112097.0, + "otelcol_exporter_send_failed_log_records": 31979.0, + "otelcol_exporter_send_failed_metric_points": 1570.0, + "otelcol_exporter_send_failed_spans": 44902.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "collector_counters_before": { + "otelcol_exporter_enqueue_failed_spans": 92611.0, + "otelcol_exporter_send_failed_log_records": 20916.0, + "otelcol_exporter_send_failed_metric_points": 1086.0, + "otelcol_exporter_send_failed_spans": 28355.0, + "otelcol_receiver_refused_log_records": 0.0, + "otelcol_receiver_refused_metric_points": 0.0, + "otelcol_receiver_refused_spans": 0.0 + }, + "load_passed": true, + "smoke_passed": true, + "smoke_tail": [ + "python3 tests/smoke.py --base-url \"http://127.0.0.1:8080\" ", + "PASS: lab6 public API scenario; infrastructure and grading evidence remain separate." + ], + "started": "2026-10-02T14:01:35+00:00", + "stats_after_35s": [ + { + "BlockIO": "2.23GB / 0B", + "CPUPerc": "1.21%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "30.20%", + "MemUsage": "116MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 37.1MB", + "PIDs": "10" + }, + { + "BlockIO": "40.6MB / 0B", + "CPUPerc": "14.25%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "32.39%", + "MemUsage": "82.91MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.6MB / 29.5MB", + "PIDs": "10" + }, + { + "BlockIO": "11.3MB / 0B", + "CPUPerc": "0.09%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.12%", + "MemUsage": "54.67MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "334kB / 453kB", + "PIDs": "8" + } + ], + "stats_before": [ + { + "BlockIO": "2.21GB / 0B", + "CPUPerc": "0.19%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "18.44%", + "MemUsage": "70.82MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "110MB / 36.8MB", + "PIDs": "10" + }, + { + "BlockIO": "35.6MB / 0B", + "CPUPerc": "8.60%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "22.86%", + "MemUsage": "58.52MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "1.13MB / 1.78MB", + "PIDs": "10" + }, + { + "BlockIO": "10.8MB / 0B", + "CPUPerc": "0.15%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.09%", + "MemUsage": "54.44MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "188kB / 267kB", + "PIDs": "8" + } + ], + "stats_during": [ + { + "BlockIO": "2.23GB / 0B", + "CPUPerc": "2.40%", + "Container": "mtusi2026lab6-otel-collector-1", + "ID": "3bedc51054f5", + "MemPerc": "27.14%", + "MemUsage": "104.2MiB / 384MiB", + "Name": "mtusi2026lab6-otel-collector-1", + "NetIO": "126MB / 37MB", + "PIDs": "10" + }, + { + "BlockIO": "40.6MB / 0B", + "CPUPerc": "7.68%", + "Container": "mtusi2026lab6-api-1", + "ID": "2ad72f3266ec", + "MemPerc": "32.65%", + "MemUsage": "83.58MiB / 256MiB", + "Name": "mtusi2026lab6-api-1", + "NetIO": "22.5MB / 29.3MB", + "PIDs": "13" + }, + { + "BlockIO": "11.3MB / 0B", + "CPUPerc": "0.14%", + "Container": "mtusi2026lab6-worker-1", + "ID": "88e67b14961c", + "MemPerc": "7.12%", + "MemUsage": "54.67MiB / 768MiB", + "Name": "mtusi2026lab6-worker-1", + "NetIO": "325kB / 431kB", + "PIDs": "8" + } + ], + "stores": [ + "prometheus", + "loki", + "tempo" + ], + "stores_healthy": true, + "stores_restored": true, + "stores_restored_at": "2026-10-02T14:03:01+00:00", + "stores_stopped": "2026-10-02T14:01:48+00:00" +} diff --git a/evidence/telemetry-outage-http-load-2026-10-02.json b/evidence/telemetry-outage-http-load-2026-10-02.json new file mode 100644 index 0000000..a81e211 --- /dev/null +++ b/evidence/telemetry-outage-http-load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.017, + "concurrency": 4, + "successful_requests": 17876, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 595.531, + "successful_latency_ms": { + "p50": 4.844, + "p95": 16.108, + "p99": 49.174 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/tempo-recovery-load-2026-10-02.json b/evidence/tempo-recovery-load-2026-10-02.json new file mode 100644 index 0000000..ec0c08d --- /dev/null +++ b/evidence/tempo-recovery-load-2026-10-02.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.011, + "concurrency": 4, + "successful_requests": 18860, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 628.445, + "successful_latency_ms": { + "p50": 4.677, + "p95": 14.469, + "p99": 44.625 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/scripts/measure-image-pipeline.py b/scripts/measure-image-pipeline.py index 6b27caa..4a744fe 100644 --- a/scripts/measure-image-pipeline.py +++ b/scripts/measure-image-pipeline.py @@ -41,6 +41,7 @@ def main(): parser.add_argument("--base-url", default="http://127.0.0.1:8080") parser.add_argument("--concurrency", type=int, default=4) parser.add_argument("--per-size", type=int, default=4) + parser.add_argument("--profile", choices=("small_320x200", "noise_800x600")) parser.add_argument("--output", required=True) args = parser.parse_args() if not 1 <= args.concurrency <= 16 or not 1 <= args.per_size <= 100: @@ -52,6 +53,8 @@ def main(): "small_320x200": (ROOT / "tests/fixtures/sample.png").read_bytes(), "noise_800x600": png_noise(800, 600), } + if args.profile: + images = {args.profile: images[args.profile]} items = [(name, data) for name, data in images.items() for _ in range(args.per_size)] def work(item): diff --git a/student-tests/test_observability_failure.py b/student-tests/test_observability_failure.py new file mode 100644 index 0000000..dd49af8 --- /dev/null +++ b/student-tests/test_observability_failure.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Prove business API survives a bounded outage of all telemetry stores.""" + +import json +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode +from urllib.request import urlopen + + +ROOT = Path(__file__).resolve().parents[1] +FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + FILES += ["-f", "compose.local.yaml"] +FILES += ["-f", "compose.telemetry.yaml"] +COMPOSE = ["docker", "compose", *FILES, "--env-file", ".env"] +COUNTERS = ( + "otelcol_exporter_enqueue_failed_spans", + "otelcol_exporter_send_failed_spans", + "otelcol_exporter_send_failed_log_records", + "otelcol_exporter_send_failed_metric_points", + "otelcol_receiver_refused_spans", + "otelcol_receiver_refused_log_records", + "otelcol_receiver_refused_metric_points", +) + + +def run(*args): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True) + + +def stats(): + result = run("docker", "stats", "--no-stream", "--format", "{{json .}}", + "mtusi2026lab6-otel-collector-1", "mtusi2026lab6-api-1", "mtusi2026lab6-worker-1") + return [json.loads(line) for line in result.stdout.splitlines()] if result.returncode == 0 else [] + + +def metric(name): + url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": f"sum({name})"}) + try: + with urlopen(url, timeout=5) as response: + rows = json.load(response)["data"]["result"] + except (OSError, ValueError): + return None + return float(rows[0]["value"][1]) if rows else None + + +def stores_up(): + url = "http://127.0.0.1:9090/api/v1/query?" + urlencode({"query": "up{job=~\"prometheus|loki|tempo\"}"}) + try: + with urlopen(url, timeout=5) as response: + rows = json.load(response)["data"]["result"] + except (OSError, ValueError): + return False + return len(rows) == 3 and all(row["value"][1] == "1" for row in rows) + + +def main(): + output = ROOT / "evidence/telemetry-backend-outage-2026-10-02.json" + data = {"started": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "stores": ["prometheus", "loki", "tempo"], "stats_before": stats(), + "collector_counters_before": {name: metric(name) for name in COUNTERS}} + stopped = False + try: + result = run(*COMPOSE, "stop", "prometheus", "loki", "tempo") + if result.returncode: + raise RuntimeError(result.stderr[-500:]) + stopped = True + data["stores_stopped"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + smoke = run("make", "test", "BASE_URL=http://127.0.0.1:8080") + data["smoke_passed"] = smoke.returncode == 0 + data["smoke_tail"] = (smoke.stdout + smoke.stderr).splitlines()[-5:] + load = run("python3", "tests/load.py", "--base-url", "http://127.0.0.1:8080", + "--duration", "30", "--concurrency", "4", + "--output", "evidence/telemetry-outage-http-load-2026-10-02.json") + data["load_passed"] = load.returncode == 0 + data["stats_during"] = stats() + # Longer than Collector's 30-second exporter retry horizon. + time.sleep(35) + data["stats_after_35s"] = stats() + finally: + if stopped: + restored = run(*COMPOSE, "up", "-d", "prometheus", "loki", "tempo") + data["stores_restored"] = restored.returncode == 0 + data["stores_restored_at"] = datetime.now(timezone.utc).isoformat(timespec="seconds") + healthy_since = None + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if stores_up(): + healthy_since = healthy_since or time.monotonic() + if time.monotonic() - healthy_since >= 30: + break + else: + healthy_since = None + time.sleep(5) + data["stores_healthy"] = (healthy_since is not None and + time.monotonic() - healthy_since >= 30 and stores_up()) + data["collector_counters_after"] = {name: metric(name) for name in COUNTERS} + data["collector_counter_delta"] = { + name: (data["collector_counters_after"][name] - data["collector_counters_before"][name] + if data["collector_counters_before"][name] is not None and + data["collector_counters_after"][name] is not None else None) + for name in COUNTERS + } + output.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"smoke_passed": data.get("smoke_passed"), + "load_passed": data.get("load_passed"), + "stores_restored": data.get("stores_restored"), + "collector_counter_delta": data["collector_counter_delta"], + "output": str(output)}, sort_keys=True)) + if not all(data.get(key) for key in ("smoke_passed", "load_passed", "stores_restored", "stores_healthy")): + raise SystemExit("business path or telemetry recovery failed") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_queue_incident.py b/student-tests/test_queue_incident.py new file mode 100644 index 0000000..d112841 --- /dev/null +++ b/student-tests/test_queue_incident.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Stop worker, build a real durable backlog, then verify queue-lag recovery.""" + +import json +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) +from client import Client # noqa: E402 +from slo_watch import snapshot # noqa: E402 + + +FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + FILES += ["-f", "compose.local.yaml"] +FILES += ["-f", "compose.telemetry.yaml"] + + +def compose(*args): + subprocess.run(["docker", "compose", *FILES, "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + + +def now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def main(): + client = Client("http://127.0.0.1:8080") + alice, _bob, _eve, chat = client.bootstrap() + image = (ROOT / "tests/fixtures/sample.png").read_bytes() + samples = [] + stop = threading.Event() + + def watch(): + while not stop.is_set(): + samples.append(snapshot("http://127.0.0.1:9090")) + stop.wait(5) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + times = {"started": now()} + worker_stopped = False + try: + compose("stop", "worker") + worker_stopped = True + times["worker_stopped"] = now() + attachments = [] + for _ in range(8): + response = client.call( + "POST", f"/api/v1/chats/{chat['id']}/attachments?filename=queue-probe.png", + 202, actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment", + ) + attachments.append(response["json"]) + times["uploads_accepted"] = now() + time.sleep(35) + compose("up", "-d", "worker") + worker_stopped = False + times["worker_started"] = now() + for item in attachments: + client.wait_ready(alice, item, wait_seconds=120) + times["all_ready"] = now() + time.sleep(40) + finally: + if worker_stopped: + compose("up", "-d", "worker") + stop.set() + watcher.join(timeout=6) + states = [item["alerts"].get("Lab6QueueLag") for item in samples] + data = {"times": times, "queued_uploads": len(attachments), + "ready_uploads": len(attachments), "prometheus_samples": samples, + "queue_alert_states": sorted(set(states))} + path = ROOT / "evidence/queue-worker-stop-2026-10-02.json" + path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"queued_uploads": len(attachments), "states": data["queue_alert_states"], + "times": times, "output": str(path)}, sort_keys=True)) + if "firing" not in states or states[-1] != "inactive": + raise SystemExit("queue lag alert did not fire and resolve") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_slo_alerts.py b/student-tests/test_slo_alerts.py new file mode 100644 index 0000000..2818276 --- /dev/null +++ b/student-tests/test_slo_alerts.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Reproduce a DB outage and capture Prometheus burn-rate transitions.""" + +import argparse +import json +import subprocess +import sys +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE_FILES = ["-f", "compose.yaml"] +if (ROOT / "compose.local.yaml").exists(): + COMPOSE_FILES += ["-f", "compose.local.yaml"] +COMPOSE_FILES += ["-f", "compose.telemetry.yaml"] +sys.path.insert(0, str(ROOT / "scripts")) +from slo_watch import snapshot # noqa: E402 + + +PROBE = r''' +import base64, json, sys, time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +end = time.monotonic() + int(sys.argv[1]) +events = [] +headers = {"Authorization": "Basic " + base64.b64encode(b"lab6_probe:synthetic-invalid-password").decode()} +def run_one(_): + result = [] + while time.monotonic() < end: + req = Request("http://127.0.0.1:8080/api/v1/auth/sessions", data=b"", headers=headers, method="POST") + try: + with urlopen(req, timeout=30) as response: + code = response.status + except HTTPError as exc: + code = exc.code + except OSError: + code = 0 + result.append([datetime.now(timezone.utc).isoformat(timespec="seconds"), code]) + time.sleep(0.4) + return result +with ThreadPoolExecutor(max_workers=1) as pool: + for batch in pool.map(run_one, range(1)): + events.extend(batch) +print(json.dumps({"events": sorted(events)})) +''' + + +def utc_now(): + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def compose(*args): + subprocess.run( + ["docker", "compose", *COMPOSE_FILES, "--env-file", ".env", *args], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", choices=("short", "long"), required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + pre, outage, post = (15, 50, 90) if args.case == "short" else (15, 180, 130) + duration = pre + outage + post + 10 + samples, watch_errors, times = [], [], {"started": utc_now()} + stop_watching = threading.Event() + + def watch(): + while not stop_watching.is_set(): + try: + samples.append(snapshot("http://127.0.0.1:9090")) + except (OSError, ValueError, KeyError) as exc: + watch_errors.append({"timestamp": utc_now(), "type": type(exc).__name__}) + stop_watching.wait(5) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + command = [ + "docker", "compose", *COMPOSE_FILES, "--env-file", ".env", "exec", "-T", + "api", "python", "-u", "-", str(duration), + ] + probe = subprocess.Popen(command, cwd=ROOT, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + probe.stdin.write(PROBE) + probe.stdin.close() + db_stopped = False + try: + time.sleep(pre) + times["stop_requested"] = utc_now() + compose("stop", "postgres") + db_stopped = True + times["db_stopped"] = utc_now() + time.sleep(outage) + times["restart_requested"] = utc_now() + compose("up", "-d", "--wait", "postgres") + db_stopped = False + times["db_healthy"] = utc_now() + time.sleep(post) + finally: + if db_stopped: + compose("up", "-d", "--wait", "postgres") + times["db_healthy_after_cleanup"] = utc_now() + stop_watching.set() + watcher.join(timeout=6) + + output = probe.stdout.read() + error = probe.stderr.read() + probe.wait(timeout=15) + result = json.loads(output) if probe.returncode == 0 else {"error": error[-500:]} + result.update({"case": args.case, "planned_seconds": {"pre": pre, "outage": outage, "post": post}, + "times": times, "prometheus_samples": samples, "watch_errors": watch_errors}) + path = Path(args.output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + states = {name: {sample["alerts"].get(name) for sample in samples} + for name in ("Lab6HighHttpErrorRate", "Lab6ApiBurnShort", "Lab6ApiBurnLong")} + codes = {event[1] for event in result.get("events", [])} + print(json.dumps({"case": args.case, "codes": sorted(codes), + "states": {name: sorted(values) for name, values in states.items()}, + "samples": len(samples), "output": str(path)}, sort_keys=True)) + expected = {"Lab6HighHttpErrorRate", "Lab6ApiBurnShort"} + if args.case == "long": + expected.add("Lab6ApiBurnLong") + if probe.returncode or not {401, 503} <= codes or any("firing" not in states[name] for name in expected): + raise SystemExit("incident did not demonstrate the expected HTTP SLO transitions") + if any(sample["alerts"].get(name) != "inactive" for name in expected for sample in samples[-3:]): + raise SystemExit("alerts did not resolve after database recovery") + + +if __name__ == "__main__": + main() -- 2.54.0 From 4384098c56311c7738a47ddaa56a6c4de4736dde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 17:07:23 +0300 Subject: [PATCH 12/13] Document lab6 results and verify application in CI --- .github/workflows/application.yml | 35 +++++++++ REPORT.md | 114 ++++++++++++++++++++++++------ 2 files changed, 128 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/application.yml diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 0000000..87430cc --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,35 @@ +name: Lab 6 application and telemetry +on: [push, pull_request] +permissions: + contents: read +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install -r requirements-dev.txt + - run: make check validate PYTHON=python + - name: Start disposable stack + run: | + cp .env.example .env + python scripts/setup-local-secrets.py + python scripts/init_local_broker.py + docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env up -d --build --wait --scale api=2 + - name: Verify API and telemetry rules + run: | + make test + python - <<'PY' + import json, urllib.request + url = 'http://127.0.0.1:9090/api/v1/rules' + data = json.load(urllib.request.urlopen(url, timeout=10)) + assert data['status'] == 'success' + names = {rule['name'] for group in data['data']['groups'] for rule in group['rules']} + assert {'Lab6HighHttpErrorRate', 'Lab6QueueLag'} <= names, names + PY + - name: Stop stack + if: always() + run: docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env down || true diff --git a/REPORT.md b/REPORT.md index b97ae42..46c4fda 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,109 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: Самохвалов Андрей Ремович, БПИ-2302. +- Целевая оценка: 5; трек 2 «SLO и обнаружение инцидентов». Уровни 3 и 4 проверяются отдельно. +- Ветка: `solution/bpi2302-samohvalov`. Локальные `docs/plans/` и `LAB_NOTES.md` не публикуются. +- Источники: `README.md`, `COURSE.md`, `contracts/observability.md`, публичные проверки и собственная lab5. Помощь ИИ: Codex; результаты ниже получены на синтетических данных. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose и Python 3.9+. На хост открыты только loopback-порты API 8080, Grafana 3000 и Prometheus 9090. Секреты создаются локально с правами 0600 и не попадают в Git. -## Архитектура и решения +```sh +cp .env.example .env +python3 scripts/setup-local-secrets.py +python3 scripts/init_local_broker.py +docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env config --quiet +docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env up -d --build --wait --scale api=2 +make check +make test BASE_URL=http://127.0.0.1:8080 +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`migrate`, `s3-volume-init`, `s3-bootstrap`, `broker-init` — одноразовые роли, их успешный `Exited (0)` штатен. Остановка с сохранением томов: `docker compose -f compose.yaml -f compose.telemetry.yaml --env-file .env down`. Намеренное удаление **только синтетических локальных** данных: та же команда с `-v`. Не запускать `-v` для существующего стенда, который нужно сохранить. Миграции 001–008 применяются под PostgreSQL advisory lock. -## Доказательства критериев +Образы: Python 3.12.12, PostgreSQL 17.6, RabbitMQ 4.1.8, HAProxy 3.0.11, MinIO `RELEASE.2025-10-15T17-29-55Z` (digest), MinIO Client `2025.7.21` (digest); OTel Collector contrib 0.128.0, Prometheus 3.5.0, Loki 3.5.0, Tempo 2.9.5, Grafana 12.0.2. Приложение: FastAPI 0.118.3, OTel SDK/API/OTLP 1.45.0, semantic conventions 0.66b0, psycopg 3.2.12, Pillow 11.3.0, pika 1.4.4. Локальный Colima/arm64: 2 CPU, 2 GiB RAM. На этом хосте Docker build обходил неисправный DNS PyPI через ignored `compose.local.yaml`; обычному окружению файл не нужен. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +## Архитектура и конфигурация + +Собственный стек lab5 перенесён без замены контрактов: HAProxy → две API-реплики → PostgreSQL, приватный MinIO, долговечный RabbitMQ и отдельные publisher/worker/cleanup. API возвращает `202 queued` только после сохранения проверенного staging-объекта и одной транзакции attachment/job/outbox. Publisher подтверждает AMQP-публикацию, worker подтверждает delivery после устойчивого результата. Повторы безопасны по version/lease и уникальности эффектов. Реплика API, воркер или broker могут перезапускаться без потери принятой job; единственные DB/S3/broker/host остаются точками отказа. + +| Роль | Версия, health, данные, пределы | Доступ | +| --- | --- | --- | +| `api ×2` | Python/FastAPI; `/health/ready`; 0,5 CPU/256 MiB на реплику | DB и ограниченный S3 secret; API через LB | +| `lb` | HAProxy 3.0.11; backend health; без тома; 0,5 CPU/128 MiB | `127.0.0.1:8080` | +| `postgres` | 17.6; `pg_isready`; `postgres_data` | только внутренняя сеть, secret | +| `s3` | MinIO, digest; HTTP readiness; `s3_data`; 0,5 CPU/384 MiB | приватный bucket, раздельные app/maintenance secrets | +| `broker` | RabbitMQ 4.1.8; diagnostics ping; `broker_data` | durable quorum queue, AMQP secret | +| `publisher` | Python 3.12.12; DB outbox и broker confirms | DB/AMQP secrets, нет S3 | +| `worker` | Python 3.12.12; serial consumer, 0,5 CPU/768 MiB | DB/AMQP/S3 secrets | +| `staging-cleanup` | Python 3.12.12; периодическая сверка | DB и отдельный maintenance S3 secret | +| `otel-collector` | contrib 0.128.0; internal metrics 8888; 384 MiB | OTLP 4317/4318 только Compose | +| `prometheus` | 3.5.0; `prometheus_data`, retention 7d | `127.0.0.1:9090` | +| `loki` / `tempo` | 3.5.0 / 2.9.5; `loki_data` / `tempo_data`; Tempo 768 MiB, блоки до 5 MB/1 мин | только Compose | +| `grafana` | 12.0.2; `grafana_data`, provisioning из Git | `127.0.0.1:3000`, password secret | + +`.env.example` задаёт локальные порты/пределы; Compose передаёт файлы секретов через `/run/secrets`. OTLP endpoint внутри сети `otel-collector:4318`. Имена `APP_VERSION=lab6`, instance UUID и `deployment.environment.name=lab` записаны как resource attributes. Grafana импортирует три datasource и dashboard из `infra/grafana/`. Prometheus загружает rules из `infra/prometheus/rules.yml`. + +OTel SDK экспортирует асинхронно раз в 5 секунд с 2-секундным timeout и очередями 512 spans/records. Collector применяет memory limiter 256/64 MiB, batch 512/2s и конечные exporter queues: metrics 1000, logs/traces по 64 batches, retry не дольше 30 секунд. Отказ хранилища наблюдаемости не входит в business readiness. Потери сигналов при долгом отказе допустимы и измеряются отдельно. + +## Сигналы и запросы + +| Сигнал | Native Prometheus metric / Loki / Tempo | Единица и labels | +| --- | --- | --- | +| HTTP count, duration | `messenger_http_requests_total`, `messenger_http_duration_seconds_bucket` | запросы, секунды; method/route template/status_class | +| Jobs count, duration | `messenger_jobs_processed_total`, `messenger_jobs_duration_seconds_bucket` | jobs, секунды; job_type/outcome | +| Backlog | `messenger_queue_depth`, `messenger_queue_oldest_age_seconds` | jobs, секунды; queue_name | +| Pool | `messenger_db_pool_used`, `messenger_db_pool_waiting` | connections; service_name из resource | +| Worker | `messenger_worker_busy`, `messenger_worker_capacity` | slots; capacity=1 на serial worker | +| CPU/RAM | `messenger_process_cpu_time_seconds_total`, `messenger_process_rss_bytes` | CPU-секунды, bytes текущего RSS | +| До ready | `messenger_attachments_ready_duration_seconds_bucket` | секунды от durable job creation до result COMMIT | +| Логи/трассы | Loki `{service_name="messenger-api"}` / `messenger-worker`; Tempo trace ID | timestamp, severity, event, trace/span IDs, server/producer/consumer spans | + +Dashboard `infra/grafana/dashboards/telemetry.json` содержит сохранённые запросы и единицы. RPS = `sum(rate(messenger_http_requests_total{route!~"/health/.*"}[1m]))`; HTTP 5xx SLI = `sum(rate(...{status_class="5xx"}[1m])) / sum(rate(...[1m]))` для бизнес-запросов. 4xx остаются в знаменателе, отдельно показаны. p50/p95/p99 = `histogram_quantile(q,sum by(le)(rate(*_bucket[5m])))` по всем репликам; средние p95 реплик не используются. Queue age рассчитывается из старейшей queued job в PostgreSQL и не заменяет время обработки. После перезапуска publisher устаревшие remote-write ряды отфильтрованы по sample age <15 с. + +Контекст W3C `traceparent/tracestate` сохраняется в job/outbox и передаётся через брокер; consumer span является потомком producer span. Один upload trace проходит API → publisher → worker → S3/render/DB. Логи API/worker сохраняют `trace_id` и `span_id`; `X-Request-Id` коррелирует HTTP-ошибку. Метки метрик не содержат ID, filename или raw URL. Тела запросов, изображения, токены, пароли, Authorization, signed URLs в telemetry не пишутся. Loki индексирует только service/environment/instance; trace ID хранится как metadata. + +## Проверки критериев + +| Критерий | Команда или источник | Наблюдаемый результат | +| --- | --- | --- | +| Бизнес-контракт с telemetry | `make test`; `student-tests/test_pipeline_faults.py`; `test_publish_confirm_gap.py` | smoke PASS, шесть fault-сценариев PASS, повтор после broker confirm дал один логический ready-эффект | +| Связь трёх сигналов | `student-tests/test_telemetry.py`; Loki и Tempo по одному trace ID | API upload → publisher producer → worker consumer → S3/render/DB; API и worker logs содержат тот же trace ID | +| Метрики и dashboard | Grafana API `GET /api/dashboards/uid/lab6-telemetry`; каждый PromQL из dashboard через `/api/v1/query` | dashboard импортирован, 16 панелей; запросы к реально существующим рядам возвращают `success`; inactive alert не создаёт ряд firing | +| Alert и worker stop | `student-tests/test_queue_incident.py` | 8/8 accepted→ready, queue age peak 34,47 с, `Lab6QueueLag` firing через 35 с, inactive через 12 с после запуска worker; `evidence/queue-worker-stop-2026-10-02.json` | +| Alert и DB outage | `student-tests/test_slo_alerts.py --case short/long` | 12/46 ответов 503; короткое и длинное API burn rules прошли firing→inactive; `evidence/slo-short-db-2026-10-02.json`, `slo-long-db-2026-10-02.json` | +| Отказ telemetry stores | `student-tests/test_observability_failure.py` | остановлены Prometheus/Loki/Tempo; smoke PASS, 17 876 POST без ошибки, 595,531 RPS; Collector RSS 70,82→116 MiB при лимите 384 MiB; все stores `up=1` непрерывно 30 с после восстановления; `evidence/telemetry-backend-outage-2026-10-02.json` | +| Сохранность и отказ реплики | `make test-persistence ACTION_SCRIPT=scripts/restart.sh`; `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | существующая сессия и подтверждённые записи сохранились; после остановки одной реплики 113 успешных записей, один неопределённый ответ, первая успешная запись через 3,685 с | +| CI | `.github/workflows/application.yml` | отдельный job создаёт локальные secrets, собирает Compose, ждёт готовность, выполняет `make test` и проверяет rules API; удалённый CI ещё не запускался | + +Публичный `make test` проверяет бизнес-API, но сам по себе не доказывает полноту наблюдаемости. Публичные контракты и тесты не изменялись. Отдельные тесты лежат в `student-tests/`, исходные агрегаты — в `evidence/`. Пять базовых ролей lab5 с fault matrix сохранены, migration 008 добавляет только допустимый trace carrier. + +## SLO и эксперимент трека 5 + +Пороги зафиксированы **до SLO-инъекций** в `evidence/slo-thresholds-2026-10-02.md` и коммитах `5deecdf`, `43bcde7`. Baseline на двух API-репликах: 30,010 с closed-loop, concurrency=4, 18 260/18 260 успешных POST, 608,462 RPS, p50/p95/p99 успешных ответов 4,289/14,966/59,664 мс. Для 40 PNG (20×320×200 и 20×800×600, concurrency=4) — 40/40 ready за 13,029 с; p95 от начала upload до ready 1,305/1,618 с. Raw JSON: `evidence/lab6-http-baseline-2api-2026-10-02.json`, `lab6-image-baseline-2api-2026-10-02.json`. На спокойном 25-секундном наблюдении 6 правил × 6 samples = 36 состояний `inactive`, ложных firing нет (`evidence/slo-quiet-baseline-2026-10-02.jsonl`). Closed-loop тест имеет coordinated omission и клиентские накладные расходы; p95 изображений включает polling по 0,3 с. + +- API SLO: **99%** non-health ответов без 5xx на скользящих 10 мин; 4xx в знаменателе, но не в числителе. В опыте запросы отправлены прямо в живой API-контейнер: HAProxy после потери readiness может сам вернуть 503, а SDK внутри API не видит такие ответы. Это измеряемое ограничение покрытия SLI. +- Ready SLO: **95%** завершённых image jobs дошли до ready за ≤5 с от durable job creation на скользящих 10 мин. Время upload до `202` исключено; ещё не завершённая job не входит в histogram, поэтому отдельный queue age alert обязателен. +- Burn-rate: short сочетает окна 15/30 с, >14,4× бюджета, `for:10s`; long — 60/120 с, >6×, `for:90s`. Для API error ratio это >14,4% и >6%; для ready slow ratio >72% и >30%. Оценка каждые 5 с. Формулы с фактическими buckets `le="5"` находятся в `infra/prometheus/rules.yml`; `promtool check rules` нашёл 6 правил. Время обнаружения считается от команды воздействия до первого сохранённого `firing`, восстановление — от healthy зависимости до первого `inactive`. Дискретизация 5 с даёт погрешность ±5 с, плюс задержка экспорта. + +| Опыт | Воздействие и выборка | Обнаружение | Восстановление и ложные срабатывания | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Короткий DB | остановка PostgreSQL на 50 с; 12 × 503, 243 × 401 в фиктивном auth-probe | HTTP error и API short firing через 35 с | API short inactive через 22 с после DB healthy; HTTP error через 37 с; API long остался pending и затем inactive, firing не было | +| Длинный DB | остановка PostgreSQL на 180 с; 46 × 503, 234 × 401 | HTTP error 30 с; API short 36 с; API long 141 с | после DB healthy: short 68 с, long 98 с, HTTP error 78 с до inactive; данных не потеряно | +| Worker stop | 8 real uploads при остановленном worker | queue lag firing через 35 с, возраст достиг 34,47 с | 8/8 ready; queue lag inactive через 12 с после запуска; ready histogram был слеп до завершения jobs | +| Умеренный CPU throttle | worker 0,1 CPU, 40 image jobs | ready p95 больших PNG 4,757 с, peak slow ratio 11,36% | пороги burn-rate не достигнуты; 30/30 samples ready short inactive | +| Сильный CPU throttle | worker 0,05 CPU, 30 больших PNG | 30/30 ready за 72,988 с, p95 10,360 с, peak slow ratio 100%; ready short firing с 13:36:26 UTC | лимит возвращён 0,5 CPU; short/long в конце окна inactive; long дал задержанное firing после завершения нагрузки из-за 120-секундной памяти окна и `for:90s` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Пробник с четырьмя одновременными клиентами в коротком DB-опыте сам создавал клиентские timeout вместо устойчивого ряда 503; перед финальными сериями его сменили на один равномерный поток после отдельной проверки `503` через живой API. Пороги не меняли. Файлы финальных серий содержат UTC timestamps, все состояния rules и обезличенные коды ответов. Сильный CPU throttle показал известный компромисс длинного окна: сигнал может сработать уже после прекращения воздействия, пока бюджет ещё расходуется по историческим завершениям. Это одно запаздывающее оповещение, а не новая ошибка после восстановления. -## Эксперименты и отказы +## Отказ наблюдаемости и сохранённые сигналы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +В финальном опыте с остановкой всех трёх storage public smoke прошёл, 30-секундная нагрузка дала 17 876 POST без ошибок, 595,531 RPS, latency успехов p50/p95/p99 4,844/16,108/49,174 мс. Collector имел RSS 70,82 MiB до, 104,2 MiB во время, 116 MiB через 35 с; контейнерный предел 384 MiB. Дельта Collector counters: `otelcol_exporter_enqueue_failed_spans` +19 486, `send_failed_spans` +16 547, `send_failed_log_records` +11 063, `send_failed_metric_points` +484; receiver refused counters остались 0. `send_failed_*` считают неудачные попытки экспорта и не равны потерянным элементам; `enqueue_failed_spans` означает отказ поставить spans в конечную очередь. API не блокировался, но часть traces потеряна. При пропаже telemetry отсутствующая точка не считается нулём ошибок. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +В предварительном опыте `compose up` вернул код 0, но Tempo завершился по OOM при flush накопленного блока 14 MiB; первоначальная проверка восстановления была недостаточной. WAL этого синтетического блока сохранён в локальной копии `/private/tmp/lab6-tempo-wal-backup-2026-10-02`, проблемный блок изолирован внутри тома, остальные сохранённые traces доступны. Tempo получил лимит 768 MiB и блоки до 5 MB/1 мин с одним одновременным flush. Повторная нагрузка с работающей telemetry дала 18 860/18 860 успешных POST, 628,445 RPS, и Tempo оставался `up=1` после flush (`evidence/tempo-recovery-load-2026-10-02.json`). Финальный отказ хранилищ после исправления прошёл; тест восстановления требует 30 секунд непрерывного `up=1` у Prometheus, Loki и Tempo. -## Ограничения и следующие шаги +Grafana импортировала обновлённый dashboard через provisioning; Prometheus сохранил rule states, Loki — структурные API/worker logs, Tempo — один trace с API upload, publisher, worker и дочерними S3/render/DB spans. Пример доказанной цепочки в локальном хранилище: trace ID `5626970bc7d05d13fb839033d5f9b410`, worker consumer span `8850acdc538bf1ae`, API root `0ee38f12aafbf091`, publisher producer `9ef172904a867da6`. Trace ID — случайный технический идентификатор синтетического запроса; тела, credentials и URL не сохранены в evidence. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Пробный слепой разбор runbook выполнил отдельный агент только по сохранённым Prometheus/Loki/Tempo сигналам, без сценариев инъекций и Docker stdout. Для окна 13:16:20–13:21:15 UTC он нашёл первые 5xx в 13:16:30, рост ожидания DB pool до 418 и HTTP p95 до 9,75 с; предположил деградацию БД/пула, но не мог различить их без DB child span. Для 13:23:45–13:24:50 UTC он увидел восемь `job_published`, queue depth 8, рост возраста до 34,5 с, отсутствие свежей worker capacity и восемь `job_processed` после восстановления; предположил недоступность worker. Во время разбора `up{job="tempo"}=0` из-за описанного OOM, поэтому trace не был доступен, а причина остановки worker не доказывалась одними сохранёнными сигналами. Это проверка понятности runbook другим агентом, не независимая проверка студентом. + +Runbook: `infra/runbooks/incident.md`. При пропаже сигнала оператор сначала проверяет `up` storage и счётчики Collector. Локальный стенд не гарантирует доступности при потере одного PostgreSQL/S3/RabbitMQ/host, а serial worker ограничивает throughput изображений. Синтетические тесты не доказывают production SLO или безошибочность внешнего клиента. -- 2.54.0 From d1f8c9cfcee2aa74b596cb353db869a9d3002013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Sat, 7 Nov 2026 09:53:50 +0300 Subject: [PATCH 13/13] fix: preserve NUL characters in persisted messages Store message text as UTF-8 bytes and migrate existing rows without loss. Cover HTTP round trips, restart persistence, Unicode limits and idempotent retries. Adapt direct database tools to the binary storage format. (cherry picked from commit 386276532730b8758298080b49db63738ca300a4) --- migrations/009_message_text_utf8.sql | 6 + src/store.py | 4 +- student-tests/test_message_unicode.py | 156 ++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 migrations/009_message_text_utf8.sql create mode 100644 student-tests/test_message_unicode.py diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql new file mode 100644 index 0000000..86196ee --- /dev/null +++ b/migrations/009_message_text_utf8.sql @@ -0,0 +1,6 @@ +-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload. +-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); +ALTER TABLE messages ADD CONSTRAINT messages_text_check + CHECK (octet_length(text) <= 16000); diff --git a/src/store.py b/src/store.py index 0a701b0..db6425e 100644 --- a/src/store.py +++ b/src/store.py @@ -84,7 +84,7 @@ def chat_view(row: dict) -> dict: def message_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), - "sender_id": str(row["sender_id"]), "text": row["text"], + "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"), "created_at": timestamp(row["created_at"]), "attachment_ids": [str(attachment_id) for attachment_id in row.get("attachment_ids", [])], } @@ -265,7 +265,7 @@ class Store: row = conn.execute( "INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) " "RETURNING id,chat_id,sender_id,text,created_at", - (uuid4(), chat_uuid, actor_uuid, text), + (uuid4(), chat_uuid, actor_uuid, text.encode("utf-8")), ).fetchone() for position, attachment_id in enumerate(ids): conn.execute( diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py new file mode 100644 index 0000000..4ff79d8 --- /dev/null +++ b/student-tests/test_message_unicode.py @@ -0,0 +1,156 @@ +"""Run with DATABASE_URL pointing to a disposable PostgreSQL database. + +Checks real HTTP writes, persisted history, retries, and migration of old rows. +""" + +import base64 +import json +import os +from pathlib import Path +import socket +import subprocess +import sys +import time +import unittest +import urllib.error +import urllib.parse +import urllib.request +from uuid import uuid4 + +import psycopg +from psycopg.conninfo import make_conninfo + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text())["number"] +URL = os.environ.get("DATABASE_URL") + + +@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database") +class MessageUnicodeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + cls.port = sock.getsockname()[1] + cls.start_server() + cls.addClassCleanup(cls.stop_server) + + @classmethod + def start_server(cls): + cls.server = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"}, + ) + for _ in range(100): + if cls.server.poll() is not None: + raise AssertionError("API exited during startup") + try: + cls.call("GET", "/health/live") + return + except (AssertionError, urllib.error.URLError): + time.sleep(0.05) + cls.stop_server() + raise AssertionError("API did not become ready") + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=5) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait(timeout=5) + + @classmethod + def call(cls, method, path, body=None, headers=None, expected=200): + request = urllib.request.Request( + f"http://127.0.0.1:{cls.port}{path}", method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Content-Type": "application/json", **(headers or {})}, + ) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as exc: + response = exc + with response: + payload = response.read() + assert response.status == expected, (method, path, response.status, expected, payload) + return json.loads(payload) if payload else None + + def test_http_round_trip_after_restart_and_replay(self): + username, password = "u_" + uuid4().hex[:16], "Test-password-123" + user = self.call("POST", "/api/v1/users", { + "username": username, "display_name": "Unicode", "password": password, + }, expected=201) + session = self.call("POST", "/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(), + }, expected=201) + auth = {"Authorization": "Bearer " + session["token"]} + chat = self.call("POST", "/api/v1/chats", { + "title": "Unicode", "member_ids": [user["id"]], + }, auth, 201) + path = f"/api/v1/chats/{chat['id']}/messages" + texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000] + messages, keys = [], [] + for text in texts: + key = uuid4().hex + message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(message["text"], text) + messages.append(message) + keys.append(key) + for text in ("", "\0" * 4001, "🙂" * 4001): + self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400) + self.stop_server() + self.start_server() + found, cursor = [], None + while True: + page = self.call("GET", path + "?limit=1" + ( + "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else "" + ), headers=auth) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(found, messages) + if LAB >= 3: + for text, message, key in zip(texts, messages, keys): + replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201) + self.assertEqual(replay, message) + self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409) + self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages)) + + def test_upgrade_preserves_old_text_and_is_repeatable(self): + schema = "unicode_" + uuid4().hex + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'CREATE SCHEMA "{schema}"') + isolated = make_conninfo(URL, options=f"-c search_path={schema}") + try: + old_text = "Старое 🙂 \\ literal \\u0000" + with psycopg.connect(isolated) as conn: + conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)") + for migration in sorted((ROOT / "migrations").glob("*.sql")): + version = int(migration.name.split("_", 1)[0]) + if version >= 9: + continue + conn.execute(migration.read_text()) + conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,)) + user, chat, message = uuid4(), uuid4(), uuid4() + conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,)) + conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,)) + conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user)) + conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text)) + for _ in range(2): + subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True) + with psycopg.connect(isolated) as conn: + conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text()) + saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone() + self.assertEqual(saved.decode("utf-8"), old_text) + finally: + with psycopg.connect(URL, autocommit=True) as conn: + conn.execute(f'DROP SCHEMA "{schema}" CASCADE') + + +if __name__ == "__main__": + unittest.main() -- 2.54.0