From 0497b0c2178959746c74705622f14cdb2464709d Mon Sep 17 00:00:00 2001 From: da1zy3r Date: Mon, 16 Nov 2026 15:13:01 +0300 Subject: [PATCH 1/2] Add 5 lab code --- .env.example | 33 ++-- .gitattributes | 2 + Dockerfile | 24 +++ compose.yaml | 228 ++++++++++++++++++++++++ infra/nginx.conf | 38 ++++ infra/readiness.sh | 38 ++++ infra/s3-bootstrap.sh | 9 + infra/s3-policy.json | 17 ++ migrations/001_initial.sql | 30 ++++ migrations/002_history_version.sql | 1 + migrations/003_message_idempotency.sql | 9 + migrations/004_attachments.sql | 18 ++ migrations/005_attachment_jobs.sql | 23 +++ migrations/006_job_error.sql | 1 + migrations/007_queue_measurements.sql | 3 + requirements.txt | 19 ++ src/__init__.py | 0 src/__main__.py | 13 ++ src/admission.py | 21 +++ src/attachments.py | 87 ++++++++++ src/auth.py | 68 ++++++++ src/cache.py | 40 +++++ src/cleanup.py | 86 ++++++++++ src/database.py | 51 ++++++ src/dependencies.py | 31 ++++ src/errors.py | 60 +++++++ src/image_processor.py | 116 +++++++++++++ src/jobs.py | 180 +++++++++++++++++++ src/limits.py | 99 +++++++++++ src/main.py | 48 ++++++ src/middleware.py | 43 +++++ src/models.py | 108 ++++++++++++ src/object_storage.py | 23 +++ src/routers/__init__.py | 0 src/routers/attachments.py | 90 ++++++++++ src/routers/auth.py | 34 ++++ src/routers/chats.py | 49 ++++++ src/routers/health.py | 23 +++ src/routers/users.py | 19 ++ src/schemas/__init__.py | 0 src/schemas/chat.py | 35 ++++ src/schemas/common.py | 17 ++ src/schemas/message.py | 39 +++++ src/schemas/user.py | 34 ++++ src/store.py | 229 +++++++++++++++++++++++++ src/worker.py | 76 ++++++++ 46 files changed, 2196 insertions(+), 16 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/nginx.conf create mode 100644 infra/readiness.sh create mode 100644 infra/s3-bootstrap.sh create mode 100644 infra/s3-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_history_version.sql create mode 100644 migrations/003_message_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_attachment_jobs.sql create mode 100644 migrations/006_job_error.sql create mode 100644 migrations/007_queue_measurements.sql create mode 100644 requirements.txt create mode 100644 src/__init__.py create mode 100644 src/__main__.py create mode 100644 src/admission.py create mode 100644 src/attachments.py create mode 100644 src/auth.py create mode 100644 src/cache.py create mode 100644 src/cleanup.py create mode 100644 src/database.py create mode 100644 src/dependencies.py create mode 100644 src/errors.py create mode 100644 src/image_processor.py create mode 100644 src/jobs.py create mode 100644 src/limits.py create mode 100644 src/main.py create mode 100644 src/middleware.py create mode 100644 src/models.py create mode 100644 src/object_storage.py create mode 100644 src/routers/__init__.py create mode 100644 src/routers/attachments.py create mode 100644 src/routers/auth.py create mode 100644 src/routers/chats.py create mode 100644 src/routers/health.py create mode 100644 src/routers/users.py create mode 100644 src/schemas/__init__.py create mode 100644 src/schemas/chat.py create mode 100644 src/schemas/common.py create mode 100644 src/schemas/message.py create mode 100644 src/schemas/user.py create mode 100644 src/store.py create mode 100644 src/worker.py diff --git a/.env.example b/.env.example index 2b32655..8cd24a0 100644 --- a/.env.example +++ b/.env.example @@ -1,28 +1,29 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 +# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses. APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +MAX_PENDING_ATTACHMENTS=64 WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_MEMORY_MB=768 STAGING_RETENTION_SECONDS=86400 + +# Choose credentials locally. +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace-with-local-password +CACHE_TTL_SECONDS=30 +MAX_REQUEST_BYTES=65536 +MAX_INFLIGHT=32 +MINIO_ROOT_USER=local-admin +MINIO_ROOT_PASSWORD=replace-with-admin-password +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=replace-with-api-password + +# Reserved for lab6 observability setup. OTEL_SERVICE_NAME=messenger-api # Worker must set a distinct service.name. OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..49819eb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Shell scripts are mounted into Linux containers, including on Windows hosts. +*.sh text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..564995c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src ./src +COPY migrations ./migrations + +RUN useradd --create-home --shell /usr/sbin/nologin appuser +USER appuser + +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()" + +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src"] diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..b95a457 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,228 @@ +x-api: &api + image: mtusi2026lab6-api:local + dns_opt: + - timeout:1 + - attempts:1 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + POSTGRES_HOST: postgres + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30} + MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536} + MAX_INFLIGHT: ${MAX_INFLIGHT:-32} + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_PENDING_ATTACHMENTS: ${MAX_PENDING_ATTACHMENTS:-64} + BROKER_URL: redis://broker:6379/0 + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + networks: [backend] + cpus: 1.0 + mem_limit: 512m + stop_grace_period: 10s + +services: + broker: + image: redis:7.4.2-bookworm + command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction] + volumes: + - broker_data:/data + networks: [backend] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 256m + + dispatcher: + <<: *api + command: [python, -m, src.jobs, dispatch] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + mem_limit: 128m + + worker: + <<: *api + command: [python, -m, src.worker] + init: true + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + broker: + condition: service_healthy + s3-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 2.0 + mem_limit: 2g + pids_limit: 64 + + api: + <<: *api + build: . + deploy: + replicas: 2 + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + s3-init: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"] + interval: 2s + timeout: 2s + start_period: 5s + retries: 2 + + migrate: + <<: *api + command: [python, -m, src.database] + healthcheck: + disable: true + restart: 'no' + depends_on: + postgres: + condition: service_healthy + + s3: + # RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: [server, /data, --console-address, '127.0.0.1:9001'] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + MINIO_BROWSER: 'off' + volumes: + - s3_data:/data + networks: [backend] + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + + s3-init: + # The pinned server image also contains mc; no second image is needed. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + entrypoint: [sh, /bootstrap.sh] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + volumes: + - ./infra/s3-bootstrap.sh:/bootstrap.sh:ro + - ./infra/s3-policy.json:/policy.json:ro + depends_on: + s3: + condition: service_healthy + networks: [backend] + restart: 'no' + cpus: 0.25 + mem_limit: 128m + + cleanup: + <<: *api + command: [python, -m, src.cleanup] + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + cpus: 0.25 + mem_limit: 128m + + nginx: + image: nginx:1.28.0-alpine + entrypoint: [sh, /etc/nginx/readiness.sh] + dns_opt: + - timeout:1 + - attempts:1 + ports: + - '127.0.0.1:${APP_PORT:-8080}:8080' + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/readiness.sh:/etc/nginx/readiness.sh:ro + networks: [edge, backend] + depends_on: + api: + condition: service_healthy + healthcheck: + test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready'] + interval: 2s + timeout: 2s + retries: 3 + cpus: 0.5 + mem_limit: 128m + stop_grace_period: 10s + + postgres: + image: postgres:17.4-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + + redis: + image: redis:7.4.2-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"] + volumes: + - redis_data:/data + networks: [backend] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 256m + +networks: + edge: + backend: + internal: true +volumes: + broker_data: + postgres_data: + redis_data: + s3_data: diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 0000000..1cc308b --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,38 @@ +worker_processes 1; +worker_shutdown_timeout 8s; +events { worker_connections 1024; } +http { + upstream messenger { + zone messenger 64k; + include /etc/nginx/ready-backends.conf; + } + log_format upstream '$request_method $uri $status upstream=$upstream_addr ' + 'upstream_status=$upstream_status time=$request_time'; + access_log /var/log/nginx/access.log upstream; + server { + listen 8080; + # API enforces the byte limit and returns the contracted JSON error. + client_max_body_size 0; + client_body_timeout 5s; + client_header_timeout 5s; + keepalive_timeout 5s; + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_request_buffering off; + proxy_buffering off; + proxy_set_header Host $host; + proxy_set_header Connection close; + proxy_connect_timeout 1s; + proxy_send_timeout 35s; + proxy_read_timeout 120s; + # A 503 is an API response (for example, S3 is unavailable), so + # preserve it for the client instead of marking every replica down. + proxy_next_upstream error timeout http_502 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 8s; + # Never transparently retry a POST already sent to an API. + # A client retries messages using the original Idempotency-Key. + } + } +} diff --git a/infra/readiness.sh b/infra/readiness.sh new file mode 100644 index 0000000..9a2e627 --- /dev/null +++ b/infra/readiness.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Only ready API addresses enter the upstream, including at nginx startup. +update_backends() { + next=/etc/nginx/ready-backends.conf.next + : > "$next" + for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do + if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then + echo "server $address:8080;" >> "$next" + fi + done + # If every running API temporarily loses a shared dependency (for example + # S3), retain the last addresses. The API can then return the contracted + # 503 instead of Nginx replacing it with a 502. At initial startup there + # is no previous healthy set, so use an inert placeholder. + if [ ! -s "$next" ]; then + if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then + rm -f "$next" + return + fi + echo 'server 127.0.0.1:9 down;' > "$next" + fi + if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then + mv "$next" /etc/nginx/ready-backends.conf + if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then + nginx -s reload + fi + else + rm -f "$next" + fi +} + +update_backends +( + while sleep 2; do + update_backends reload + done +) & +exec nginx -g 'daemon off;' diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100644 index 0000000..c0baf62 --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID" +mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null +mc mb --ignore-existing local/messenger-attachments +mc anonymous set none local/messenger-attachments +mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null +mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json +mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID" diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 0000000..8e77546 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,17 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/originals/*", + "arn:aws:s3:::messenger-attachments/staging/*", + "arn:aws:s3:::messenger-attachments/variants/*"] + } + ] +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..aee31d8 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE INDEX chats_order_idx ON chats (created_at, id); diff --git a/migrations/002_history_version.sql b/migrations/002_history_version.sql new file mode 100644 index 0000000..7b7972c --- /dev/null +++ b/migrations/002_history_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0; diff --git a/migrations/003_message_idempotency.sql b/migrations/003_message_idempotency.sql new file mode 100644 index 0000000..434b1e2 --- /dev/null +++ b/migrations/003_message_idempotency.sql @@ -0,0 +1,9 @@ +-- Additive migration: no rewrite of existing lab2 tables or messages. +CREATE TABLE message_idempotency ( + actor_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (actor_id, chat_id, key) +); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..77317cf --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + object_key text NOT NULL UNIQUE, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); + +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0); +ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10); diff --git a/migrations/005_attachment_jobs.sql b/migrations/005_attachment_jobs.sql new file mode 100644 index 0000000..1e5e88c --- /dev/null +++ b/migrations/005_attachment_jobs.sql @@ -0,0 +1,23 @@ +ALTER TABLE attachments + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + ADD COLUMN status text NOT NULL DEFAULT 'ready' + CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + ADD COLUMN staging_key text, + ADD COLUMN variants jsonb NOT NULL DEFAULT '[]', + ADD COLUMN error_code text; +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1), + attempt bigint NOT NULL DEFAULT 0, + failures bigint NOT NULL DEFAULT 0, + available_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + lease_until timestamptz, + lease_token uuid, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + replay_audit jsonb NOT NULL DEFAULT '[]' +); +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at); diff --git a/migrations/006_job_error.sql b/migrations/006_job_error.sql new file mode 100644 index 0000000..f721d2e --- /dev/null +++ b/migrations/006_job_error.sql @@ -0,0 +1 @@ +ALTER TABLE attachment_jobs ADD COLUMN last_error text; diff --git a/migrations/007_queue_measurements.sql b/migrations/007_queue_measurements.sql new file mode 100644 index 0000000..1c57b13 --- /dev/null +++ b/migrations/007_queue_measurements.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz; +CREATE INDEX attachments_pending_idx ON attachments(status) + WHERE status IN ('queued', 'processing'); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..86eff95 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +pydantic==2.13.5 +starlette==0.41.3 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +colorama==0.4.6; sys_platform == "win32" +h11==0.16.0 +idna==3.20 +pydantic_core==2.46.5 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +psycopg[binary]==3.2.6 +SQLAlchemy==2.0.41 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.40.70 +Pillow==12.0.0 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 0000000..d5a3407 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,13 @@ +import os + +import uvicorn + + +if __name__ == '__main__': + uvicorn.run( + 'src.main:app', + host=os.getenv('APP_HOST', '0.0.0.0'), + port=int(os.getenv('APP_PORT', '8080')), + workers=1, + timeout_graceful_shutdown=5, + ) diff --git a/src/admission.py b/src/admission.py new file mode 100644 index 0000000..6b76e99 --- /dev/null +++ b/src/admission.py @@ -0,0 +1,21 @@ +"""A shared admission limit for unfinished attachments across all API replicas.""" +import os + +from sqlalchemy import func, select + +from .errors import APIError +from .models import Attachment + +MAX_PENDING = int(os.getenv('MAX_PENDING_ATTACHMENTS', '64')) +if MAX_PENDING < 1: + raise ValueError('MAX_PENDING_ATTACHMENTS must be positive') + + +def reserve_capacity(session): + # Keep the lock until the new queued row commits. Counting without this lock + # would allow simultaneous uploads on different replicas to exceed the limit. + session.execute(select(func.pg_advisory_xact_lock(202605))) + count = session.scalar(select(func.count()).select_from(Attachment).where( + Attachment.status.in_(['queued', 'processing']))) + if count >= MAX_PENDING: + raise APIError(429, 'rate_limited', 'Очередь обработки заполнена; повторите загрузку позже') diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..10c6233 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,87 @@ +import os +import unicodedata +from uuid import uuid4 + +from PIL import Image, UnidentifiedImageError +from sqlalchemy import func, select + +from .errors import APIError +from .admission import reserve_capacity +from .models import Attachment, AttachmentJob, ChatMember +from .object_storage import BUCKET, PREFIX, s3_client +from .store import ensure_uuid, serialize, store + +MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760) +MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000) +if MAX_BYTES < 1 or MAX_PIXELS < 1: + raise ValueError('Upload limits must be positive') + + +def validate_filename(filename): + if (not filename or not filename.strip() or len(filename) > 255 + or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)): + raise APIError(400, 'invalid_request', 'Неверное имя файла') + + +def validate_image(source, content_type): + try: + source.seek(0) + with Image.open(source, formats=['PNG', 'JPEG']) as picture: + if picture.width * picture.height > MAX_PIXELS: + raise APIError(413, 'payload_too_large', 'Слишком много пикселей') + if Image.MIME[picture.format] != content_type: + raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type') + picture.verify() + source.seek(0) + with Image.open(source, formats=['PNG', 'JPEG']) as picture: + picture.load() # verify alone does not decode JPEG or detect truncated pixel data. + except Image.DecompressionBombError as exc: + raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc: + raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc + source.seek(0) + + +def lock_object(session, key): + # Shared by upload and cleanup: no deletion while a COMMIT is in flight. + session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0)))) + + +def metadata(attachment): + result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename', + 'content_type', 'size_bytes', 'sha256', 'created_at', + 'updated_at', 'status', 'error_code') + return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'} + for item in attachment.variants]} + + +def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest): + validate_image(source, content_type) + attachment_id = str(uuid4()) + key = PREFIX + attachment_id + staging_key = 'staging/' + attachment_id + with store.sessions.begin() as session: + store._chat(session, chat_id, actor_id) + reserve_capacity(session) + lock_object(session, staging_key) + s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source, + ContentLength=size, ContentType=content_type) + attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id, + object_key=key, filename=filename, content_type=content_type, + size_bytes=size, sha256=digest, staging_key=staging_key) + session.add(attachment) + session.flush() + session.add(AttachmentJob(attachment_id=attachment_id)) + result = metadata(attachment) + # An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later. + return result + + +def get_attachment(attachment_id, actor_id): + with store.sessions() as session: + attachment = session.scalar(select(Attachment).join( + ChatMember, ChatMember.chat_id == Attachment.chat_id).where( + Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id)) + if attachment is None: + raise APIError(404, 'not_found', 'Объект не найден') + return attachment diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 0000000..ab93a9f --- /dev/null +++ b/src/auth.py @@ -0,0 +1,68 @@ +import hashlib +import json +import os +import re +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from redis import Redis + +from .errors import APIError + +password_hasher = PasswordHasher() +dummy_hash = password_hasher.hash(secrets.token_urlsafe(32)) + + +def verify_password(encoded, password): + try: + return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None + except VerificationError: + return False + + +def unauthorized(): + return APIError(401, 'unauthorized', 'Нет действительной идентификации') + + +class Sessions: + def __init__(self): + self.client = Redis.from_url( + os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=1, socket_timeout=1, max_connections=20, + retry_on_timeout=False, + ) + self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600')) + if self.ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + + @staticmethod + def key(token): + if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise unauthorized() + return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest() + + def create(self, user): + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl) + expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z') + self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}), + pxat=int(expires.timestamp() * 1000)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user} + + def user_id(self, token): + raw = self.client.get(self.key(token)) + if raw is None: + raise unauthorized() + session = json.loads(raw) + if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc): + raise unauthorized() + return session['user_id'] + + def delete(self, token): + if not self.client.delete(self.key(token)): + raise unauthorized() + + +sessions = Sessions() diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 0000000..bde2077 --- /dev/null +++ b/src/cache.py @@ -0,0 +1,40 @@ +"""History pages are disposable; PostgreSQL owns both data and their version.""" +import hashlib +import json +import os + +from redis.exceptions import RedisError + +from .auth import sessions + + +class HistoryCache: + def __init__(self, client, ttl=30, prefix='history'): + self.client = client + self.ttl = ttl + self.prefix = prefix + if ttl < 1: + raise ValueError('CACHE_TTL_SECONDS must be positive') + + def key(self, chat_id, version, limit, after): + page = json.dumps([limit, after], separators=(',', ':')) + digest = hashlib.sha256(page.encode()).hexdigest() + return f'{self.prefix}:{chat_id}:{version}:{digest}' + + def get_or_load(self, chat_id, version, limit, after, load): + key = self.key(chat_id, version, limit, after) + try: + cached = self.client.get(key) + except RedisError: + return load() + if cached is not None: + return json.loads(cached) + page = load() + try: + self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl) + except RedisError: + pass + return page + + +history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4') diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 0000000..0d5eeb3 --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,86 @@ +"""Remove old S3 objects without committed metadata. Never delete on a DB error.""" +import argparse +import logging +import os +import time +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.orm import sessionmaker + +from .attachments import lock_object +from .database import create_db_engine +from .models import Attachment +from .object_storage import BUCKET, s3_client + +GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400')))) + + +def cleanup(sessions, client, now=None): + cutoff = (now or datetime.now(timezone.utc)) - GRACE + deleted = 0 + for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET): + for obj in page.get('Contents', []): + key = obj['Key'] + if not key.startswith(('staging/', 'originals/', 'variants/')): + continue + with sessions.begin() as session: + lock_object(session, key) + if key.startswith('staging/'): + attachment = session.scalar(select(Attachment).where( + Attachment.staging_key == key).with_for_update()) + if attachment: + if attachment.status in ('queued', 'processing'): + continue + if attachment.status == 'failed' and attachment.updated_at >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + attachment.staging_key = None + deleted += 1 + continue + else: + # All final keys belong to a server-generated attachment UUID. + from uuid import UUID + try: + attachment_id = str(UUID(key.split('/')[1])) + except (ValueError, IndexError): + continue + attachment = session.get(Attachment, attachment_id, with_for_update=True) + if attachment: + if attachment.status != 'failed' or attachment.updated_at >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + deleted += 1 + continue + # Recheck age after taking the same lock as the uploader. + if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff: + continue + client.delete_object(Bucket=BUCKET, Key=key) + deleted += 1 + return deleted + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--once', action='store_true') + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + try: + while True: + try: + count = cleanup(sessionmaker(engine), s3_client()) + logging.info('Cleanup completed: deleted=%d', count) + except Exception as exc: + logging.error('Cleanup deferred: %s', type(exc).__name__) + if args.once: + raise SystemExit(1) from None + if args.once: + break + time.sleep(60) + finally: + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/src/database.py b/src/database.py new file mode 100644 index 0000000..e98b4d4 --- /dev/null +++ b/src/database.py @@ -0,0 +1,51 @@ +import os +from pathlib import Path + +from sqlalchemy import create_engine, func, insert, select +from sqlalchemy.engine import URL, make_url + +from .models import SchemaMigration + + +def database_url(): + if os.getenv('DATABASE_URL'): + url = make_url(os.environ['DATABASE_URL']) + if url.drivername in ('postgres', 'postgresql'): + url = url.set(drivername='postgresql+psycopg') + if url.drivername != 'postgresql+psycopg': + raise ValueError('DATABASE_URL must use PostgreSQL with psycopg') + return url + return URL.create( + 'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'), + database=os.getenv('POSTGRES_DB', 'messenger'), + username=os.getenv('POSTGRES_USER', 'messenger'), + password=os.environ['POSTGRES_PASSWORD'], + ) + + +def create_db_engine(): + return create_engine( + database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True, + connect_args={'connect_timeout': 2, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}, + ) + + +def migrate(engine): + with engine.begin() as conn: + conn.execute(select(func.pg_advisory_xact_lock(202602))) + SchemaMigration.__table__.create(conn, checkfirst=True) + applied = set(conn.scalars(select(SchemaMigration.version))) + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if path.name not in applied: + conn.exec_driver_sql(path.read_text(encoding='utf-8')) + conn.execute(insert(SchemaMigration).values(version=path.name)) + + +if __name__ == '__main__': + engine = create_db_engine() + try: + migrate(engine) + print('Migrations applied') + finally: + engine.dispose() diff --git a/src/dependencies.py b/src/dependencies.py new file mode 100644 index 0000000..03aac92 --- /dev/null +++ b/src/dependencies.py @@ -0,0 +1,31 @@ +from typing import Optional + +from fastapi import Depends, Header, Query + +from .auth import sessions, unauthorized +from .errors import APIError +from .store import store + + +def page_limit(limit: str = Query(default="50")) -> int: + if not limit.isascii() or not limit.isdecimal() or len(limit) > 3: + raise APIError(400, "invalid_request", "Неверный limit") + number = int(limit) + if not 1 <= number <= 100: + raise APIError(400, "invalid_request", "Неверный limit") + return number + + + +def bearer_token(authorization: Optional[str] = Header(default=None)) -> str: + parts = (authorization or '').split() + if len(parts) != 2 or parts[0].lower() != 'bearer': + raise unauthorized() + return parts[1] + + +def current_user(token: str = Depends(bearer_token)) -> dict: + user = store.get_user(sessions.user_id(token)) + if user is None: + raise unauthorized() + return user diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..d02425b --- /dev/null +++ b/src/errors.py @@ -0,0 +1,60 @@ +from fastapi import Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException +from .middleware import INSTANCE_ID + + +class APIError(Exception): + def __init__(self, status_code: int, code: str, message: str): + self.status_code = status_code + self.code = code + self.message = message + + +def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=status_code, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + ) + + +def api_error_handler(request: Request, exc: APIError) -> JSONResponse: + response = error_response(request, exc.status_code, exc.code, exc.message) + if exc.status_code == 429: + response.headers['Retry-After'] = '1' + if exc.status_code == 401: + response.headers['WWW-Authenticate'] = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"' + ) + return response + + +def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse: + return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно') + + +def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Неверный запрос") + + +def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + if exc.status_code == 404: + return error_response(request, 404, "not_found", "Объект не найден") + if exc.status_code == 405: + return error_response(request, 400, "invalid_request", "Метод не поддерживается") + else: + return error_response(request, exc.status_code, "invalid_request", "Неверный запрос") + + +def internal_error_handler(request: Request, exc: Exception) -> JSONResponse: + return error_response(request, 500, "internal_error", "Внутренняя ошибка") diff --git a/src/image_processor.py b/src/image_processor.py new file mode 100644 index 0000000..019eb9a --- /dev/null +++ b/src/image_processor.py @@ -0,0 +1,116 @@ +"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC.""" +import hashlib +import io +import os +import sys + +from PIL import Image, ImageOps, UnidentifiedImageError +from sqlalchemy.orm import sessionmaker + +from .attachments import MAX_BYTES, MAX_PIXELS +from .database import create_db_engine +from .jobs import now +from .models import Attachment, AttachmentJob +from .object_storage import BUCKET, s3_client + + +def fit_size(size, bound): + width, height = size + if max(size) <= bound: + return size + return max(1, width * bound // max(size)), max(1, height * bound // max(size)) + + +def variants(data): + with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original: + if original.width * original.height > MAX_PIXELS: + raise ValueError('pixel limit') + original.load() + oriented = ImageOps.exif_transpose(original) + rgba = oriented.convert('RGBA') + rgb = Image.new('RGB', rgba.size, 'white') + rgb.paste(rgba, mask=rgba.getchannel('A')) + rgba.close() + oriented.close() + rgb.info.clear() + try: + for name, image, fmt, mime, quality in ( + ('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85), + ('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85), + ('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80), + ): + with image, io.BytesIO() as output: + image.save(output, format=fmt, quality=quality) + blob = output.getvalue() + yield {'name': name, 'content_type': mime, 'width': image.width, + 'height': image.height, 'size_bytes': len(blob), + 'sha256': hashlib.sha256(blob).hexdigest()}, blob + finally: + rgb.close() + + +def process(sessions, job_id, token): + with sessions() as session: + job = session.get(AttachmentJob, job_id) + if job.lease_token != token: + return + attachment = session.get(Attachment, job.attachment_id) + staging_key = attachment.staging_key + client = s3_client() + response = client.get_object(Bucket=BUCKET, Key=staging_key) + with response['Body'] as body: + data = body.read(MAX_BYTES + 1) + if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256: + raise ValueError('staging checksum mismatch') + try: + rendered = list(variants(data)) + except OSError as exc: + raise ValueError('invalid image') from exc + # Job lock fences the final writes, cleanup and publication against lease recovery. + # Work before this transaction is CPU-only. Keys are stable across retries. + with sessions.begin() as session: + job = session.get(AttachmentJob, job_id, with_for_update=True) + if job.lease_token != token or job.lease_until <= now(): + return + current = session.get(Attachment, job.attachment_id, with_for_update=True) + if current.status != 'processing': + return + client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data, + ContentType=current.content_type) + saved = [] + for meta, blob in rendered: + key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}' + client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type']) + saved.append({**meta, 'object_key': key}) + # Test-only crash injection; never enabled by compose.yaml. + if os.getenv('JOB_FAULT') == 'after_objects': + os._exit(91) + current.variants = saved + current.status, current.error_code = 'ready', None + current.updated_at = now() + job.lease_token = job.lease_until = None + if os.getenv('JOB_FAULT') == 'after_commit': + os._exit(92) + + +def main(): + # Linux worker: bounded address space as well as the container memory limit. + import resource + memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + engine = create_db_engine() + try: + process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2]) + except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError): + return 20 + except MemoryError: + return 21 + except Exception: + return 22 # No credentials or decoder/database internals in logs. + finally: + engine.dispose() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..a1e1b98 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,180 @@ +"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state.""" +import argparse +import json +import logging +import os +import random +import time +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from redis import Redis +from redis.exceptions import ResponseError +from sqlalchemy import and_, or_, select +from sqlalchemy.orm import sessionmaker + +from .database import create_db_engine +from .admission import reserve_capacity +from .models import Attachment, AttachmentJob +from .object_storage import BUCKET, s3_client + +STREAM = 'attachment-jobs' +GROUP = 'image-workers' +MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5'))) +TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))) +LEASE_SECONDS = TIMEOUT + 15 + + +def now(): + return datetime.now(timezone.utc) + + +def broker(): + return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'), + decode_responses=True, socket_connect_timeout=2, socket_timeout=5) + + +def ensure_group(client): + try: + client.xgroup_create(STREAM, GROUP, id='0', mkstream=True) + except ResponseError as exc: + if not str(exc).startswith('BUSYGROUP'): + raise + + +def envelope(job): + return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1', + 'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version, + 'attempt': job.attempt + 1, + 'created_at': job.created_at.isoformat().replace('+00:00', 'Z')} + + +def fail(session, job, attachment, code, permanent=False): + job.failures += 1 + job.last_error = code + job.lease_token = job.lease_until = job.published_at = None + attachment.updated_at = now() + if permanent or job.failures >= MAX_ATTEMPTS: + attachment.status = 'failed' + attachment.error_code = code if permanent else 'retry_exhausted' + else: + attachment.status = 'queued' + attachment.error_code = None + delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5) + job.available_at = now() + timedelta(seconds=delay) + logging.info('job=%s attempt=%s result=%s code=%s', job.id, job.attempt, attachment.status, code) + + +def dispatch_once(sessions, client): + # A crash after XADD but before COMMIT deliberately results in redelivery. + with sessions.begin() as session: + jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where( + or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)), + and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())), + AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at) + .with_for_update(skip_locked=True, of=AttachmentJob).limit(50))) + for job in jobs: + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment.status == 'processing': + if job.lease_until <= now(): + fail(session, job, attachment, 'lease_expired') + continue + if job.published_at is None: + client.xadd(STREAM, {'job': json.dumps(envelope(job))}) + job.published_at = now() + + +def claim(sessions, message): + if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1' + or message.get('pipeline_version') != 1): + raise ValueError('unsupported job envelope') + job_id = str(UUID(message['job_id'])) + with sessions.begin() as session: + job = session.get(AttachmentJob, job_id, with_for_update=True) + if job is None or job.attachment_id != message.get('attachment_id'): + raise ValueError('unknown job') + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment.status != 'queued' or job.available_at > now(): + return None + job.attempt += 1 + if job.first_started_at is None: + job.first_started_at = now() + job.lease_token = str(uuid4()) + job.lease_until = now() + timedelta(seconds=LEASE_SECONDS) + attachment.status = 'processing' + attachment.updated_at = now() + return job.id, job.lease_token + + +def record_failure(sessions, job_id, token, code, permanent=False): + with sessions.begin() as session: + job = session.get(AttachmentJob, job_id, with_for_update=True) + if job.lease_token == token: + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + fail(session, job, attachment, code, permanent) + + +def replay(sessions, job_id, reason): + if not reason.strip() or len(reason) > 200: + raise ValueError('Give a nonempty reason of at most 200 characters') + with sessions.begin() as session: + job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True) + if job is None: + raise ValueError('Job not found') + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment.status != 'failed': + raise ValueError('Replay is allowed only for failed jobs') + if not attachment.staging_key: + raise ValueError('Staging retention expired; upload the file again') + s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key) + reserve_capacity(session) + job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason, + 'after_attempt': job.attempt, + 'last_error': job.last_error}] + job.failures = 0 + job.available_at = now() + job.published_at = job.lease_until = job.lease_token = None + attachment.status, attachment.error_code = 'queued', None + attachment.updated_at = now() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + sub.add_parser('dispatch') + sub.add_parser('failed') + retry = sub.add_parser('replay') + retry.add_argument('job_id') + retry.add_argument('--reason', required=True) + args = parser.parse_args() + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + sessions = sessionmaker(engine, expire_on_commit=False) + try: + if args.command == 'failed': + with sessions() as session: + for job, att in session.execute(select(AttachmentJob, Attachment).join( + Attachment, Attachment.id == AttachmentJob.attachment_id) + .where(Attachment.status == 'failed')): + print(json.dumps({'job_id': job.id, 'attachment_id': att.id, + 'attempt': job.attempt, 'error_code': att.error_code, + 'last_error': job.last_error, + 'replay_available': bool(att.staging_key)})) + elif args.command == 'replay': + replay(sessions, args.job_id, args.reason) + print('Job queued for replay') + else: + client = broker() + while True: + try: + ensure_group(client) + dispatch_once(sessions, client) + except Exception as exc: + logging.warning('Dispatch deferred: %s', type(exc).__name__) + time.sleep(1) + finally: + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/src/limits.py b/src/limits.py new file mode 100644 index 0000000..facdf11 --- /dev/null +++ b/src/limits.py @@ -0,0 +1,99 @@ +import asyncio +import os +from dataclasses import dataclass + +from starlette.requests import Request + +from .errors import error_response +from .middleware import is_upload + + +@dataclass(frozen=True) +class Limits: + max_users: int = 1000 + max_chats: int = 1000 + max_messages: int = 10000 + max_text_bytes: int = 16 * 1024 * 1024 + max_request_bytes: int = 64 * 1024 + max_inflight: int = 32 + message_rate: int = 1000 + message_burst: int = 1000 + + @classmethod + def from_env(cls): + values = {} + for name, default in cls().__dict__.items(): + value = int(os.getenv(name.upper(), str(default))) + if value < 1: + raise ValueError(f'{name.upper()} must be positive') + values[name] = value + return cls(**values) + + +limits = Limits.from_env() + + +class ResourceLimitMiddleware: + def __init__(self, app, settings=limits): + self.app = app + self.settings = settings + self.active = 0 + + async def __call__(self, scope, receive, send): + if scope['type'] != 'http' or scope['path'].startswith('/health/'): + return await self.app(scope, receive, send) + + request = Request(scope) + + async def reject(status, code, message): + response = error_response(request, status, code, message) + if status == 503: + response.headers['Retry-After'] = '1' + await response(scope, receive, send) + + if self.active >= self.settings.max_inflight: + return await reject(503, 'unavailable', 'Слишком много одновременных запросов') + self.active += 1 + try: + # The upload endpoint authenticates and bounds its stream itself. + if is_upload(scope['method'], scope['path']): + return await self.app(scope, receive, send) + length = request.headers.get('content-length') + if length is not None: + try: + size = int(length) + if size < 0: + raise ValueError() + except ValueError: + return await reject(400, 'invalid_request', 'Неверный Content-Length') + if size > self.settings.max_request_bytes: + return await reject(413, 'payload_too_large', 'Тело запроса слишком большое') + + body = bytearray() + try: + async with asyncio.timeout(5): + while True: + message = await receive() + if message['type'] == 'http.disconnect': + return + chunk = message.get('body', b'') + if len(body) + len(chunk) > self.settings.max_request_bytes: + return await reject(413, 'payload_too_large', 'Тело запроса слишком большое') + body.extend(chunk) + if not message.get('more_body', False): + break + except TimeoutError: + return await reject(503, 'unavailable', 'Истекло время чтения запроса') + + delivered = False + + async def buffered_receive(): + nonlocal delivered + if not delivered: + delivered = True + return {'type': 'http.request', 'body': bytes(body), 'more_body': False} + return await receive() + + await self.app(scope, buffered_receive, send) + finally: + self.active -= 1 diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..fc96b18 --- /dev/null +++ b/src/main.py @@ -0,0 +1,48 @@ +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException +from contextlib import asynccontextmanager +from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout +from sqlalchemy.orm import sessionmaker +from redis.exceptions import RedisError +from botocore.exceptions import BotoCoreError, ClientError + +from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler +from .database import create_db_engine +from .store import store +from .auth import sessions +from .middleware import RequestIdMiddleware +from .limits import ResourceLimitMiddleware +from .routers.health import router as health_router +from .routers.users import router as users_router +from .routers.chats import router as chats_router +from .routers.auth import router as auth_router +from .routers.attachments import router as attachments_router + + +@asynccontextmanager +async def lifespan(app): + engine = create_db_engine() + try: + store.sessions = sessionmaker(engine, expire_on_commit=False) + yield + finally: + engine.dispose() + sessions.client.close() + + +app = FastAPI(title="MTUSI Messenger", lifespan=lifespan) +app.add_middleware(ResourceLimitMiddleware) +app.add_middleware(RequestIdMiddleware) +app.add_exception_handler(APIError, api_error_handler) +app.add_exception_handler(RequestValidationError, validation_error_handler) +app.add_exception_handler(HTTPException, http_error_handler) +app.add_exception_handler(Exception, internal_error_handler) +for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError): + app.add_exception_handler(error, dependency_error_handler) + +app.include_router(health_router) +app.include_router(users_router) +app.include_router(chats_router) +app.include_router(auth_router) +app.include_router(attachments_router) diff --git a/src/middleware.py b/src/middleware.py new file mode 100644 index 0000000..c2b73a1 --- /dev/null +++ b/src/middleware.py @@ -0,0 +1,43 @@ +from uuid import uuid4 +from socket import gethostname +from starlette.responses import JSONResponse +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +import re + + +def is_upload(method, path): + return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None + +INSTANCE_ID = gethostname() + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4()) + request.state.request_id = request_id + + if (request.method in {"POST", "PUT", "PATCH"} + and request.url.path.startswith("/api/") + and not is_upload(request.method, request.url.path) + and request.url.path != '/api/v1/auth/sessions'): + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + response = JSONResponse( + status_code=415, + content={ + "error": { + "code": "unsupported_media_type", + "message": "Ожидается Content-Type application/json", + "request_id": request_id, + } + }, + ) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response diff --git a/src/models.py b/src/models.py new file mode 100644 index 0000000..e425570 --- /dev/null +++ b/src/models.py @@ -0,0 +1,108 @@ +from datetime import datetime +from uuid import uuid4 + +from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from sqlalchemy.dialects.postgresql import ARRAY, JSONB + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = 'users' + __table_args__ = ( + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + username: Mapped[str] = mapped_column(Text, unique=True) + display_name: Mapped[str] = mapped_column(Text) + password_hash: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class Chat(Base): + __tablename__ = 'chats' + __table_args__ = ( + CheckConstraint('char_length(title) BETWEEN 1 AND 100'), + Index('chats_order_idx', 'created_at', 'id'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + title: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + history_version: Mapped[int] = mapped_column(BigInteger, server_default='0') + + +class ChatMember(Base): + __tablename__ = 'chat_members' + __table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True) + + +class Message(Base): + __tablename__ = 'messages' + __table_args__ = ( + CheckConstraint('char_length(text) <= 4000'), + CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"), + Index('messages_history_idx', 'chat_id', 'created_at', 'id'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) + sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) + text: Mapped[str] = mapped_column(Text) + attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}') + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class SchemaMigration(Base): + __tablename__ = 'schema_migrations' + version: Mapped[str] = mapped_column(Text, primary_key=True) + applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True) + + +class Attachment(Base): + __tablename__ = 'attachments' + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) + owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) + object_key: Mapped[str] = mapped_column(Text, unique=True) + filename: Mapped[str] = mapped_column(Text) + content_type: Mapped[str] = mapped_column(Text) + size_bytes: Mapped[int] = mapped_column(BigInteger) + sha256: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + status: Mapped[str] = mapped_column(Text, server_default='queued') + staging_key: Mapped[str | None] = mapped_column(Text) + variants: Mapped[list] = mapped_column(JSONB, server_default='[]') + error_code: Mapped[str | None] = mapped_column(Text) + + +class AttachmentJob(Base): + """The job row is also the transactional outbox and failed-job registry.""" + __tablename__ = 'attachment_jobs' + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True) + pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1') + attempt: Mapped[int] = mapped_column(BigInteger, server_default='0') + failures: Mapped[int] = mapped_column(BigInteger, server_default='0') + last_error: Mapped[str | None] = mapped_column(Text) + first_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]') + + +class IdempotencyRecord(Base): + __tablename__ = 'message_idempotency' + actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + key: Mapped[str] = mapped_column(Text, primary_key=True) + message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE')) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) diff --git a/src/object_storage.py b/src/object_storage.py new file mode 100644 index 0000000..d3cb252 --- /dev/null +++ b/src/object_storage.py @@ -0,0 +1,23 @@ +import os +from functools import lru_cache + +import boto3 +from botocore.config import Config + +BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') +PREFIX = 'originals/' + + +@lru_cache(maxsize=1) +def s3_client(): + return boto3.client( + 's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), + region_name=os.getenv('S3_REGION', 'us-east-1'), + aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'], + aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'], + config=Config(connect_timeout=2, read_timeout=5, + retries={'total_max_attempts': 1}, + s3={'addressing_style': 'path'}, + request_checksum_calculation='when_required', + response_checksum_validation='when_required'), + ) diff --git a/src/routers/__init__.py b/src/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/routers/attachments.py b/src/routers/attachments.py new file mode 100644 index 0000000..f5eb354 --- /dev/null +++ b/src/routers/attachments.py @@ -0,0 +1,90 @@ +import asyncio +import hashlib +from tempfile import TemporaryFile +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Query, Request +from starlette.background import BackgroundTask +from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect +from starlette.responses import StreamingResponse + +from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename +from ..dependencies import current_user +from ..errors import APIError +from ..object_storage import BUCKET, s3_client +from ..store import ensure_uuid, store + +router = APIRouter(prefix='/api/v1', tags=['attachments']) +# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk. +decode_slot = asyncio.Semaphore(1) + + +@router.post('/chats/{chat_id}/attachments', status_code=202) +async def upload(chat_id: str, request: Request, filename: str = Query(...), + user: dict = Depends(current_user)): + chat_id = ensure_uuid(chat_id) + await run_in_threadpool(store.get_chat_for_user, chat_id, user['id']) + validate_filename(filename) + content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower() + if content_type not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG') + length = request.headers.get('content-length') + if length is not None: + try: + length = int(length) + if length < 0: + raise ValueError() + except ValueError as exc: + raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Файл слишком большой') + size = 0 + digest = hashlib.sha256() + with TemporaryFile() as source: + try: + async with asyncio.timeout(30): + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Файл слишком большой') + digest.update(chunk) + await run_in_threadpool(source.write, chunk) + except (TimeoutError, ClientDisconnect) as exc: + raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc + if length is not None and size != length: + raise APIError(400, 'invalid_request', 'Неполное тело запроса') + async with decode_slot: + return await run_in_threadpool(save_attachment, source, chat_id, user['id'], + filename, content_type, size, digest.hexdigest()) + + +@router.get('/attachments/{attachment_id}') +def attachment_info(attachment_id: str, user: dict = Depends(current_user)): + return metadata(get_attachment(attachment_id, user['id'])) + + +@router.get('/attachments/{attachment_id}/content') +def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)): + attachment = get_attachment(attachment_id, user['id']) + if attachment.status != 'ready': + raise APIError(409, 'conflict', 'Вложение ещё не готово') + key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes + if variant != 'original': + item = next((v for v in attachment.variants if v['name'] == variant), None) + if item is None: + raise APIError(404, 'not_found', 'Вариант не найден') + key, content_type, size = item['object_key'], item['content_type'], item['size_bytes'] + body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body'] + + def chunks(): + try: + yield from body.iter_chunks(chunk_size=64 * 1024) + finally: + body.close() + + return StreamingResponse(chunks(), media_type=content_type, + headers={'Content-Length': str(size), + 'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''), + 'X-Content-Type-Options': 'nosniff'}, + background=BackgroundTask(body.close)) diff --git a/src/routers/auth.py b/src/routers/auth.py new file mode 100644 index 0000000..bb4beaa --- /dev/null +++ b/src/routers/auth.py @@ -0,0 +1,34 @@ +import base64 + +from fastapi import APIRouter, Depends, Header, Request, Response +from starlette.concurrency import run_in_threadpool + +from ..auth import sessions, unauthorized +from ..dependencies import bearer_token, current_user +from ..errors import APIError +from ..store import store + +router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth']) + + +@router.post('', status_code=201) +async def create_session(request: Request, authorization: str = Header(default='')): + if await request.body(): + raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса') + try: + scheme, encoded = authorization.split() + if scheme.lower() != 'basic': + raise ValueError() + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username: + raise ValueError() + except (ValueError, UnicodeError): + raise unauthorized() from None + user = await run_in_threadpool(store.login, username, password) + return await run_in_threadpool(sessions.create, user) + + +@router.delete('/current', status_code=204) +def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)): + sessions.delete(token) + return Response(status_code=204) diff --git a/src/routers/chats.py b/src/routers/chats.py new file mode 100644 index 0000000..58aa1c7 --- /dev/null +++ b/src/routers/chats.py @@ -0,0 +1,49 @@ +from typing import Optional +import re + +from fastapi import APIRouter, Depends, Header, Query, status +from ..errors import APIError + +from ..dependencies import current_user, page_limit +from ..schemas.chat import Chat, ChatPage, CreateChat +from ..schemas.message import CreateMessage, Message, MessagePage +from ..store import store + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED) +def create_chat(payload: CreateChat, user: dict = Depends(current_user)): + return store.create_chat(user["id"], payload.title, payload.member_ids) + + +@router.get("", response_model=ChatPage) +def list_chats( + limit: int = Depends(page_limit), + cursor: Optional[str] = Query(default=None, min_length=1), + user: dict = Depends(current_user), +): + return store.list_chats(user["id"], limit, cursor) + + +@router.get("/{chat_id}", response_model=Chat) +def get_chat(chat_id: str, user: dict = Depends(current_user)): + return store.get_chat_for_user(chat_id, user["id"]) + + +@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED) +def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user), + idempotency_key: Optional[str] = Header(default=None)): + if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key): + raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key') + return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids) + + +@router.get("/{chat_id}/messages", response_model=MessagePage) +def list_messages( + chat_id: str, + limit: int = Depends(page_limit), + cursor: Optional[str] = Query(default=None, min_length=1), + user: dict = Depends(current_user), +): + return store.list_messages(chat_id, user["id"], limit, cursor) diff --git a/src/routers/health.py b/src/routers/health.py new file mode 100644 index 0000000..f2015d3 --- /dev/null +++ b/src/routers/health.py @@ -0,0 +1,23 @@ +from fastapi import APIRouter +from sqlalchemy import select + +from ..schemas.common import Health +from ..auth import sessions +from ..store import store +from ..object_storage import BUCKET, s3_client + +router = APIRouter(prefix='/health') + + +@router.get("/live", response_model=Health) +def health_live(): + return {"status": "ok"} + + +@router.get("/ready", response_model=Health) +def health_ready(): + with store.sessions() as session: + session.execute(select(1)) + sessions.client.ping() + s3_client().head_bucket(Bucket=BUCKET) + return {"status": "ok"} diff --git a/src/routers/users.py b/src/routers/users.py new file mode 100644 index 0000000..2c8b69c --- /dev/null +++ b/src/routers/users.py @@ -0,0 +1,19 @@ +from fastapi import APIRouter, Depends, status + +from ..dependencies import current_user +from ..schemas.user import CreateUser, User +from ..store import store + + + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + + +@router.post("", response_model=User, status_code=status.HTTP_201_CREATED) +def create_user(payload: CreateUser): + return store.create_user(payload.username, payload.display_name, payload.password) + + +@router.get("/me", response_model=User) +def get_current_user(user: dict = Depends(current_user)): + return user diff --git a/src/schemas/__init__.py b/src/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/schemas/chat.py b/src/schemas/chat.py new file mode 100644 index 0000000..b1a3629 --- /dev/null +++ b/src/schemas/chat.py @@ -0,0 +1,35 @@ +from typing import List, Optional + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +class CreateChat(ContractModel): + title: StrictStr = Field(min_length=1, max_length=100) + member_ids: List[StrictStr] = Field(min_length=1, max_length=100) + + @field_validator("title") + def validate_title(cls, value): + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + def validate_member_ids(cls, value): + if len(value) != len(set(value)): + raise ValueError("duplicate member_ids") + return value + + +class Chat(ContractModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class ChatPage(ContractModel): + items: List[Chat] + next_cursor: Optional[str] diff --git a/src/schemas/common.py b/src/schemas/common.py new file mode 100644 index 0000000..06a7bc2 --- /dev/null +++ b/src/schemas/common.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel, ConfigDict, field_validator + + +class ContractModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator('*') + def validate_text_encoding(cls, value, info): + if isinstance(value, str): + value.encode('utf-8') + if info.field_name != 'password' and '\x00' in value: + raise ValueError('NUL is not supported in PostgreSQL text') + return value + + +class Health(ContractModel): + status: str diff --git a/src/schemas/message.py b/src/schemas/message.py new file mode 100644 index 0000000..75268c5 --- /dev/null +++ b/src/schemas/message.py @@ -0,0 +1,39 @@ +from typing import List, Optional + +from pydantic import Field, StrictStr, field_validator, model_validator +from uuid import UUID + +from .common import ContractModel + + + +class CreateMessage(ContractModel): + text: StrictStr = Field(max_length=4000) + attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10) + + @field_validator('attachment_ids') + def validate_ids(cls, value): + ids = [str(UUID(item)) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('duplicate attachments') + return ids + + @model_validator(mode='after') + def validate_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError('empty message') + return self + + +class Message(ContractModel): + id: str + chat_id: str + sender_id: str + text: str + attachment_ids: List[str] + created_at: str + + +class MessagePage(ContractModel): + items: List[Message] + next_cursor: Optional[str] diff --git a/src/schemas/user.py b/src/schemas/user.py new file mode 100644 index 0000000..bd18d5b --- /dev/null +++ b/src/schemas/user.py @@ -0,0 +1,34 @@ +import re + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class CreateUser(ContractModel): + username: StrictStr + display_name: StrictStr = Field(min_length=1, max_length=100) + password: StrictStr = Field(min_length=12, max_length=128) + + @field_validator("username") + def validate_username(cls, value): + if not USERNAME_RE.fullmatch(value): + raise ValueError("invalid username") + return value + + @field_validator("display_name") + def validate_display_name(cls, value): + if not value.strip(): + raise ValueError("blank display_name") + return value + + +class User(ContractModel): + id: str + username: str + display_name: str = Field(min_length=1, max_length=100) + created_at: str diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..b273f4c --- /dev/null +++ b/src/store.py @@ -0,0 +1,229 @@ +import base64 +import json +from datetime import datetime, timezone +from typing import Tuple +from uuid import UUID + +from sqlalchemy import select, tuple_, update +from sqlalchemy.exc import IntegrityError + +from .auth import password_hasher, verify_password, unauthorized +from .errors import APIError +from .cache import history_cache +from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment + + +def ensure_uuid(value): + try: + if not isinstance(value, str): + raise ValueError() + return str(UUID(value)) + except (TypeError, ValueError) as exc: + raise APIError(400, 'invalid_request', 'Неверный UUID') from exc + + +def serialize(model, *fields): + if model is None: + return None + result = {} + for field in fields: + value = getattr(model, field) + if isinstance(value, datetime): + value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + result[field] = value + return result + + +USER_FIELDS = ('id', 'username', 'display_name', 'created_at') +CHAT_FIELDS = ('id', 'title', 'created_at') +MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids') + + +def after_condition(model, after): + return tuple_(model.created_at, model.id) > ( + datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1]) + + +def history_query(chat_id, limit, after): + query = select(Message).where(Message.chat_id == chat_id) + if after: + query = query.where(after_condition(Message, after)) + return query.order_by(Message.created_at, Message.id).limit(limit + 1) + + +class PostgresStore: + def __init__(self, cache=history_cache): + self.sessions = None + self.cache = cache + + def create_user(self, username, display_name, password): + user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password)) + try: + with self.sessions.begin() as session: + session.add(user) + session.flush() + result = serialize(user, *USER_FIELDS) + return result + except IntegrityError as exc: + if getattr(exc.orig, 'sqlstate', None) == '23505': + raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc + raise + + def get_user(self, user_id): + with self.sessions() as session: + return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS) + + def login(self, username, password): + with self.sessions() as session: + user = session.scalar(select(User).where(User.username == username)) + encoded = user.password_hash if user else None + result = serialize(user, *USER_FIELDS) + if not verify_password(encoded, password): + raise unauthorized() + return result + + def create_chat(self, actor_id, title, member_ids): + members = [ensure_uuid(member) for member in member_ids] + if len(members) != len(set(members)) or actor_id not in members: + raise APIError(400, 'invalid_request', 'Неверный список участников') + try: + with self.sessions.begin() as session: + chat = Chat(title=title) + session.add(chat) + session.flush() + session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members]) + result = serialize(chat, *CHAT_FIELDS) + result['member_ids'] = sorted(members) + return result + except IntegrityError as exc: + if getattr(exc.orig, 'sqlstate', None) == '23503': + raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc + raise + + @staticmethod + def _chat(session, chat_id, actor_id): + chat = session.scalar(select(Chat).join(ChatMember).where( + Chat.id == chat_id, ChatMember.user_id == actor_id)) + if chat is None: + raise APIError(404, 'not_found', 'Объект не найден') + return chat + + @staticmethod + def _members(session, chat): + result = serialize(chat, *CHAT_FIELDS) + result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where( + ChatMember.chat_id == chat.id).order_by(ChatMember.user_id))) + return result + + def get_chat_for_user(self, chat_id, actor_id): + with self.sessions() as session: + return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id)) + + def list_chats(self, actor_id, limit, cursor): + after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None + query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id) + if after: + query = query.where(after_condition(Chat, after)) + query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1) + with self.sessions() as session: + chats = list(session.scalars(query)) + items = [self._members(session, chat) for chat in chats[:limit]] + return self._page(items, len(chats) > limit, 'chats', actor_id) + + def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None): + chat_id = ensure_uuid(chat_id) + attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])] + with self.sessions.begin() as session: + self._chat(session, chat_id, actor_id) + if attachment_ids: + found = list(session.scalars(select(Attachment).where( + Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id) + .order_by(Attachment.id).with_for_update())) + if len(found) != len(attachment_ids): + raise APIError(404, 'not_found', 'Объект не найден') + if idempotency_key is not None: + # Serialize writers to this chat before looking up the key. + # The same row is updated below for history cache invalidation. + session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update()) + record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key)) + if record is not None: + previous = session.get(Message, record.message_id) + if previous.text != text or previous.attachment_ids != attachment_ids: + raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения') + return serialize(previous, *MESSAGE_FIELDS) + if attachment_ids and any(item.status == 'failed' for item in found): + raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой') + session.execute(update(Chat).where(Chat.id == chat_id).values( + history_version=Chat.history_version + 1)) + message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids) + session.add(message) + session.flush() + if idempotency_key is not None: + session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id, + key=idempotency_key, message_id=message.id)) + result = serialize(message, *MESSAGE_FIELDS) + return result + + def list_messages(self, chat_id, actor_id, limit, cursor): + chat_id = ensure_uuid(chat_id) + after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None + with self.sessions() as session: + version = self._chat(session, chat_id, actor_id).history_version + return self.cache.get_or_load( + chat_id, version, limit, after, + lambda: self._load_messages(chat_id, limit, after), + ) + + def _load_messages(self, chat_id, limit, after): + with self.sessions() as session: + messages = list(session.scalars(history_query(chat_id, limit, after))) + items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]] + return self._page(items, len(messages) > limit, 'messages', chat_id) + + def _page(self, items, has_more, resource, owner_id): + cursor = None + if has_more: + last = items[-1] + cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id']) + return {'items': items, 'next_cursor': cursor} + + @staticmethod + def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str: + payload = { + "resource": resource, + "owner_id": owner_id, + "created_at": created_at, + "id": item_id, + } + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + @staticmethod + def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8")) + except (ValueError, UnicodeError, json.JSONDecodeError) as exc: + raise APIError(400, "invalid_request", "Неверный cursor") from exc + + if not isinstance(payload, dict): + raise APIError(400, "invalid_request", "Неверный cursor") + + if payload.get("resource") != resource or payload.get("owner_id") != owner_id: + raise APIError(400, "invalid_request", "Неверный cursor") + + item_id = ensure_uuid(payload.get("id")) + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise APIError(400, "invalid_request", "Неверный cursor") + try: + parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc: + raise ValueError("Expected UTC timestamp") + created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z") + except ValueError as exc: + raise APIError(400, "invalid_request", "Неверный cursor") from exc + return created_at, item_id + + +store = PostgresStore() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..0708885 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,76 @@ +"""Bounded consumers, one unacknowledged message per consumer, hard process timeout.""" +import json +import logging +import os +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +from sqlalchemy.orm import sessionmaker + +from .database import create_db_engine +from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure + + +def handle(sessions, client, message_id, fields): + try: + claimed = claim(sessions, json.loads(fields['job'])) + except (ValueError, KeyError, TypeError): + logging.error('Invalid broker envelope: id=%s', message_id) + claimed = None + if claimed: + job_id, token = claimed + try: + result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token], + timeout=TIMEOUT, check=False) + code = result.returncode + except subprocess.TimeoutExpired: + code = 23 + if code in (91, 92): + # Simulate a lost worker before ack: leave the Redis pending entry and lease. + return + if code: + record_failure(sessions, job_id, token, + {20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'} + .get(code, 'processing_failed'), permanent=code in (20, 21)) + # Only after a durable ready/failed/retry (or an existing durable lease). + with client.pipeline(transaction=True) as pipe: + pipe.xack(STREAM, GROUP, message_id) + pipe.xdel(STREAM, message_id) + pipe.execute() + + +def consume(sessions): + client = broker() + consumer = str(uuid4()) + cursor = '0-0' + while True: + try: + ensure_group(client) + pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000, + start_id=cursor, count=1) + cursor, messages = pending[0], pending[1] + if not messages: + batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000) + messages = batches[0][1] if batches else [] + for message_id, fields in messages: + handle(sessions, client, message_id, fields) + except Exception as exc: + logging.warning('Worker deferred: %s', type(exc).__name__) + time.sleep(1) + + +def main(): + logging.basicConfig(level=logging.INFO) + engine = create_db_engine() + sessions = sessionmaker(engine, expire_on_commit=False) + concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2')))) + with ThreadPoolExecutor(max_workers=concurrency) as pool: + for task in [pool.submit(consume, sessions) for _ in range(concurrency)]: + task.result() + + +if __name__ == '__main__': + main() -- 2.54.0 From 442595b37c6791cc0e86bc10ef7c443806fb0330 Mon Sep 17 00:00:00 2001 From: da1zy3r Date: Mon, 16 Nov 2026 16:08:15 +0300 Subject: [PATCH 2/2] Grade 4 --- .env.example | 9 +- .github/pull_request_template.md | 42 +- REPORT.md | 63 +- compose.yaml | 87 +- evidence/load.json | 17 + evidence/observability.json | 1001 +++++++++++++++++ evidence/verification.txt | 5 + infra/grafana/dashboards/messenger.json | 716 ++++++++++++ .../provisioning/dashboards/default.yaml | 7 + .../provisioning/datasources/sources.yaml | 41 + infra/otel/README.md | 141 ++- infra/otel/alerts.yaml | 28 + infra/otel/collector.yaml | 69 ++ infra/otel/loki.yaml | 37 + infra/otel/prometheus.yaml | 12 + infra/otel/tempo.yaml | 23 + migrations/008_trace_context.sql | 1 + requirements.txt | 4 + src/__main__.py | 1 + src/attachments.py | 3 +- src/database.py | 59 +- src/image_processor.py | 8 +- src/jobs.py | 27 +- src/main.py | 7 + src/models.py | 1 + src/object_storage.py | 12 +- src/telemetry.py | 206 ++++ src/worker.py | 76 +- student-tests/observability.py | 290 +++++ 29 files changed, 2930 insertions(+), 63 deletions(-) create mode 100644 evidence/load.json create mode 100644 evidence/observability.json create mode 100644 evidence/verification.txt create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/default.yaml create mode 100644 infra/grafana/provisioning/datasources/sources.yaml create mode 100644 infra/otel/alerts.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/otel/loki.yaml create mode 100644 infra/otel/prometheus.yaml create mode 100644 infra/otel/tempo.yaml create mode 100644 migrations/008_trace_context.sql create mode 100644 src/telemetry.py create mode 100644 student-tests/observability.py diff --git a/.env.example b/.env.example index 8cd24a0..30eb7da 100644 --- a/.env.example +++ b/.env.example @@ -23,9 +23,6 @@ MINIO_ROOT_PASSWORD=replace-with-admin-password S3_ACCESS_KEY_ID=messenger-api S3_SECRET_ACCESS_KEY=replace-with-api-password -# Reserved for lab6 observability setup. -OTEL_SERVICE_NAME=messenger-api -# Worker must set a distinct service.name. -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 -OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf -OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab +# Compose sets service.name separately for API, dispatcher and worker. +OTEL_SDK_DISABLED=false +GRAFANA_ADMIN_PASSWORD=replace-with-grafana-password diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 59c9f4e..41e8ce3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,32 @@ ## Лабораторная 6 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Ромашко Владимир, БВТ2301. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Целевая оценка (3/4/5): **4**. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Что реализовано и что перенесено: приложение lab5 с API-репликами, БД, S3, брокером и worker сохранено. Добавлены OTLP metrics/logs/traces, Collector, Prometheus/Loki/Tempo, Grafana provisioning, сквозной trace context с retry, SLO и два алерта. + +Команды запуска из чистого clone: + +```sh +cp .env.example .env +# Задать локальные пароли в .env. +docker compose up -d --build +curl http://127.0.0.1:8080/health/ready # дождаться 200 +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 student-tests/observability.py --scenario all +``` + +Результаты проверок: check, публичный smoke и проверка сигналов — PASS; сценарии worker/БД/S3/Tempo пройдены, алерты firing → resolved. Нагрузка: 2389 запросов за 30s, без ошибок. [REPORT.md](../REPORT.md), [вывод](../evidence/verification.txt), [сценарии](../evidence/observability.json), [нагрузка](../evidence/load.json). + +Ограничения / известные ошибки: один узел, без HA/TLS и внешней доставки алертов; при длительном отказе backend допустима потеря телеметрии. Блокирующих ошибок в проверенных сценариях не выявлено. + +Источники и использование ИИ: контракт лабораторной, документация OpenTelemetry и Loki; Codex — реализация, тестирование и оформление. Ссылки в REPORT.md. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены эквивалентными Python-командами; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index b97ae42..ff92ee6 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,72 @@ # Отчёт — лабораторная 6 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Ромашко Владимир, БВТ2301. +- Целевая оценка: **4**. +- Источники: [контракт](contracts/observability.md), [OpenTelemetry](https://opentelemetry.io/docs/collector/configuration/), [Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/). Библиотеки: FastAPI, SQLAlchemy, Redis, boto3, Pillow, OpenTelemetry SDK, psutil. Codex использовался для реализации, проверок и оформления. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker с Compose и Python 3.9+ на хосте; свободные порты 8080, 3000, 9090, 3100, 3200. В контейнерах Python 3.12.8; PostgreSQL 17.4, Redis 7.4.2, Nginx 1.28.0; MinIO закреплён digest. Телеметрия: SDK 1.38.0 / semantic conventions 0.59b0, Collector 0.136.0, Prometheus 3.6.0, Loki 3.5.7, Tempo 2.9.0, Grafana 12.2.0. + +```sh +cp .env.example .env +# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD, +# S3_SECRET_ACCESS_KEY и GRAFANA_ADMIN_PASSWORD. +docker compose config --quiet +docker compose up -d --build +curl http://127.0.0.1:8080/health/ready # дождаться 200 {"status":"ok"} +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 student-tests/observability.py --scenario all +python -X utf8 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +docker compose down # безопасная остановка, volumes остаются +``` + +Миграции (включая 008 с trace context) и создание S3 bucket выполняются автоматически. Grafana: http://localhost:3000, `admin`, пароль из `.env`; dashboard **Lab 6 / Messenger / Lab 6** импортируется автоматически. Python-команды check/smoke эквивалентны `make check` / `make test`; на проверявшемся Windows-хосте make отсутствует. Для **намеренного удаления всех учебных данных**: `docker compose down -v` (при проверках не выполнялось). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab5 сохранены Nginx → две API-реплики, PostgreSQL, Redis для сессий/кеша, S3, durable Redis Streams, outbox/dispatcher и worker. PostgreSQL — источник истины для заданий; attachment и outbox фиксируются одной транзакцией после записи staging, ack выполняется после устойчивого результата. + +Добавлены API/worker/dispatcher → OTLP → Collector → Prometheus (метрики), Loki (логи), Tempo (трассы) → Grafana. История хранится в volumes, retention 72 часа. W3C context сохраняется в outbox и переносится через broker в worker и image process; retry остаётся в той же трассе. Экспортируются шаблоны маршрутов и технические поля, без SQL-параметров, токенов и содержимого сообщений. + +Метрики: `http_requests_total` и `jobs_processed_total` — счётчики; `http_duration_seconds`, `job_duration_seconds`, `attachment_time_to_ready_seconds` — histogram в секундах; `queue_depth`, `db_pool_used/waiting`, `worker_busy/capacity` — количества; `queue_oldest_age_seconds` — секунды; `container_memory_bytes` / `container_cpu_seconds_total` — память/CPU. Точные имена, labels и запросы: [описание сигналов](infra/otel/README.md#сигналы), [dashboard JSON](infra/grafana/dashboards/messenger.json). + +Переменные `.env.example` подставляются в Compose: пароли — соответствующим сервисам, `MAX_*`, `JOB_*`, `WORKER_CONCURRENCY` — приложению, `APP_PORT` — Nginx. `OTEL_SDK_DISABLED=false` включает телеметрию; service.name и внутренние адреса заданы отдельно по ролям. + +SLI = 1 − 5xx / все запросы без health за **30 минут**, SLO ≥ **99%**. `ApiHighErrorRate`: >5% ошибок за 2m, минимум 20 запросов, `for: 20s`. `QueueLagHigh`: возраст queued >30s при свежем sample, `for: 15s`. Правила и [инструкции диагностики](infra/otel/README.md#queue-lag) сохранены в репозитории. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Структура и публичный smoke | Команды check/smoke выше | PASS при включённой телеметрии | [verification.txt](evidence/verification.txt) | +| Минимум lab5 и общий стенд | `docker compose up -d --build`; smoke | API, upload 202, ready, оригинал и thumbnail работают; хранилища имеют volumes | [compose.yaml](compose.yaml), verification.txt | +| OTLP: метрики, логи, трассы API/worker | `observability.py --scenario smoke` | Все обязательные метрики найдены; связанные логи трёх сервисов | [observability.json: smoke](evidence/observability.json) | +| Импортируемый dashboard и безопасные атрибуты | Provisioning; сценарий smoke | 18 панелей, 3 datasource; проверены labels и отсутствие приватных входных данных | [provisioning](infra/grafana/provisioning), observability.json | +| Корреляция HTTP/job → log/trace | Сценарий smoke | Найдены request_id, trace_id и parent-связи до S3/DB | observability.json: smoke | +| W3C propagation, включая retry | `observability.py --scenario retry` | Попытки 1 и 2 в одной трассе, tracestate сохранён | observability.json: retry | +| p50/p95/p99, очередь, DB pool, worker, CPU/RAM, time-to-ready | Сценарий smoke; dashboard | Метрики существуют; percentile вычисляется по сумме histogram buckets | dashboard JSON, observability.json | +| SLI/SLO и два проверенных алерта | `--scenario queue`, `--scenario database` | Оба алерта firing → resolved после восстановления | [alerts.yaml](infra/otel/alerts.yaml), observability.json | +| Ограниченная буферизация и отказ telemetry backend | `--scenario backend` | API продолжает работу; очередь ≤32 batches, RSS ≤199.03 MiB | [collector.yaml](infra/otel/collector.yaml), observability.json | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Здесь `observability.py` — `python -X utf8 student-tests/observability.py`. Сценарии останавливают реальные контейнеры текущего Compose-проекта и восстанавливают их в `finally`; volumes не удаляют. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверки 16.11.2026. Гипотеза: телеметрия показывает причину отказа, а её backend не блокирует бизнес-операции. Docker Desktop: 8 CPU, 7.76 GiB RAM; лимиты API — по 1 CPU/512 MiB, worker — 2 CPU/2 GiB, Collector — 0.5 CPU/256 MiB. + +Нагрузка: один прогон closed-loop POST сообщений, 4 клиента, 30 секунд, без отдельного прогрева и автоматических retry. **2389 успешных запросов, 0 ошибок, 79.219 RPS; p50/p95/p99 = 40.784/99.520/129.314 ms**. Исходная сводка: [load.json](evidence/load.json). Это локальная проверка, не исследование предельной производительности; CPU во время прогона отдельно не фиксировался, RSS Collector измерен в сценарии отказа ниже. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановить worker | Рост возраста очереди и QueueLagHigh | Depth=1, возраст 55s; producer есть, consumer появляется после запуска; затем ready, depth=0 | firing → resolved: 20.3s; принятое вложение сохранено | +| Остановить PostgreSQL | 503 и ApiHighErrorRate | Error ratio 42.6%; в trace `db.acquire`/`OperationalError` и связанный лог — отказ БД, а не медленный handler | После запуска запросы успешны; firing → resolved: 109.1s с учётом окна 2m | +| Временно остановить S3 | Retry без потери задания | Первая попытка failed, вторая ready, общий trace | Задание успешно завершено | +| Остановить Tempo на 60s | Приложение работает, буферы ограничены | 165 записей и обработка вложения успешны; очередь ≤32, RSS 198.52–199.02 MiB; после запуска новая трасса найдена | Потеря 808 принятых Collector spans, бизнес-операции успешны | + +Порядок диагностики: dashboard → лог по request_id/trace_id → ошибочный или отсутствующий span зависимости → восстановление сервиса → проверка ready/успешных запросов и resolved. Для Tempo потеря рассчитана как Δaccepted − Δsent после опустошения очереди; повторы отправки не считаются отдельными потерянными spans. Подробные timestamps и ID — в [observability.json](evidence/observability.json). ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Один локальный узел, без HA и TLS; алерты только локальные. SDK ограничивает очереди 512 записями, Collector использует memory limiter/batch и retry до 15s: длительный отказ может терять телеметрию. Потери до приёма Collector указанной дельтой не измеряются. История удаляется по retention; метрики не заменяют журнал транзакций. При недоступной БД API возвращает 503, после восстановления работает снова. Треки оценки 5 не реализовывались. diff --git a/compose.yaml b/compose.yaml index b95a457..aa5b2de 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,9 +1,10 @@ x-api: &api image: mtusi2026lab6-api:local + pull_policy: never dns_opt: - timeout:1 - attempts:1 - environment: + environment: &app-env APP_HOST: 0.0.0.0 APP_PORT: 8080 POSTGRES_HOST: postgres @@ -29,6 +30,9 @@ x-api: &api JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768} STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false} networks: [backend] cpus: 1.0 mem_limit: 512m @@ -51,6 +55,9 @@ services: dispatcher: <<: *api + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-dispatcher command: [python, -m, src.jobs, dispatch] healthcheck: disable: true @@ -64,6 +71,9 @@ services: worker: <<: *api + environment: + <<: *app-env + OTEL_SERVICE_NAME: messenger-worker command: [python, -m, src.worker] init: true healthcheck: @@ -101,6 +111,9 @@ services: migrate: <<: *api + environment: + <<: *app-env + OTEL_SDK_DISABLED: 'true' command: [python, -m, src.database] healthcheck: disable: true @@ -150,6 +163,9 @@ services: cleanup: <<: *api + environment: + <<: *app-env + OTEL_SDK_DISABLED: 'true' command: [python, -m, src.cleanup] healthcheck: disable: true @@ -217,6 +233,71 @@ services: cpus: 0.5 mem_limit: 256m + otel-collector: + image: otel/opentelemetry-collector-contrib:0.136.0 + command: [--config=/etc/otel/collector.yaml] + volumes: + - ./infra/otel/collector.yaml:/etc/otel/collector.yaml:ro + networks: [backend] + mem_limit: 256m + cpus: 0.5 + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.6.0 + command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=3d] + volumes: + - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro + - prometheus_data:/prometheus + ports: ['127.0.0.1:9090:9090'] + networks: [edge, backend] + mem_limit: 512m + cpus: 0.5 + restart: unless-stopped + + loki: + image: grafana/loki:3.5.7 + command: [-config.file=/etc/loki/local.yaml] + volumes: + - ./infra/otel/loki.yaml:/etc/loki/local.yaml:ro + - loki_data:/loki + ports: ['127.0.0.1:3100:3100'] + networks: [edge, backend] + mem_limit: 384m + cpus: 0.5 + restart: unless-stopped + + tempo: + image: grafana/tempo:2.9.0 + command: [-config.file=/etc/tempo.yaml] + volumes: + - ./infra/otel/tempo.yaml:/etc/tempo.yaml:ro + - tempo_data:/var/tempo + ports: ['127.0.0.1:3200:3200'] + networks: [edge, backend] + mem_limit: 384m + cpus: 0.5 + restart: unless-stopped + + grafana: + image: grafana/grafana:12.2.0 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} + GF_USERS_ALLOW_SIGN_UP: 'false' + GF_ANALYTICS_REPORTING_ENABLED: 'false' + GF_ANALYTICS_CHECK_FOR_UPDATES: 'false' + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + ports: ['127.0.0.1:3000:3000'] + networks: [edge, backend] + mem_limit: 384m + cpus: 0.5 + restart: unless-stopped + networks: edge: backend: @@ -226,3 +307,7 @@ volumes: postgres_data: redis_data: s3_data: + prometheus_data: + loki_data: + tempo_data: + grafana_data: diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..de736cf --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 6, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.157, + "concurrency": 4, + "successful_requests": 2389, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 79.219, + "successful_latency_ms": { + "p50": 40.784, + "p95": 99.52, + "p99": 129.314 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/observability.json b/evidence/observability.json new file mode 100644 index 0000000..9121fd4 --- /dev/null +++ b/evidence/observability.json @@ -0,0 +1,1001 @@ +{ + "smoke": { + "checked_at": "2026-11-16T12:49:07.787515+00:00", + "trace_id": "1c23b22062884843928ab9e73cfe8aa2", + "span_count": 38, + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "attachment-jobs process", + "attachment-jobs publish", + "db.INSERT", + "db.SELECT", + "db.UPDATE", + "db.acquire", + "image.process", + "s3.GetObject", + "s3.PutObject" + ], + "log_services": [ + "messenger-api", + "messenger-dispatcher", + "messenger-worker" + ], + "attempts": [ + 1 + ], + "parent_chain_verified": true, + "tracestate_preserved": true, + "request_id": "af182a84-f41f-435f-ab16-577992401501", + "private_inputs_absent": true, + "metrics_present": [ + "http_requests_total", + "http_duration_seconds_bucket", + "jobs_processed_total", + "job_duration_seconds_bucket", + "attachment_time_to_ready_seconds_bucket", + "queue_depth", + "queue_oldest_age_seconds", + "db_pool_used", + "db_pool_waiting", + "worker_busy", + "worker_capacity", + "container_cpu_seconds_total", + "container_memory_bytes" + ] + }, + "queue": { + "checked_at": "2026-11-16T12:35:01.506971+00:00", + "started_at": "2026-11-16T12:35:01.506989+00:00", + "firing_at": "2026-11-16T12:36:02.269300+00:00", + "depth": 1.0, + "oldest_age_seconds": 55.03155, + "resolved_at": "2026-11-16T12:36:22.588083+00:00", + "ready": "ready", + "depth_after": 0.0, + "pipeline": { + "trace_id": "4519e3ff9aa14cbf9bfdf43bfa44ee80", + "span_count": 38, + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "attachment-jobs process", + "attachment-jobs publish", + "db.INSERT", + "db.SELECT", + "db.UPDATE", + "db.acquire", + "image.process", + "s3.GetObject", + "s3.PutObject" + ], + "log_services": [ + "messenger-api", + "messenger-dispatcher", + "messenger-worker" + ], + "attempts": [ + 1 + ], + "parent_chain_verified": true, + "request_id": "73749068-b3ee-477b-bcad-db0de3c75ffc", + "private_inputs_absent": true + } + }, + "database": { + "checked_at": "2026-11-16T12:36:22.633344+00:00", + "started_at": "2026-11-16T12:36:22.633359+00:00", + "failed_requests": 18, + "firing_at": "2026-11-16T12:36:57.404789+00:00", + "error_ratio": 0.4259189297239447, + "db_error_spans": [ + { + "name": "db.acquire", + "error_type": "OperationalError", + "duration_seconds": 0.002054823 + } + ], + "trace_id": "fcbd996a6908494da6179cd128c2bd28", + "correlated_log_streams": 4, + "resolved_at": "2026-11-16T12:38:46.516570+00:00" + }, + "retry": { + "checked_at": "2026-11-16T12:41:29.322339+00:00", + "trace_id": "aba0e6312bd9484e856da069aef38c3f", + "span_count": 55, + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "attachment-jobs process", + "attachment-jobs publish", + "db.INSERT", + "db.SELECT", + "db.UPDATE", + "db.acquire", + "image.process", + "s3.GetObject", + "s3.PutObject" + ], + "log_services": [ + "messenger-api", + "messenger-dispatcher", + "messenger-worker" + ], + "attempts": [ + 1, + 2 + ], + "parent_chain_verified": true, + "tracestate_preserved": true, + "request_id": "407272a6-ed22-4758-9dc7-2cd814530cfe", + "private_inputs_absent": true + }, + "backend": { + "checked_at": "2026-11-16T12:41:54.031393+00:00", + "started_at": "2026-11-16T12:42:02.082124+00:00", + "before": { + "accepted": 985.0, + "sent": 985.0 + }, + "after": { + "accepted": 2743.0, + "sent": 1935.0 + }, + "lost_accepted_spans": 808.0, + "successful_writes": 165, + "attachment_ready_during_outage": true, + "samples": [ + { + "at": "2026-11-16T12:42:08.591984+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:08.974244+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:09.311115+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:09.667765+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:10.012255+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:10.344207+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:10.677829+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:11.006673+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 2.0 + }, + { + "at": "2026-11-16T12:42:11.337125+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:11.682832+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:12.032278+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:12.388323+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:12.727095+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:13.073830+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:13.430820+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:13.787635+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:14.134485+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:14.472759+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:14.810031+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:15.153426+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:15.518049+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:15.852954+00:00", + "collector_rss_bytes": 208166912.0, + "queue_batches": 7.0 + }, + { + "at": "2026-11-16T12:42:16.189126+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:16.540109+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:16.888590+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:17.264900+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:17.606336+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:17.942601+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:18.282908+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:18.619872+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:18.955992+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:19.287349+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:19.621891+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:19.953901+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:20.296420+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:20.635024+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:20.975548+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 11.0 + }, + { + "at": "2026-11-16T12:42:21.322299+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:21.657104+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:22.007058+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:22.361829+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:22.694943+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:23.042808+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:23.365271+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:23.704060+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:24.025263+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:24.362116+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:24.727485+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:25.062355+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:25.391615+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:25.725268+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:26.059632+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 15.0 + }, + { + "at": "2026-11-16T12:42:26.407145+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:26.758432+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:27.107084+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:27.473353+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:27.872791+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:28.203113+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:28.529960+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:28.878642+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:29.208243+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:29.543834+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:29.875588+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:30.211687+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:30.546138+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:30.900656+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 20.0 + }, + { + "at": "2026-11-16T12:42:31.230567+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:31.557914+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:31.885712+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:32.209573+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:32.535012+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:32.866421+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:33.188624+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:33.512070+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:33.833272+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:34.153833+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:34.474766+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:34.797676+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:35.146195+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:35.474055+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:35.795893+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:36.118999+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 23.0 + }, + { + "at": "2026-11-16T12:42:36.444385+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:36.777861+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:37.119616+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:37.450904+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:37.776880+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:38.098072+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:38.421305+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:38.747441+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:39.101583+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:39.427953+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:39.768280+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:40.094431+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:40.421707+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:40.743114+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:41.068845+00:00", + "collector_rss_bytes": 208297984.0, + "queue_batches": 28.0 + }, + { + "at": "2026-11-16T12:42:41.414541+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:41.739391+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:42.064533+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:42.393109+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:42.817591+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:43.141096+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:43.466521+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:43.792473+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:44.134164+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:44.455961+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:44.783997+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:45.110729+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:45.441249+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:45.780397+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:46.105040+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:46.428666+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:46.758928+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:47.100672+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:47.463968+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:47.792560+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:48.118544+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:48.449748+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:48.779273+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:49.106702+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:49.570723+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:49.921607+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:50.251106+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:50.578716+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:50.903551+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:51.235684+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:51.586873+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:51.931420+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:52.276665+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:52.604758+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:52.925887+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:53.247947+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:53.571512+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:53.897356+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:54.224932+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:54.558029+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:54.896329+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:55.227368+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:55.556902+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:55.891872+00:00", + "collector_rss_bytes": 208429056.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:42:56.222832+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:56.569495+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:56.914263+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:57.269966+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:57.623303+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:57.963659+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:58.296148+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:58.649295+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:58.989947+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:59.312764+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:42:59.657822+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:43:00.012611+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:43:00.344175+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:43:00.663982+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:43:01.022241+00:00", + "collector_rss_bytes": 208560128.0, + "queue_batches": 31.0 + }, + { + "at": "2026-11-16T12:43:01.348759+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:01.673975+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:02.028605+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:02.361265+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:02.686924+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:03.008412+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:03.329930+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:03.659190+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + }, + { + "at": "2026-11-16T12:43:03.981153+00:00", + "collector_rss_bytes": 208691200.0, + "queue_batches": 32.0 + } + ], + "recovered_at": "2026-11-16T12:43:17.130052+00:00", + "recovered_pipeline": { + "trace_id": "693a11c881224cba967fe2b954b707a9", + "span_count": 38, + "span_names": [ + "POST /api/v1/chats/{chat_id}/attachments", + "attachment-jobs process", + "attachment-jobs publish", + "db.INSERT", + "db.SELECT", + "db.UPDATE", + "db.acquire", + "image.process", + "s3.GetObject", + "s3.PutObject" + ], + "log_services": [ + "messenger-api", + "messenger-dispatcher", + "messenger-worker" + ], + "attempts": [ + 1 + ], + "parent_chain_verified": true, + "tracestate_preserved": true, + "request_id": "8105ec49-e42a-4cff-91e0-4192fe3b0fc7", + "private_inputs_absent": true + } + } +} diff --git a/evidence/verification.txt b/evidence/verification.txt new file mode 100644 index 0000000..0144c83 --- /dev/null +++ b/evidence/verification.txt @@ -0,0 +1,5 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +PASS: lab6 public API scenario; infrastructure and grading evidence remain separate. +Running smoke +PASS: smoke +PASS: Grafana provisioning (18 panels, 3 datasources), all dashboard queries, no active alerts; local secrets absent from Git candidates; assignment/contracts/public tests unchanged. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..0bcd64d --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,716 @@ +{ + "uid": "messenger-lab6", + "title": "Messenger / Lab 6", + "schemaVersion": 41, + "version": 1, + "refresh": "5s", + "time": { + "from": "now-30m", + "to": "now" + }, + "tags": [ + "lab6" + ], + "panels": [ + { + "id": 1, + "title": "API requests / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + }, + "description": "Business requests only; health probes are excluded.", + "targets": [ + { + "refId": "A", + "expr": "sum by (method,route) (rate(http_requests_total{service_name=\"messenger-api\"}[2m]))", + "legendFormat": "{{method}} {{route}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 2, + "title": "API server error ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + }, + "description": "5xx / all non-health requests, rolling 2m. 4xx are not server errors.", + "targets": [ + { + "refId": "A", + "expr": "messenger:http_error_ratio:2m", + "legendFormat": "5xx / all" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 3, + "title": "API duration p50 / p95 / p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 8 + }, + "description": "Quantiles from merged histogram buckets across API replicas. Never averages of per-instance percentiles.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_duration_seconds_bucket{service_name=\"messenger-api\"}[2m])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 4, + "title": "Job attempts / second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 8, + "w": 12, + "h": 8 + }, + "description": "One count per claimed attempt; failed includes retryable failures, skipped is a fenced attempt.", + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (rate(jobs_processed_total[2m]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 5, + "title": "Worker processing duration p50 / p95 / p99", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 16, + "w": 12, + "h": 8 + }, + "description": "Includes isolated image process and durable result check; excludes queue wait.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(job_duration_seconds_bucket[2m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(job_duration_seconds_bucket[2m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(job_duration_seconds_bucket[2m])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 6, + "title": "Time from accepted upload to ready", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 16, + "w": 12, + "h": 8 + }, + "description": "Database timestamps: durable attachment creation to ready update. Includes queue wait and retries; observed on successful attempts.", + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.5, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p50" + }, + { + "refId": "B", + "expr": "histogram_quantile(0.95, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p95" + }, + { + "refId": "C", + "expr": "histogram_quantile(0.99, sum by (le) (rate(attachment_time_to_ready_seconds_bucket[5m])))", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 7, + "title": "Durable queue depth", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + }, + "description": "PostgreSQL queued attachments, including unpublished outbox/retries; excludes processing jobs.", + "targets": [ + { + "refId": "A", + "expr": "max(queue_depth)", + "legendFormat": "queued" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 8, + "title": "Oldest queued attachment age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + }, + "description": "Age since acceptance. A stale sample is not zero; check queue sample age below.", + "targets": [ + { + "refId": "A", + "expr": "max(queue_oldest_age_seconds)", + "legendFormat": "age" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 9, + "title": "DB pool used / acquiring", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "description": "Per long-lived process. Acquiring includes connection creation and pool wait; max pool size is 10.", + "targets": [ + { + "refId": "A", + "expr": "db_pool_used", + "legendFormat": "{{service_name}} {{service_instance_id}} used" + }, + { + "refId": "B", + "expr": "db_pool_waiting", + "legendFormat": "{{service_name}} {{service_instance_id}} acquiring" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 10, + "title": "Worker saturation", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "description": "Busy consumers / configured concurrency. Missing capacity after 30s indicates stopped worker.", + "targets": [ + { + "refId": "A", + "expr": "sum(worker_busy)", + "legendFormat": "busy" + }, + { + "refId": "B", + "expr": "sum(worker_capacity)", + "legendFormat": "capacity" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 11, + "title": "Container memory (includes image children)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "description": "cgroup v2 memory.current. Includes child processes and charged cache.", + "targets": [ + { + "refId": "A", + "expr": "container_memory_bytes", + "legendFormat": "{{service_name}} {{service_instance_id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 12, + "title": "Container CPU", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "description": "cgroup v2 CPU seconds / second; 1.0 is one fully used CPU core.", + "targets": [ + { + "refId": "A", + "expr": "rate(container_cpu_seconds_total[2m])", + "legendFormat": "{{service_name}} {{service_instance_id}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 13, + "title": "API availability SLI / SLO", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "description": "Target: >=99% non-5xx business requests in a rolling 30m window. No traffic is not evidence that SLO is met.", + "targets": [ + { + "refId": "A", + "expr": "1 - ((sum(increase(http_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[30m])) or vector(0)) / clamp_min(sum(increase(http_requests_total{service_name=\"messenger-api\"}[30m])), 1))", + "legendFormat": "SLI over 30m" + }, + { + "refId": "B", + "expr": "vector(0.99)", + "legendFormat": "SLO" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 14, + "title": "Local alerts", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "description": "1 means pending/firing. Series disappears after resolution. Rules and runbooks are stored in infra/otel.", + "targets": [ + { + "refId": "A", + "expr": "ALERTS{alertname=~\"ApiHighErrorRate|QueueLagHigh\"}", + "legendFormat": "{{alertname}} {{alertstate}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 15, + "title": "Collector memory", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "description": "", + "targets": [ + { + "refId": "A", + "expr": "otelcol_process_memory_rss_bytes", + "legendFormat": "RSS" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 16, + "title": "Collector exporter queue / capacity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "description": "Bounded queues: 32 batches per backend, up to 256 records per batch.", + "targets": [ + { + "refId": "A", + "expr": "otelcol_exporter_queue_size", + "legendFormat": "{{exporter}} queued" + }, + { + "refId": "B", + "expr": "otelcol_exporter_queue_capacity", + "legendFormat": "{{exporter}} capacity" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 17, + "title": "Collector failed sends / rejected enqueue", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + }, + "description": "Send failures may include retries, not unique lost spans. Exact experiment loss is measured by accepted/sent deltas after draining.", + "targets": [ + { + "refId": "A", + "expr": "rate(otelcol_exporter_send_failed_spans_total[2m])", + "legendFormat": "{{exporter}} span send failures" + }, + { + "refId": "B", + "expr": "rate(otelcol_exporter_send_failed_log_records_total[2m])", + "legendFormat": "{{exporter}} log send failures" + }, + { + "refId": "C", + "expr": "rate(otelcol_exporter_enqueue_failed_spans_total[2m])", + "legendFormat": "{{exporter}} span queue rejections" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 18, + "title": "Queue sample age", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "x": 12, + "y": 64, + "w": 12, + "h": 8 + }, + "description": "Above 15s: queue measurements are stale (e.g. DB outage).", + "targets": [ + { + "refId": "A", + "expr": "time() - max(queue_sample_timestamp_seconds)", + "legendFormat": "seconds since DB sample" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + } + ], + "links": [ + { + "title": "Explore logs", + "url": "/explore?left=%7B%22datasource%22%3A%22loki%22%7D" + }, + { + "title": "Explore traces", + "url": "/explore?left=%7B%22datasource%22%3A%22tempo%22%7D" + } + ] +} diff --git a/infra/grafana/provisioning/dashboards/default.yaml b/infra/grafana/provisioning/dashboards/default.yaml new file mode 100644 index 0000000..5be8c9d --- /dev/null +++ b/infra/grafana/provisioning/dashboards/default.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: messenger + folder: Lab 6 + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/sources.yaml b/infra/grafana/provisioning/datasources/sources.yaml new file mode 100644 index 0000000..2a3dc75 --- /dev/null +++ b/infra/grafana/provisioning/datasources/sources.yaml @@ -0,0 +1,41 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 5s + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + datasourceUid: tempo + url: '$${__value.raw}' + urlDisplayLabel: View trace + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: '-5m' + spanEndTimeShift: '5m' + tags: [{key: service.name, value: service_name}] + filterByTraceID: true + filterBySpanID: false + nodeGraph: + enabled: true diff --git a/infra/otel/README.md b/infra/otel/README.md index c46fe89..7e72e78 100644 --- a/infra/otel/README.md +++ b/infra/otel/README.md @@ -1,7 +1,140 @@ -# Конфигурация наблюдаемости — реализовать +# Лаба 6: запуск и диагностика -Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend. +```sh +cp .env.example .env +# Задать локальные пароли в .env (в Git не добавлять). +docker compose up -d --build +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +python -X utf8 student-tests/observability.py --scenario all +``` -Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`. +Последняя команда создаёт синтетические данные и временно останавливает **только +сервисы текущего Compose-проекта**. В `finally` запускает остановленные сервисы. +Не запускать параллельно с другими проверками этого стенда. Volumes не удаляются. +Санитизированные результаты сохраняются в `evidence/observability.json`. -Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён. +Grafana: http://localhost:3000, пользователь `admin`, пароль `GRAFANA_ADMIN_PASSWORD` +из `.env`. Dashboard **Lab 6 / Messenger / Lab 6** и три datasource загружаются +автоматически. Prometheus: http://localhost:9090; Loki API: порт 3100; Tempo API: +3200. Все опубликованные порты слушают только loopback, OTLP наружу не опубликован. + +Collector принимает OTLP/HTTP от API, dispatcher и worker. Prometheus хранит +метрики, Loki — логи, Tempo — трассы; Grafana обращается к этим хранилищам. +История сохраняется в именованных volumes, retention — 72 часа. +Версии закреплены в Compose и requirements.txt: OTel Python 1.38.0, semantic +conventions 0.59b0 (зависимость SDK); наши имена метрик перечислены ниже. +Это явная инструментация: auto-instrumentation, сбор SQL и HTTP headers отключены. + +## Сигналы + +| Prometheus name | Единицы и смысл | +| --- | --- | +| `http_requests_total` | Число бизнес-запросов; method, route template, status_class | +| `http_duration_seconds` | Histogram полного HTTP-запроса, включая отдачу stream | +| `jobs_processed_total` | Попытки обработки; job_type, outcome=ready/failed/skipped | +| `job_duration_seconds` | Histogram времени попытки, без ожидания в очереди | +| `attachment_time_to_ready_seconds` | Histogram от записи attachment до durable ready, включая retry | +| `queue_depth` | Количество queued в PostgreSQL, включая outbox и отложенные retry | +| `queue_oldest_age_seconds` | Возраст старейшего queued от момента приёма, не время обработки | +| `queue_sample_timestamp_seconds` | Unix-время последнего успешного опроса очереди | +| `db_pool_used`, `db_pool_waiting` | Занятые соединения / потоки, получающие соединение (ожидание + reconnect) | +| `worker_busy`, `worker_capacity` | Число занятых consumer / лимит concurrency | +| `process_memory_bytes`, `process_cpu_seconds_total` | RSS и CPU процесса | +| `container_memory_bytes`, `container_cpu_seconds_total` | cgroup v2, включая дочерние процессы обработки | + +Histogram buckets, секунды: .005, .01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, +30, 60, 120, 300, +Inf. Dashboard содержит PromQL, единицы и описания каждого +графика. p50/p95/p99 считаются `histogram_quantile` по сумме buckets всех реплик. +Метрики очереди отдаёт только dispatcher; это не длина Redis Stream (в нём +ещё могут быть уже взятые сообщения). При недоступной БД sample становится +устаревшим; возраст sample выведен отдельно, QueueLagHigh не использует stale data. + +Resource: service.name, service.version=1.6.0, service.instance.id=hostname, +deployment.environment.name=lab. API: messenger-api; worker и его дочерний +процесс: messenger-worker; publisher: messenger-dispatcher. Дочерний процесс +экспортирует только traces/logs, чтобы не создавать отдельные краткоживущие +ряды метрик. Parent измеряет длительность попытки и результат после COMMIT. +Метрики наблюдательные: авария после COMMIT до записи счётчика может потерять +измерение, но не результат обработки. Предметных ID в labels метрик нет. + +## Связь сигналов + +SERVER span API → traceparent/tracestate в transactional outbox → PRODUCER span +dispatcher → W3C context в JSON сообщения Redis → CONSUMER span worker → context +через окружение изолированного image process → S3/DB CLIENT spans. Retry публикует +новый producer/consumer в той же трассе, с `job.attempt`. Baggage не переносится. +Context сохранён в миграции 008, поэтому рестарт API не разрывает причинную связь. + +Логи содержат timestamp/severity/service.name/event и trace_id/span_id активного +span. `http_request` содержит ограниченный `request_id`, совпадающий с +X-Request-Id ответа. Loki сохраняет технические поля как structured metadata; +индексирует только service.name и окружение. Для корреляции достаточно request_id, +trace_id и номера попытки: имена файлов и предметные ID не экспортируются. + +В Grafana Explore / Loki: + +```logql +{service_name="messenger-api"} | request_id="ЗНАЧЕНИЕ_X_REQUEST_ID" +{service_name=~"messenger-.*"} | trace_id="TRACE_ID" +``` + +Поле trace_id открывает Tempo; из Tempo доступен переход к логам. В трассе upload +видны `POST /api/v1/chats/{chat_id}/attachments`, `attachment-jobs publish`, +`attachment-jobs process`, `image.process`, `s3.GetObject`, `s3.PutObject`, +`db.SELECT`/`db.UPDATE`. Исключения содержат только тип, без message/stack/SQL. + +## SLI/SLO и alerts + +SLI доступности = 1 − число 5xx / число всех бизнес-запросов за скользящие **30 минут**. +SLO: **не менее 99%**. Health исключён из числителя и знаменателя; 4xx не считается +отказом сервера. Без запросов SLI не оцениваем. Это учебное окно, а не месячный SLO. + +`ApiHighErrorRate`: 5xx > 5% за 2 минуты, минимум 20 запросов, `for: 20s`. +`QueueLagHigh`: queued > 0 и возраст > 30s, свежий sample, `for: 15s`. +Пороги короткие для воспроизводимой локальной демонстрации; алерты не являются +burn-rate исследованием. Rules вычисляет Prometheus; firing/pending видны в +http://localhost:9090/alerts и на dashboard. После восстановления alert пропадает +из instant `ALERTS`; история переходов остаётся в range query. Доставка наружу не настроена. + +## Queue lag + +1. Открыть queue depth/age и worker capacity/busy. Растущая очередь при отсутствии + capacity означает остановку worker; свежий sample подтверждает доступность БД. +2. В Loki найти `job_published`, затем открыть trace: producer есть, consumer ещё нет. +3. `docker compose start worker`; дождаться `job_processed` outcome=ready, depth=0. +4. Проверить, что QueueLagHigh больше не firing и в той же трассе появился consumer. + +## API errors + +1. При ApiHighErrorRate найти http_request со status_class=5xx и взять trace_id. +2. В Tempo проверить `db.acquire`/`db.SELECT`: error.type и длительность покажут + отказ подключения/БД. В Loki есть `db_error`/`operation_failed`. Медленный handler + без отказа зависимости не имел бы ошибочного DB span. Посмотреть db_pool_waiting. +3. Проверить PostgreSQL: `docker compose ps postgres`; восстановить + `docker compose start postgres`, дождаться readiness и успешного бизнес-запроса. +4. Через окно 2m error ratio снизится, alert перейдёт в resolved. Sample очереди + тоже должен стать свежим. Сценарий проверки сохраняет трассу ошибочного запроса. + +## Недоступность telemetry backend + +SDK: очереди по 512 spans/logs, batch 64, HTTP timeout 1s; metric export каждые 5s. +Отправка выполняется в фоновых потоках. Collector: memory limiter 192 MiB, +spike 48 MiB, container limit 256 MiB; batch ≤256 записей; отдельная очередь +32 batches / 2 consumers на Loki и Tempo; retry до 15s, timeout 2s, overflow +не блокирует приём. Конечные ограничения защищают память, но допускают потерю +телеметрии при длительном отказе. API и pipeline не зависят от готовности backend. + +Сценарий `backend` останавливает Tempo, продолжает создавать сообщения и +обрабатывать изображения, снимает RSS/очередь Collector, затем запускает Tempo +и проверяет новую трассу. `send_failed_spans` может считать повторные попытки: +его нельзя выдавать за число уникально потерянных spans. Для отдельного окна +тест вычисляет accepted − sent после опустошения очереди (без рестартов Collector) +и сохраняет эти дельты вместе с наблюдаемой памятью. Это потери уже принятых +Collector spans; отброшенные до приёма SDK/receiver в этот счёт не входят. +Prometheus всё это время +работает, поэтому сохраняется история отказа и восстановления. + +Первичные источники: [Collector](https://opentelemetry.io/docs/collector/configuration/), +[native Loki OTLP](https://grafana.com/docs/loki/latest/send-data/otel/), +[bounded exporter queues](https://github.com/open-telemetry/opentelemetry-collector/blob/v0.136.0/exporter/exporterhelper/README.md). diff --git a/infra/otel/alerts.yaml b/infra/otel/alerts.yaml new file mode 100644 index 0000000..b4701d9 --- /dev/null +++ b/infra/otel/alerts.yaml @@ -0,0 +1,28 @@ +groups: + - name: messenger + rules: + - record: messenger:http_error_ratio:2m + expr: | + (sum(rate(http_requests_total{service_name="messenger-api",status_class="5xx"}[2m])) or vector(0)) + / clamp_min(sum(rate(http_requests_total{service_name="messenger-api"}[2m])), 0.001) + - alert: ApiHighErrorRate + expr: messenger:http_error_ratio:2m > 0.05 and sum(increase(http_requests_total{service_name="messenger-api"}[2m])) >= 20 + for: 20s + labels: + severity: warning + annotations: + summary: 'API: more than 5% server errors over 2 minutes' + description: 'SLI is 5xx / all non-health requests. At least 20 requests in the window.' + runbook: 'infra/otel/README.md#api-errors' + - alert: QueueLagHigh + expr: | + max(queue_oldest_age_seconds) > 30 + and max(queue_depth) > 0 + and time() - max(queue_sample_timestamp_seconds) < 15 + for: 15s + labels: + severity: warning + annotations: + summary: 'Oldest queued attachment waits more than 30 seconds' + description: 'Check worker capacity/busy and job errors; backlog is sampled by dispatcher.' + runbook: 'infra/otel/README.md#queue-lag' diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..529a107 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,69 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 192 + spike_limit_mib: 48 + batch: + timeout: 1s + send_batch_size: 128 + send_batch_max_size: 256 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + enable_open_metrics: true + metric_expiration: 30s + translation_strategy: UnderscoreEscapingWithSuffixes + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 2s + sending_queue: + queue_size: 32 + num_consumers: 2 + block_on_overflow: false + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 15s + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 2s + sending_queue: + queue_size: 32 + num_consumers: 2 + block_on_overflow: false + retry_on_failure: + initial_interval: 1s + max_interval: 5s + max_elapsed_time: 15s + +service: + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml new file mode 100644 index 0000000..b5cf04f --- /dev/null +++ b/infra/otel/loki.yaml @@ -0,0 +1,37 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +limits_config: + allow_structured_metadata: true + retention_period: 72h + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: [service.name, deployment.environment.name] +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem +analytics: + reporting_enabled: false diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml new file mode 100644 index 0000000..5613d18 --- /dev/null +++ b/infra/otel/prometheus.yaml @@ -0,0 +1,12 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s +rule_files: + - /etc/prometheus/alerts.yaml +scrape_configs: + - job_name: messenger + static_configs: + - targets: ['otel-collector:8889'] + - job_name: collector + static_configs: + - targets: ['otel-collector:8888'] diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml new file mode 100644 index 0000000..9653bf2 --- /dev/null +++ b/infra/otel/tempo.yaml @@ -0,0 +1,23 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 +ingester: + trace_idle_period: 10s + max_block_duration: 5m +compactor: + compaction: + block_retention: 72h +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks +usage_report: + reporting_enabled: false diff --git a/migrations/008_trace_context.sql b/migrations/008_trace_context.sql new file mode 100644 index 0000000..28c6fe5 --- /dev/null +++ b/migrations/008_trace_context.sql @@ -0,0 +1 @@ +ALTER TABLE attachment_jobs ADD COLUMN trace_context jsonb NOT NULL DEFAULT '{}'; diff --git a/requirements.txt b/requirements.txt index 86eff95..01c2cf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,7 @@ redis==5.2.1 argon2-cffi==23.1.0 boto3==1.40.70 Pillow==12.0.0 +opentelemetry-api==1.38.0 +opentelemetry-sdk==1.38.0 +opentelemetry-exporter-otlp-proto-http==1.38.0 +psutil==7.1.2 diff --git a/src/__main__.py b/src/__main__.py index d5a3407..6846b7f 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -9,5 +9,6 @@ if __name__ == '__main__': host=os.getenv('APP_HOST', '0.0.0.0'), port=int(os.getenv('APP_PORT', '8080')), workers=1, + access_log=False, # Structured logs use route templates, never query strings. timeout_graceful_shutdown=5, ) diff --git a/src/attachments.py b/src/attachments.py index 10c6233..aa85a58 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -10,6 +10,7 @@ from .admission import reserve_capacity from .models import Attachment, AttachmentJob, ChatMember from .object_storage import BUCKET, PREFIX, s3_client from .store import ensure_uuid, serialize, store +from .telemetry import carrier MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760) MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000) @@ -71,7 +72,7 @@ def save_attachment(source, chat_id, actor_id, filename, content_type, size, dig size_bytes=size, sha256=digest, staging_key=staging_key) session.add(attachment) session.flush() - session.add(AttachmentJob(attachment_id=attachment_id)) + session.add(AttachmentJob(attachment_id=attachment_id, trace_context=carrier())) result = metadata(attachment) # An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later. return result diff --git a/src/database.py b/src/database.py index e98b4d4..0c0fda4 100644 --- a/src/database.py +++ b/src/database.py @@ -1,8 +1,12 @@ import os from pathlib import Path -from sqlalchemy import create_engine, func, insert, select +from sqlalchemy import create_engine, event, func, insert, select from sqlalchemy.engine import URL, make_url +from sqlalchemy.pool import QueuePool +from threading import Lock + +from . import telemetry as otel from .models import SchemaMigration @@ -23,12 +27,63 @@ def database_url(): ) +class ObservedPool(QueuePool): + """Count threads acquiring a connection, including pool wait and reconnect time.""" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.waiting = 0 + self.wait_lock = Lock() + + def connect(self): + with self.wait_lock: + self.waiting += 1 + try: + with otel.dependency_span('db.acquire', **{'db.system': 'postgresql'}): + return super().connect() + finally: + with self.wait_lock: + self.waiting -= 1 + + def create_db_engine(): - return create_engine( + engine = create_engine( database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True, + poolclass=ObservedPool, hide_parameters=True, connect_args={'connect_timeout': 2, 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}, ) + # Only operation names; SQL text, bound values and connection URLs are never exported. + @event.listens_for(engine, 'before_cursor_execute') + def before(conn, cursor, statement, parameters, context, executemany): + if not otel.trace.get_current_span().get_span_context().is_valid: + return + operation = statement.lstrip().split(None, 1)[0].upper() if statement.strip() else 'OTHER' + if operation not in {'SELECT', 'INSERT', 'UPDATE', 'DELETE'}: + operation = 'OTHER' + context.otel_span = otel.tracer.start_span('db.' + operation, + kind=otel.SpanKind.CLIENT, attributes={'db.system': 'postgresql', 'db.operation.name': operation}) + + @event.listens_for(engine, 'after_cursor_execute') + def after(conn, cursor, statement, parameters, context, executemany): + current = getattr(context, 'otel_span', None) + if current: + current.end() + + @event.listens_for(engine, 'handle_error') + def error(context): + current = getattr(context.execution_context, 'otel_span', None) + if current: + current.set_status(otel.Status(otel.StatusCode.ERROR)) + current.set_attribute('error.type', type(context.original_exception).__name__) + current.end() + otel.event('db_error', severity=otel.logging.ERROR, error_type=type(context.original_exception).__name__) + + if os.getenv('OTEL_CHILD_PROCESS') != 'true': + otel.meter.create_observable_gauge('db_pool_used', callbacks=[ + lambda _: [otel.metrics.Observation(engine.pool.checkedout())]]) + otel.meter.create_observable_gauge('db_pool_waiting', callbacks=[ + lambda _: [otel.metrics.Observation(engine.pool.waiting)]]) + return engine def migrate(engine): diff --git a/src/image_processor.py b/src/image_processor.py index 019eb9a..8533d88 100644 --- a/src/image_processor.py +++ b/src/image_processor.py @@ -12,6 +12,7 @@ from .database import create_db_engine from .jobs import now from .models import Attachment, AttachmentJob from .object_storage import BUCKET, s3_client +from . import telemetry as otel def fit_size(size, bound): @@ -98,9 +99,13 @@ def main(): import resource memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024 resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + otel.setup(child=True) engine = create_db_engine() try: - process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2]) + with otel.span('image.process', context=otel.propagator.extract({ + 'traceparent': os.getenv('TRACEPARENT', ''), 'tracestate': os.getenv('TRACESTATE', '')})): + process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2]) + otel.event('image_completed') except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError): return 20 except MemoryError: @@ -109,6 +114,7 @@ def main(): return 22 # No credentials or decoder/database internals in logs. finally: engine.dispose() + otel.shutdown() return 0 diff --git a/src/jobs.py b/src/jobs.py index a1e1b98..0757b2c 100644 --- a/src/jobs.py +++ b/src/jobs.py @@ -10,13 +10,14 @@ from uuid import UUID, uuid4 from redis import Redis from redis.exceptions import ResponseError -from sqlalchemy import and_, or_, select +from sqlalchemy import and_, or_, select, func from sqlalchemy.orm import sessionmaker from .database import create_db_engine from .admission import reserve_capacity from .models import Attachment, AttachmentJob from .object_storage import BUCKET, s3_client +from . import telemetry as otel STREAM = 'attachment-jobs' GROUP = 'image-workers' @@ -62,7 +63,8 @@ def fail(session, job, attachment, code, permanent=False): attachment.error_code = None delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5) job.available_at = now() + timedelta(seconds=delay) - logging.info('job=%s attempt=%s result=%s code=%s', job.id, job.attempt, attachment.status, code) + otel.event('job_retry' if attachment.status == 'queued' else 'job_failed', + attempt=job.attempt, outcome=attachment.status, error_code=code) def dispatch_once(sessions, client): @@ -80,10 +82,24 @@ def dispatch_once(sessions, client): fail(session, job, attachment, 'lease_expired') continue if job.published_at is None: - client.xadd(STREAM, {'job': json.dumps(envelope(job))}) + with otel.span('attachment-jobs publish', kind=otel.SpanKind.PRODUCER, + context=otel.propagator.extract(job.trace_context), attributes={ + 'messaging.system': 'redis', 'messaging.destination.name': STREAM, + 'messaging.operation.type': 'publish', 'job.attempt': job.attempt + 1}): + client.xadd(STREAM, {'job': json.dumps({**envelope(job), **otel.carrier()})}) + otel.event('job_published', attempt=job.attempt + 1) job.published_at = now() +def sample_queue(sessions): + # Database is the durable backlog: includes outbox rows not yet in Redis and retries. + with sessions() as session: + depth, oldest = session.execute(select(func.count(), func.min(Attachment.created_at)) + .select_from(Attachment).where(Attachment.status == 'queued')).one() + otel.queue_state.update(depth=depth, age=max(0, (now() - oldest).total_seconds()) if oldest else 0, + updated=time.time()) + + def claim(sessions, message): if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1' or message.get('pipeline_version') != 1): @@ -148,6 +164,7 @@ def main(): retry.add_argument('--reason', required=True) args = parser.parse_args() logging.basicConfig(level=logging.INFO) + otel.setup() engine = create_db_engine() sessions = sessionmaker(engine, expire_on_commit=False) try: @@ -169,11 +186,13 @@ def main(): try: ensure_group(client) dispatch_once(sessions, client) + sample_queue(sessions) except Exception as exc: - logging.warning('Dispatch deferred: %s', type(exc).__name__) + otel.event('dispatch_deferred', error_type=type(exc).__name__) time.sleep(1) finally: engine.dispose() + otel.shutdown() if __name__ == '__main__': diff --git a/src/main.py b/src/main.py index fc96b18..cfc43e6 100644 --- a/src/main.py +++ b/src/main.py @@ -18,10 +18,13 @@ from .routers.users import router as users_router from .routers.chats import router as chats_router from .routers.auth import router as auth_router from .routers.attachments import router as attachments_router +from . import telemetry @asynccontextmanager async def lifespan(app): + telemetry.setup() + telemetry.seed_http(app.routes) engine = create_db_engine() try: store.sessions = sessionmaker(engine, expire_on_commit=False) @@ -29,6 +32,7 @@ async def lifespan(app): finally: engine.dispose() sessions.client.close() + telemetry.shutdown() app = FastAPI(title="MTUSI Messenger", lifespan=lifespan) @@ -46,3 +50,6 @@ app.include_router(users_router) app.include_router(chats_router) app.include_router(auth_router) app.include_router(attachments_router) + +# Outside FastAPI's error middleware: includes generated 500 responses and full streams. +app = telemetry.HttpTelemetry(app) diff --git a/src/models.py b/src/models.py index e425570..d9e0212 100644 --- a/src/models.py +++ b/src/models.py @@ -97,6 +97,7 @@ class AttachmentJob(Base): lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False)) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]') + trace_context: Mapped[dict] = mapped_column(JSONB, server_default='{}') class IdempotencyRecord(Base): diff --git a/src/object_storage.py b/src/object_storage.py index d3cb252..924a837 100644 --- a/src/object_storage.py +++ b/src/object_storage.py @@ -3,6 +3,7 @@ from functools import lru_cache import boto3 from botocore.config import Config +from .telemetry import dependency_span BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments') PREFIX = 'originals/' @@ -10,7 +11,7 @@ PREFIX = 'originals/' @lru_cache(maxsize=1) def s3_client(): - return boto3.client( + client = boto3.client( 's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), region_name=os.getenv('S3_REGION', 'us-east-1'), aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'], @@ -21,3 +22,12 @@ def s3_client(): request_checksum_calculation='when_required', response_checksum_validation='when_required'), ) + # Instrument SDK operations without object keys, signed URLs, headers or bytes. + original = client._make_api_call + + def call(operation, parameters): + with dependency_span('s3.' + operation, **{'rpc.system': 'aws-api', 'rpc.service': 'S3', + 'rpc.method': operation}): + return original(operation, parameters) + client._make_api_call = call + return client diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..bc285bf --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,206 @@ +"""Small, explicit OTel instrumentation. Only allowlisted technical fields leave the app.""" +import json +import logging +import os +import re +import socket +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +import psutil +from opentelemetry import metrics, trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.metrics import MeterProvider, Histogram +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + +tracer = trace.get_tracer('messenger', '1.6.0') +meter = metrics.get_meter('messenger', '1.6.0') +propagator = TraceContextTextMapPropagator() # No baggage: it may contain user data. +logger = logging.getLogger('messenger.events') +logger.propagate = False +logger.setLevel(logging.INFO) +providers = [] +LOG_FIELDS = {'method', 'route', 'status_class', 'status', 'duration_seconds', 'request_id', + 'operation', 'error_type', 'attempt', 'outcome', 'error_code', 'job_type'} +http_requests = meter.create_counter('http_requests') +http_duration = meter.create_histogram('http_duration', unit='s') +jobs_processed = meter.create_counter('jobs_processed') +job_duration = meter.create_histogram('job_duration', unit='s') +time_to_ready = meter.create_histogram('attachment_time_to_ready', unit='s') +worker_busy = meter.create_up_down_counter('worker_busy') +queue_state = {'depth': 0, 'age': 0, 'updated': 0} + + +class JsonFormatter(logging.Formatter): + def format(self, record): + context = trace.get_current_span().get_span_context() + return json.dumps({ + 'timestamp': datetime.fromtimestamp(record.created, timezone.utc).isoformat(), + 'severity': record.levelname, 'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'), + 'event': record.getMessage(), + 'trace_id': format(context.trace_id, '032x') if context.is_valid else '', + 'span_id': format(context.span_id, '016x') if context.is_valid else '', + **{key: record.__dict__[key] for key in LOG_FIELDS if key in record.__dict__}, + }) + + +def event(name, *, severity=logging.INFO, **fields): + # Call sites pass fixed event names, safe enums/numbers and technical correlation IDs. + logger.log(severity, name, extra={'event': name, **{k: v for k, v in fields.items() if k in LOG_FIELDS}}) + + +def setup(*, child=False): + if providers or os.getenv('OTEL_SDK_DISABLED', 'false').lower() == 'true': + return + resource = Resource({ + 'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api'), + 'service.version': '1.6.0', 'service.instance.id': socket.gethostname(), + 'deployment.environment.name': 'lab', + }) + endpoint = os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://otel-collector:4318').rstrip('/') + traces = TracerProvider(resource=resource) + traces.add_span_processor(BatchSpanProcessor( + OTLPSpanExporter(endpoint=endpoint + '/v1/traces', timeout=1), + max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000)) + trace.set_tracer_provider(traces) + logs = LoggerProvider(resource=resource) + logs.add_log_record_processor(BatchLogRecordProcessor( + OTLPLogExporter(endpoint=endpoint + '/v1/logs', timeout=1), + max_queue_size=512, max_export_batch_size=64, schedule_delay_millis=1000)) + console = logging.StreamHandler() + console.setFormatter(JsonFormatter()) + logger.addHandler(console) + logger.addHandler(LoggingHandler(logger_provider=logs)) + providers.extend([traces, logs]) + if child: + return # Short-lived image processes emit spans/logs; parent owns worker metrics. + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=endpoint + '/v1/metrics', timeout=1), + export_interval_millis=5000, export_timeout_millis=2000) + buckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 10, 30, 60, 120, 300] + metric_provider = MeterProvider(resource=resource, metric_readers=[reader], views=[ + View(instrument_type=Histogram, + aggregation=ExplicitBucketHistogramAggregation(buckets))]) + metrics.set_meter_provider(metric_provider) + providers.append(metric_provider) + process = psutil.Process() + meter.create_observable_gauge('process_memory', unit='By', callbacks=[ + lambda _: [metrics.Observation(process.memory_info().rss)]]) + meter.create_observable_counter('process_cpu', unit='s', callbacks=[ + lambda _: [metrics.Observation(sum(process.cpu_times()[:2]))]]) + cgroup = Path('/sys/fs/cgroup') + if (cgroup / 'memory.current').exists(): + meter.create_observable_gauge('container_memory', unit='By', callbacks=[ + lambda _: [metrics.Observation(int((cgroup / 'memory.current').read_text()))]]) + def cpu_usage(_): + values = dict(line.split() for line in (cgroup / 'cpu.stat').read_text().splitlines()) + return [metrics.Observation(int(values['usage_usec']) / 1_000_000)] + meter.create_observable_counter('container_cpu', unit='s', callbacks=[cpu_usage]) + if os.getenv('OTEL_SERVICE_NAME') == 'messenger-worker': + capacity = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2')))) + meter.create_observable_gauge('worker_capacity', callbacks=[ + lambda _: [metrics.Observation(capacity)]]) + worker_busy.add(0) + if os.getenv('OTEL_SERVICE_NAME') == 'messenger-dispatcher': + for name, key, unit in [('queue_depth', 'depth', ''), ('queue_oldest_age', 'age', 's'), + ('queue_sample_timestamp', 'updated', 's')]: + meter.create_observable_gauge(name, unit=unit, callbacks=[ + lambda _, key=key: [metrics.Observation(queue_state[key], {'queue': 'attachment-jobs'})]]) + + +def shutdown(): + for provider in reversed(providers): + provider.shutdown() + providers.clear() + + +def seed_http(routes): + # Export zero before the first failure: Prometheus rate needs a prior sample. + for route in routes: + if route.path.startswith('/health/'): + continue + for method in route.methods or (): + for status in ('2xx', '3xx', '4xx', '5xx'): + http_requests.add(0, {'method': method, 'route': route.path, 'status_class': status}) + + +def carrier(): + result = {} + propagator.inject(result) + return result + + +@contextmanager +def span(name, *, kind=SpanKind.INTERNAL, context=None, attributes=None): + # Never export exception messages/stack traces (SQL errors can embed parameters). + with tracer.start_as_current_span(name, kind=kind, context=context, attributes=attributes, + record_exception=False, set_status_on_exception=False) as current: + try: + yield current + except Exception as exc: + current.set_status(Status(StatusCode.ERROR)) + current.set_attribute('error.type', type(exc).__name__) + event('operation_failed', severity=logging.ERROR, operation=name, error_type=type(exc).__name__) + raise + + +@contextmanager +def dependency_span(name, **attributes): + # Background health probes and queue polling must not create disconnected trace noise. + if not trace.get_current_span().get_span_context().is_valid: + yield + else: + with span(name, kind=SpanKind.CLIENT, attributes=attributes): + yield + + +class HttpTelemetry: + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope['type'] != 'http' or scope['path'].startswith('/health/'): + return await self.app(scope, receive, send) + headers = {k.decode('latin1'): v.decode('latin1') for k, v in scope['headers'] + if k in (b'traceparent', b'tracestate')} + method = scope['method'] if scope['method'] in {'GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS'} else 'OTHER' + status = 500 + started = time.monotonic() + with span('HTTP ' + method, kind=SpanKind.SERVER, context=propagator.extract(headers)) as current: + async def response(message): + nonlocal status + if message['type'] == 'http.response.start': + status = message['status'] + await send(message) + try: + await self.app(scope, receive, response) + finally: + route = getattr(scope.get('route'), 'path', 'unmatched') + labels = {'method': method, 'route': route, 'status_class': f'{status // 100}xx'} + elapsed = time.monotonic() - started + http_requests.add(1, labels) + http_duration.record(elapsed, labels) + current.update_name(method + ' ' + route) + current.set_attributes({'http.request.method': method, 'http.route': route, + 'http.response.status_code': status}) + if status >= 500: + current.set_status(Status(StatusCode.ERROR)) + request_id = scope.get('state', {}).get('request_id', '') + if not re.fullmatch(r'[A-Za-z0-9_-]{1,64}', request_id): + request_id = '' + current.set_attribute('request_id', request_id) + event('http_request', severity=logging.ERROR if status >= 500 else logging.INFO, + **labels, status=status, duration_seconds=elapsed, + request_id=request_id) diff --git a/src/worker.py b/src/worker.py index 0708885..118c885 100644 --- a/src/worker.py +++ b/src/worker.py @@ -12,30 +12,75 @@ from sqlalchemy.orm import sessionmaker from .database import create_db_engine from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure +from .models import Attachment, AttachmentJob +from . import telemetry as otel def handle(sessions, client, message_id, fields): try: - claimed = claim(sessions, json.loads(fields['job'])) + message = json.loads(fields['job']) + if not isinstance(message, dict): + raise ValueError('invalid envelope') except (ValueError, KeyError, TypeError): - logging.error('Invalid broker envelope: id=%s', message_id) + otel.event('invalid_job') + acknowledge(client, message_id) + return + with otel.span('attachment-jobs process', kind=otel.SpanKind.CONSUMER, + context=otel.propagator.extract(message), attributes={ + 'messaging.system': 'redis', 'messaging.destination.name': STREAM, + 'messaging.operation.type': 'process', 'job.attempt': message.get('attempt', 0)}): + process_message(sessions, client, message_id, message) + + +def process_message(sessions, client, message_id, message): + try: + claimed = claim(sessions, message) + except (ValueError, KeyError, TypeError): + otel.event('invalid_job') claimed = None if claimed: job_id, token = claimed + started = time.monotonic() + outcome = 'failed' + otel.worker_busy.add(1) try: - result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token], - timeout=TIMEOUT, check=False) - code = result.returncode - except subprocess.TimeoutExpired: - code = 23 - if code in (91, 92): - # Simulate a lost worker before ack: leave the Redis pending entry and lease. - return - if code: - record_failure(sessions, job_id, token, - {20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'} - .get(code, 'processing_failed'), permanent=code in (20, 21)) + environment = {**os.environ, 'OTEL_CHILD_PROCESS': 'true'} + environment.update({key.upper(): value for key, value in otel.carrier().items()}) + try: + result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token], + timeout=TIMEOUT, check=False, env=environment) + code = result.returncode + except subprocess.TimeoutExpired: + code = 23 + if code in (91, 92): + # Simulate a lost worker before ack: leave the pending entry and lease. + return + if code: + record_failure(sessions, job_id, token, + {20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'} + .get(code, 'processing_failed'), permanent=code in (20, 21)) + otel.trace.get_current_span().set_status(otel.Status(otel.StatusCode.ERROR)) + else: + with sessions() as session: + job = session.get(AttachmentJob, job_id) + attachment = session.get(Attachment, job.attachment_id) + outcome = 'ready' if attachment.status == 'ready' else 'skipped' + if outcome == 'ready': + otel.time_to_ready.record((attachment.updated_at - attachment.created_at).total_seconds(), + {'job_type': 'attachment.process.v1'}) + finally: + otel.worker_busy.add(-1) + labels = {'job_type': 'attachment.process.v1', 'outcome': outcome} + elapsed = time.monotonic() - started + otel.jobs_processed.add(1, labels) + otel.job_duration.record(elapsed, labels) + otel.event('job_processed', severity=logging.ERROR if outcome == 'failed' else logging.INFO, + **labels, duration_seconds=elapsed) # Only after a durable ready/failed/retry (or an existing durable lease). + acknowledge(client, message_id) + + +def acknowledge(client, message_id): with client.pipeline(transaction=True) as pipe: pipe.xack(STREAM, GROUP, message_id) pipe.xdel(STREAM, message_id) @@ -58,12 +103,13 @@ def consume(sessions): for message_id, fields in messages: handle(sessions, client, message_id, fields) except Exception as exc: - logging.warning('Worker deferred: %s', type(exc).__name__) + otel.event('worker_deferred', error_type=type(exc).__name__) time.sleep(1) def main(): logging.basicConfig(level=logging.INFO) + otel.setup() engine = create_db_engine() sessions = sessionmaker(engine, expire_on_commit=False) concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2')))) diff --git a/student-tests/observability.py b/student-tests/observability.py new file mode 100644 index 0000000..28755ec --- /dev/null +++ b/student-tests/observability.py @@ -0,0 +1,290 @@ +"""Lab 6 acceptance: real signals, two incidents, retry propagation and bounded outage. + +Uses only the standard library. Stops services only in this Compose project and +always starts them again; never deletes volumes. Run separately from other tests. +""" +import argparse +import json +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + +PROM = 'http://127.0.0.1:9090' +LOKI = 'http://127.0.0.1:3100' +TEMPO = 'http://127.0.0.1:3200' + + +def stamp(): + return datetime.now(timezone.utc).isoformat() + + +def get(url, **params): + if params: + url += '?' + urllib.parse.urlencode(params) + with urllib.request.urlopen(urllib.request.Request(url, headers={'Accept': 'application/json'}), timeout=10) as r: + return json.load(r) + + +def query(expr): + data = get(PROM + '/api/v1/query', query=expr) + check(data['status'] == 'success', 'PromQL failed') + return data['data']['result'] + + +def number(expr): + values = query(expr) + return float(values[0]['value'][1]) if values else 0.0 + + +def wait(label, fn, timeout=90): + end = time.monotonic() + timeout + last = None + while time.monotonic() < end: + try: + value = fn() + if value: + return value + except (urllib.error.URLError, TimeoutError, AssertionError) as exc: + last = type(exc).__name__ + time.sleep(2) + raise AssertionError(f'Timeout: {label} ({last})') + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=90) + + +def trace_data(trace_id): + data = get(TEMPO + '/api/traces/' + trace_id) + spans = [] + for batch in data.get('batches', data.get('resourceSpans', [])): + resource = {a['key']: a['value'].get('stringValue') for a in batch.get('resource', {}).get('attributes', [])} + for scope in batch.get('scopeSpans', batch.get('instrumentationLibrarySpans', [])): + for span in scope.get('spans', []): + spans.append({**span, 'service': resource.get('service.name')}) + return data, spans + + +def logs(trace_id): + return get(LOKI + '/loki/api/v1/query_range', + query='{service_name=~"messenger-.*"} | trace_id="' + trace_id + '"', + start=str(time.time_ns() - 3_600_000_000_000), limit=1000)['data']['result'] + + +def attrs(span): + return {a['key']: next(iter(a['value'].values())) for a in span.get('attributes', [])} + + +def pipeline(trace_id, *, retry=False): + data, spans = trace_data(trace_id) + names = {s['name'] for s in spans} + required = {'POST /api/v1/chats/{chat_id}/attachments', 'attachment-jobs publish', + 'attachment-jobs process', 'image.process', 's3.GetObject', 's3.PutObject', 'db.UPDATE'} + if not required <= names: + return None + by_id = {s['spanId']: s for s in spans} + for name, parent_name in [('attachment-jobs publish', 'POST /api/v1/chats/{chat_id}/attachments'), + ('attachment-jobs process', 'attachment-jobs publish'), + ('image.process', 'attachment-jobs process')]: + for s in (s for s in spans if s['name'] == name): + check(by_id.get(s.get('parentSpanId'), {}).get('name') == parent_name, + 'Broken trace parent for ' + name) + check(s.get('traceState') == 'lab=check', 'W3C tracestate lost at ' + name) + attempts = sorted({int(attrs(s)['job.attempt']) for s in spans + if s['name'] == 'attachment-jobs process'}) + if retry and len(attempts) < 2: + return None + records = logs(trace_id) + services = {r['stream'].get('service_name') for r in records} + if not {'messenger-api', 'messenger-worker', 'messenger-dispatcher'} <= services: + return None + return {'trace_id': trace_id, 'span_count': len(spans), 'span_names': sorted(names), + 'log_services': sorted(services), 'attempts': attempts, 'parent_chain_verified': True, + 'tracestate_preserved': True, + '_raw': json.dumps([data, records])} + + +class Scenario: + def __init__(self, base_url): + self.client = Client(base_url) + self.alice, _, _, self.chat = self.client.bootstrap() + self.private_markers = [self.alice['token'], self.alice['registration']['password']] + + def upload(self): + trace_id, request_id = uuid.uuid4().hex, str(uuid.uuid4()) + filename = 'private_filename_' + uuid.uuid4().hex + '.png' + self.private_markers.append(filename) + result = self.client.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=' + filename, + 202, actor=self.alice, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), + headers={'Content-Type': 'image/png', 'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01', + 'tracestate': 'lab=check', 'X-Request-Id': request_id}, schema='Attachment') + check(result['headers']['X-Request-Id'] == request_id, 'Request ID changed') + return result['json'], trace_id, request_id + + def check_pipeline(self, trace_id, request_id, retry=False): + result = wait('pipeline spans/logs', lambda: pipeline(trace_id, retry=retry)) + raw = result.pop('_raw') + check(all(value not in raw for value in self.private_markers), 'Private input leaked into telemetry') + check(request_id in raw, 'Request ID is missing from trace/log correlation') + result['request_id'] = request_id + result['private_inputs_absent'] = True + return result + + def business(self, expected=200, trace_id=None): + headers = {'traceparent': f'00-{trace_id}-{uuid.uuid4().hex[:16]}-01'} if trace_id else {} + return self.client.call('GET', '/api/v1/chats/' + self.chat['id'], expected, + actor=self.alice, headers=headers, schema='Chat' if expected == 200 else None) + + def smoke(self): + att, trace_id, request_id = self.upload() + self.client.wait_ready(self.alice, att) + result = self.check_pipeline(trace_id, request_id) + names = ['http_requests_total', 'http_duration_seconds_bucket', 'jobs_processed_total', + 'job_duration_seconds_bucket', 'attachment_time_to_ready_seconds_bucket', + 'queue_depth', 'queue_oldest_age_seconds', 'db_pool_used', 'db_pool_waiting', + 'worker_busy', 'worker_capacity', 'container_cpu_seconds_total', 'container_memory_bytes'] + for name in names: + wait(name, lambda name=name: query(name)) + series = get(PROM + '/api/v1/series', **{'match[]': '{job="messenger"}', 'start': str(time.time() - 300)})['data'] + forbidden = {'user_id', 'chat_id', 'message_id', 'attachment_id', 'request_id', 'filename', 'url'} + check(all(not forbidden.intersection(row) for row in series), 'High-cardinality metric label') + check(all(self.chat['id'] not in json.dumps(row) for row in series), 'Raw route ID in metrics') + result['metrics_present'] = names + return result + + def queue(self): + result = {'started_at': stamp()} + compose('stop', 'worker') + try: + att, trace_id, request_id = self.upload() + self.client.message(self.alice, self.chat, 'queued attachment', attachments=[att['id']]) + wait('QueueLagHigh firing', lambda: number('ALERTS{alertname="QueueLagHigh",alertstate="firing"}') == 1) + result.update(firing_at=stamp(), depth=number('max(queue_depth)'), + oldest_age_seconds=number('max(queue_oldest_age_seconds)')) + finally: + compose('start', 'worker') + ready = self.client.wait_ready(self.alice, att) + wait('queue alert resolved', lambda: not query('ALERTS{alertname="QueueLagHigh"}')) + result.update(resolved_at=stamp(), ready=ready['status'], depth_after=number('max(queue_depth)'), + pipeline=self.check_pipeline(trace_id, request_id)) + return result + + def database(self): + result = {'started_at': stamp()} + trace_id = uuid.uuid4().hex + compose('stop', 'postgres') + try: + errors = 0 + def load_until_firing(): + nonlocal errors + self.business(expected=503, trace_id=trace_id if errors == 0 else None) + errors += 1 + return number('ALERTS{alertname="ApiHighErrorRate",alertstate="firing"}') == 1 + wait('ApiHighErrorRate firing under sustained traffic', load_until_firing, timeout=120) + result['failed_requests'] = errors + result.update(firing_at=stamp(), error_ratio=number('messenger:http_error_ratio:2m')) + def failed_trace(): + _, spans = trace_data(trace_id) + failed = [s for s in spans if s['name'].startswith('db.') + and s.get('status', {}).get('code') in (2, 'STATUS_CODE_ERROR')] + return [{'name': s['name'], 'error_type': attrs(s).get('error.type'), + 'duration_seconds': (int(s['endTimeUnixNano']) - int(s['startTimeUnixNano'])) / 1e9} + for s in failed] + result['db_error_spans'] = wait('failed DB span', failed_trace) + result['trace_id'] = trace_id + result['correlated_log_streams'] = len(wait('DB error logs', lambda: logs(trace_id))) + finally: + compose('start', 'postgres') + wait('API recovered', self.business) + for _ in range(10): + self.business() + wait('API alert resolved', lambda: not query('ALERTS{alertname="ApiHighErrorRate"}'), timeout=160) + result['resolved_at'] = stamp() + return result + + def retry(self): + failures = number('sum(jobs_processed_total{outcome="failed"})') + compose('stop', 'worker') + try: + att, trace_id, request_id = self.upload() + compose('stop', 's3') + # `compose start worker` can also start its dependency graph, including S3. + compose('up', '-d', '--no-deps', '--no-recreate', 'worker') + wait('failed processing attempt', lambda: number('sum(jobs_processed_total{outcome="failed"})') > failures, timeout=40) + finally: + compose('start', 's3', 'worker') + wait('S3/API ready', lambda: self.client.call('GET', '/health/ready', schema='Health')) + self.client.wait_ready(self.alice, att) + return self.check_pipeline(trace_id, request_id, retry=True) + + def backend(self): + accepted = 'sum(otelcol_receiver_accepted_spans_total)' + sent = 'sum(otelcol_exporter_sent_spans_total{exporter="otlphttp/tempo"})' + queue = 'sum(otelcol_exporter_queue_size{exporter="otlphttp/tempo"})' + # A quiet interval on both ends makes accepted-minus-sent a loss count, + # not a count of failed export attempts or currently buffered spans. + time.sleep(8) + wait('trace queue drained', lambda: number(queue) == 0) + before = {'accepted': number(accepted), 'sent': number(sent)} + result = {'started_at': stamp(), 'before': before} + samples, writes = [], 0 + compose('stop', 'tempo') + try: + end = time.monotonic() + 60 + att, _, _ = self.upload() + self.client.wait_ready(self.alice, att) + while time.monotonic() < end: + body = 'private_message_' + uuid.uuid4().hex + self.private_markers.append(body) + self.client.message(self.alice, self.chat, body) + writes += 1 + samples.append({'at': stamp(), 'collector_rss_bytes': number('otelcol_process_memory_rss_bytes'), + 'queue_batches': number(queue)}) + time.sleep(0.3) + check(writes > 10, 'Not enough writes during telemetry outage') + check(max(s['queue_batches'] for s in samples) <= 32, 'Collector queue is unbounded') + check(0 < max(s['collector_rss_bytes'] for s in samples) < 256 * 1024 * 1024, + 'Collector exceeded its memory budget') + finally: + compose('start', 'tempo') + wait('trace queue drained', lambda: number(queue) == 0) + time.sleep(10) # Include final Prometheus scrape after drain. + after = {'accepted': number(accepted), 'sent': number(sent)} + lost = (after['accepted'] - before['accepted']) - (after['sent'] - before['sent']) + check(lost >= 0, 'Loss window was not quiescent; rerun backend separately') + result.update(after=after, lost_accepted_spans=lost, successful_writes=writes, + attachment_ready_during_outage=True, samples=samples, recovered_at=stamp()) + att, trace_id, request_id = self.upload() + self.client.wait_ready(self.alice, att) + result['recovered_pipeline'] = self.check_pipeline(trace_id, request_id) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--scenario', choices=['smoke', 'queue', 'database', 'retry', 'backend', 'all'], default='smoke') + parser.add_argument('--base-url', default='http://127.0.0.1:8080') + args = parser.parse_args() + scenario = Scenario(args.base_url) + path = ROOT / 'evidence/observability.json' + evidence = json.loads(path.read_text(encoding='utf-8')) if path.exists() else {} + for name in (['smoke', 'queue', 'database', 'retry', 'backend'] if args.scenario == 'all' else [args.scenario]): + print('Running ' + name, flush=True) + evidence[name] = {'checked_at': stamp(), **getattr(scenario, name)()} + path.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') + print('PASS: ' + name, flush=True) + + +if __name__ == '__main__': + main() -- 2.54.0