From ac2d74b74abd8ac81921d41ebcc06a379178f4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D0=B0=D1=81=D1=82=D0=B0=D1=81=D0=B8=D1=8F=20?= =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D0=B8=D0=BD=D0=B5=D1=81?= Date: Fri, 20 Nov 2026 19:12:20 +0300 Subject: [PATCH 1/2] Add observability metrics logs and Grafana provisioning --- .gitattributes | 5 + Dockerfile | 14 + REPORT.md | 53 ++- compose.yaml | 163 +++++++++ infra/grafana/dashboards/messenger.json | 1 + infra/grafana/dashboards/provider.yml | 6 + infra/grafana/datasources/prometheus.yml | 7 + infra/haproxy.cfg | 34 ++ infra/prometheus.yml | 7 + infra/s3-init.sh | 16 + infra/s3-policy.json | 1 + migrations/001_initial.sql | 34 ++ migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 23 ++ migrations/004_jobs.sql | 3 + requirements.txt | 22 ++ scripts/action.py | 21 ++ scripts/bootstrap.py | 26 ++ scripts/restart.cmd | 3 + scripts/restart.sh | 4 + scripts/stop-one.cmd | 3 + scripts/stop-one.sh | 4 + scripts/verify.py | 79 +++++ src/__init__.py | 0 src/app.py | 433 +++++++++++++++++++++++ src/attachments.py | 123 +++++++ src/cleanup.py | 37 ++ src/db.py | 52 +++ src/storage.py | 10 + src/telemetry.py | 22 ++ src/worker.py | 60 ++++ student-tests/attachments.py | 60 ++++ student-tests/replication.py | 101 ++++++ 33 files changed, 1408 insertions(+), 29 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/dashboards/provider.yml create mode 100644 infra/grafana/datasources/prometheus.yml create mode 100644 infra/haproxy.cfg create mode 100644 infra/prometheus.yml create mode 100755 infra/s3-init.sh create mode 100644 infra/s3-policy.json create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_jobs.sql create mode 100644 requirements.txt create mode 100644 scripts/action.py create mode 100644 scripts/bootstrap.py create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh create mode 100644 scripts/verify.py create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/attachments.py create mode 100644 src/cleanup.py create mode 100644 src/db.py create mode 100644 src/storage.py create mode 100644 src/telemetry.py create mode 100644 src/worker.py create mode 100644 student-tests/attachments.py create mode 100644 student-tests/replication.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f49c19e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..806a906 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.12.8-slim-bookworm +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app +COPY src/ src/ +COPY migrations/ migrations/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json . +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 +USER app +EXPOSE 8080 +CMD ["gunicorn", "--bind", "0.0.0.0:8080", "--workers", "1", "--threads", "4", "--timeout", "30", "--graceful-timeout", "5", "src.app:app"] diff --git a/REPORT.md b/REPORT.md index b97ae42..760525b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,32 @@ -# Отчёт — лабораторная 6 +# Лабораторная 6 — отчёт -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Мартинес Анастасия, БПИ2303. Целевая оценка: 4. Трек 5 не заявлен. -## Запуск из чистого clone +Lab6 переносит API, worker, S3, outbox и HAProxy из lab5 и добавляет +Prometheus/Grafana provisioning. API публикует /metrics с RPS, status, +histogram duration; structured JSON logs содержат request_id и route. +Dashboard содержит RPS и p95 histogram quantile. Cardinality ограничена +методом, route и status; user/chat/message IDs в labels не используются. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Запуск: -## Архитектура и решения + python scripts/bootstrap.py + docker compose config --quiet + docker compose up -d --build + python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Grafana доступна на 127.0.0.1:3001, Prometheus на 127.0.0.1:9090. Конфигурации +и provisioning лежат в infra/. Telemetry backend не является зависимостью API: +остановка Prometheus или Grafana не должна ломать запись сообщений/attachments. +Volumes promdata/grafanadata сохраняют историю и dashboard state. -## Доказательства критериев +Проверено compileall, contract check и перенесённый lab5 smoke. Полный pull +образов Grafana/Prometheus зависит от доступности Docker registry и запускается +тем же compose без изменения приложения. Нагрузка lab5 сохраняет evidence; +metrics endpoint проверяется отдельно curl GET /metrics. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Ограничения: один Collector/backend host, sampling и alert delivery наружу не +заявлены; это учебный локальный стенд. Среднее latency не используется вместо +p95/p99. Секреты, тела сообщений и изображения в telemetry не записываются. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. - -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Помощь ИИ: Codex подготовил код, конфигурации, проверку и отчёт. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..ff0b7ce --- /dev/null +++ b/compose.yaml @@ -0,0 +1,163 @@ +name: vp-lab6 +x-api: &api + build: . + image: vp-lab4-api + environment: &app-env + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://minio:9000 + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + volumes: + - ./secrets/database_url:/run/secrets/database_url:ro + - ./secrets/s3_access_key:/run/secrets/s3_access_key:ro + - ./secrets/s3_secret_key:/run/secrets/s3_secret_key:ro + depends_on: + migrate: + condition: service_completed_successfully + s3init: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"] + interval: 3s + timeout: 5s + retries: 20 + stop_grace_period: 8s + mem_limit: 384m + cpus: 1 +services: + api1: + <<: *api + environment: + <<: *app-env + INSTANCE_ID: api1 + api2: + <<: *api + environment: + <<: *app-env + INSTANCE_ID: api2 + migrate: + build: . + image: vp-lab4-api + command: [python, -m, src.db] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + volumes: + - ./secrets/database_url:/run/secrets/database_url:ro + depends_on: + db: + condition: service_healthy + mem_limit: 128m + cpus: 0.5 + lb: + image: haproxy:3.0.6-alpine + ports: ["127.0.0.1:8080:8080", "127.0.0.1:9000:9000"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + mem_limit: 128m + cpus: 0.5 + minio: + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: [server, /data, --console-address, ':9001'] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + volumes: + - s3data:/data + - ./secrets/s3_root_user:/run/secrets/s3_root_user:ro + - ./secrets/s3_root_password:/run/secrets/s3_root_password:ro + mem_limit: 512m + cpus: 1 + s3init: + image: minio/mc@sha256:a7fe349ef4bd8521fb8497f55c6042871b2ae640607cf99d9bede5e9bdf11727 + entrypoint: [/bin/sh, /bootstrap/s3-init.sh] + volumes: + - ./infra/s3-init.sh:/bootstrap/s3-init.sh:ro + - ./infra/s3-policy.json:/bootstrap/policy.json:ro + - ./secrets/s3_root_user:/run/secrets/s3_root_user:ro + - ./secrets/s3_root_password:/run/secrets/s3_root_password:ro + - ./secrets/s3_access_key:/run/secrets/s3_access_key:ro + - ./secrets/s3_secret_key:/run/secrets/s3_secret_key:ro + depends_on: [minio] + mem_limit: 128m + cpus: 0.5 + cleanup: + <<: *api + command: [python, -m, src.cleanup, --apply, --loop] + healthcheck: + test: [CMD, python, -c, "import sys; sys.exit(0)"] + depends_on: + lb: + condition: service_started + mem_limit: 128m + worker: + <<: *api + command: [python, -m, src.worker] + healthcheck: + test: [CMD, python, -c, "import os; os.kill(os.getpid(), 0)"] + depends_on: + lb: + condition: service_started + mem_limit: 512m + cpus: 1 + prometheus: + image: prom/prometheus:v3.1.0 + ports: ["127.0.0.1:9090:9090"] + volumes: + - ./infra/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - promdata:/prometheus + mem_limit: 256m + cpus: 0.5 + grafana: + image: grafana/grafana:11.4.0 + ports: ["127.0.0.1:3001:3000"] + volumes: + - grafanadata:/var/lib/grafana + - ./infra/grafana:/etc/grafana/provisioning:ro + depends_on: [prometheus] + mem_limit: 256m + cpus: 0.5 + db: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + volumes: + - pgdata:/var/lib/postgresql/data + - ./secrets/db_password:/run/secrets/db_password:ro + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 512m + cpus: 1 + redis: + image: redis:7.4.1-bookworm + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + mem_limit: 128m + cpus: 0.5 +volumes: + pgdata: + s3data: + promdata: + grafanadata: +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url + s3_root_user: + file: ./secrets/s3_root_user + s3_root_password: + file: ./secrets/s3_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..3ba6859 --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1 @@ +{"title":"Messenger API","schemaVersion":39,"version":1,"panels":[{"type":"timeseries","title":"RPS","targets":[{"expr":"sum(rate(messenger_http_requests_total[1m]))"}]},{"type":"timeseries","title":"p95 latency","targets":[{"expr":"histogram_quantile(0.95,sum(rate(messenger_http_request_duration_seconds_bucket[5m])) by (le))"}]}]} diff --git a/infra/grafana/dashboards/provider.yml b/infra/grafana/dashboards/provider.yml new file mode 100644 index 0000000..ff3fb29 --- /dev/null +++ b/infra/grafana/dashboards/provider.yml @@ -0,0 +1,6 @@ +apiVersion: 1 +providers: + - name: messenger + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/infra/grafana/datasources/prometheus.yml b/infra/grafana/datasources/prometheus.yml new file mode 100644 index 0000000..0eddf26 --- /dev/null +++ b/infra/grafana/datasources/prometheus.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..2ed8140 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,34 @@ +global + maxconn 512 +defaults + mode http + timeout connect 1s + timeout client 30s + timeout server 30s + timeout check 1s + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s +frontend ingress + bind :8080 + default_backend api +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server api1 api1:8080 check + server api2 api2:8080 check +frontend object_ingress + bind :9000 + default_backend objects +backend objects + option httpchk GET /minio/health/live + http-check expect status 200 + server storage minio:9000 check inter 1s fall 1 rise 1 resolvers docker init-addr libc,none diff --git a/infra/prometheus.yml b/infra/prometheus.yml new file mode 100644 index 0000000..66a28e0 --- /dev/null +++ b/infra/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 5s +scrape_configs: + - job_name: api + metrics_path: /metrics + static_configs: + - targets: ['api1:8080','api2:8080'] diff --git a/infra/s3-init.sh b/infra/s3-init.sh new file mode 100755 index 0000000..87fdc40 --- /dev/null +++ b/infra/s3-init.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +root_user=$(cat /run/secrets/s3_root_user) +root_password=$(cat /run/secrets/s3_root_password) +app_user=$(cat /run/secrets/s3_access_key) +app_password=$(cat /run/secrets/s3_secret_key) +attempt=0 +until mc alias set local http://minio:9000 "$root_user" "$root_password" >/dev/null 2>&1; do + attempt=$((attempt+1)); [ "$attempt" -lt 60 ] || exit 1; sleep 1 +done +mc mb --ignore-existing local/messenger >/dev/null +mc anonymous set none local/messenger >/dev/null +mc admin user add local "$app_user" "$app_password" >/dev/null +mc admin policy create local messenger-app /bootstrap/policy.json >/dev/null +mc admin policy attach local messenger-app --user "$app_user" >/dev/null +echo 'Private bucket and scoped application identity ready' diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 0000000..5467996 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation"],"Resource":["arn:aws:s3:::messenger"]},{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger/original/*","arn:aws:s3:::messenger/staging/*","arn:aws:s3:::messenger/variants/*"]}]} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 0000000..093ce41 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (length(trim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(trim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX members_user_chat ON members(user_id, chat_id); +CREATE INDEX chats_history ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL CHECK (length(trim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id, user_id) +); +CREATE INDEX messages_history ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..f23ab30 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key varchar(128) NOT NULL, + request jsonb NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL DEFAULT (clock_timestamp() + interval '24 hours'), + PRIMARY KEY (user_id,chat_id,key) +); +CREATE INDEX idempotency_expiry ON idempotency(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..c52b7f2 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,23 @@ +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (length(trim(text)) > 0 OR cardinality(attachment_ids) > 0); +CREATE TABLE object_intents ( + object_key text PRIMARY KEY, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL, + status text NOT NULL DEFAULT 'ready', + variants jsonb NOT NULL DEFAULT '[]', + error_code text, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX attachments_chat ON attachments(chat_id); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 0000000..d705789 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,3 @@ +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; +CREATE TABLE attachment_jobs (attachment_id uuid PRIMARY KEY REFERENCES attachments(id), pipeline_version integer NOT NULL DEFAULT 1, attempts integer NOT NULL DEFAULT 0, available_at timestamptz NOT NULL DEFAULT clock_timestamp(), locked_until timestamptz, last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), completed_at timestamptz); +CREATE INDEX attachment_jobs_ready ON attachment_jobs(available_at) WHERE completed_at IS NULL; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5bcdf1b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +Flask==3.1.0 +gunicorn==23.0.0 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 +# Transitive dependencies pinned to the verified Linux/Python 3.12 environment. +argon2-cffi-bindings==26.2.0 +blinker==1.9.0 +cffi==2.1.1 +click==8.5.1 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.4 +packaging==26.3 +psycopg-binary==3.2.3 +psycopg-pool==3.3.3 +pycparser==3.12 +typing_extensions==4.16.0 +Werkzeug==3.1.9 +boto3==1.35.99 +Pillow==11.1.0 +prometheus-client==0.21.1 diff --git a/scripts/action.py b/scripts/action.py new file mode 100644 index 0000000..8c29f02 --- /dev/null +++ b/scripts/action.py @@ -0,0 +1,21 @@ +"""Explicit local Compose failure hooks. Never deletes volumes.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +mode = sys.argv[1] +if mode == 'restart': + services = subprocess.check_output(['docker', 'compose', 'ps', '--services', '--status', 'running'], text=True).split() + subprocess.run(['docker', 'compose', 'restart', *services], check=True) +elif mode == 'stop-one': + alias = os.environ['TARGET_INSTANCE_ID'] + if alias not in ('api1', 'api2'): + raise SystemExit('Unknown API alias') + Path('evidence/last-stopped.json').write_text(json.dumps({'service': alias}), encoding='utf-8') + subprocess.run(['docker', 'compose', 'stop', '-t', '2', alias], check=True) +else: + raise SystemExit('Unknown action') diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..9990814 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,26 @@ +"""Create local synthetic secrets once; never overwrite existing credentials.""" +import secrets +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(exist_ok=True) +password_file = directory / 'db_password' +if not password_file.exists(): + with password_file.open('x', encoding='utf-8') as file: + file.write(secrets.token_urlsafe(32)) +password = password_file.read_text(encoding='utf-8').strip() +url_file = directory / 'database_url' +if not url_file.exists(): + with url_file.open('x', encoding='utf-8') as file: + file.write('postgresql://messenger:' + password + '@db:5432/messenger') +env = root / '.env' +if not env.exists(): + with env.open('x', encoding='utf-8') as file: + file.write('SESSION_TTL_SECONDS=3600\n') +print('Local configuration ready; credentials are not printed.') +for name in ('s3_root_user','s3_root_password','s3_access_key','s3_secret_key'): + path = directory / name + if not path.exists(): + with path.open('x', encoding='utf-8') as file: + file.write(secrets.token_hex(16)) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..82d4fb9 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts/action.py restart diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..2651a49 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/action.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..97bf88d --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts/action.py stop-one diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..2448345 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/action.py stop-one diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 0000000..a5b1dad --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,79 @@ +"""Run public and student acceptance, preserving sanitized evidence.""" +import json +import os +import platform +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +EVIDENCE = ROOT / 'evidence' +ENV = {**os.environ, 'PYTHONUTF8': '1', 'BASE_URL': os.getenv('BASE_URL', 'http://127.0.0.1:8080')} + + +def run(args, artifact): + result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, + encoding='utf-8', errors='replace', env=ENV) + output = result.stdout + result.stderr + (EVIDENCE / artifact).write_text(output, encoding='utf-8') + print(output, end='', flush=True) + if result.returncode: + raise SystemExit(result.returncode) + return output + + +def containers(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], text=True).split() + raw = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True)) + return [{'name': x['Name'], 'id': x['Id'], 'started_at': x['State']['StartedAt'], + 'image': x['Config']['Image']} for x in raw] + + +def main(): + run([sys.executable, 'scripts/check_contract.py'], 'contract.txt') + run([sys.executable, 'tests/smoke.py'], 'smoke.txt') + before = containers() + suffix = '.cmd' if os.name == 'nt' else '.sh' + run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart' + suffix], 'persistence.txt') + after = containers() + old = {x['id']: x for x in before} + assert all(x['id'] in old and x['started_at'] != old[x['id']]['started_at'] for x in after) + (EVIDENCE / 'restart.json').write_text(json.dumps({'before':before,'after':after},indent=2),encoding='utf-8') + try: + run([sys.executable, 'tests/resilience.py', '--mode', 'failover', + '--action-script', 'scripts/stop-one' + suffix], 'failover.txt') + finally: + subprocess.run(['docker','compose','up','-d','--no-deps','--wait','api1','api2'], check=True) + time.sleep(2) + run([sys.executable, 'student-tests/replication.py'], 'replication.txt') + if (ROOT / 'student-tests/attachments.py').exists(): + run([sys.executable, 'student-tests/attachments.py'], 'attachments.txt') + if (ROOT / 'student-tests/pipeline.py').exists(): + run([sys.executable, 'student-tests/pipeline.py'], 'pipeline.txt') + if (ROOT / 'student-tests/observability.py').exists(): + run([sys.executable, 'student-tests/observability.py'], 'observability.txt') + process = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4', + '--output','evidence/load.json'], stdout=subprocess.PIPE,stderr=subprocess.STDOUT, + text=True,encoding='utf-8',env=ENV) + ids = [x['id'] for x in containers()] + samples=[] + while process.poll() is None: + stats=subprocess.check_output(['docker','stats','--no-stream','--format', + '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}',*ids],text=True) + samples.append({'at':datetime.now(timezone.utc).isoformat(),'stats':stats.strip()}) + time.sleep(2) + output=process.communicate()[0] + (EVIDENCE/'load.txt').write_text(output,encoding='utf-8') + (EVIDENCE/'resources.json').write_text(json.dumps(samples,indent=2),encoding='utf-8') + assert process.returncode==0, output + (EVIDENCE/'environment.json').write_text(json.dumps({'platform':platform.platform(), + 'cpu':platform.processor(),'logical_cpus':os.cpu_count(),'observed_at':datetime.now(timezone.utc).isoformat()},indent=2),encoding='utf-8') + print(output,flush=True) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/app.py b/src/app.py new file mode 100644 index 0000000..1826f61 --- /dev/null +++ b/src/app.py @@ -0,0 +1,433 @@ +"""Lab 2 API: PostgreSQL is authoritative; Redis caches immutable public profiles.""" +import base64 +import binascii +import hashlib +import json +import os +import re +import secrets +import time +import uuid +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout +from datetime import datetime +from threading import BoundedSemaphore + +import psycopg +import redis +from botocore.exceptions import BotoCoreError, ClientError +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from flask import Flask, g, jsonify, request +from psycopg_pool import PoolTimeout, TooManyRequests +from psycopg.types.json import Jsonb +from werkzeug.exceptions import HTTPException + +from src.db import make_pool, setting +from src.telemetry import HTTP_REQUESTS, HTTP_DURATION, metrics_response, setup + +app = Flask(__name__) +setup() +app.config['MAX_CONTENT_LENGTH'] = 10485760 +pool = make_pool() +cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.2, + socket_timeout=0.2, decode_responses=True) +cache_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix='cache') +cache_slots = BoundedSemaphore(2) +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +dummy_hash = hasher.hash(secrets.token_urlsafe(32)) +SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600')) +if SESSION_TTL < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') +USER_FIELDS = 'id, username, display_name, created_at' +INSTANCE_ID = os.getenv('INSTANCE_ID', uuid.uuid4().hex) + + +class APIError(Exception): + def __init__(self, status, code, message, challenge=None): + self.status, self.code, self.message, self.challenge = status, code, message, challenge + + +def invalid(): + raise APIError(400, 'invalid_request', 'Invalid request') + + +def unauthorized(basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + raise APIError(401, 'unauthorized', 'Authentication required', challenge) + + +def serial(value): + if isinstance(value, dict): + return {key: serial(item) for key, item in value.items()} + if isinstance(value, list): + return [serial(item) for item in value] + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + return value.isoformat().replace('+00:00', 'Z') + return value + + +def response(value, status=200): + return jsonify(serial(value)), status + + +@app.before_request +def identify_request(): + g.request_id = uuid.uuid4().hex + g.started_at = time.monotonic() + + +@app.after_request +def headers(result): + result.headers['X-Request-Id'] = g.request_id + result.headers['X-Instance-Id'] = INSTANCE_ID + result.headers['Cache-Control'] = 'no-store' + route = request.url_rule.rule if request.url_rule else 'unknown' + HTTP_REQUESTS.labels(request.method, route, result.status_code).inc() + HTTP_DURATION.labels(request.method, route).observe(time.monotonic() - g.started_at) + app.logger.info('http_request method=%s route=%s status=%s request_id=%s', request.method, route, result.status_code, g.request_id) + return result + +@app.get('/metrics') +def metrics(): + return metrics_response() + + +@app.errorhandler(Exception) +def error(exc): + if isinstance(exc, APIError): + problem = exc + elif isinstance(exc, psycopg.errors.UniqueViolation): + problem = APIError(409, 'conflict', 'Username already exists') + elif isinstance(exc, psycopg.errors.ForeignKeyViolation): + problem = APIError(400, 'invalid_request', 'Invalid member list') + elif isinstance(exc, (BotoCoreError, ClientError, psycopg.OperationalError, PoolTimeout, TooManyRequests, + psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable)): + problem = APIError(503, 'unavailable', 'Database temporarily unavailable') + elif isinstance(exc, HTTPException): + status = exc.code if exc.code in (400, 404, 413, 415) else 400 + code = {400: 'invalid_request', 404: 'not_found', 413: 'payload_too_large', + 415: 'unsupported_media_type'}[status] + problem = APIError(status, code, 'Request cannot be processed') + else: + app.logger.error('Unhandled error type=%s request_id=%s', type(exc).__name__, g.request_id) + problem = APIError(500, 'internal_error', 'Internal error') + result = jsonify(error={'code': problem.code, 'message': problem.message, + 'request_id': g.request_id}) + result.status_code = problem.status + if problem.challenge: + result.headers['WWW-Authenticate'] = problem.challenge + if problem.status == 429: + result.headers['Retry-After'] = '1' + return result + + +def body(fields): + if request.mimetype != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Expected application/json') + value = request.get_json() + if not isinstance(value, dict) or set(value) != set(fields): + invalid() + return value + + +def string(value, minimum, maximum): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + if '\x00' in value: + invalid() + return value + + +def uid(value): + try: + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value): + invalid() + return uuid.UUID(value) + except ValueError: + invalid() + + +def cached_operation(operation, *args, **kwargs): + # Socket timeouts do not bound DNS resolution when a Compose service is down. + # At most two background operations per worker, no unbounded queue. + if not cache_slots.acquire(blocking=False): + return None + future = cache_executor.submit(operation, *args, **kwargs) + future.add_done_callback(lambda _: cache_slots.release()) + try: + return future.result(timeout=0.25) + except (redis.RedisError, FutureTimeout): + return None + + +def profile(conn, user_id): + key = 'profile:v1:' + str(user_id) + cached = cached_operation(cache.get, key) + if cached: + try: + return json.loads(cached) + except ValueError: + pass + user = serial(conn.execute('SELECT ' + USER_FIELDS + ' FROM users WHERE id=%s', + (user_id,)).fetchone()) + if user is None: + unauthorized() + cached_operation(cache.set, key, json.dumps(user), ex=60) + return user + + +def actor(conn): + auth = request.headers.get('Authorization', '').split() + if len(auth) != 2 or auth[0].lower() != 'bearer' or len(auth[1]) > 256: + unauthorized() + digest = hashlib.sha256(auth[1].encode()).hexdigest() + session = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s ' + 'AND expires_at > clock_timestamp()', (digest,)).fetchone() + if not session: + unauthorized() + g.token_hash = digest + return session['user_id'] + + +def chat_access(conn, chat_id, user_id): + chat = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id ' + 'WHERE c.id=%s AND m.user_id=%s', (chat_id, user_id)).fetchone() + if not chat: + raise APIError(404, 'not_found', 'Resource not found') + return chat + + +def chat_members(conn, chat): + chat['member_ids'] = [row['user_id'] for row in conn.execute( + 'SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))] + return chat + + +def page_args(scope): + limit = request.args.get('limit', '50') + if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100: + invalid() + cursor = request.args.get('cursor') + after = None + if cursor is not None: + try: + if len(cursor) > 1024: + invalid() + data = json.loads(base64.b64decode(cursor, altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + invalid() + date = datetime.fromisoformat(data[1]) + if date.tzinfo is None: + invalid() + after = (date, uid(data[2])) + except (ValueError, TypeError, binascii.Error): + invalid() + return int(limit), after + + +def page(rows, limit, scope): + more = len(rows) > limit + items = rows[:limit] + cursor = None + if more: + last = serial(items[-1]) + cursor = base64.urlsafe_b64encode(json.dumps( + [scope, last['created_at'], last['id']]).encode()).decode() + return {'items': items, 'next_cursor': cursor} + + +@app.get('/health/live') +def live(): + return response({'status': 'ok'}) + + +@app.get('/health/ready') +def ready(): + with pool.connection() as conn: + conn.execute('SELECT 1 FROM schema_migrations LIMIT 1') + return response({'status': 'ok'}) + + +@app.post('/api/v1/users') +def register(): + data = body(('username', 'display_name', 'password')) + name = string(data['username'], 3, 32) + if not re.fullmatch('[a-z0-9_]{3,32}', name): + invalid() + display = string(data['display_name'], 1, 100) + password = string(data['password'], 12, 128) + encoded = hasher.hash(password) + with pool.connection() as conn: + user = conn.execute('INSERT INTO users (id,username,display_name,password_hash) ' + 'VALUES (%s,%s,%s,%s) RETURNING ' + USER_FIELDS, + (uuid.uuid4(), name, display, encoded)).fetchone() + return response(user, 201) + + +@app.post('/api/v1/auth/sessions') +def login(): + auth = request.headers.get('Authorization', '').split() + if len(auth) != 2 or auth[0].lower() != 'basic' or len(auth[1]) > 2048: + unauthorized(True) + try: + name, password = base64.b64decode(auth[1], validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError, binascii.Error): + unauthorized(True) + if not re.fullmatch('[a-z0-9_]{3,32}', name) or not 12 <= len(password) <= 128: + unauthorized(True) + if request.get_data(): + invalid() + with pool.connection() as conn: + user = conn.execute('SELECT * FROM users WHERE username=%s', (name,)).fetchone() + try: + hasher.verify(user['password_hash'] if user else dummy_hash, password) + except VerificationError: + unauthorized(True) + if not user: + unauthorized(True) + token = secrets.token_urlsafe(32) + with pool.connection() as conn: + expiry = conn.execute('INSERT INTO sessions VALUES (%s,%s,clock_timestamp() ' + '+ %s * interval \'1 second\') RETURNING expires_at', + (hashlib.sha256(token.encode()).hexdigest(), user['id'], + SESSION_TTL)).fetchone()['expires_at'] + user.pop('password_hash') + return response({'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user}, 201) + + +@app.delete('/api/v1/auth/sessions/current') +def logout(): + with pool.connection() as conn: + actor(conn) + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (g.token_hash,)) + return '', 204 + + +@app.get('/api/v1/users/me') +def me(): + with pool.connection() as conn: + user = profile(conn, actor(conn)) + return response(user) + + +@app.post('/api/v1/chats') +def create_chat(): + with pool.connection() as conn: + user = actor(conn) + data = body(('title', 'member_ids')) + title = string(data['title'], 1, 100) + if not isinstance(data['member_ids'], list) or not 1 <= len(data['member_ids']) <= 100: + invalid() + members = [uid(item) for item in data['member_ids']] + if len(set(members)) != len(members) or user not in members: + invalid() + chat = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *', + (uuid.uuid4(), title)).fetchone() + for member in members: + conn.execute('INSERT INTO members VALUES (%s,%s)', (chat['id'], member)) + chat['member_ids'] = sorted(members) + return response(chat, 201) + + +@app.get('/api/v1/chats') +def list_chats(): + with pool.connection() as conn: + user = actor(conn) + scope = 'chats:' + str(user) + limit, after = page_args(scope) + sql = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s' + params = [user] + if after: + sql += ' AND (c.created_at,c.id) > (%s,%s)' + params.extend(after) + rows = conn.execute(sql + ' ORDER BY c.created_at,c.id LIMIT %s', params + [limit+1]).fetchall() + result = page(rows, limit, scope) + for chat in result['items']: + chat_members(conn, chat) + return response(result) + + +@app.get('/api/v1/chats/') +def get_chat(chat_id): + with pool.connection() as conn: + user = actor(conn) + result = chat_members(conn, chat_access(conn, uid(chat_id), user)) + return response(result) + + +@app.route('/api/v1/chats//messages', methods=['GET', 'POST']) +def messages(chat_id): + with pool.connection() as conn: + user = actor(conn) + chat_id = uid(chat_id) + chat_access(conn, chat_id, user) + if request.method == 'POST': + if request.mimetype != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Expected application/json') + data = request.get_json() + if not isinstance(data, dict) or 'text' not in data or set(data) - {'text', 'attachment_ids'}: + invalid() + text = data['text'] + if not isinstance(text, str) or len(text) > 4000 or '\x00' in text: + invalid() + try: + text.encode('utf-8') + except UnicodeError: + invalid() + attachments = data.get('attachment_ids', []) + if not isinstance(attachments, list) or len(attachments) > 10: + invalid() + attachments = [uid(item) for item in attachments] + if len(set(attachments)) != len(attachments) or not (text.strip() or attachments): + invalid() + for attachment_id in attachments: + attachment = conn.execute('SELECT status FROM attachments WHERE id=%s AND chat_id=%s', + (attachment_id, chat_id)).fetchone() + if not attachment: + raise APIError(404, 'not_found', 'Attachment not found') + if attachment['status'] == 'failed': + raise APIError(409, 'conflict', 'Attachment failed') + key = request.headers.get('Idempotency-Key', '') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + canonical = {'text': text, 'attachment_ids': [str(item) for item in attachments]} + lock = int.from_bytes(hashlib.sha256(f'{user}:{chat_id}:{key}'.encode()).digest()[:8], + 'big', signed=True) + conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,)) + conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s ' + 'AND expires_at<=clock_timestamp()', (user, chat_id, key)) + prior = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s', + (user, chat_id, key)).fetchone() + if prior: + if prior['request'] != canonical: + raise APIError(409, 'conflict', 'Idempotency key already used with a different message') + return response(prior['response'], 201) + result = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) ' + 'VALUES (%s,%s,%s,%s,%s) RETURNING *', + (uuid.uuid4(), chat_id, user, text, attachments)).fetchone() + conn.execute('INSERT INTO idempotency(user_id,chat_id,key,request,response) ' + 'VALUES (%s,%s,%s,%s,%s)', + (user, chat_id, key, Jsonb(canonical), Jsonb(serial(result)))) + status = 201 + else: + scope = 'messages:' + str(chat_id) + limit, after = page_args(scope) + sql = 'SELECT * FROM messages WHERE chat_id=%s' + params = [chat_id] + if after: + sql += ' AND (created_at,id) > (%s,%s)' + params.extend(after) + rows = conn.execute(sql + ' ORDER BY created_at,id LIMIT %s', params + [limit+1]).fetchall() + result, status = page(rows, limit, scope), 200 + return response(result, status) + + +from src.attachments import register_routes +register_routes(app, pool, actor, chat_access, uid, response, APIError, invalid) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 0000000..f071f71 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,123 @@ +import hashlib +import tempfile +import uuid +import warnings +from threading import BoundedSemaphore + +from flask import Response, request, stream_with_context +from PIL import Image, UnidentifiedImageError +from src.storage import BUCKET, s3 + +MAX_BYTES = 10 * 1024 * 1024 +Image.MAX_IMAGE_PIXELS = 40_000_000 +# Each API worker is bounded by Gunicorn threads; this semaphore prevents an +# unbounded number of uploads without serializing the required concurrent case. +slots = BoundedSemaphore(16) + + +def public(row): + return {k: v for k, v in row.items() if k != 'object_key'} + + +def register_routes(app, pool, actor, chat_access, uid, response, APIError, invalid): + @app.post('/api/v1/chats//attachments') + def upload(chat_id): + with pool.connection() as conn: + user = actor(conn) + chat_id = uid(chat_id) + chat_access(conn, chat_id, user) + filename = request.args.get('filename', '') + if not 1 <= len(filename) <= 255 or not filename.strip() or any( + ord(c) < 32 or ord(c) == 127 or c in '/\\' for c in filename): + invalid() + if request.mimetype not in ('image/png', 'image/jpeg'): + raise APIError(415, 'unsupported_media_type', 'Expected PNG or JPEG') + if request.content_length and request.content_length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Image too large') + if not slots.acquire(timeout=5): + raise APIError(429, 'rate_limited', 'Upload capacity reached') + try: + with tempfile.SpooledTemporaryFile(max_size=1024*1024) as file: + length, digest = 0, hashlib.sha256() + while True: + chunk = request.stream.read(65536) + if not chunk: + break + length += len(chunk) + if length > MAX_BYTES: + raise APIError(413, 'payload_too_large', 'Image too large') + digest.update(chunk) + file.write(chunk) + file.seek(0) + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + with Image.open(file) as image: + if image.width * image.height > 40_000_000: + raise APIError(413, 'payload_too_large', 'Too many pixels') + if Image.MIME.get(image.format) != request.mimetype: + raise APIError(415, 'unsupported_media_type', 'Image type mismatch') + image.verify() + file.seek(0) + with Image.open(file) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise APIError(413, 'payload_too_large', 'Too many pixels') from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, 'unsupported_media_type', 'Invalid image') from None + file.seek(0) + identifier = uuid.uuid4() + key = 'original/' + str(identifier) + with pool.connection() as conn: + conn.execute('INSERT INTO object_intents(object_key) VALUES (%s)', (key,)) + s3.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=length, ContentType=request.mimetype) + # Intent survives a crash or failed DB commit; the janitor removes old unreferenced objects. + with pool.connection() as conn: + row = conn.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,status) " + "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING *", + (identifier,chat_id,user,filename,request.mimetype,length,digest.hexdigest(),key)).fetchone() + conn.execute('INSERT INTO attachment_jobs(attachment_id) VALUES (%s)', (identifier,)) + conn.execute('DELETE FROM object_intents WHERE object_key=%s', (key,)) + return response(public(row),202) + finally: + slots.release() + + def authorized(attachment_id): + with pool.connection() as conn: + user = actor(conn) + identifier = uid(attachment_id) + row = conn.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id ' + 'WHERE a.id=%s AND m.user_id=%s', (identifier,user)).fetchone() + if not row: + raise APIError(404,'not_found','Attachment not found') + return row + + @app.get('/api/v1/attachments/') + def metadata(attachment_id): + return response(public(authorized(attachment_id))) + + @app.get('/api/v1/attachments//content') + def download(attachment_id): + row = authorized(attachment_id) + if row['status'] != 'ready': + raise APIError(409,'conflict','Attachment not ready') + variant = request.args.get('variant','original') + key, content_type, size_bytes = row['object_key'], row['content_type'], row['size_bytes'] + if variant != 'original': + found = next((item for item in row['variants'] if item['name'] == variant), None) + if not found: + raise APIError(404,'not_found','Variant not found') + key = row['object_key'] + '.' + variant + content_type, size_bytes = found['content_type'], found['size_bytes'] + obj = s3.get_object(Bucket=BUCKET,Key=key) + def chunks(): + try: + while True: + chunk = obj['Body'].read(65536) + if not chunk: + break + yield chunk + finally: + obj['Body'].close() + return Response(stream_with_context(chunks()),content_type=content_type, + headers={'Content-Length':str(size_bytes),'X-Content-Type-Options':'nosniff'}) diff --git a/src/cleanup.py b/src/cleanup.py new file mode 100644 index 0000000..2afc7bd --- /dev/null +++ b/src/cleanup.py @@ -0,0 +1,37 @@ +"""Old upload intent reconciliation; dry run unless --apply, never deletes fresh objects.""" +import argparse +import json +import time +from src.db import make_pool +from src.storage import s3, BUCKET + + +def sweep(apply=False): + pool = make_pool() + with pool.connection() as conn: + rows = conn.execute("SELECT i.object_key FROM object_intents i LEFT JOIN attachments a " + "ON a.object_key=i.object_key WHERE a.id IS NULL " + "AND i.created_at < clock_timestamp()-interval '1 hour' FOR UPDATE OF i SKIP LOCKED").fetchall() + for row in rows: + if apply: + s3.delete_object(Bucket=BUCKET,Key=row['object_key']) + conn.execute('DELETE FROM object_intents WHERE object_key=%s',(row['object_key'],)) + pool.close() + return {'old_unreferenced_intents':len(rows),'applied':apply} + + +if __name__ == '__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--apply',action='store_true') + parser.add_argument('--loop',action='store_true') + args=parser.parse_args() + while True: + try: + print(json.dumps(sweep(args.apply)),flush=True) + except Exception as exc: + print(json.dumps({'cleanup_error':type(exc).__name__}),flush=True) + if not args.loop: + raise SystemExit(1) + if not args.loop: + break + time.sleep(60) diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..19cf69c --- /dev/null +++ b/src/db.py @@ -0,0 +1,52 @@ +"""Bounded PostgreSQL pool and serialized, versioned migrations.""" +import hashlib +import os +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def setting(name): + path = os.getenv(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def migrate(): + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(202602)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(version text PRIMARY KEY, checksum text NOT NULL)') + for file in sorted(Path('migrations').glob('*.sql')): + sql = file.read_text(encoding='utf-8') + digest = hashlib.sha256(sql.encode()).hexdigest() + row = conn.execute('SELECT checksum FROM schema_migrations WHERE version=%s', + (file.name,)).fetchone() + if row: + if row[0] != digest: + raise RuntimeError('Applied migration changed') + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations VALUES (%s,%s)', (file.name, digest)) + + +def make_pool(): + return ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=8, + timeout=3, max_waiting=32, + check=ConnectionPool.check_connection, + kwargs={'connect_timeout': 3, 'row_factory': dict_row, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}) + + +if __name__ == '__main__': + for attempt in range(30): + try: + migrate() + break + except psycopg.OperationalError: + if attempt == 29: + raise RuntimeError('Database did not become available') from None + time.sleep(2) + print('Migrations applied successfully') diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000..240cd6d --- /dev/null +++ b/src/storage.py @@ -0,0 +1,10 @@ +import os +import boto3 +from botocore.config import Config +from src.db import setting + +BUCKET = os.getenv('S3_BUCKET', 'messenger') +s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://lb:9000'), + aws_access_key_id=setting('S3_ACCESS_KEY'), aws_secret_access_key=setting('S3_SECRET_KEY'), + region_name='us-east-1', config=Config(connect_timeout=2, read_timeout=5, + retries={'max_attempts': 0}, s3={'addressing_style': 'path'})) diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..fac3e95 --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,22 @@ +import json +import logging +import time +from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST + +HTTP_REQUESTS = Counter('messenger_http_requests_total','HTTP requests',['method','route','status']) +HTTP_DURATION = Histogram('messenger_http_request_duration_seconds','HTTP duration',['method','route']) +JOBS = Counter('messenger_attachment_jobs_total','Attachment jobs',['status']) +QUEUE_AGE = Gauge('messenger_attachment_oldest_job_age_seconds','Oldest pending job age') + +class JsonFormatter(logging.Formatter): + def format(self, record): + return json.dumps({'ts': self.formatTime(record), 'level': record.levelname, + 'message': record.getMessage(), 'trace_id': getattr(record,'trace_id',None)}) + +def setup(): + handler=logging.StreamHandler(); handler.setFormatter(JsonFormatter()) + logging.getLogger().handlers[:] = [handler]; logging.getLogger().setLevel(logging.INFO) + +def metrics_response(): + from flask import Response + return Response(generate_latest(), content_type=CONTENT_TYPE_LATEST) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..d691786 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,60 @@ +import hashlib +import io +import json +import random +import time +from PIL import Image, ImageOps +from src.db import make_pool +from src.storage import BUCKET, s3 + +MAX_ATTEMPTS = 5 + +def make_variant(image, name): + image = ImageOps.exif_transpose(image) + if name == 'thumbnail': + image.thumbnail((256, 256), Image.Resampling.LANCZOS); target = image + elif name == 'crop': + target = ImageOps.fit(image, (128, 128), method=Image.Resampling.LANCZOS) + else: + image.thumbnail((1280, 1280), Image.Resampling.LANCZOS); target = image + if target.mode not in ('RGB', 'L'): + bg = Image.new('RGB', target.size, 'white') + bg.paste(target, mask=target.getchannel('A') if 'A' in target.getbands() else None) + target = bg + out = io.BytesIO() + content_type = 'image/webp' if name == 'optimized' else 'image/jpeg' + target.convert('RGB').save(out, format='WEBP' if name == 'optimized' else 'JPEG', quality=82, exif=b'') + raw = out.getvalue() + return raw, content_type, target.width, target.height + +def process(row, pool): + raw = s3.get_object(Bucket=BUCKET, Key=row['object_key'])['Body'].read() + with Image.open(io.BytesIO(raw)) as image: + variants=[] + for name in ('thumbnail','crop','optimized'): + data, content_type, width, height = make_variant(image.copy(), name) + s3.put_object(Bucket=BUCKET, Key=row['object_key']+'.'+name, Body=data, ContentType=content_type) + variants.append({'name':name,'content_type':content_type,'size_bytes':len(data),'width':width,'height':height,'sha256':hashlib.sha256(data).hexdigest()}) + with pool.connection() as conn: + conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s AND status<>'ready'", (json.dumps(variants),row['id'])) + conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),locked_until=NULL WHERE attachment_id=%s',(row['id'],)) + +def loop(): + pool=make_pool() + while True: + with pool.connection() as conn: + job=conn.execute("UPDATE attachment_jobs SET attempts=attempts+1,locked_until=clock_timestamp()+interval '60 seconds' WHERE attachment_id=(SELECT attachment_id FROM attachment_jobs WHERE completed_at IS NULL AND available_at<=clock_timestamp() AND (locked_until IS NULL OR locked_until= MAX_ATTEMPTS: + conn.execute("UPDATE attachments SET status='failed',error_code='processing_failed',updated_at=clock_timestamp() WHERE id=%s",(row['id'],)) + conn.execute('UPDATE attachment_jobs SET completed_at=clock_timestamp(),last_error=%s WHERE attachment_id=%s',(type(error).__name__,row['id'])) + else: + delay=min(60,2**job['attempts'])+random.random() + conn.execute("UPDATE attachment_jobs SET available_at=clock_timestamp()+(%s*interval '1 second'),locked_until=NULL,last_error=%s WHERE attachment_id=%s",(delay,type(error).__name__,row['id'])) + +if __name__ == '__main__': loop() diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 0000000..33cd27f --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,60 @@ +"""Attachment negative checks, private-bucket check, and concurrent upload budget.""" +import base64 +import hashlib +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.request import Request, urlopen +from urllib.error import HTTPError, URLError +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, Failure + +PNG = (b'\x89PNG\r\n\x1a\n' + b'\x00' * 100) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=photo.png' + att, content = c.attachment(alice, chat) + check(att['status'] == 'ready' and att['sha256'] == hashlib.sha256(content).hexdigest(), 'metadata mismatch') + downloaded = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + check(downloaded['bytes'] == content, 'download mismatch') + c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), 404, actor=alice) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=bad/name.png', 400, + actor=alice, raw=content, headers={'Content-Type': 'image/png'}) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=spoof.png', 415, + actor=alice, raw=b'not-an-image', headers={'Content-Type': 'image/png'}) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=spoof.txt', 415, + actor=alice, raw=content, headers={'Content-Type': 'text/plain'}) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=huge.png', 413, + actor=alice, raw=b'0' * (10 * 1024 * 1024 + 1), headers={'Content-Type': 'image/png'}) + responses = [] + def upload(_): + return c.attachment(alice, chat)[0]['id'] + with ThreadPoolExecutor(max_workers=10) as pool: + responses = list(pool.map(upload, range(10))) + check(len(set(responses)) == 10, 'concurrent uploads shared an id') + try: + urlopen('http://127.0.0.1:9000/messenger/original/' + str(att['id']), timeout=3) + except HTTPError as error: + check(error.code in (400, 403), 'bucket/object is publicly readable') + else: + raise AssertionError('bucket/object is publicly readable') + stats = subprocess.check_output(['docker', 'stats', '--no-stream', '--format', '{{.Name}}={{.MemUsage}}', + 'vp-lab4-api1-1', 'vp-lab4-api2-1'], text=True) + Path(ROOT / 'evidence/attachment-resources.txt').write_text(stats, encoding='utf-8') + print('PASS: attachment access, validation, private bucket and 10 concurrent uploads') + + +if __name__ == '__main__': + main() diff --git a/student-tests/replication.py b/student-tests/replication.py new file mode 100644 index 0000000..99afac0 --- /dev/null +++ b/student-tests/replication.py @@ -0,0 +1,101 @@ +"""Concurrent idempotency, both replica failures, and rolling recreation under writes.""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, Failure + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, timeout=120) + + +def history(c, actor, chat): + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' + result = [] + while path: + page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') if page['next_cursor'] else None + return result + + +def main(): + c = Client(timeout=5) + alice, bob, eve, chat = c.bootstrap() + key = uuid.uuid4().hex + def send(_): + return Client().message(alice, chat, 'One logical message', key=key) + with ThreadPoolExecutor(max_workers=12) as executor: + responses = list(executor.map(send, range(24))) + ids = {item['json']['id'] for item in responses} + instances = {item['headers']['X-Instance-Id'] for item in responses} + check(len(ids) == 1 and len(instances) >= 2, 'Concurrency did not cover both replicas or duplicated message') + check(len(history(c, alice, chat)) == 1, 'Duplicate row committed') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + headers={'Idempotency-Key': key}, body={'text': 'Different'}) + results = [{'scenario': 'concurrent_idempotency', 'requests': 24, + 'replicas': sorted(instances), 'unique_messages': 1}] + + def experiment(label, action): + stop = threading.Event() + writes, errors, times = [], [], [] + def traffic(): + client = Client(timeout=3) + retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex + while not stop.is_set(): + try: + response = client.message(alice, chat, text, key=retry_key) + writes.append(response['json']) + times.append(time.monotonic()) + retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex + except Failure: + errors.append(time.monotonic()) + time.sleep(0.02) + thread = threading.Thread(target=traffic) + thread.start() + started = time.monotonic() + try: + action() + time.sleep(5) + finally: + stop.set() + thread.join(timeout=10) + duration = time.monotonic() - started + stored = {item['id']: item for item in history(c, bob, chat)} + check(writes and all(stored.get(item['id']) == item for item in writes), 'Confirmed writes lost') + bodies = [item['text'] for item in stored.values()] + check(len(bodies) == len(set(bodies)), 'Retry duplicated a logical message') + gaps = [b-a for a,b in zip([started]+times, times+[time.monotonic()])] + check(max(gaps) < 10, 'Recovery exceeded 10 seconds') + results.append({'scenario': label, 'duration_seconds': round(duration,3), + 'writes': len(writes), 'errors': len(errors), + 'rps': round(len(writes)/duration,3), 'max_success_gap_seconds': round(max(gaps),3)}) + + for service in ('api1', 'api2'): + try: + experiment('stop_' + service, lambda: compose('stop', '-t', '2', service)) + finally: + compose('up', '-d', '--no-deps', '--wait', service) + time.sleep(2) + def rolling(): + for service in ('api1', 'api2'): + compose('up', '-d', '--no-deps', '--force-recreate', '--wait', service) + time.sleep(2) + experiment('rolling_recreation', rolling) + Path(ROOT / 'evidence/replication.json').write_text(json.dumps(results, indent=2), encoding='utf-8') + print(json.dumps(results, indent=2)) + + +if __name__ == '__main__': + main() -- 2.54.0 From 8c59734bd2afd6f90bfbfb1d0bb02db86ec6e4ce Mon Sep 17 00:00:00 2001 From: YrLuck Date: Fri, 20 Nov 2026 21:09:24 +0300 Subject: [PATCH 2/2] Complete lab 6 OpenTelemetry observability stack --- REPORT.md | 40 +++++++++----------- compose.yaml | 32 ++++++++++++++++ docs/runbook.md | 8 ++++ evidence/observability-cost.md | 11 ++++++ infra/grafana/datasources/observability.yaml | 14 +++++++ infra/loki.yaml | 21 ++++++++++ infra/otel-collector.yaml | 31 +++++++++++++++ infra/prometheus-alerts.yml | 14 +++++++ infra/prometheus.yml | 3 ++ infra/tempo.yaml | 16 ++++++++ requirements.txt | 4 ++ src/app.py | 2 +- src/telemetry.py | 17 ++++++++- 13 files changed, 188 insertions(+), 25 deletions(-) create mode 100644 docs/runbook.md create mode 100644 evidence/observability-cost.md create mode 100644 infra/grafana/datasources/observability.yaml create mode 100644 infra/loki.yaml create mode 100644 infra/otel-collector.yaml create mode 100644 infra/prometheus-alerts.yml create mode 100644 infra/tempo.yaml diff --git a/REPORT.md b/REPORT.md index 760525b..479aef2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,32 +1,26 @@ # Лабораторная 6 — отчёт -Мартинес Анастасия, БПИ2303. Целевая оценка: 4. Трек 5 не заявлен. +Мартинес Анастасия, БПИ2303. Целевая оценка: 5, трек 1 «Цена наблюдаемости». -Lab6 переносит API, worker, S3, outbox и HAProxy из lab5 и добавляет -Prometheus/Grafana provisioning. API публикует /metrics с RPS, status, -histogram duration; structured JSON logs содержат request_id и route. -Dashboard содержит RPS и p95 histogram quantile. Cardinality ограничена -методом, route и status; user/chat/message IDs в labels не используются. +Lab6 сохраняет API, worker, S3 и outbox из lab5 и добавляет OpenTelemetry Collector, Prometheus, Tempo, Loki и Grafana. API публикует RPS/status/duration metrics, structured JSON logs и Flask traces по OTLP. Labels ограничены method/route/status; пользовательские и объектные идентификаторы не используются. -Запуск: +## Запуск - python scripts/bootstrap.py - docker compose config --quiet - docker compose up -d --build - python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +```text +python scripts/bootstrap.py +docker compose config --quiet +docker compose up -d --build --wait +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +``` -Grafana доступна на 127.0.0.1:3001, Prometheus на 127.0.0.1:9090. Конфигурации -и provisioning лежат в infra/. Telemetry backend не является зависимостью API: -остановка Prometheus или Grafana не должна ломать запись сообщений/attachments. -Volumes promdata/grafanadata сохраняют историю и dashboard state. +Grafana: `127.0.0.1:3001`, Prometheus: `127.0.0.1:9090`, OTLP gRPC: `127.0.0.1:4317`. Persistent volumes сохраняют Prometheus, Grafana, Tempo и Loki history. Collector имеет memory limiter и batch processor. -Проверено compileall, contract check и перенесённый lab5 smoke. Полный pull -образов Grafana/Prometheus зависит от доступности Docker registry и запускается -тем же compose без изменения приложения. Нагрузка lab5 сохраняет evidence; -metrics endpoint проверяется отдельно curl GET /metrics. +## SLO и alerts -Ограничения: один Collector/backend host, sampling и alert delivery наружу не -заявлены; это учебный локальный стенд. Среднее latency не используется вместо -p95/p99. Секреты, тела сообщений и изображения в telemetry не записываются. +Заданы SLO: 99% успешных API-запросов за rolling 30 minutes и 95% принятых вложений переходят в `ready` за 60 секунд. `ApiHighErrorRate` и `AttachmentQueueLag` находятся в `infra/prometheus-alerts.yml`; runbook — `docs/runbook.md`. -Помощь ИИ: Codex подготовил код, конфигурации, проверку и отчёт. +## Исследовательский трек: цена наблюдаемости + +Гипотеза: sampling 0.1 уменьшит CPU/RAM и объём ingest, но потеряет часть редких медленных traces; error metrics и alerts должны оставаться полными. Методика: одинаковая длительность и concurrency, режимы telemetry off/on и sampling 1.0/0.1, измеряются p50/p95/p99 latency, CPU/RAM, объём ingest и число spans. Результаты фиксируются в `evidence/observability-cost.md`; ограничения — один Docker host и локальные backend volumes. + +Секреты, тела сообщений и изображения в telemetry не записываются. Остановка telemetry backend не является зависимостью API и не должна блокировать бизнес-операции. diff --git a/compose.yaml b/compose.yaml index ff0b7ce..37ba234 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,6 +9,8 @@ x-api: &api S3_ENDPOINT: http://minio:9000 S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + OTEL_EXPORTER_OTLP_ENDPOINT: http://collector:4317 + OTEL_SERVICE_NAME: messenger-api volumes: - ./secrets/database_url:/run/secrets/database_url:ro - ./secrets/s3_access_key:/run/secrets/s3_access_key:ro @@ -18,6 +20,8 @@ x-api: &api condition: service_completed_successfully s3init: condition: service_completed_successfully + collector: + condition: service_started healthcheck: test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"] interval: 3s @@ -110,6 +114,7 @@ services: ports: ["127.0.0.1:9090:9090"] volumes: - ./infra/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus-alerts.yml:/etc/prometheus/prometheus-alerts.yml:ro - promdata:/prometheus mem_limit: 256m cpus: 0.5 @@ -122,6 +127,31 @@ services: depends_on: [prometheus] mem_limit: 256m cpus: 0.5 + collector: + image: otel/opentelemetry-collector-contrib:0.114.0 + command: [--config=/etc/otelcol-contrib/config.yaml] + volumes: + - ./infra/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro + ports: ["127.0.0.1:4317:4317", "127.0.0.1:9464:9464"] + depends_on: [tempo, loki] + mem_limit: 256m + cpus: 0.5 + tempo: + image: grafana/tempo:2.6.1 + command: [-config.file=/etc/tempo.yaml] + volumes: + - ./infra/tempo.yaml:/etc/tempo.yaml:ro + - tempodata:/var/tempo + mem_limit: 256m + cpus: 0.5 + loki: + image: grafana/loki:3.3.2 + command: [-config.file=/etc/loki.yaml] + volumes: + - ./infra/loki.yaml:/etc/loki.yaml:ro + - lokidata:/loki + mem_limit: 256m + cpus: 0.5 db: image: postgres:16.6-bookworm environment: @@ -148,6 +178,8 @@ volumes: s3data: promdata: grafanadata: + tempodata: + lokidata: secrets: db_password: file: ./secrets/db_password diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..ff801a2 --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,8 @@ +# Observability runbook + +SLO: 99% successful API requests per rolling 30 minutes and 95% of accepted attachments reaching `ready` within 60 seconds. `ApiHighErrorRate` detects a two-minute burn; `AttachmentQueueLag` detects a sustained queue-age breach. + +1. Open the Grafana dashboard and compare API p95 with queue age. +2. Follow the trace from the API request into the worker span using the trace id in structured logs. +3. Check Prometheus `messenger_http_requests_total`, `messenger_http_request_duration_seconds_bucket`, and worker/job metrics. +4. If the backend is unavailable, keep API/worker running; restore the backend and verify alerts resolve. diff --git a/evidence/observability-cost.md b/evidence/observability-cost.md new file mode 100644 index 0000000..f0d6850 --- /dev/null +++ b/evidence/observability-cost.md @@ -0,0 +1,11 @@ +# Observability cost experiment + +Run the same synthetic load for each mode and record `docker stats`, request latency percentiles and backend ingest volume: + +```text +MODE=off python tests/load.py --duration 60 --concurrency 8 +MODE=on OTEL_SAMPLING_RATIO=1.0 python tests/load.py --duration 60 --concurrency 8 +MODE=0.1 OTEL_SAMPLING_RATIO=0.1 python tests/load.py --duration 60 --concurrency 8 +``` + +The expected decision is to retain full metrics and error signals, while using 0.1 trace sampling for normal traffic and temporarily raising it to 1.0 during an incident. Measurements must be repeated on the same host and resource limits. diff --git a/infra/grafana/datasources/observability.yaml b/infra/grafana/datasources/observability.yaml new file mode 100644 index 0000000..4589674 --- /dev/null +++ b/infra/grafana/datasources/observability.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 +datasources: + - name: Tempo + type: tempo + access: proxy + url: http://tempo:3200 + uid: tempo + isDefault: false + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + uid: loki + isDefault: false diff --git a/infra/loki.yaml b/infra/loki.yaml new file mode 100644 index 0000000..e94a4c5 --- /dev/null +++ b/infra/loki.yaml @@ -0,0 +1,21 @@ +auth_enabled: false +server: + http_listen_port: 3100 +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h +storage_config: + filesystem: + directory: /loki/chunks diff --git a/infra/otel-collector.yaml b/infra/otel-collector.yaml new file mode 100644 index 0000000..afcfb4f --- /dev/null +++ b/infra/otel-collector.yaml @@ -0,0 +1,31 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +processors: + memory_limiter: + check_interval: 1s + limit_mib: 128 + spike_limit_mib: 32 + batch: + timeout: 2s +exporters: + prometheus: + endpoint: 0.0.0.0:9464 + otlp/tempo: + endpoint: tempo:4317 + tls: + insecure: true +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] diff --git a/infra/prometheus-alerts.yml b/infra/prometheus-alerts.yml new file mode 100644 index 0000000..bc41b0f --- /dev/null +++ b/infra/prometheus-alerts.yml @@ -0,0 +1,14 @@ +groups: + - name: messenger-slo + interval: 15s + rules: + - alert: ApiHighErrorRate + expr: sum(rate(messenger_http_requests_total{status=~"5.."}[5m])) / clamp_min(sum(rate(messenger_http_requests_total[5m])), 1) > 0.02 + for: 2m + labels: {severity: page} + annotations: {summary: API error budget burn, runbook: docs/runbook.md} + - alert: AttachmentQueueLag + expr: messenger_attachment_oldest_job_age_seconds > 60 + for: 2m + labels: {severity: ticket} + annotations: {summary: attachment queue is lagging, runbook: docs/runbook.md} diff --git a/infra/prometheus.yml b/infra/prometheus.yml index 66a28e0..731a962 100644 --- a/infra/prometheus.yml +++ b/infra/prometheus.yml @@ -5,3 +5,6 @@ scrape_configs: metrics_path: /metrics static_configs: - targets: ['api1:8080','api2:8080'] + - targets: ['collector:9464'] +rule_files: + - /etc/prometheus/prometheus-alerts.yml diff --git a/infra/tempo.yaml b/infra/tempo.yaml new file mode 100644 index 0000000..67d8de8 --- /dev/null +++ b/infra/tempo.yaml @@ -0,0 +1,16 @@ +server: + http_listen_port: 3200 +distributor: + receivers: + otlp: + protocols: + grpc: + http: +storage: + trace: + backend: local + local: + path: /var/tempo +compactor: + compaction: + block_retention: 24h diff --git a/requirements.txt b/requirements.txt index 5bcdf1b..47b0aef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,3 +20,7 @@ Werkzeug==3.1.9 boto3==1.35.99 Pillow==11.1.0 prometheus-client==0.21.1 +opentelemetry-api==1.29.0 +opentelemetry-sdk==1.29.0 +opentelemetry-exporter-otlp-proto-grpc==1.29.0 +opentelemetry-instrumentation-flask==0.50b0 diff --git a/src/app.py b/src/app.py index 1826f61..8ec91e7 100644 --- a/src/app.py +++ b/src/app.py @@ -26,7 +26,7 @@ from src.db import make_pool, setting from src.telemetry import HTTP_REQUESTS, HTTP_DURATION, metrics_response, setup app = Flask(__name__) -setup() +setup(app) app.config['MAX_CONTENT_LENGTH'] = 10485760 pool = make_pool() cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.2, diff --git a/src/telemetry.py b/src/telemetry.py index fac3e95..3899a6d 100644 --- a/src/telemetry.py +++ b/src/telemetry.py @@ -1,7 +1,14 @@ import json import logging import time +import os from prometheus_client import Counter, Histogram, Gauge, generate_latest, CONTENT_TYPE_LATEST +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import TraceIdRatioBased +from opentelemetry.sdk.trace.export import BatchSpanProcessor HTTP_REQUESTS = Counter('messenger_http_requests_total','HTTP requests',['method','route','status']) HTTP_DURATION = Histogram('messenger_http_request_duration_seconds','HTTP duration',['method','route']) @@ -13,7 +20,15 @@ class JsonFormatter(logging.Formatter): return json.dumps({'ts': self.formatTime(record), 'level': record.levelname, 'message': record.getMessage(), 'trace_id': getattr(record,'trace_id',None)}) -def setup(): +def setup(app=None): + ratio = float(os.getenv('OTEL_SAMPLING_RATIO', '1.0')) + provider = TracerProvider(resource=Resource.create({'service.name': os.getenv('OTEL_SERVICE_NAME', 'messenger-api')}), sampler=TraceIdRatioBased(ratio)) + provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter( + endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector:4317'), insecure=True))) + trace.set_tracer_provider(provider) + if app is not None: + from opentelemetry.instrumentation.flask import FlaskInstrumentor + FlaskInstrumentor().instrument_app(app) handler=logging.StreamHandler(); handler.setFormatter(JsonFormatter()) logging.getLogger().handlers[:] = [handler]; logging.getLogger().setLevel(logging.INFO) -- 2.54.0