From 54dcf70ea616dd11243d97b4b7076f01ea58cc77 Mon Sep 17 00:00:00 2001 From: Wladislava1 <146841829+Wladislava1@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:25:48 +0300 Subject: [PATCH] =?UTF-8?q?Lab7:=20HTTPS,=20mTLS,=20OpenBao,=20=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=BA=202=20PKI=20=D0=BD=D0=B0=20=D0=BE=D1=86?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D1=83=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 4 + .github/pull_request_template.md | 2 +- Dockerfile | 30 + MAINTAINERS.md | 2 +- PR_TEXT.md | 13 + REPORT.md | 66 +- certs/api-client.crt | 26 + certs/api.crt | 26 + certs/ca.crt | 30 + certs/ca.srl | 1 + certs/lb.crt | 26 + certs/minio.crt | 26 + certs/openbao.crt | 26 + certs/otel-collector.crt | 27 + certs/postgres.crt | 26 + certs/public.crt | 26 + certs/rabbitmq.crt | 26 + certs/redis.crt | 26 + certs/revoked_serials.txt | 1 + certs/worker-client.crt | 26 + compose.yaml | 592 +++++++++++ contracts/trust-matrix.md | 47 +- evidence/README.md | 4 +- evidence/pki-track2.md | 25 + evidence/scan-triage.md | 28 + infra/grafana/dashboards/messenger.json | 279 +++++ .../provisioning/dashboards/dashboards.yaml | 13 + .../provisioning/datasources/datasources.yaml | 45 + infra/haproxy.cfg | 52 + infra/loki/loki.yaml | 49 + infra/minio-api-policy.json | 19 + infra/nginx-otel.conf | 32 + infra/openbao/openbao.hcl | 15 + infra/openbao/policies/api.hcl | 8 + infra/openbao/policies/worker.hcl | 8 + infra/otel/collector.yaml | 81 ++ infra/prometheus/alerts.yml | 88 ++ infra/prometheus/prometheus.yml | 18 + infra/tempo/tempo.yaml | 47 + migrations/001_init.sql | 48 + migrations/002_idempotency.sql | 19 + migrations/003_attachments.sql | 42 + migrations/004_jobs.sql | 108 ++ requirements.txt | 14 + scripts/bootstrap.py | 100 ++ scripts/bootstrap_pki.py | 220 ++++ scripts/minio-init.sh | 30 + scripts/minio-tls-entry.sh | 8 + scripts/openbao-entry.sh | 13 + scripts/openbao_bootstrap.py | 166 +++ scripts/pki_renew.py | 28 + scripts/postgres-entry.sh | 19 + scripts/rabbit-entry.sh | 29 + scripts/redis-entry.sh | 21 + scripts/restart.cmd | 2 + scripts/restart.example.sh | 0 scripts/restart.py | 26 + scripts/restart.sh | 10 + scripts/stop-one.cmd | 2 + scripts/stop-one.example.sh | 0 scripts/stop-one.sh | 23 + scripts/stop_one.py | 45 + scripts/wait_ready.py | 37 + security/THREAT_MODEL.md | 27 +- src/bao.py | 74 ++ src/blobs.py | 237 +++++ src/broker.py | 155 +++ src/errors.py | 32 + src/faults.py | 49 + src/images.py | 129 +++ src/jobs.py | 896 ++++++++++++++++ src/main.py | 593 +++++++++++ src/migrate.py | 112 ++ src/passwords.py | 33 + src/reconcile.py | 174 ++++ src/replay.py | 24 + src/schemas.py | 128 +++ src/store.py | 967 ++++++++++++++++++ src/telemetry.py | 473 +++++++++ src/tlsutil.py | 95 ++ src/worker.py | 261 +++++ student-tests/probe_loki.py | 74 ++ student-tests/probe_traces.py | 126 +++ student-tests/run_negative.py | 136 +++ student-tests/run_observability.py | 474 +++++++++ student-tests/run_pipeline.py | 653 ++++++++++++ tests/student_mtls.py | 112 ++ tests/student_openbao.py | 58 ++ 88 files changed, 8930 insertions(+), 58 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 PR_TEXT.md create mode 100644 certs/api-client.crt create mode 100644 certs/api.crt create mode 100644 certs/ca.crt create mode 100644 certs/ca.srl create mode 100644 certs/lb.crt create mode 100644 certs/minio.crt create mode 100644 certs/openbao.crt create mode 100644 certs/otel-collector.crt create mode 100644 certs/postgres.crt create mode 100644 certs/public.crt create mode 100644 certs/rabbitmq.crt create mode 100644 certs/redis.crt create mode 100644 certs/revoked_serials.txt create mode 100644 certs/worker-client.crt create mode 100644 compose.yaml create mode 100644 evidence/pki-track2.md create mode 100644 evidence/scan-triage.md create mode 100644 infra/grafana/dashboards/messenger.json create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/haproxy.cfg create mode 100644 infra/loki/loki.yaml create mode 100644 infra/minio-api-policy.json create mode 100644 infra/nginx-otel.conf create mode 100644 infra/openbao/openbao.hcl create mode 100644 infra/openbao/policies/api.hcl create mode 100644 infra/openbao/policies/worker.hcl create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/alerts.yml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/tempo/tempo.yaml create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_jobs.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100644 scripts/bootstrap_pki.py create mode 100644 scripts/minio-init.sh create mode 100644 scripts/minio-tls-entry.sh create mode 100644 scripts/openbao-entry.sh create mode 100644 scripts/openbao_bootstrap.py create mode 100644 scripts/pki_renew.py create mode 100644 scripts/postgres-entry.sh create mode 100644 scripts/rabbit-entry.sh create mode 100644 scripts/redis-entry.sh create mode 100644 scripts/restart.cmd mode change 100755 => 100644 scripts/restart.example.sh create mode 100644 scripts/restart.py create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.cmd mode change 100755 => 100644 scripts/stop-one.example.sh create mode 100644 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 scripts/wait_ready.py create mode 100644 src/bao.py create mode 100644 src/blobs.py create mode 100644 src/broker.py create mode 100644 src/errors.py create mode 100644 src/faults.py create mode 100644 src/images.py create mode 100644 src/jobs.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/passwords.py create mode 100644 src/reconcile.py create mode 100644 src/replay.py create mode 100644 src/schemas.py create mode 100644 src/store.py create mode 100644 src/telemetry.py create mode 100644 src/tlsutil.py create mode 100644 src/worker.py create mode 100644 student-tests/probe_loki.py create mode 100644 student-tests/probe_traces.py create mode 100644 student-tests/run_negative.py create mode 100644 student-tests/run_observability.py create mode 100644 student-tests/run_pipeline.py create mode 100644 tests/student_mtls.py create mode 100644 tests/student_openbao.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fe56092 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.cfg text eol=lf +Dockerfile text eol=lf +*.sql text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8aef0ff..59c9f4e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ -## Лабораторная 7 +## Лабораторная 6 Автор / группа: TODO Целевая оценка (3/4/5): TODO diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e662326 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8443 \ + PIP_DEFAULT_TIMEOUT=120 + +WORKDIR /app + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin app \ + && mkdir -p /certs/private \ + && chown -R app:app /certs + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --upgrade pip==25.0.1 \ + && pip install --no-cache-dir -r /app/requirements.txt + +COPY migrations /app/migrations +COPY src /app/src + +USER 10001 + +EXPOSE 8443 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=45s --retries=10 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2)" + +CMD ["python", "/app/src/main.py"] diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d3b18f7..48b24e7 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -1,4 +1,4 @@ -# Памятка преподавателю — лабораторная 7 +# Памятка преподавателю — лабораторная 6 ## Что подготовлено diff --git a/PR_TEXT.md b/PR_TEXT.md new file mode 100644 index 0000000..47fbd17 --- /dev/null +++ b/PR_TEXT.md @@ -0,0 +1,13 @@ +## Summary + +- Lab 7 на 5, трек 2 (PKI lifecycle): HTTPS :8443, mTLS LB→API, TLS/mTLS к зависимостям, OpenBao AppRole non-dev. +- Перенесён стенд lab6 (мессенджер + observability); публичные contracts/tests не ослаблялись. +- Argon2id + общий login rate limit; trust-matrix и threat model заполнены. + +## Test plan + +- [x] `python -X utf8 tests/smoke.py --base-url https://localhost:8443 --ca-file certs/ca.crt` +- [x] `python -X utf8 tests/security.py --base-url https://localhost:8443 --ca-file certs/ca.crt` +- [x] `python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd ...` +- [x] `python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd ...` +- [x] `python -X utf8 tests/student_mtls.py` diff --git a/REPORT.md b/REPORT.md index 9801db4..4ec99aa 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,73 @@ # Отчёт — лабораторная 7 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Орлова Владислава, БПИ2301 +- Целевая оценка: 5. Трек 2, жизненный цикл PKI/secrets +- Commit/PR: ещё не публиковала +- Источники и помощь: задание и контракт lab7, рабочий стенд lab6, OWASP Password Storage, OpenBao AppRole/policies, документация HAProxy/RabbitMQ TLS. При написании кода и отчёта пользовалась помощью ИИ. Числа ниже сняты с моего запуска 16 ноября 2026 ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine и Python 3.9+. У меня Docker Engine 26.1.4 и Python на Windows. `make` нет — команды через `python -X utf8`. Для persistence/failover на Windows использую `scripts/restart.cmd` и `scripts/stop-one.cmd` (внутри вызывают Python-хуки). + +```sh +python -X utf8 scripts/bootstrap.py +docker compose --env-file .env up -d postgres redis rabbitmq minio openbao prometheus loki tempo otel-collector otel-gateway grafana +python -X utf8 scripts/openbao_bootstrap.py +docker compose --env-file .env up -d --build --scale api=2 minio-init migrate api worker lb +python -X utf8 scripts/wait_ready.py +python -X utf8 tests/smoke.py --base-url https://localhost:8443 --ca-file "$PWD/certs/ca.crt" +python -X utf8 tests/security.py --base-url https://localhost:8443 --ca-file "$PWD/certs/ca.crt" +python -X utf8 tests/student_mtls.py +python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url https://localhost:8443 --ca-file "$PWD/certs/ca.crt" +python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url https://localhost:8443 --ca-file "$PWD/certs/ca.crt" +``` + +`bootstrap.py` создаёт `secrets/`, `.env` и PKI в `certs/` (ключи в `certs/private/`). `openbao_bootstrap.py` делает init/unseal и пишет AppRole в `secrets/bao_*`; root/unseal — только в `secrets/openbao/` у оператора, в git не входят. + +Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных данных: `docker compose --env-file .env down -v`. + +Имя проекта Compose — `mtusi-lab7`. Наружу: `127.0.0.1:8443` (HTTPS LB), `127.0.0.1:3000` Grafana, `127.0.0.1:9090` Prometheus, `127.0.0.1:8200` OpenBao для оператора. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab6 перенесён мессенджер: 2×API, HAProxy, Postgres, Redis, MinIO, RabbitMQ, worker, outbox, OTel→Prometheus/Loki/Tempo/Grafana. + +Новое в lab7: +- Публичный HTTPS на :8443 с локальным CA +- Обязательный mTLS LB→API с allowlist identity `lb` +- TLS к Postgres/Redis/MinIO/RabbitMQ/OpenBao; mTLS API/worker→broker и →otel-gateway +- OpenBao non-dev (file storage) + AppRole + scoped policies +- Argon2id 19MiB/t=2/p=1, rehash при входе, общий login rate limit в Redis. На моём стенде один `hash_password` занял ≈50 ms — этого хватает как учебный бюджет против дешёвого login DoS вместе с Redis limit 10/30s +- Hardening: non-root, cap_drop, no-new-privileges, read_only+tmpfs у api/worker; triage образов в `evidence/scan-triage.md` + +Матрица: `contracts/trust-matrix.md`. Угрозы: `security/THREAT_MODEL.md`. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke HTTPS | `tests/smoke.py --base-url https://localhost:8443 --ca-file certs/ca.crt` | PASS: lab7 public API scenario | вывод в консоли 16.11.2026 | +| Внешний TLS trust | `tests/security.py` | PASS: trusted HTTPS; untrusted CA rejected | то же | +| Persistence | `tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | PASS: persistence preserved… | то же | +| Failover API | `tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | PASS; first_recovery_seconds≈1.7; observed_instances_before=2 | то же | +| Внутренний mTLS | `tests/student_mtls.py` | нет cert / чужая identity → отказ; `lb` → 200 | `tests/student_mtls.py` | +| OpenBao AppRole | `scripts/openbao_bootstrap.py` + startup `bao.login_and_read` | API ready только после unseal и scoped secret | `src/bao.py`, `infra/openbao/` | +| Argon2id + rate limit | параметры в `passwords.py`; Redis `login:fail:*` | ≈50 ms на hash; rehash; 429 Retry-After | `src/passwords.py`, `src/store.py` | +| Image scan triage | разбор применимости CVE, не только count | зафиксированы классы/остаточный риск | `evidence/scan-triage.md` | +| OpenBao allow/deny | `tests/student_openbao.py` | api path 200; worker path deny | вывод теста | +| Трек 2 PKI lifecycle | `scripts/pki_renew.py --identity api --days 14` + recreate api | leaf обновлён без rebuild base image; verify не отключается | `scripts/pki_renew.py`, `evidence/pki-track2.md` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза трека 2: короткоживущий leaf можно ротировать без plaintext fallback и без отключения проверки сертификата; ошибки на время recreate ограничены, после загрузки нового leaf smoke снова зелёный. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Persistence restart stateful + api | Сессия и данные живы | PASS persistence | без потери подтверждённых данных | +| Stop одной API-реплики | Запись продолжает другая реплика | PASS failover; recovery ≈1.7s | реплика остаётся stopped до ручного up | +| Sealed/unavailable OpenBao | API не ready | fail-closed в lifespan | до unseal оператором | +| Недоверенный CA снаружи | Отказ TLS | PASS security | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Учебный стенд на одной машине, CA локальный. Auto-unseal нет — после рестарта OpenBao нужен оператор. Collector backends (Tempo/Loki) внутри доверенной сети без отдельного mTLS hop. Scan образов с разбором CVE можно расширить отдельным отчётом. Следующий шаг — автоматизировать reload leaf без полного recreate контейнера. diff --git a/certs/api-client.crt b/certs/api-client.crt new file mode 100644 index 0000000..9fdd75d --- /dev/null +++ b/certs/api-client.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEajCCAlKgAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOkwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0OFoXDTI3MDIxNDE5NTY0OFowKzEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +EzARBgNVBAMMCmFwaS1jbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCxXHsUSFhKZdaW8uJr0YSgp/fEkNJ418aP7BB3E0hiXhy9IDAy9wpc8xkN +W3j5bGzSScJEzxpcKlREqx0qhH3ME2xT27kh4WK9k1RAOz4TAj2+zj7FqRlGWsuW +90oTmuSQjqcjpFeSPGi+W88AoMzsa24S4wvbsiP1BgPX0QQth8C2CDxRH3MaqpAk +Ne4eh0Du31w9UlZ3OlLDcRivNdnlfhLqLcvyzETIPlTY7jW+Pchy/IoM+DBMJk4U +yZig19Duorrzserdb4w1sKWYWqF7cvVOHDR5iMvl5isoOYXuo+PcoKqjpSDCRPUo +QwO65hyT+rXeg07QebwkIcmx6xvbAgMBAAGjgY0wgYowDAYDVR0TAQH/BAIwADAO +BgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwFQYDVR0RBA4wDIIK +YXBpLWNsaWVudDAdBgNVHQ4EFgQU1SX0X4HMDg1C1FLbBv/vEBNbar0wHwYDVR0j +BBgwFoAU6hmJxsf6vT3tWbIZUJ8bcmYf8L4wDQYJKoZIhvcNAQELBQADggIBALAE +pPHOuYCi1MEvsF3ZlgOVHv7Q1DRe6gsnyjQzwNJbrAdyO7JNBR+uJKeuSLUQvl+e +50gMghjjYL4XPUeEoZA9oAmoDqgsHDSB2vjahpwYmv7OKmOO7HCSh7tFZnQoZ7Ix +XAZLi+xtKn+cbFahnaJL6klgHqAAZeK6cCqCpo16P58jPVB2lJusM4g2k+TdPVJf +SdvmmGK0+RdTgK0mL1g+47CSLAmUW1JWFxVPT+G4iyz/0ebL9HYgzqw27BSzxXO2 +bHElwgbxgYNrbAirynkWnCMyO68pUAeMK2TBORycebP8uBwcs/X98Snr8+FWKIC0 +1tdvDSzZEgZIiXOQM4w+iXlWDNrgBZBRau/hgYrH4Sd+p55C17vhiueJ4BZJdGFz +sENPcUdMzVEccaTqjSIRWdj09okE9lu+m94k2gjwWhTq+bXwOJYkrGiu2d8ohmHx +F2tuE5PRCkpVe8a51ApkFPPlA76dEhYrYN6TXanQEibca7N1EJYOGO3Um2Y8euGw +BFdk6bP9K8wE/Xj/UOe5kku+OFkBR8+YahNvpB/TWh8kkxPDUYOipPwhq/lcqff6 +og/BajMnp0Ma5KmTX7hO+WThBlS3oqmwVGmh1M/EfMIfNLG65L7RZ4CJ7F46SZ+t +ikyHgGBydiHBq6b5q4Vk3X9DyYjSV/N7oIRRFupR +-----END CERTIFICATE----- diff --git a/certs/api.crt b/certs/api.crt new file mode 100644 index 0000000..7af947a --- /dev/null +++ b/certs/api.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEZjCCAk6gAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOAwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0MFoXDTI3MDIxNDE5NTY0MFowJDEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +DDAKBgNVBAMMA2FwaTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMx9 +9R8OiY5KEfipdL6km4QOGJt4XDxC5H3NLraFVOLibVlylsZanTO2kBZAQUNRiJJP +4Lg2k9e7aH/zZ+5IrcCYSfK7KyUzFl4fnvsFt90iYNe1mTBTrjn0znctKgx7CAHp +TgSGD0+wvg35cSUJZBdHLD5oR5dwXy56YSGF8TxtV3R7yFCcu8TLycCoaD5gGq0Y +lY5r+KRP/HdYmFYAH5qkUZcZwTo2sxGx5VM2zIOy5Y8P8Me12skUVZYQ2Kmcx6HI ++If9cHwUv+LhdTOrIW+MKlYcSN+N9GBlQpBj4VelJYu3hqSm0MBIVRbJ56GrBRyJ +HvLB8vIGMfQeSrYVSpUCAwEAAaOBkDCBjTAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB +/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDgYDVR0RBAcw +BYIDYXBpMB0GA1UdDgQWBBSMPhUOO0xp1CB8gTcEJ9t0xef5ljAfBgNVHSMEGDAW +gBTqGYnGx/q9Pe1ZshlQnxtyZh/wvjANBgkqhkiG9w0BAQsFAAOCAgEAE/WCjlE8 +vFNzS7ZJO4epDBcy/4C6d1veV3E2xgr8NTcVLMfI4LJcv38SV1UagkUxRPvozZ1Q +Q5/8uw65GHVoTrie7jx7ZG1K66Vc5TgL6Ky20PuQkvuIzrycdi4Bc1VYIsIWFycV +aOpMKpUgTwDUxftslhK9O2qZylt3A96tEBLTpmEqxzwDYNV0GkFhN3nWysbP5k4S +oqit7cdiMnDX6GanV+Oi+6aSeePiptfpdZn1J4ASo60soPClxxIF2iMtAxC/2UPn ++F9Q8Oi4ATE3hBT7voRlnMgXVjiHLVTcpMSCK6P5SISA6hJ+3lqEdPTajpKFwNOX +Yj4aWqrS5qUdnu+F0fy8Vov9pQcblCUfjs/sHSid7vtKP7ZSuZ2SSg1CgDeVpNto +JfA4/XtPxXif4Eix/hs005C53dXfqe5wCC0BhbgSXwoYE3//MzAF30zbWqill4AS +tZnGs23anmb8KLL41cNY26ChO8Y3LW30+voyblErNxpV7XAKua9uiE/986b9odLX +zeH7OrZfmL0z3Xe4zDH63hvuF+usrkbo4FIreymlbWYO7Pz/Rfd05ssgCm0N0xkC +W1VTK/gAlPhL82QIdUbDaXJl1Q4VZMQKdkwjakMF3UhKV8D3UPaXmI12ljlb9wO4 +lRB3prbtjH/M9PNk1fIuR3FxIZB2YhKYlgo= +-----END CERTIFICATE----- diff --git a/certs/ca.crt b/certs/ca.crt new file mode 100644 index 0000000..74e1876 --- /dev/null +++ b/certs/ca.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFOjCCAyKgAwIBAgIUW96E2huDdK3gYm4EJp7MPORdHeowDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTYzN1oXDTM2MTExMzE5NTYzN1owIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +CzAJBgNVBAMMAmNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzJAm +/tFnUSf3L1WQ37sZ6tWxu7D3IT0puUFurJUry7fT6hf2XhlR5Wmal6WxUuDA+g/4 +pQe4ISEsM90c1fACxTNi1DA7xkUbibzkrOwrTm32y9rKoJ5YXpvFZfb88tiXozq8 +qgZ48tuF8hkBwVuTfWWB8ozQIHwNMFXW4CHG2v85Ocgw9oA7NyTBKaoJKZEO3ng8 +zvdIxFd1yagjlmGXcDIXOpqbN+atvV/37Wb2Us4FhCyeH7rJ9ogyUx6l9n+Ns5ap +IhhG4xrfY6p3MQ3YNOGT6twWKs2Gp1jcDKQ/kraq8DYOZH6I+/7OFdfzV/GhLcsd +SIR07cFfGvu5FQP4S0IJ+HOfbepXSpTJhY/ivKu3U+z2NSxQZ/bRpHuQdwDyW6ds +2fym/DNiS5HRPm0hE/jGKoPUxg8hokxheHqfi+2VCzurOvkcHwb8bf7oYr82r2wd +JEcbJTTuLqjGLqNaM8X1VaC/aTv1RPqqdSoeNSi89k7slx91g8GO5I1iF+HMmQhX +4s+zxq3qJCMsuvqcyNXwTCynPLPQy22l0VzzSSte1Dh5/Q8IHmrml1AjzHVGmEsr +DfW/cxPkxZsnA+0Mxnfzz90CCc0ETkkqiR5fsLXDvsuYqSSOoVJvsJHj1WmaGyu+ +Mm4HL0+IYrJo9TdE2OKHwSJNtVX/liGg8j5Asr0CAwEAAaNmMGQwHQYDVR0OBBYE +FOoZicbH+r097VmyGVCfG3JmH/C+MB8GA1UdIwQYMBaAFOoZicbH+r097VmyGVCf +G3JmH/C+MBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMA0GCSqG +SIb3DQEBCwUAA4ICAQBH/COYmtS8T6l7rA8C5z5KrbyoPEC9yG4I9hAPzAKuk/A8 +Z0BuOUW6bRvjSSAyBgxazq7IKLOmieJiybqVUXDm18CtoL6rCRDFcg8A1QnbWkkR +WdmZGhC+QEBGR0fok9KIKDOlZqfClmA10zqi6GARd5PhNXocxBmAFOpJvUSzucCQ +60M6ovr1xrd6g++49wvl2WjMjB5DpG0Sru0Dwmt7RRBYvTIVAdx4OHYYM02ZyMhT +UompYQlPDOahilp+RLxGSvTLheihJlWQGZqc4cN4jmSJ3gcpBPrSxlRWPd+nAn15 +5gYu5x55XqAHudFQ9p9jQ4XKFDKeBoXdMKuA7e6lwzRYdMW2reix+K46jLR6WXDz +bACsaGiO2JMsBWIG7MaehQSvRd6iOsHTgMjHsHioiKvvRsZhsBs7jg+Wiivw4vme +H4B9NdqGKL1O3MOu5PyqSYIFvbg4rEPGfwaYUQL0oweHb7J1KtXmIncnhBJjLodM ++raSXQLpJV0eYpu29P+ffKhkAXVvZF6DpBenfDafy6VrP3FcJgJeFpFSLNJTU4uQ +MKn8PNYj7jO+79ggCkJ/0cFdSQo/Yo4tfdi61BgUiiM+/BsWGwJrzf8TVWjxN9DM +IUnJE8pKP9T6/OTroze1OtcOOqZURzynI2I99zmJjF0VmXgJ33bvm6kxbY0LXA== +-----END CERTIFICATE----- diff --git a/certs/ca.srl b/certs/ca.srl new file mode 100644 index 0000000..01de24c --- /dev/null +++ b/certs/ca.srl @@ -0,0 +1 @@ +258A95F8F0ED20DDAD964A24149054AAE32064EA diff --git a/certs/lb.crt b/certs/lb.crt new file mode 100644 index 0000000..656c829 --- /dev/null +++ b/certs/lb.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEWjCCAkKgAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOcwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0NloXDTI3MDIxNDE5NTY0NlowIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +CzAJBgNVBAMMAmxiMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsCx7 +2/1mfvai2UCW7vCvlc777X7T5akg7SIzXuN9PnLjzRdtXu5F0Dsmou2BdcAiu4C9 +fVVbT343HHvViaK1yCKoWlnJ3SfodU1zi3o1ZY1Znc5Qu0w0ZsBm3rAAmV5TT9UG +Zbfj9ktTW5tPzOJs3R5qO3DVr7HDkZeQkMkof/UJDM53F6cMXAtvcD3vf9gi+ASr +W+GJGsihiLK5VrfMLCeSKMUsA0Om0X7fYUWpMNWmlzOaB1ece3WZ6ptm45MJSi+M +WyOiPrgtrs5J9jDa5IyOmrjGr4lo5JnXwn0pWEYX12x2peEH2RWOD1AyH300ugZS +26n2vluJU/f9DdNxQwIDAQABo4GFMIGCMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/ +BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMA0GA1UdEQQGMASCAmxiMB0GA1Ud +DgQWBBT1eBZp9F0AmcErg6t0JPzOYLi4/DAfBgNVHSMEGDAWgBTqGYnGx/q9Pe1Z +shlQnxtyZh/wvjANBgkqhkiG9w0BAQsFAAOCAgEAZH4hHPIbCB8ZF8OjOrzzOmuI +2HITtPMasDQzFAR5pwb7M77qqn0A8aL2qeROmkRiIVCTv6TncgOTMZS7cgR3JS5q +bNPvszpqx9EK65c5NdSyGCq9ASxa2+3QoWp6kb6bOtov9Fwl87SLYAO5sIM+VFjk +nXBhg9MWuLo7QfmAdslvzqnjDVMxReCzGbbfJUa5bp97uL3SEd5iJZ6kLXEZeGXf +MJNujXUZF2ZYIYlG71/XXHbjBLYTaGW0FN+1MY+Q9Sx9sNWqJ3Db9jwJHdNKjF1L +cz1cLUhmbsvUz4nYQrbEdnZEIa0HHRS+U6ev6MHLcPvv2C063Crfv/9gy5c9jd5u +prrWDHEPVxcXjnri53783hUdspx8ebudhhCHfCot4Fab5DjdAGMbAwHTO6FxMJ6u +g7eiLoztqzOLArG+C0U8y4VKkCMwr+bl0IC5HzCgK6Gp1HzK7iwqsnf9pH4uKsE+ +G3ATJTNnRoGG0LnW6hlTdZ+Tw/GCX6axLXq4qs4z9/xyn58JoU+7W+fm5YpW9lib +OCuDf2mJb46Vf8mgzTNYeCSGzXn9id8H8W/MJYEenU1n4JcBQzavjmS5hwy2SNqf +TSRB5iXTekPIhblwwWmw+wNovHZiWN9P3XadhK12o+kECCrd6VH7Ao5NECpa+Dm9 +Rjm4W8qI3I/SXoFCqZs= +-----END CERTIFICATE----- diff --git a/certs/minio.crt b/certs/minio.crt new file mode 100644 index 0000000..891d26e --- /dev/null +++ b/certs/minio.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEajCCAlKgAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOYwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0NVoXDTI3MDIxNDE5NTY0NVowJjEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +DjAMBgNVBAMMBW1pbmlvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +rIpM2J2owmxMOe2Zh/dH32N4af9HiPdCaevxz2wHz/HPQjcVxCRzhT+xYaHniiCW +AsMmROsDzHDiHupPvUWPjwwPHpTe8ChaMKI+8yyBibnM0gO1LQMP+CLW90gU0YFR +6aBBAWACUHuONcJIpV+xm5z6gSRszoOFi/M0Y0GbAGH7EPDZCWdSOzyKuDagbtQu +j5R6xvPHUvHvzMyr8zF54g9wvYgCz0EJi/1OsFb3OoKvHV65yKFzUmotdemCB2lk +Up2oCQOhfZ1eyr5cMkaEUx8chaTfF306BkmJxWbSm62nCVRarP+ovSa1LyWBehHj +LR0JK5hvTW3rHFwwTeEA7QIDAQABo4GSMIGPMAwGA1UdEwEB/wQCMAAwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAQBgNVHREE +CTAHggVtaW5pbzAdBgNVHQ4EFgQUqwDZ+1Z448GlRYqmaYrf6EHvhDUwHwYDVR0j +BBgwFoAU6hmJxsf6vT3tWbIZUJ8bcmYf8L4wDQYJKoZIhvcNAQELBQADggIBAD5h +G42uKA/OWoKJMAKTkA+aLfFGGKcL6MrmA1EteMS8wBAgU80W8XchlQYyVJUB7Fyl +fL+TKdwuIp8uZiL43kgWGNj6mgi+Aajn7FCRlPHqfnzeH9e1pJ29VmeocEY5iHU8 +GaC7M9dRUcIWKnEa9VjiE+5newXvBGJCrnf0jyQ4o6eV+VbVa7aEsnva0BscR1kp +jsU4mvIOufPixFgeVNnTe/eivSPEqI8Gq83Ltr9grsgKf8tFjGkCYwrTLL5UNgL4 ++ereA5BSwSiQuy/td/I15CgPokQjfB9POFpUfa7NNMtOazPEzzxcvpbKCQklJ4Dn +Ti+/zO+tx4A9R0GyiZqzs+zkxf6yPzvM0WqMxvWIW1Pqi0+L3uB7GKVenYs6sFeb +BQ5HCfflf0WpSsVD0SExk9A9OPbxU40bY3mMCRphG+fMAsPezpj+SQPEB8UPcpZf +3toiuV6iDY7DDzo9j9oseUFkdsw5KMEDe7AqbRgILGP3rgfPmFR62E5rRFwPvOH8 +ObPpbEHCj1CWGKmi+KP/riI87rnWgd6OcuiT+CIAvQ5+bWKR/pPGOtqVgAOoj74R +cZKQIIzcQ21HS9jBWxJoH5zjZqcBu8He7krv0bwpKLXRRS3O4coBV3ghD7qr2R+/ +22GyVgj/HLRjMBj0+GYD3f+KA+gAipHhv2RXc5jn +-----END CERTIFICATE----- diff --git a/certs/openbao.crt b/certs/openbao.crt new file mode 100644 index 0000000..98c4be2 --- /dev/null +++ b/certs/openbao.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEdDCCAlygAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOMwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0MloXDTI3MDIxNDE5NTY0MlowKDEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +EDAOBgNVBAMMB29wZW5iYW8wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDWljdF+89fksslEpnesGmrErHXTStgRZ5mYlKfp+156KPpNRGG0njPXlsJoeVC +SPyU5YNaYPofh/LgLOF40GBMyOLF1pDvrmjm0S9L75vLE26iFw1T7XPMxCMMJ8hc +yfGrsy3M/Rqxuskq+MTeroPOgUoi4Zu8bpHEqTV5SJ6x7vnIZ6POLsXTj7hkES50 +Zpt1XmiWSXliUmC8Xr7JHpCBbUerf2CUuqaOYOrRRbaBCaCiPwOd8z6Td/tf3WQf +9NHdBHuLfiB+Kcqi09WMzEqk4vF9l0xMkyLfXRXttdZ4WC1tMageOgYrEm1tPFmc +S73uLxPFSmW5+jpWHxVhb1F7AgMBAAGjgZowgZcwDAYDVR0TAQH/BAIwADAOBgNV +HQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBgGA1Ud +EQQRMA+CB29wZW5iYW+HBH8AAAEwHQYDVR0OBBYEFFup+tHiqwiSyak3jWdAkvqc +uP7BMB8GA1UdIwQYMBaAFOoZicbH+r097VmyGVCfG3JmH/C+MA0GCSqGSIb3DQEB +CwUAA4ICAQCKSXpuSj7pqQPe805lcPwlS0+zet/abSeNi2Ub2CQhbYub5JE+lzqH +DEM8deOUQnP8WnNIsoXi1X7CRDq8N0GXH2rKj4L/YK4ml9fnLdxolk0+RSRDORw7 +VQw0grWL3/kARJjzaI6chKcUuHxZcu31uMc1luv/XeVL3PCg8xuV6FEt14AOqZMM +vEHbPDV107JeM+u8J6TirCGXs+UkxpDHZ62f2YEL1DIHUaQgQNtLCc+ALMxsRj73 +vumXKJIUPGdzf6XXYRASnzPzdh/sK0OdNsayCMkQyw2KNLkKIqMYa/toNLOa/RZy +Kqvvd/xQyw0N6ypHIGJz/sdQ+6PKe1AAb2A19B+M/w7uDFnPLajVORmwz/D4EaJt +CUuMB5uSR4crXHd59CfOFNnHHPVZm5Id0EiDvmb9xiMTMIwFSB5r2pxu88m2CNBc +y+3GOrinh+4KDYhaIurOn/IWr6hAeaVVdwqzZgMJmce555pIzmCjnZMoTMv6fx8F +Chdm613lU5X1ghMO1Aq1OccJe0+Q5/yco7fot2CQ3oQ7/f4ML9nNcW4STnr3AH1B +anRbz68zb6XYkFy7Jzu5oA2+8DNQAcCmnaS73jqJFG2XOt0s/jYjTT+rzAo9J2xs +9BzfoMj2oLXvZ8VDa5naDRXFxrhUPufxgqc6QHn1X8txWvlwHWHBiQ== +-----END CERTIFICATE----- diff --git a/certs/otel-collector.crt b/certs/otel-collector.crt new file mode 100644 index 0000000..33c9f6c --- /dev/null +++ b/certs/otel-collector.crt @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEijCCAnKgAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOowDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjIwMjYzNVoXDTI3MDIxNDIwMjYzNVowLzEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +FzAVBgNVBAMMDm90ZWwtY29sbGVjdG9yMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAkMdAA7r+P1p+D9madJzy1G+upOUctJPJ5bXHs0x8T/Oex3BTzqsp +L45nXR/N8yO7JLDXq2Lltu57lQk1RFxbzaXY0/gqx9O0LtoZajTYueQs5xqTxqAt +vCGwf6KnZO89foky0inXndrBJoYBFCZlXBYqdmrfn7xiTKLVWzd11ho6yAr0WfYk +itFJas2x896p7AVHSFzv0N1D7XoAkdYRfKBJHaNmK4+0LgD0BEj83FZnojoNwSj8 +3ZHddwci1KhZE1n/yqaAnQbzSm2+0QDY+DR+aduv0gQ6/97q6qN22ddWckuSxk93 +mKxe9PvibsZNma2ttTf+tlxj1Gka16ApeQIDAQABo4GpMIGmMAwGA1UdEwEB/wQC +MAAwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcD +AjAnBgNVHREEIDAegg5vdGVsLWNvbGxlY3RvcoIMb3RlbC1nYXRld2F5MB0GA1Ud +DgQWBBQl6YNrdlX9oOBqIdTvYH1joXDJpDAfBgNVHSMEGDAWgBTqGYnGx/q9Pe1Z +shlQnxtyZh/wvjANBgkqhkiG9w0BAQsFAAOCAgEAXxnMZ9eUFUoY3+017I63AQ32 +ng9aUEPmxuDnNtOfOemaH04w0xsWUS6cvquTb30hYF9JB/WW4V/aRGrB/RvRDB8C +5KhniFNRJGYF1GRI658emfZ0GetDz34b2ruiMP7qAEQRXe7uv5GV7ofZVNVZZPox +zplEfYmSQjAP0dxkoad6bZFKC8qv8+wwI7xIF13UJUNTGLfzgYBLEZVqPchO/QtG +QJcaDpHegB08Owobj2wDj54aB1103bThCEdDwbJmmtZ/9UhuBpgTgcWJRjO4Oa1U +W8wpk8DxolLdtxiKtb0HPlihdAC8QVH2cf3THgE8xnUYpID5+tHa3jSZRBtsdtuv +ZqtP4/ALJ1R7iaJNMQwdfaAWFzGtOhH1yHEcj5re7MtZkvIG5una16O2NyjVqWvL +syhQGdKmU6Fc/sLWxaA6+8g0Cq/VTiOFwqMwV1MIkhlwoaBHb8zNAb2tzcLX00uU +6ec0hljA6NvchrXho0dJ/FuAl91pCP2e31Fars4UnvTtS20AF3eRI+Mso7wXbxLf +Zc+y8gvvQlHBrR83K21YuiS6DR4JhHUkFn5MEZvfyEkxiHSrcKY5hqThYHKzLXFP +hpQkfgD1MYCOzYJNPHIYUWX8lXbITSsQcmR+vSKFVr5ILz+a7306dGgnOgdt6rI6 +0MJzrXoDsxRrsK5iIAA= +-----END CERTIFICATE----- diff --git a/certs/postgres.crt b/certs/postgres.crt new file mode 100644 index 0000000..99df340 --- /dev/null +++ b/certs/postgres.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEcDCCAligAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOQwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0NFoXDTI3MDIxNDE5NTY0NFowKTEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +ETAPBgNVBAMMCHBvc3RncmVzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEA3P8WmhCPtIA3qgEFDqKAo8uzlpTKQ19htm0tXBmSz9pRUfJePfuHP3XxZgEf +f1fSVdZw6Uvmc2GWx+/So2BsfWqpkhFELeG4Wt0KFyvadqs2MHCZUrgJmVJSW00W +HElndLcpAW0YMHovclEZp3EjonlQQZxoWKnrYZRmHcRKO1C+fYDK4CIXrAMijOax +0NCYdfj3/BMvpgWmSGmvGQ4ssvWNsJ3dmard2bYe8d8J3FtKqK0nRUfA55QpPN5g +voCb2Tbjo68I2kZs/IxDtYAA6ZwoziBI26E67l2aOFPxM2C/7wiNtivuKjdNk+v0 +yra84joZHJYBbQkpWtbXD+T5jQIDAQABo4GVMIGSMAwGA1UdEwEB/wQCMAAwDgYD +VR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjATBgNV +HREEDDAKgghwb3N0Z3JlczAdBgNVHQ4EFgQUyjxBqErsHzmCoZNXfgjX/lULGucw +HwYDVR0jBBgwFoAU6hmJxsf6vT3tWbIZUJ8bcmYf8L4wDQYJKoZIhvcNAQELBQAD +ggIBAGlWIfZQjHILoTUxfmU8xSwItEM4kwLcYJPYvcSzUppXjUEcqnSmlFGX5ti3 +37TAtoXy+N1EX94nZleLNnoGYtuM9b6Ew0D4c4PXONSDVtrlFLEwsefGq2XMdXmK +axi9TkAN0BItfN3iSwyYWk1s0TwC/17jOHqC1tMGKNGIXz8aNmU4BkYxj61Dk7cD +oTXwBw8Yzy28lbcQfkilr2bPk2vNxV85aUox/00JVNkvmbWvzlX7LZ5YDaMcRzKG +pjLsx0aLZM2m6ahXtc+uK6sU3nGGp+y+8lS3+CD5QmBctLLgzZe4r6RdPEhrQOsE +chddJSwwPQk8NAU4ND02fnOeD7UIBwDfRzSUjBzroYkrJ1EkHktp0E/0fSC8aH4X +c+Lp6PkS++jTDYkILTosQaLTRm61sKW5zGJapGLkyZ2cIRzby8bm7xIrPMkgHPqj +B3PgfWDF5y8b76lbjadwGByTsDSBS1+5vpG5hEH2+K+pI9mgC+tyZC8vAwZFBRaE +5vm0Dljif99jLM8zenp9i71q2EOFYxjFblx4qB6dju/EFjLLqysn5w0aboA1RKY2 +wAF++RdhL+/o5Of0+nF0/PptqO0XfR0T9Z1OSOeAWH7juAP+J05jX/YBwwaIp+0J +qIXI8PACWpjQ1jl72Z53Fypp10NLwMV+mUstDWnBq3RDNiPO +-----END CERTIFICATE----- diff --git a/certs/public.crt b/certs/public.crt new file mode 100644 index 0000000..5222a8a --- /dev/null +++ b/certs/public.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEazCCAlOgAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZN8wDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTYzOVoXDTI3MDIxNDE5NTYzOVowJzEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +DzANBgNVBAMMBnB1YmxpYzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AMtSmA5nkC5dlLKQFlGyVDxmp3paKarFa4faXESHee+Hs6vjatvWdScK4dtsME5l +2Rm9TVG2ck5DlR7+PSPeXOj1cBAWwkR0IsKovH6urya9VudMR12r94g4RebUF2Pd +rBps1UidKHSuz/XiRj2xRxbAJv821tlzG7NAX8UbRlUqBst3/e3u9akJy15qOM4n +hQC9u8SPgsOfxMor5qSsUf987xHVN7Q0GRRvLH1dVquNQjVHSu4449uTZ90CTUhm +jCmp94PTLU6DlFqRlSi/chQ/WSBbDTAVmuIKUh71av7xAJ88BcAsbMqrSr7QojC+ +OYb2Y9pZHOpMY2DzcJvu7eECAwEAAaOBkjCBjzAMBgNVHRMBAf8EAjAAMA4GA1Ud +DwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAaBgNVHREEEzARgglsb2Nh +bGhvc3SHBH8AAAEwHQYDVR0OBBYEFDyeBoZS+54MRbqGYAjKWLrsfArTMB8GA1Ud +IwQYMBaAFOoZicbH+r097VmyGVCfG3JmH/C+MA0GCSqGSIb3DQEBCwUAA4ICAQBZ +Sd4iy4dfPklL6b0snIx8iW1d2g7/OmJN9iq+u0SaWuXO7svYQLAjNCnsbxmXJjzT +ZXf/70hRo3SvQAAHmOruktvAGTfXaPEKUGrfJH3HGSCNdpvsMxESPfHFBljMq2CE +qSQvgZyV++Rv0HZtwiveViLYvr9JO4Mze3UikxLud6Qvxz2zVUxOCraZUAFIth13 +x4Jcamlxtja2ai68P6w6Cbg5YDf6aMwhQ7sbIJanKhFMgyUJEeLEUBozF5QrD/Uo ++fdw3ktwc1+g/G/XKFSPCTlPRgrHH5SPogrDnSpgDrzXgtxvLV2J12rkASjTnnAT +CYIWtQpHHdBHS99pYiqZnyrwcvLAOh16Ig/3QaMJPy3jNfJNS9odG5l212ZSKGG7 +1hukHsryfvVLLnDC2hWZ/dVX1OzlkLJ/st9bJZchu3ynUtVIMJCQOWdSJWwRHfkI +Xbk39joEluxCsQqUwjBBrUnkDg8/YuN8NAe2NQmP95+ajmzWHhHB6UGo1hKNXFb6 +Y3pEcR19az718ks0ob527KfmZPSdscvj6X64Ig/u0+LRhxy6/5TMcdBMfiStCBpL +Dscg5jLPV40opOlQsAAc1geObGi+mn2k9DGkrJjJPO3+5K9HIX1Xo/LhDrOdAsH9 +7YyNGH5URgZgxWgryC9x+lvDGkdOq13p2ty6txpczA== +-----END CERTIFICATE----- diff --git a/certs/rabbitmq.crt b/certs/rabbitmq.crt new file mode 100644 index 0000000..cb92f0f --- /dev/null +++ b/certs/rabbitmq.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEcDCCAligAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOEwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0MFoXDTI3MDIxNDE5NTY0MFowKTEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +ETAPBgNVBAMMCHJhYmJpdG1xMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC +AQEAsTD3aJyFKXVUhxGuBwPmLVG4OosI0eH9y9IqVBQw2wEjlIwXh1Z6CtpuYgxe +8LFX2eIdybcNjHqHZm39gcNmNIxHxVZtIN7wjmHJK3aV9ONZ5X40fXL1SfBsJMdx +H9rKywakFP4pMWY5pFzmnHgaMiXobmFu3MUVYkhIL3/gtIalHHUlu2xsdeSyH0XA +DwNzLyHqlcS/O2e/3RPGkaKJ3lzT3qZ8I/VpEKSqY+QrckwnLbz1rG4OC3/ffRZy +VOQ58kJCus5Uz1iKGa/DKg5lUn3nP2a9Zmz5b/x4vjEeaNBCrtU2VYb56UeKBR8k +/zWSWVhkIyC/wJH9JhKPqBf0/wIDAQABo4GVMIGSMAwGA1UdEwEB/wQCMAAwDgYD +VR0PAQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjATBgNV +HREEDDAKgghyYWJiaXRtcTAdBgNVHQ4EFgQUf4CzJ5g382KvhMfof2uZK2q6hp4w +HwYDVR0jBBgwFoAU6hmJxsf6vT3tWbIZUJ8bcmYf8L4wDQYJKoZIhvcNAQELBQAD +ggIBAHlaM6hD4to3Bud2cTFpVgh59dgJZi7EMK0AiX02os5ApGmbkpXvKfQthbIF +q7kdeOxz64HS2R4yRqXQlX2zWfpDZkqKfQEk3qHCiwMp5cgO1VdIpvmaNCQ+GvUM +MVYKmrzrjhSnNE13h8rA4B4i50CCoUmRZ/5YWrm1s+S5z7EnrEJPYAGr7B7kGFGD +1j19bAUhH+VAQE245ydrHwoyMmhXg7igyp7owehjKlPzoopOpdlEwEvFY+7BOcmp +a7B1R94Lrx74v0JYqGmMiKf7g5XrAqi6pVQOTusfxbBrbVgoAS1aQaNT6zKy5WTZ +hqBtGlG2jUqjiAks8hPNucVIw6qTcBEVnlHsnDBk84qZAUf0VhnrPRkSy7bf9+ty +3YA7+RZMmvSeHF8f99/nmUnHRk7kK28GVcuOKZ58FscaiBaqohmFl/S/RXaAWEpS +GYw1PXumSIDruVECv0cz9W+SrFHeb6k1Y4WTrePtX8+j2d9H9sAYYEQnunPuDDZq +TOt1KHE/B5RC7aHvjLSA6VZr4Qs493SYzzK29K7fa84p3Lpuul0zDvbLIXVhslM9 +Am6HJb+owQVoFYofWlj72O+sxUA3UACr1veVhgZehTAWMDkM0+S/79GX4pTlTDW5 +axL4NGfN2lpfhaOKNcNkRRnVsAzAHvm0o4WosHuVe0T7tw1+ +-----END CERTIFICATE----- diff --git a/certs/redis.crt b/certs/redis.crt new file mode 100644 index 0000000..336d08b --- /dev/null +++ b/certs/redis.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEajCCAlKgAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOUwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0NFoXDTI3MDIxNDE5NTY0NFowJjEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +DjAMBgNVBAMMBXJlZGlzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA +m0R3mzLtvoWR288Q1LcRVbEdyg7YkXroW9SxFua77wrdjw0PZNcsQFWGZydoxd+d +pUtLvoq1KJMX+zMR+I9I03XXY25dosc/vB+K61qrVmlilbb4BYbi8TzRos6RjU30 +aTb/nq5DsFYCR1LU6w9gB5yb9fxRbnME0zYlBmCnY8KD3EOd+AAhUEwcBRiZIkkO +CUqrjZ1/0NeeNlQ2NEKwwFVMD67HxS8d+kc7mM+dZ16kBARdLhwF8wvEvym0nAZw +7sWq+fuap0htL47oEmE1Mlv+XX6TVvUzZQTlCjRLIZInAir5BEia0ZsPSMUc9zFx +JlW69yknWbn1oYptrM0zVQIDAQABo4GSMIGPMAwGA1UdEwEB/wQCMAAwDgYDVR0P +AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAQBgNVHREE +CTAHggVyZWRpczAdBgNVHQ4EFgQUBdlw/xNpiL67iES1tR8T6r3KIJkwHwYDVR0j +BBgwFoAU6hmJxsf6vT3tWbIZUJ8bcmYf8L4wDQYJKoZIhvcNAQELBQADggIBAHia +7gCJqvCFWenpdqCAUEIxQ/3tMehPWrfo2jPxGxeWCiR68xn20nkzXfdIUoumO/W1 +RVFelN1Sg14YPzQZXC9JAOkx3R00Lur4Yz9jPXmElxx2DrD53RE4OkWtNqAgZZXT +Ab2sE+lMMD5wJBKVqIydzSg/d7tp8HSQDkrx4pTc6JV36Ud7MY4LJhUuJWI+rr9y +R+LWXiG+ztjSJpnaU3bRci/QjL5xgeP5wCBMMeYTXV2GTm8ClSoZRtIj0egEWUUT +A7hwxw9v5/QEG4U0VWt4ImKNAqlMVvkvcof+V6NjMwcPx4z6f7Llar3Ojk8ry2SE +E7zS+9oH9u7A/8T62+cAieNjAW7TjddW35lSquv1jPfcK2DSz4snd/RFLU5DPoAA +ZoPjADY+AmIS01PXAJF16FhGpQUSP6Kfhl+UKRwC0B+AIhfYgMttbes91NTJc12x +WJZozBBHh4p896hRfPC49i5cNC6dI3Cfex8lEQOeKQqva2tR2NtfKeu0MzZZv+9c +xcA2EkZnTaqnnwrZrkw6Ds3U87PrEenEC72KAwV/L04AWIFwHLytnqN/kEkHbijH +jojrubxbcGpURrJKssE5dQ+z2bo4ExaWz0KMuhvV9M4F0sU1K/B+LM+ICIi7AUmr +merYLhgOV5/io77gt62TodlDxC2K4KXL21cH+ju6 +-----END CERTIFICATE----- diff --git a/certs/revoked_serials.txt b/certs/revoked_serials.txt new file mode 100644 index 0000000..cda33d7 --- /dev/null +++ b/certs/revoked_serials.txt @@ -0,0 +1 @@ +# one uppercase certificate serial per line diff --git a/certs/worker-client.crt b/certs/worker-client.crt new file mode 100644 index 0000000..f3f5f7c --- /dev/null +++ b/certs/worker-client.crt @@ -0,0 +1,26 @@ +-----BEGIN CERTIFICATE----- +MIIEcDCCAligAwIBAgIUJYqV+PDtIN2tlkokFJBUquMgZOgwDQYJKoZIhvcNAQEL +BQAwIzEUMBIGA1UECgwLTVRVU0kgTGFiIDcxCzAJBgNVBAMMAmNhMB4XDTI2MTEx +NjE5NTY0N1oXDTI3MDIxNDE5NTY0N1owLjEUMBIGA1UECgwLTVRVU0kgTGFiIDcx +FjAUBgNVBAMMDXdvcmtlci1jbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDAwyTrR8mXfuJ/SotIt+vcCqV/Hdrq+vQwczMQSxQZ6NyFc9HLVlYw +QEl0sM3VCp58cscJB9x9xYigvbI84wTo1khOJRqWNVdXR0NstHImoI5QSwYLuMgZ +WRkqKynzXxQGLmvxftIN0AQpi8P7eFR7WTdi47cEFLcTsmxx6qS3DHRz439GYLNC ++UnwM82lW707v2VmjTbO6z4Fx6ITyK4FpNaEwqOKb17KaCFk7A+tfJmzTBZVsqky +teROXEEx8ycPvS3StMGsEvOqMH0EjAYSHFvgSzu0z7dLJh6LbckBFqW+79rPV93o +3hKmzKPZPAklT+I/jrVjs6wc1X0E7XIrAgMBAAGjgZAwgY0wDAYDVR0TAQH/BAIw +ADAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwIwGAYDVR0RBBEw +D4INd29ya2VyLWNsaWVudDAdBgNVHQ4EFgQU4oMNPIY7h3V9fHy3Qeyxvhjn/zow +HwYDVR0jBBgwFoAU6hmJxsf6vT3tWbIZUJ8bcmYf8L4wDQYJKoZIhvcNAQELBQAD +ggIBAMwt0OAPZU/PW0LSOn/WWE4+yYdmsdaqXsLTmKYZxs0+9Z81riSz8kaiaBEv +bSpwjoY+YhdS8RWYpbRD2kYzbNk3MXBfcH7B5s4/3ikYygnFg7SN+FMFeLl8GEGe +DWTzXm/cyyg/v1UmCcf/ylEhwCIXbzvHL/+nXdX5GF7z37AmJhgoN/eZFgfPoS+4 +7n9dOq0eqGa9Xaa0SA82ousNFB2h7FF37kdN6EM2AKjeh+AZrDj5W4MYx6WK0VUZ +b1G5UfwDfJSAghPjm2wOdItcL25fFYhfiDVEJ1DynuaU6a6fKbL/LLFRkRQ1n3id +UbF/jQIfIazogvkbaQ6abnVNndu1Uoiz9O2vfp2j4UZOp9F11enA6mf1//A5U4vG +7KmIKdaS72794tZEVDcsWfAdvoiOEA28JM9j7VDPSuWuQt9+yfriotyT+pWlvbU0 +WqfKCAViTkBsH5ileHyxweWNwVX/zbltVzk9mY8dDc+MoW9lkFRR4Zc3Bew/IOVT +3BPqa3qUcL22smDfy1REWmyocxOUvQ8gcvLYrT9EFx+XF1RFLm/N1aG08+m23Ece +5yHUzP3nP09syEnLs7hpZwWd3OOkryYFexAb2wKRBb+Hi2sW8a8ROAo5HlH1Sf+X +sJ3qzuxpj+zR3OQJmGiIPnWDxTmj4/xhU/2syMaZ+QHquI+H +-----END CERTIFICATE----- diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..90a9d9a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,592 @@ +name: mtusi-lab7 + +x-app-env: &app-env + APP_HOST: 0.0.0.0 + APP_PORT: "8443" + HEALTH_HOST: 127.0.0.1 + HEALTH_PORT: "8081" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + POOL_MAX: ${POOL_MAX:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: minio:9000 + S3_BUCKET: messenger-attachments + S3_SECURE: "true" + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + TLS_CA_FILE: /certs/ca.crt + TLS_CERT_FILE: /certs/api.crt + TLS_KEY_FILE: /certs/private/api.key + MTLS_ALLOWED_CLIENTS: lb + BROKER_CLIENT_CERT: /certs/api-client.crt + BROKER_CLIENT_KEY: /certs/private/api-client.key + BAO_ADDR: https://openbao:8200 + BAO_ROLE_ID_FILE: /run/secrets/bao_api_role_id + BAO_SECRET_ID_FILE: /run/secrets/bao_api_secret_id + BAO_SECRET_PATH: secret/data/messenger/api + OTEL_SERVICE_NAME: messenger-api + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-gateway:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_CERTIFICATE: /certs/ca.crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /certs/api-client.crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /certs/private/api-client.key + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + SERVICE_VERSION: lab7 + ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456} + ARGON2_ITERATIONS: ${ARGON2_ITERATIONS:-2} + ARGON2_PARALLELISM: ${ARGON2_PARALLELISM:-1} + +services: + postgres: + image: postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74 + entrypoint: ["/bin/sh", "/init/postgres-entry.sh"] + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + volumes: + - postgres-data:/var/lib/postgresql/data + - ./scripts/postgres-entry.sh:/init/postgres-entry.sh:ro + - ./certs/postgres.crt:/tls/postgres.crt:ro + - ./certs/private/postgres.key:/tls/postgres.key:ro + - ./certs/ca.crt:/tls/ca.crt:ro + secrets: + - postgres_password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 15s + cpus: "1.00" + mem_limit: 512m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + redis: + image: redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9 + entrypoint: ["/bin/sh", "/init/redis-entry.sh"] + volumes: + - redis-data:/data + - ./scripts/redis-entry.sh:/init/redis-entry.sh:ro + - ./certs/redis.crt:/tls/redis.crt:ro + - ./certs/private/redis.key:/tls/redis.key:ro + - ./certs/ca.crt:/tls/ca.crt:ro + healthcheck: + test: ["CMD", "redis-cli", "--tls", "--cacert", "/tls-run/ca.crt", "ping"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 8s + cpus: "0.25" + mem_limit: 128m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + rabbitmq: + image: rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc + hostname: rabbitmq + entrypoint: ["/bin/sh", "/init/rabbit-entry.sh"] + volumes: + - rabbitmq-data:/var/lib/rabbitmq + - ./scripts/rabbit-entry.sh:/init/rabbit-entry.sh:ro + - ./certs/rabbitmq.crt:/tls/rabbitmq.crt:ro + - ./certs/private/rabbitmq.key:/tls/rabbitmq.key:ro + - ./certs/ca.crt:/tls/ca.crt:ro + secrets: + - rabbit_user + - rabbit_password + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 10s + retries: 12 + start_period: 180s + cpus: "1.00" + mem_limit: 1024m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + minio: + image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + entrypoint: ["/bin/sh", "/init/minio-tls-entry.sh"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + volumes: + - minio-data:/data + - ./scripts/minio-tls-entry.sh:/init/minio-tls-entry.sh:ro + - ./certs/minio.crt:/tls/minio.crt:ro + - ./certs/private/minio.key:/tls/minio.key:ro + - ./certs/ca.crt:/tls/ca.crt:ro + secrets: + - minio_root_user + - minio_root_password + healthcheck: + test: ["CMD", "curl", "-fk", "https://127.0.0.1:9000/minio/health/live"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + cpus: "0.50" + mem_limit: 512m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + minio-init: + image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + entrypoint: ["/bin/sh", "/init/minio-init.sh"] + environment: + MC_HOST_local: "" + S3_SECURE: "true" + volumes: + - ./scripts/minio-init.sh:/init/minio-init.sh:ro + - ./infra/minio-api-policy.json:/init/policy.json:ro + - ./certs/ca.crt:/certs/ca.crt:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key + - s3_secret_key + depends_on: + minio: + condition: service_healthy + restart: "no" + networks: + - lab + + openbao: + image: openbao/openbao:2.1.0 + entrypoint: ["/bin/sh", "/init/openbao-entry.sh"] + volumes: + - ./infra/openbao/openbao.hcl:/openbao/config/openbao.hcl:ro + - ./scripts/openbao-entry.sh:/init/openbao-entry.sh:ro + - ./certs/openbao.crt:/openbao/tls/openbao.crt:ro + - ./certs/private/openbao.key:/openbao/tls/openbao.key:ro + - ./certs/ca.crt:/openbao/tls/ca.crt:ro + - openbao-data:/openbao/file + environment: + BAO_ADDR: https://127.0.0.1:8200 + BAO_CACERT: /openbao/tls-run/ca.crt + healthcheck: + test: + [ + "CMD", + "wget", + "--no-check-certificate", + "-qO-", + "https://127.0.0.1:8200/v1/sys/health?sealedcode=200&uninitcode=200&standbycode=200", + ] + interval: 5s + timeout: 5s + retries: 20 + start_period: 10s + cpus: "0.50" + mem_limit: 256m + security_opt: + - no-new-privileges:true + ports: + - "127.0.0.1:8200:8200" + restart: unless-stopped + networks: + - lab + + migrate: + build: . + image: mtusi-lab7-api:local + user: "10001:10001" + command: ["python", "/app/src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + TLS_CA_FILE: /certs/ca.crt + volumes: + - ./certs/ca.crt:/certs/ca.crt:ro + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: "0.50" + mem_limit: 256m + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp + restart: "no" + networks: + - lab + + api: + build: . + image: mtusi-lab7-api:local + user: "10001:10001" + environment: + <<: *app-env + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-120} + volumes: + - ./certs/ca.crt:/certs/ca.crt:ro + - ./certs/api.crt:/certs/api.crt:ro + - ./certs/private/api.key:/certs/private/api.key:ro + - ./certs/api-client.crt:/certs/api-client.crt:ro + - ./certs/private/api-client.key:/certs/private/api-client.key:ro + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key + - s3_secret_key + - bao_api_role_id + - bao_api_secret_id + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + openbao: + condition: service_healthy + otel-gateway: + condition: service_started + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/health/ready', timeout=2)", + ] + interval: 5s + timeout: 3s + retries: 20 + start_period: 45s + init: true + stop_grace_period: 10s + cpus: "0.50" + mem_limit: 512m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + tmpfs: + - /tmp + deploy: + replicas: 2 + restart: unless-stopped + networks: + - lab + + worker: + build: . + image: mtusi-lab7-api:local + user: "10001:10001" + command: ["python", "/app/src/worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: minio:9000 + S3_BUCKET: messenger-attachments + S3_SECURE: "true" + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + TLS_CA_FILE: /certs/ca.crt + BROKER_CLIENT_CERT: /certs/worker-client.crt + BROKER_CLIENT_KEY: /certs/private/worker-client.key + BAO_ADDR: https://openbao:8200 + BAO_ROLE_ID_FILE: /run/secrets/bao_worker_role_id + BAO_SECRET_ID_FILE: /run/secrets/bao_worker_secret_id + BAO_SECRET_PATH: secret/data/messenger/worker + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + OTEL_SERVICE_NAME: messenger-worker + OTEL_EXPORTER_OTLP_ENDPOINT: https://otel-gateway:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_EXPORTER_OTLP_CERTIFICATE: /certs/ca.crt + OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE: /certs/worker-client.crt + OTEL_EXPORTER_OTLP_CLIENT_KEY: /certs/private/worker-client.key + OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab + OTEL_TRACES_SAMPLER_ARG: ${OTEL_TRACES_SAMPLER_ARG:-1.0} + SERVICE_VERSION: lab7 + volumes: + - ./certs/ca.crt:/certs/ca.crt:ro + - ./certs/worker-client.crt:/certs/worker-client.crt:ro + - ./certs/private/worker-client.key:/certs/private/worker-client.key:ro + secrets: + - database_url + - broker_url + - s3_access_key + - s3_secret_key + - bao_worker_role_id + - bao_worker_secret_id + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + openbao: + condition: service_healthy + otel-gateway: + condition: service_started + healthcheck: + disable: true + init: true + stop_grace_period: 20s + cpus: "1.00" + mem_limit: 768m + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + read_only: true + tmpfs: + - /tmp + restart: unless-stopped + networks: + - lab + + lb: + image: haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./certs/ca.crt:/etc/certs/ca.crt:ro + - ./certs/private/public.pem:/etc/certs/private/public.pem:ro + - ./certs/private/lb.pem:/etc/certs/private/lb.pem:ro + ports: + - "127.0.0.1:8443:8443" + depends_on: + api: + condition: service_healthy + cpus: "0.25" + mem_limit: 128m + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp + restart: unless-stopped + networks: + - lab + + otel-collector: + image: otel/opentelemetry-collector-contrib:0.114.0@sha256:37fa87091cfaaec7234a27e4e395a40c31c2bfaea97a349a4afef6d9e9681197 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + cpus: "0.50" + mem_limit: 384m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + otel-gateway: + image: nginx:1.27.2-alpine + volumes: + - ./infra/nginx-otel.conf:/etc/nginx/nginx.conf:ro + - ./certs/ca.crt:/certs/ca.crt:ro + - ./certs/otel-collector.crt:/certs/otel-collector.crt:ro + - ./certs/private/otel-collector.key:/certs/private/otel-collector.key:ro + depends_on: + - otel-collector + cpus: "0.10" + mem_limit: 64m + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp + - /var/cache/nginx + - /var/run + restart: unless-stopped + networks: + - lab + + prometheus: + image: prom/prometheus:v2.54.1@sha256:f6639335d34a77d9d9db382b92eeb7fc00934be8eae81dbc03b31cfe90411a94 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=48h + - --web.enable-remote-write-receiver + - --web.enable-lifecycle + volumes: + - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro + - prometheus-data:/prometheus + ports: + - "127.0.0.1:9090:9090" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + cpus: "0.50" + mem_limit: 512m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + loki: + image: grafana/loki:3.2.1@sha256:09a53b4a4ff81ffcd8f13886df19d33fac7a8d3aaf952e3c7e66cbade5b2fc31 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - loki-data:/loki + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3100/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + cpus: "0.50" + mem_limit: 512m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + tempo: + image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempo-data:/var/tempo + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3200/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + cpus: "0.50" + mem_limit: 512m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + + grafana: + image: grafana/grafana:11.2.2@sha256:d5133220d770aba5cb655147b619fa8770b90f41d8489a821d33b1cd34d16f89 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: labadmin + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SERVER_HTTP_PORT: "3000" + GF_PATHS_PROVISIONING: /etc/grafana/provisioning + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: + - "127.0.0.1:3000:3000" + depends_on: + prometheus: + condition: service_started + loki: + condition: service_started + tempo: + condition: service_started + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 20s + cpus: "0.50" + mem_limit: 384m + security_opt: + - no-new-privileges:true + restart: unless-stopped + networks: + - lab + +networks: + lab: + name: mtusi-lab7-net + +volumes: + postgres-data: + redis-data: + minio-data: + rabbitmq-data: + prometheus-data: + loki-data: + tempo-data: + grafana-data: + openbao-data: + +secrets: + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + postgres_password: + file: ./secrets/postgres_password + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + rabbit_user: + file: ./secrets/rabbit_user + rabbit_password: + file: ./secrets/rabbit_password + broker_url: + file: ./secrets/broker_url + bao_api_role_id: + file: ./secrets/bao_api_role_id + bao_api_secret_id: + file: ./secrets/bao_api_secret_id + bao_worker_role_id: + file: ./secrets/bao_worker_role_id + bao_worker_secret_id: + file: ./secrets/bao_worker_secret_id diff --git a/contracts/trust-matrix.md b/contracts/trust-matrix.md index 0c2a977..c778e65 100644 --- a/contracts/trust-matrix.md +++ b/contracts/trust-matrix.md @@ -1,37 +1,20 @@ -# Матрица доверия — заполнить в решении lab7 +# Матрица доверия — Lab 7 (БПИ2301, Орлова) -Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать. +TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно ограничивается разрешённая identity. | Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг | | --- | --- | --- | --- | -| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO | -| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO | -| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO | -| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO | -| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO | -| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO | -| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO | -| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO | -| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO | -| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO | -| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO | -| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO | +| Клиент → ingress/LB | TLS :8443, CA `certs/ca.crt`, SAN localhost; Bearer | Login rate limit в Redis (429+Retry-After), logout/expiry | `tests/security.py`, `tests/smoke.py`; HAProxy `infra/haproxy.cfg` | +| LB → API (все реплики) | mTLS: client `lb`, server `api`, verify CA + allowlist CN/SAN `lb` | Leaf API/LB без пересборки образов (`scripts/bootstrap_pki.py --force`) | `tests/student_mtls.py`; HAProxy `crt lb.pem` + API `tlsutil.server_ssl_context` | +| API/worker → PostgreSQL | TLS verify-full, `sslrootcert=/certs/ca.crt`, роль messenger | Ротация пароля через secrets + restart | `scripts/postgres-entry.sh`, `secrets/database_url` | +| API/worker → S3 | HTTPS MinIO, scoped user `messenger-api` policy | Denied prefix в policy; ротация access key в secrets | `scripts/minio-tls-entry.sh`, `infra/minio-api-policy.json` | +| API/worker → broker | amqps:5671, verify_peer + client certs | Разные identities `api-client` / `worker-client` | `scripts/rabbit-entry.sh`, `BROKER_CLIENT_*` | +| API/worker → Collector | mTLS через `otel-gateway` (nginx), identities api/worker-client | Чужой leaf того же CA (например `lb`) → 403 | `infra/nginx-otel.conf`, `tests/student_mtls.py` (паттерн) | +| Collector → telemetry backends | Закрытый Docker network; Tempo/Loki HTTP внутри доверенной сети | N/A (бэкенды не публикуют бизнес-API) | `infra/otel/collector.yaml` | +| Grafana → data sources | Локальный publish 127.0.0.1:3000, provisioning | Viewer anonymous без admin write | `infra/grafana/provisioning` | +| API/worker/agent → OpenBao | TLS :8200, AppRole, policies `messenger-api`/`messenger-worker` | TTL 5m, fail-closed при sealed | `scripts/openbao_bootstrap.py`, `src/bao.py` | +| Оператор → OpenBao/Grafana/S3 admin | OpenBao 127.0.0.1:8200; root/unseal только в `secrets/openbao/` | Разделение operator vs AppRole | REPORT.md bootstrap | +| API → Redis/Valkey | rediss:// + CA verify | Общий login rate limit по репликам | `scripts/redis-entry.sh`, `store.login_rate_*` | +| Replication/кластерные каналы | N/A — одиночные инстансы stateful | — | — | -Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход. - -## Обязательные отрицательные проверки - -1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен». -2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех. -3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity. -4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity. -5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3. -6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить. - -## Bootstrap и эксплуатация - -В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу. - -OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии. - -Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap. +Публичный обход API-порта отсутствует: наружу только `127.0.0.1:8443` (LB), admin Grafana/Prometheus/OpenBao на localhost. Health API на `127.0.0.1:8081` внутри контейнера, не опубликован. diff --git a/evidence/README.md b/evidence/README.md index 8df7001..adbc543 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,5 @@ # Доказательства -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Храните небольшие обезличенный результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. укажите дату, commit, аппаратные лимиты и профиль нагрузки. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +Не добавляйте .env, session tokens, cookies, приватный ключи, credentials, реальный сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скринѢота без методики недостаточно. diff --git a/evidence/pki-track2.md b/evidence/pki-track2.md new file mode 100644 index 0000000..fda66d9 --- /dev/null +++ b/evidence/pki-track2.md @@ -0,0 +1,25 @@ +# Track 2 — PKI lifecycle (sanitized) + +Дата: 2026-11-16 + +## Гипотеза + +Короткоживущие leaf можно перевыпускать локальным CA без пересборки базового образа приложения и без fallback на plaintext / verify off. + +## Методика + +1. `python -X utf8 scripts/bootstrap_pki.py` — исходный CA и leaf. +2. Поднять стенд, `tests/smoke.py` и `tests/security.py` по HTTPS с `certs/ca.crt`. +3. `python -X utf8 scripts/pki_renew.py --identity api --days 14` — новый leaf `api` без печати ключа. +4. `docker compose --env-file .env up -d --force-recreate --scale api=2 api` — подхватить файлы сертификатов (образ тот же tag `mtusi-lab7-api:local`, меняется только bind-mount leaf). +5. Повторный smoke/security и `tests/student_mtls.py`. + +## Результаты + +- После renew + recreate API снова healthy. +- Публичный клиент по-прежнему проверяет CA; `tests/security.py` с пустым trust store остаётся отрицательным. +- Fallback `verify=False` / HTTP на публичном входе отсутствует (LB только :8443 ssl). + +## Ограничения + +На учебном стенде rollover делается recreate контейнера, не hot-reload in-process. CRL/OCSP не внедрялись; отзыв моделируется заменой leaf и deny старой identity через отсутствие файла в mount. diff --git a/evidence/scan-triage.md b/evidence/scan-triage.md new file mode 100644 index 0000000..ea4e229 --- /dev/null +++ b/evidence/scan-triage.md @@ -0,0 +1,28 @@ +# Dependency / image scan triage (lab7) + +Дата: 2026-11-16. Сканер: `docker scout` / ручной разбор по закреплённым digest из `compose.yaml`. Секреты и токены в вывод не попадали. + +## Состав поверхности + +| Образ | Зачем | Примечание | +| --- | --- | --- | +| `mtusi-lab7-api:local` (python:3.12.10-slim) | API/worker | non-root 10001, read_only+tmpfs, cap_drop ALL | +| `postgres:16.10-bookworm@sha256:38471…` | БД | TLS, volume; privileged нет | +| `redis:7.4.2-bookworm@sha256:fbdbae…` | сессии | TLS-only порт | +| `rabbitmq:3.13.7-alpine@sha256:d7af…` | брокер | amqps + peer cert | +| `pgsty/minio@sha256:b6bfe7…` | S3 | HTTPS; root только init | +| `haproxy:2.8.14@sha256:2344d1…` | ingress | только 127.0.0.1:8443 | +| `openbao/openbao:2.1.0` | secrets | non-dev file storage, TLS | +| `otel/…:0.114.0`, Loki/Tempo/Prometheus/Grafana, nginx otel-gateway | telemetry | admin только localhost | + +## Типичные находки и применимость + +| Класс | Пример | Применимо ли к стенду | Решение / остаточный риск | +| --- | --- | --- | --- | +| CVE в клиентских утилитах base image | редко используемые CLI в slim | Низкая: процесс слушает только app/uvicorn | Оставить; обновление base при следующей лабе | +| CVE в Grafana/Prometheus UI | XSS/admin | Средняя, если открыть не-localhost | Publish только `127.0.0.1`; anon Viewer без write | +| CVE в RabbitMQ management | HTTP management | Низкая: management наружу не публикуется | Остаётся внутренним | +| Supply-chain MinIO | неофициальный mirror `pgsty/minio` | Средняя: не Docker Hub official | Закреплён digest; протокол S3 проверен smoke | +| Secrets in image | — | Нет: secrets через Docker secrets + OpenBao | Корневые/unseal только у оператора на диске | + +Итог: критичных CVE с прямым remote RCE на опубликованном :8443 в разборе не зафиксировано. Главный остаточный риск — учебный локальный CA и ручной unseal OpenBao, а не «зелёный» count CVE без triage. diff --git a/infra/grafana/dashboards/messenger.json b/infra/grafana/dashboards/messenger.json new file mode 100644 index 0000000..c3c100a --- /dev/null +++ b/infra/grafana/dashboards/messenger.json @@ -0,0 +1,279 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "title": "API RPS", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m]))", + "legendFormat": "rps", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "API error ratio (5xx / all business)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger:http_error_ratio:5m", + "legendFormat": "5m error ratio", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "API latency p50/p95/p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Jobs ready / failed / retry", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (job_outcome) (rate(messenger_jobs_processed_total[1m]))", + "legendFormat": "{{job_outcome}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "Job duration p50/p95/p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(messenger_jobs_duration_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Time to ready p50/p95/p99", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(messenger_attachment_time_to_ready_seconds_bucket[5m])) by (le))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Queue depth", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_queue_depth{messaging_destination_name=~\"attachments\\\\.process|attachments\\\\.retry\"}", + "legendFormat": "{{messaging_destination_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "Oldest job age", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "max(messenger_queue_oldest_job_age_seconds{messaging_destination_name=\"attachments.process\"})", + "legendFormat": "oldest age", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "drawStyle": "line", "fillOpacity": 10 } + } + } + }, + { + "title": "DB pool used / waiting", + "type": "timeseries", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_db_pool_used_ratio", + "legendFormat": "used {{service_name}}", + "refId": "A" + }, + { + "expr": "messenger_db_pool_waiting_ratio", + "legendFormat": "waiting {{service_name}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Worker busy / capacity", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_worker_busy_ratio", + "legendFormat": "busy", + "refId": "A" + }, + { + "expr": "messenger_worker_capacity_ratio", + "legendFormat": "capacity", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "Process RSS", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "messenger_process_rss_bytes", + "legendFormat": "{{service_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { "drawStyle": "line", "fillOpacity": 5 } + } + } + }, + { + "title": "API / worker logs", + "type": "logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 }, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "{service_name=~\"messenger-api|messenger-worker\"}", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["messenger", "lab6"], + "templating": { "list": [] }, + "time": { "from": "now-30m", "to": "now" }, + "timezone": "browser", + "title": "Messenger observability", + "uid": "messenger-lab6", + "version": 1 +} diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..7c36229 --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: messenger + orgId: 1 + folder: Messenger + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..b89f255 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,45 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + uid: prometheus + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 15s + + - name: Loki + type: loki + access: proxy + uid: loki + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([a-f0-9]+)"' + name: TraceID + url: "$${__value.raw}" + urlDisplayLabel: View Trace + + - name: Tempo + type: tempo + access: proxy + uid: tempo + url: http://tempo:3200 + editable: false + jsonData: + tracesToLogs: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + mapTagNamesEnabled: false + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: loki diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..00d6617 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,52 @@ +global + maxconn 256 + tune.bufsize 65536 + tune.maxrewrite 8192 + tune.ssl.default-dh-param 2048 + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384 + ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11 + ssl-default-server-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384 + ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 + +defaults + mode http + timeout connect 2s + timeout client 60s + timeout server 60s + timeout http-request 60s + timeout http-keep-alive 2s + timeout queue 2s + timeout check 2s + option redispatch + option dontlognull + retries 2 + retry-on conn-failure empty-response 502 503 504 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + accepted_payload_size 8192 + +frontend web + bind *:8443 ssl crt /etc/certs/private/public.pem alpn h2,http/1.1 + # Только HTTPS снаружи. HTTP-порт API не публикуется. + default_backend api + +backend api + balance roundrobin + acl upload method POST path_reg -i ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ + http-request wait-for-body time 2s if METH_POST !upload + http-request disable-l7-retry if upload + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + # mTLS к каждой реплике: клиентский сертификат LB + проверка серверного SAN api. + server-template api 8 api:8443 check inter 500ms fall 2 rise 2 resolvers docker init-addr none resolve-prefer ipv4 on-marked-down shutdown-sessions ssl verify required ca-file /etc/certs/ca.crt crt /etc/certs/private/lb.pem verifyhost api check-ssl diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..4d4798a --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,49 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + ingestion_rate_mb: 16 + ingestion_burst_size_mb: 32 + allow_structured_metadata: true + volume_enabled: true + +ruler: + alertmanager_url: http://localhost:9093 + +analytics: + reporting_enabled: false diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 0000000..cfef8ef --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/nginx-otel.conf b/infra/nginx-otel.conf new file mode 100644 index 0000000..7d7656c --- /dev/null +++ b/infra/nginx-otel.conf @@ -0,0 +1,32 @@ +worker_processes 1; +error_log /dev/stderr warn; +pid /tmp/nginx.pid; + +events { + worker_connections 64; +} + +http { + access_log off; + server { + listen 4318 ssl; + server_name otel-collector; + + ssl_certificate /certs/otel-collector.crt; + ssl_certificate_key /certs/private/otel-collector.key; + ssl_client_certificate /certs/ca.crt; + ssl_verify_client on; + ssl_protocols TLSv1.2 TLSv1.3; + + # Разрешены только api-client и worker-client (не любой leaf того же CA). + if ($ssl_client_s_dn !~* "CN=(api-client|worker-client)(,|$)") { + return 403; + } + + location / { + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_pass http://otel-collector:14318; + } + } +} diff --git a/infra/openbao/openbao.hcl b/infra/openbao/openbao.hcl new file mode 100644 index 0000000..740d1ad --- /dev/null +++ b/infra/openbao/openbao.hcl @@ -0,0 +1,15 @@ +ui = false +disable_mlock = true + +storage "file" { + path = "/openbao/file" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_cert_file = "/openbao/tls/openbao.crt" + tls_key_file = "/openbao/tls/openbao.key" +} + +api_addr = "https://openbao:8200" +cluster_addr = "https://openbao:8201" diff --git a/infra/openbao/policies/api.hcl b/infra/openbao/policies/api.hcl new file mode 100644 index 0000000..09cfb91 --- /dev/null +++ b/infra/openbao/policies/api.hcl @@ -0,0 +1,8 @@ +# API may read only its own KV v2 secrets. +path "secret/data/messenger/api" { + capabilities = ["read"] +} + +path "secret/metadata/messenger/api" { + capabilities = ["read"] +} diff --git a/infra/openbao/policies/worker.hcl b/infra/openbao/policies/worker.hcl new file mode 100644 index 0000000..0596515 --- /dev/null +++ b/infra/openbao/policies/worker.hcl @@ -0,0 +1,8 @@ +# Worker may read only its own KV v2 secrets. +path "secret/data/messenger/worker" { + capabilities = ["read"] +} + +path "secret/metadata/messenger/worker" { + capabilities = ["read"] +} diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..4a7d423 --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,81 @@ +receivers: + otlp: + protocols: + # Внешний mTLS терминирует nginx-otel; здесь только закрытый hop. + http: + endpoint: 0.0.0.0:14318 + grpc: + endpoint: 0.0.0.0:14317 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 64 + batch: + send_batch_size: 256 + send_batch_max_size: 512 + timeout: 2s + resource: + attributes: + - key: deployment.environment.name + value: lab + action: upsert + +exporters: + otlphttp/tempo: + endpoint: http://tempo:4318 + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 256 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 30s + # Pull-модель: Collector держит только последние значения в памяти, без бесконечной очереди. + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + enable_open_metrics: true + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 256 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 30s + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [otlphttp/tempo] + metrics: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch, resource] + exporters: [otlphttp/loki] + telemetry: + metrics: + level: basic + address: 0.0.0.0:8888 diff --git a/infra/prometheus/alerts.yml b/infra/prometheus/alerts.yml new file mode 100644 index 0000000..10d33a9 --- /dev/null +++ b/infra/prometheus/alerts.yml @@ -0,0 +1,88 @@ +groups: + - name: messenger-slo + rules: + - record: messenger:http_requests:rate5m + expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[5m])) or vector(0) + - record: messenger:http_5xx:rate5m + expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[5m])) or vector(0) + - record: messenger:http_error_ratio:5m + expr: | + ( + messenger:http_5xx:rate5m + / + clamp_min(messenger:http_requests:rate5m, 0.001) + ) + - record: messenger:http_requests:rate30m + expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[30m])) or vector(0) + - record: messenger:http_5xx:rate30m + expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[30m])) or vector(0) + - record: messenger:http_error_ratio:30m + expr: | + ( + messenger:http_5xx:rate30m + / + clamp_min(messenger:http_requests:rate30m, 0.001) + ) + - record: messenger:http_requests:rate1h + expr: sum(rate(messenger_http_requests_total{http_route!~"/health/.*"}[1h])) or vector(0) + - record: messenger:http_5xx:rate1h + expr: sum(rate(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[1h])) or vector(0) + - record: messenger:http_error_ratio:1h + expr: | + ( + messenger:http_5xx:rate1h + / + clamp_min(messenger:http_requests:rate1h, 0.001) + ) + + - name: messenger-alerts + rules: + - alert: HighHttpErrorRate + expr: messenger:http_error_ratio:5m > 0.05 + for: 1m + labels: + severity: page + annotations: + summary: "Доля HTTP 5xx выше 5% за 5 минут" + description: "Текущая доля {{ $value | humanizePercentage }}. Runbook: evidence/runbooks/high-error-rate.md" + runbook_url: "evidence/runbooks/high-error-rate.md" + + - alert: GrowingQueueLag + expr: max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}) > 30 + for: 30s + labels: + severity: ticket + annotations: + summary: "Возраст старейшей job в attachments.process выше 30 с" + description: "Возраст {{ $value }} с. Runbook: evidence/runbooks/queue-lag.md" + runbook_url: "evidence/runbooks/queue-lag.md" + + - alert: ErrorBudgetBurnFast + expr: | + ( + messenger:http_error_ratio:5m > (14.4 * 0.01) + and + messenger:http_error_ratio:30m > (14.4 * 0.01) + ) + for: 2m + labels: + severity: page + burn_window: fast + annotations: + summary: "Быстрый burn rate ошибки SLO (14.4x на 5м и 30м)" + description: "Кратковременный всплеск 5xx. Runbook: evidence/runbooks/burn-rate.md" + runbook_url: "evidence/runbooks/burn-rate.md" + + - alert: ErrorBudgetBurnSlow + expr: | + ( + messenger:http_error_ratio:30m > (6 * 0.01) + ) + for: 5m + labels: + severity: ticket + burn_window: slow + annotations: + summary: "Медленный burn rate ошибки SLO (6x на 30м)" + description: "Длительная деградация. Runbook: evidence/runbooks/burn-rate.md" + runbook_url: "evidence/runbooks/burn-rate.md" diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..8f7a8a2 --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: messenger-otel + metrics_path: /metrics + static_configs: + - targets: ["otel-collector:8889"] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..fc4a97b --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,47 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + trace_idle_period: 10s + max_block_bytes: 1048576 + max_block_duration: 5m + +compactor: + compaction: + block_retention: 48h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + +overrides: + defaults: + metrics_generator: + processors: [] + +querier: + max_concurrent_queries: 20 + +query_frontend: + search: + max_duration: 1h diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..994fa1d --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,48 @@ +-- Пользователи, чаты, участники и сообщения. +-- Создание чата в приложении идёт одной транзакцией: строка chats и строки chat_members. +-- Индекс messages_history_idx совпадает с keyset-запросом истории: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id +-- LIMIT $4 + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (btrim(display_name) <> ''), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_nonblank CHECK (btrim(title) <> ''), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + ordinal smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_ordinal_nonneg CHECK (ordinal >= 0) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + body text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_body_nonblank CHECK (btrim(body) <> ''), + CONSTRAINT messages_body_len CHECK (char_length(body) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 0000000..d4e557b --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,19 @@ +-- Результат успешного POST сообщения. +-- Область ключа: пользователь + метод + путь чата + Idempotency-Key. +-- Строка пишется в той же транзакции, что и messages, поэтому 5xx +-- не оставляет «занятый» ключ. Другое тело при живом ключе — отказ +-- на уровне приложения (хеш не совпал), без второй строки messages. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + request_path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, http_method, request_path, idem_key), + CONSTRAINT idempotency_method_post CHECK (http_method = 'POST'), + CONSTRAINT idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_hash_len CHECK (char_length(request_hash) = 64) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 0000000..ebb52e3 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,42 @@ +-- Метаданные вложения. Байты лежат в приватном bucket, не в этой таблице. +-- Пустой текст допустим, если к сообщению прикреплено вложение. Пробелы по-прежнему нельзя. + +ALTER TABLE messages DROP CONSTRAINT messages_body_nonblank; +ALTER TABLE messages DROP CONSTRAINT messages_body_len; +ALTER TABLE messages + ADD CONSTRAINT messages_body_len CHECK (char_length(body) <= 4000); +ALTER TABLE messages + ADD CONSTRAINT messages_body_nonblank CHECK (body = '' OR btrim(body) <> ''); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL UNIQUE, + status text NOT NULL DEFAULT 'ready', + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type_chk CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_chk CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha_chk CHECK (char_length(sha256) = 64), + CONSTRAINT attachments_status_chk CHECK (status = 'ready'), + CONSTRAINT attachments_key_chk CHECK (char_length(object_key) BETWEEN 1 AND 512) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + ordinal integer NOT NULL, + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal), + CONSTRAINT message_attachments_ordinal_chk CHECK (ordinal >= 0 AND ordinal < 10) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 0000000..f735350 --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,108 @@ +-- Задание на обработку, варианты и outbox. +-- Сообщение брокера не содержит байты и пароли: в outbox лежит только JSON задания. +-- Ключ эффекта — (attachment_id, pipeline_version, name). Повтор не создаёт вторую строку. + +ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk; + +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; + +ALTER TABLE attachments ADD COLUMN error_code text; + +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; + +ALTER TABLE attachments ADD COLUMN staging_key text; + +ALTER TABLE attachments ADD COLUMN staging_deleted_at timestamptz; + +ALTER TABLE attachments ADD CONSTRAINT attachments_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed')); + +ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_chk CHECK (pipeline_version >= 1); + +ALTER TABLE attachments ADD CONSTRAINT attachments_error_chk CHECK ( + (status = 'failed' AND error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout')) + OR (status <> 'failed' AND error_code IS NULL) +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + status text NOT NULL, + attempt integer NOT NULL, + max_attempts integer NOT NULL, + lease_owner text, + lease_until timestamptz, + next_attempt_at timestamptz, + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachment_jobs_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')), + CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt >= 1 AND attempt <= max_attempts), + CONSTRAINT attachment_jobs_max_chk CHECK (max_attempts BETWEEN 1 AND 20), + CONSTRAINT attachment_jobs_version_chk CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_error_chk CHECK ( + error_code IS NULL OR error_code IN ( + 'invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout', 'superseded' + ) + ), + CONSTRAINT attachment_jobs_one_version UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs (status, next_attempt_at); + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_name_chk CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type_chk CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size_chk CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_dim_chk CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_sha_chk CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +CREATE TABLE outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + published_at timestamptz, + CONSTRAINT outbox_attempt_chk CHECK (attempt >= 1) +); + +CREATE INDEX outbox_pending_idx ON outbox (created_at) WHERE published_at IS NULL; + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + kind text NOT NULL, + detail text, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT job_events_kind_chk CHECK (char_length(kind) BETWEEN 1 AND 40), + CONSTRAINT job_events_detail_chk CHECK (detail IS NULL OR char_length(detail) <= 64) +); + +CREATE TABLE fault_injections ( + id uuid PRIMARY KEY, + kind text NOT NULL, + attachment_id uuid, + armed boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fault_injections_kind_chk CHECK (kind IN ( + 'skip_inline_publish', 'crash_before_effect', 'crash_after_objects', + 'crash_after_commit', 'transient_once' + )) +); + +CREATE INDEX fault_injections_armed_idx ON fault_injections (kind, created_at) WHERE armed; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..be6bd23 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +minio==7.2.15 +pillow==11.1.0 +pika==1.3.2 +opentelemetry-api==1.27.0 +opentelemetry-sdk==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-semantic-conventions==0.48b0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 0000000..db9d535 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Создать локальные .env, secrets и PKI. Значения секретов в консоль не печатаются.""" +import secrets +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" +PASSWORD_FILE = SECRET_DIR / "postgres_password" +DATABASE_FILE = SECRET_DIR / "database_url" +CACHE_FILE = SECRET_DIR / "cache_url" +MINIO_USER_FILE = SECRET_DIR / "minio_root_user" +MINIO_PASSWORD_FILE = SECRET_DIR / "minio_root_password" +S3_ACCESS_FILE = SECRET_DIR / "s3_access_key" +S3_SECRET_FILE = SECRET_DIR / "s3_secret_key" +RABBIT_USER_FILE = SECRET_DIR / "rabbit_user" +RABBIT_PASSWORD_FILE = SECRET_DIR / "rabbit_password" +BROKER_FILE = SECRET_DIR / "broker_url" +BAO_FILES = ( + SECRET_DIR / "bao_api_role_id", + SECRET_DIR / "bao_api_secret_id", + SECRET_DIR / "bao_worker_role_id", + SECRET_DIR / "bao_worker_secret_id", +) +ENV_FILE = ROOT / ".env" +ENV_EXAMPLE = ROOT / ".env.example" + + +def main(): + SECRET_DIR.mkdir(exist_ok=True) + code = subprocess.call([sys.executable, "-X", "utf8", str(ROOT / "scripts" / "bootstrap_pki.py")]) + if code != 0: + return code + + files = (PASSWORD_FILE, DATABASE_FILE, CACHE_FILE) + if any(path.exists() for path in files) and not all(path.exists() for path in files): + print("secrets/ заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in files): + print("db/cache secrets already exist") + else: + password = secrets.token_hex(24) + PASSWORD_FILE.write_text(password + "\n", encoding="utf-8") + DATABASE_FILE.write_text( + "postgresql://messenger:{}@postgres:5432/messenger" + "?sslmode=verify-full&sslrootcert=/certs/ca.crt\n".format(password), + encoding="utf-8", + ) + CACHE_FILE.write_text("rediss://redis:6379/0\n", encoding="utf-8") + print("db/cache secrets created") + + s3_files = (MINIO_USER_FILE, MINIO_PASSWORD_FILE, S3_ACCESS_FILE, S3_SECRET_FILE) + if any(path.exists() for path in s3_files) and not all(path.exists() for path in s3_files): + print("secrets/ для MinIO заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in s3_files): + print("s3 secrets already exist") + else: + MINIO_USER_FILE.write_text("labroot", encoding="utf-8") + MINIO_PASSWORD_FILE.write_text(secrets.token_hex(24), encoding="utf-8") + S3_ACCESS_FILE.write_text("api" + secrets.token_hex(8), encoding="utf-8") + S3_SECRET_FILE.write_text(secrets.token_hex(24), encoding="utf-8") + print("s3 secrets created") + + broker_files = (RABBIT_USER_FILE, RABBIT_PASSWORD_FILE, BROKER_FILE) + if any(path.exists() for path in broker_files) and not all(path.exists() for path in broker_files): + print("secrets/ для брокера заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in broker_files): + print("broker secrets already exist") + else: + user = "lab" + password = secrets.token_hex(24) + RABBIT_USER_FILE.write_text(user, encoding="utf-8") + RABBIT_PASSWORD_FILE.write_text(password, encoding="utf-8") + BROKER_FILE.write_text( + "amqps://{}:{}@rabbitmq:5671/%2F\n".format(user, password), + encoding="utf-8", + ) + print("broker secrets created") + + for path in BAO_FILES: + if not path.exists(): + # Заглушка до scripts/openbao_bootstrap.py; compose требует файл. + path.write_text("pending-openbao-bootstrap\n", encoding="utf-8") + print("openbao secret placeholders ready (run openbao_bootstrap after openbao is up)") + + if ENV_FILE.exists(): + print(".env already exists") + else: + text = ENV_EXAMPLE.read_text(encoding="utf-8") + text = text.replace("APP_PORT=8080", "APP_PORT=8443") + ENV_FILE.write_text(text, encoding="utf-8") + print(".env created") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/bootstrap_pki.py b/scripts/bootstrap_pki.py new file mode 100644 index 0000000..9fe7468 --- /dev/null +++ b/scripts/bootstrap_pki.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Create the local Lab 7 CA and service certificates without printing keys.""" +from __future__ import annotations + +import argparse +import shutil +import stat +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CERT_DIR = ROOT / "certs" +PRIVATE_DIR = CERT_DIR / "private" +OPENSSL_IMAGE = "alpine/openssl:3.3.2" + +SERVER_IDENTITIES = { + "public": ["DNS:localhost", "IP:127.0.0.1"], + "api": ["DNS:api"], + "rabbitmq": ["DNS:rabbitmq"], + "otel-collector": ["DNS:otel-collector", "DNS:otel-gateway"], + "openbao": ["DNS:openbao", "IP:127.0.0.1"], + "postgres": ["DNS:postgres"], + "redis": ["DNS:redis"], + "minio": ["DNS:minio"], +} +CLIENT_IDENTITIES = ("lb", "worker-client", "api-client") + + +def _find_openssl() -> str | None: + found = shutil.which("openssl") + if found: + return found + for candidate in ( + r"C:\Program Files\Git\usr\bin\openssl.exe", + r"C:\Program Files (x86)\Git\usr\bin\openssl.exe", + ): + if Path(candidate).exists(): + return candidate + return None + + +def _container_path(path: Path) -> str: + return "/work/" + path.resolve().relative_to(ROOT.resolve()).as_posix() + + +def _run_openssl(args: list[str]) -> None: + executable = _find_openssl() + if executable: + command = [executable, *args] + else: + if not shutil.which("docker"): + raise RuntimeError("openssl and docker are both unavailable") + converted = [] + for arg in args: + try: + converted.append(_container_path(Path(arg)) if Path(arg).is_absolute() else arg) + except ValueError: + converted.append(arg) + command = [ + "docker", + "run", + "--rm", + "-v", + f"{ROOT}:/work", + "-w", + "/work", + OPENSSL_IMAGE, + *converted, + ] + subprocess.run(command, check=True, stdout=subprocess.DEVNULL) + + +def _protect(path: Path) -> None: + try: + path.chmod(stat.S_IRUSR | stat.S_IWUSR) + except OSError: + pass + + +def _write_extension(identity: str, sans: list[str], usage: str) -> Path: + handle = tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", prefix=f"{identity}-", suffix=".cnf", dir=CERT_DIR, delete=False + ) + with handle: + handle.write("[v3]\n") + handle.write("basicConstraints=critical,CA:FALSE\n") + handle.write("keyUsage=critical,digitalSignature,keyEncipherment\n") + handle.write(f"extendedKeyUsage={usage}\n") + if sans: + handle.write("subjectAltName=" + ",".join(sans) + "\n") + return Path(handle.name) + + +def issue_leaf(identity: str, days: int = 90, force: bool = False) -> tuple[Path, Path]: + if identity in SERVER_IDENTITIES: + sans = SERVER_IDENTITIES[identity] + usage = "serverAuth,clientAuth" if identity != "public" else "serverAuth" + elif identity in CLIENT_IDENTITIES or identity == "rollover-demo": + sans = [f"DNS:{identity}"] + usage = "clientAuth" + else: + raise ValueError(f"unknown PKI identity: {identity}") + + cert = CERT_DIR / f"{identity}.crt" + key = PRIVATE_DIR / f"{identity}.key" + if cert.exists() and key.exists() and not force: + return cert, key + + csr = CERT_DIR / f".{identity}.csr" + serial = CERT_DIR / "ca.srl" + extension = _write_extension(identity, sans, usage) + try: + _run_openssl( + [ + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + str(key), + "-out", + str(csr), + "-subj", + f"/O=MTUSI Lab 7/CN={identity}", + ] + ) + command = [ + "x509", + "-req", + "-in", + str(csr), + "-CA", + str(CERT_DIR / "ca.crt"), + "-CAkey", + str(PRIVATE_DIR / "ca.key"), + "-out", + str(cert), + "-days", + str(days), + "-sha256", + "-extfile", + str(extension), + "-extensions", + "v3", + ] + if serial.exists(): + command.extend(["-CAserial", str(serial)]) + else: + command.append("-CAcreateserial") + _run_openssl(command) + finally: + csr.unlink(missing_ok=True) + extension.unlink(missing_ok=True) + _protect(key) + return cert, key + + +def _combined(identity: str) -> None: + target = PRIVATE_DIR / f"{identity}.pem" + cert = CERT_DIR / f"{identity}.crt" + key = PRIVATE_DIR / f"{identity}.key" + target.write_bytes(cert.read_bytes() + key.read_bytes()) + _protect(target) + + +def bootstrap(force: bool = False, days: int = 90) -> None: + CERT_DIR.mkdir(exist_ok=True) + PRIVATE_DIR.mkdir(exist_ok=True) + ca_cert = CERT_DIR / "ca.crt" + ca_key = PRIVATE_DIR / "ca.key" + if force or not (ca_cert.exists() and ca_key.exists()): + _run_openssl( + [ + "req", + "-x509", + "-newkey", + "rsa:4096", + "-nodes", + "-keyout", + str(ca_key), + "-out", + str(ca_cert), + "-days", + "3650", + "-sha256", + "-subj", + "/O=MTUSI Lab 7/CN=ca", + "-addext", + "basicConstraints=critical,CA:TRUE,pathlen:0", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ] + ) + _protect(ca_key) + + for identity in (*SERVER_IDENTITIES, *CLIENT_IDENTITIES): + issue_leaf(identity, days=days, force=force) + _combined("public") + _combined("lb") + revoked = CERT_DIR / "revoked_serials.txt" + if not revoked.exists(): + revoked.write_text("# one uppercase certificate serial per line\n", encoding="ascii") + print("PKI ready: public certificates are in certs/; private keys remain in ignored certs/private/") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--force", action="store_true", help="replace the CA and every leaf") + parser.add_argument("--days", type=int, default=90) + args = parser.parse_args() + if args.days < 1: + parser.error("--days must be at least 1") + bootstrap(force=args.force, days=args.days) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/minio-init.sh b/scripts/minio-init.sh new file mode 100644 index 0000000..7197ab4 --- /dev/null +++ b/scripts/minio-init.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Идемпотентно создаёт приватный bucket и отдельного пользователя API. +set -eu +ROOT_USER=$(tr -d '\r\n' < /run/secrets/minio_root_user) +ROOT_PW=$(tr -d '\r\n' < /run/secrets/minio_root_password) +ACCESS=$(tr -d '\r\n' < /run/secrets/s3_access_key) +SECRET=$(tr -d '\r\n' < /run/secrets/s3_secret_key) + +export SSL_CERT_FILE=/certs/ca.crt +attempt=0 +until mc alias set local https://minio:9000 "$ROOT_USER" "$ROOT_PW" >/dev/null; do + attempt=$((attempt + 1)) + if [ "$attempt" -gt 40 ]; then + echo "minio alias failed" >&2 + exit 1 + fi + sleep 1 +done + +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null + +if ! mc admin user info local "$ACCESS" >/dev/null 2>&1; then + mc admin user add local "$ACCESS" "$SECRET" >/dev/null +fi +mc admin policy detach local messenger-api --user "$ACCESS" >/dev/null 2>&1 || true +mc admin policy rm local messenger-api >/dev/null 2>&1 || true +mc admin policy create local messenger-api /init/policy.json >/dev/null +mc admin policy attach local messenger-api --user "$ACCESS" >/dev/null +echo "minio bucket ready" diff --git a/scripts/minio-tls-entry.sh b/scripts/minio-tls-entry.sh new file mode 100644 index 0000000..c1a7f4d --- /dev/null +++ b/scripts/minio-tls-entry.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +mkdir -p /root/.minio/certs/CAs +cp /tls/minio.crt /root/.minio/certs/public.crt +cp /tls/minio.key /root/.minio/certs/private.key +cp /tls/ca.crt /root/.minio/certs/CAs/ca.crt +chmod 600 /root/.minio/certs/private.key +exec minio server /data --address :9000 --console-address 127.0.0.1:9001 diff --git a/scripts/openbao-entry.sh b/scripts/openbao-entry.sh new file mode 100644 index 0000000..72ffe9c --- /dev/null +++ b/scripts/openbao-entry.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +mkdir -p /openbao/tls-run +cp /openbao/tls/openbao.crt /openbao/tls-run/openbao.crt +cp /openbao/tls/openbao.key /openbao/tls-run/openbao.key +cp /openbao/tls/ca.crt /openbao/tls-run/ca.crt +chmod 600 /openbao/tls-run/openbao.key +chmod 644 /openbao/tls-run/openbao.crt /openbao/tls-run/ca.crt +# Подменить пути в runtime-конфиге +sed 's|/openbao/tls/openbao.crt|/openbao/tls-run/openbao.crt|; s|/openbao/tls/openbao.key|/openbao/tls-run/openbao.key|' \ + /openbao/config/openbao.hcl > /tmp/openbao.hcl +export BAO_CACERT=/openbao/tls-run/ca.crt +exec bao server -config=/tmp/openbao.hcl diff --git a/scripts/openbao_bootstrap.py b/scripts/openbao_bootstrap.py new file mode 100644 index 0000000..098610f --- /dev/null +++ b/scripts/openbao_bootstrap.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Операторский init/unseal OpenBao + AppRole/policies. Tokens/shares не печатаются.""" +from __future__ import annotations + +import json +import os +import ssl +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" +BAO_DIR = SECRET_DIR / "openbao" +CA = ROOT / "certs" / "ca.crt" +ADDR = os.environ.get("BAO_ADDR", "https://127.0.0.1:8200").rstrip("/") + + +def ctx() -> ssl.SSLContext: + context = ssl.create_default_context(cafile=str(CA)) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + # cert SAN is openbao; operator talks via 127.0.0.1 publish + context.check_hostname = False + return context + + +def call(method: str, path: str, body=None, token: str | None = None) -> dict: + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + if token: + headers["X-Vault-Token"] = token + req = urllib.request.Request(ADDR + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, context=ctx(), timeout=10) as response: + raw = response.read().decode("utf-8") + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + if exc.code >= 400: + raise RuntimeError(f"{method} {path} -> {exc.code}: {raw[:200]}") from exc + return json.loads(raw) if raw else {} + return json.loads(raw) if raw else {} + + +def write_secret(path: Path, value: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(value.strip() + "\n", encoding="utf-8") + + +def wait_up() -> None: + for _ in range(60): + try: + call("GET", "/v1/sys/health?sealedcode=200&uninitcode=200&standbycode=200") + return + except Exception: + time.sleep(2) + raise RuntimeError("openbao not reachable") + + +def main() -> int: + wait_up() + BAO_DIR.mkdir(parents=True, exist_ok=True) + root_file = BAO_DIR / "root_token" + unseal_file = BAO_DIR / "unseal_key" + + status = call("GET", "/v1/sys/seal-status") + if not status.get("initialized"): + result = call("PUT", "/v1/sys/init", {"secret_shares": 1, "secret_threshold": 1}) + keys = result.get("keys_base64") or result.get("keys") + root = result.get("root_token") + if not keys or not root: + raise RuntimeError("init failed") + write_secret(unseal_file, keys[0]) + write_secret(root_file, root) + print("OpenBao initialized (keys stored under secrets/openbao/, not printed)") + else: + if not root_file.exists() or not unseal_file.exists(): + print("OpenBao already initialized but local operator keys missing", file=sys.stderr) + return 2 + print("OpenBao already initialized") + + root = root_file.read_text(encoding="utf-8").strip() + unseal = unseal_file.read_text(encoding="utf-8").strip() + status = call("GET", "/v1/sys/seal-status") + if status.get("sealed"): + call("PUT", "/v1/sys/unseal", {"key": unseal}) + print("OpenBao unsealed") + else: + print("OpenBao already unsealed") + + # Enable secrets/auth (ignore already-exists) + try: + call("POST", "/v1/sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}, token=root) + except RuntimeError as exc: + if "already in use" not in str(exc).lower() and "existing mount" not in str(exc).lower(): + # Vault returns 400 if exists + if "400" not in str(exc): + raise + try: + call("POST", "/v1/sys/auth/approle", {"type": "approle"}, token=root) + except RuntimeError as exc: + if "path is already in use" not in str(exc).lower() and "400" not in str(exc): + raise + + api_policy = (ROOT / "infra/openbao/policies/api.hcl").read_text(encoding="utf-8") + worker_policy = (ROOT / "infra/openbao/policies/worker.hcl").read_text(encoding="utf-8") + call("PUT", "/v1/sys/policies/acl/messenger-api", {"policy": api_policy}, token=root) + call("PUT", "/v1/sys/policies/acl/messenger-worker", {"policy": worker_policy}, token=root) + + call( + "POST", + "/v1/auth/approle/role/messenger-api", + { + "token_policies": ["messenger-api"], + "token_ttl": "5m", + "token_max_ttl": "15m", + "secret_id_ttl": "720h", + }, + token=root, + ) + call( + "POST", + "/v1/auth/approle/role/messenger-worker", + { + "token_policies": ["messenger-worker"], + "token_ttl": "5m", + "token_max_ttl": "15m", + "secret_id_ttl": "720h", + }, + token=root, + ) + + call( + "POST", + "/v1/secret/data/messenger/api", + {"data": {"purpose": "api-runtime", "note": "scoped-lab7"}}, + token=root, + ) + call( + "POST", + "/v1/secret/data/messenger/worker", + {"data": {"purpose": "worker-runtime", "note": "scoped-lab7"}}, + token=root, + ) + + api_role = call("GET", "/v1/auth/approle/role/messenger-api/role-id", token=root)["data"]["role_id"] + worker_role = call("GET", "/v1/auth/approle/role/messenger-worker/role-id", token=root)["data"]["role_id"] + api_secret = call("POST", "/v1/auth/approle/role/messenger-api/secret-id", {}, token=root)["data"]["secret_id"] + worker_secret = call("POST", "/v1/auth/approle/role/messenger-worker/secret-id", {}, token=root)["data"][ + "secret_id" + ] + + write_secret(SECRET_DIR / "bao_api_role_id", api_role) + write_secret(SECRET_DIR / "bao_api_secret_id", api_secret) + write_secret(SECRET_DIR / "bao_worker_role_id", worker_role) + write_secret(SECRET_DIR / "bao_worker_secret_id", worker_secret) + print("AppRole credentials written to secrets/bao_*; recreate api/worker to pick them up") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/pki_renew.py b/scripts/pki_renew.py new file mode 100644 index 0000000..c5fca61 --- /dev/null +++ b/scripts/pki_renew.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Track 2: перевыпуск leaf без пересборки образов. Приватные ключи не печатаются.""" +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from bootstrap_pki import _combined, issue_leaf # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--identity", required=True) + parser.add_argument("--days", type=int, default=7) + args = parser.parse_args() + issue_leaf(args.identity, days=args.days, force=True) + if args.identity in {"public", "lb"}: + _combined(args.identity) + print(f"renewed {args.identity} for {args.days} days; recreate dependent containers to load new leaf") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/postgres-entry.sh b/scripts/postgres-entry.sh new file mode 100644 index 0000000..1d8dfbb --- /dev/null +++ b/scripts/postgres-entry.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -eu +mkdir -p /var/lib/postgresql/tls +cp /tls/postgres.crt /var/lib/postgresql/tls/server.crt +cp /tls/postgres.key /var/lib/postgresql/tls/server.key +cp /tls/ca.crt /var/lib/postgresql/tls/ca.crt +chown -R postgres:postgres /var/lib/postgresql/tls +chmod 600 /var/lib/postgresql/tls/server.key +chmod 644 /var/lib/postgresql/tls/server.crt /var/lib/postgresql/tls/ca.crt +exec docker-entrypoint.sh postgres \ + -c fsync=on \ + -c synchronous_commit=on \ + -c full_page_writes=on \ + -c max_connections=80 \ + -c shared_buffers=64MB \ + -c ssl=on \ + -c ssl_cert_file=/var/lib/postgresql/tls/server.crt \ + -c ssl_key_file=/var/lib/postgresql/tls/server.key \ + -c ssl_ca_file=/var/lib/postgresql/tls/ca.crt diff --git a/scripts/rabbit-entry.sh b/scripts/rabbit-entry.sh new file mode 100644 index 0000000..f5a7be5 --- /dev/null +++ b/scripts/rabbit-entry.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Пароль и TLS: конфиг только внутри контейнера, секреты не в git. +set -eu +user=$(tr -d '\r\n' < /run/secrets/rabbit_user) +pass=$(tr -d '\r\n' < /run/secrets/rabbit_password) +mkdir -p /tmp/rabbit-conf /tmp/rabbit-tls +cp /tls/rabbitmq.crt /tmp/rabbit-tls/server.crt +cp /tls/rabbitmq.key /tmp/rabbit-tls/server.key +cp /tls/ca.crt /tmp/rabbit-tls/ca.crt +# rabbitmq-пользователь должен читать key (Windows bind mount часто root-only). +chown -R rabbitmq:rabbitmq /tmp/rabbit-tls 2>/dev/null || true +chmod 640 /tmp/rabbit-tls/server.key +chmod 644 /tmp/rabbit-tls/server.crt /tmp/rabbit-tls/ca.crt +cat > /tmp/rabbit-conf/rabbitmq.conf </dev/null || true +chmod 644 /tmp/rabbit-conf/rabbitmq.conf +export RABBITMQ_CONFIG_FILE=/tmp/rabbit-conf/rabbitmq +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/scripts/redis-entry.sh b/scripts/redis-entry.sh new file mode 100644 index 0000000..bd13c9e --- /dev/null +++ b/scripts/redis-entry.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -eu +mkdir -p /tls-run +cp /tls/redis.crt /tls-run/redis.crt +cp /tls/redis.key /tls-run/redis.key +cp /tls/ca.crt /tls-run/ca.crt +chmod 600 /tls-run/redis.key +chmod 644 /tls-run/redis.crt /tls-run/ca.crt +exec redis-server \ + --tls-port 6379 \ + --port 0 \ + --tls-cert-file /tls-run/redis.crt \ + --tls-key-file /tls-run/redis.key \ + --tls-ca-cert-file /tls-run/ca.crt \ + --tls-auth-clients no \ + --appendonly yes \ + --appendfsync everysec \ + --dir /data \ + --save 60 1 \ + --bind 0.0.0.0 \ + --protected-mode no diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..6a74e62 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +python -X utf8 "%~dp0restart.py" %* diff --git a/scripts/restart.example.sh b/scripts/restart.example.sh old mode 100755 new mode 100644 diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 0000000..73f8340 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Windows-friendly persistence hook: restart stateful services, keep volumes.""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def run(args: list[str]) -> None: + subprocess.check_call(args, cwd=ROOT) + + +def main() -> int: + run(["docker", "compose", "--env-file", ".env", "restart", "postgres", "redis", "minio", "rabbitmq", "prometheus", "loki", "tempo", "openbao"]) + run(["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--no-deps", "postgres", "redis", "minio", "rabbitmq", "prometheus", "loki", "tempo", "openbao"]) + # OpenBao may be sealed after restart; operator unseal without printing keys. + subprocess.call([sys.executable, "-X", "utf8", str(ROOT / "scripts" / "openbao_bootstrap.py")], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + run(["docker", "compose", "--env-file", ".env", "restart", "api", "worker"]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..d9e36ea --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Рестарт API и хранилищ со состоянием. Volume не удаляем. +# OpenBao после рестарта может быть sealed — оператор делает unseal отдельно. +docker compose --env-file .env restart postgres redis minio rabbitmq prometheus loki tempo openbao +docker compose --env-file .env up -d --wait --no-deps postgres redis minio rabbitmq prometheus loki tempo openbao +# Если OpenBao sealed — unseal локальными ключами оператора (без печати). +python -X utf8 scripts/openbao_bootstrap.py >/tmp/bao-restart.log 2>&1 || true +docker compose --env-file .env restart api worker diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..d0a9367 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,2 @@ +@echo off +python -X utf8 "%~dp0stop_one.py" %* diff --git a/scripts/stop-one.example.sh b/scripts/stop-one.example.sh old mode 100755 new mode 100644 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..945d972 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" + +found="" +for id in $(docker ps -q \ + --filter label=com.docker.compose.project=mtusi-lab7 \ + --filter label=com.docker.compose.service=api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + short=$(printf %s "$id" | cut -c1-12) + if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then + found=$id + break + fi +done + +if [ -z "$found" ]; then + echo "api replica not found" >&2 + exit 1 +fi + +# SIGTERM, затем процесс остаётся остановленным (restart: unless-stopped). +docker stop --time 5 "$found" >/dev/null diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..675ff28 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Остановить одну API-реплику по TARGET_INSTANCE_ID (hostname контейнера).""" +from __future__ import annotations + +import os +import subprocess +import sys + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip() + if not target: + print("TARGET_INSTANCE_ID required", file=sys.stderr) + return 1 + out = subprocess.check_output( + [ + "docker", + "ps", + "-q", + "--filter", + "label=com.docker.compose.project=mtusi-lab7", + "--filter", + "label=com.docker.compose.service=api", + ], + text=True, + ) + found = None + for line in out.splitlines(): + cid = line.strip() + if not cid: + continue + host = subprocess.check_output(["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True).strip() + short = cid[:12] + if host == target or short == target: + found = cid + break + if not found: + print("api replica not found", file=sys.stderr) + return 1 + subprocess.check_call(["docker", "stop", "--time", "5", found], stdout=subprocess.DEVNULL) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 0000000..15f5bbc --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Ждать GET /health/ready через HTTPS балансировщик. Без токенов в выводе.""" +import os +import ssl +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "https://127.0.0.1:8443").rstrip("/") +URL = BASE + "/health/ready" +CA = os.environ.get("CA_FILE") or str(ROOT / "certs" / "ca.crt") + + +def main(): + context = ssl.create_default_context(cafile=CA) + deadline = time.monotonic() + 120 + last = "not-started" + while time.monotonic() < deadline: + try: + req = urllib.request.Request(URL) + response = urllib.request.urlopen(req, timeout=2, context=context) + if response.status == 200 and response.headers.get("X-Instance-Id"): + print("ready") + return 0 + last = str(response.status) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + last = type(exc).__name__ + time.sleep(1) + print("not ready: " + last, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/security/THREAT_MODEL.md b/security/THREAT_MODEL.md index 31da287..bc50fdc 100644 --- a/security/THREAT_MODEL.md +++ b/security/THREAT_MODEL.md @@ -1,19 +1,32 @@ -# Модель угроз — заполнить +# Модель угроз — Lab 7 ## Активы и акторы -TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда. +Активы: пароли/Argon2id hashes, session tokens (в Redis только sha256-ключ), сообщения и вложения, service identities/PKI, OpenBao secrets, telemetry. + +Акторы: участник чата, посторонний пользователь, внешний клиент без CA, скомпрометированный сервис с чужим leaf того же CA, оператор со стендом. + +Граница учебного стенда: Docker Compose на одной машине; публичный вход только HTTPS :8443. ## Границы доверия -TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача. +См. `contracts/trust-matrix.md`. Публичная зона — клиент↔LB. Внутренняя — LB↔API (mTLS), API/worker↔зависимости (TLS/mTLS), OpenBao (TLS+AppRole). | Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение | | --- | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | TODO | - -На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки. +| MITM к публичному API с поддельным сертификатом | Целостность/конфиденциальность клиентского канала | Публичный TLS + проверка CA/hostname | `tests/security.py` пустой trust store | Локальный CA учебного стенда | +| Прямой доступ к реплике API в обход LB | Единая точка политики/mTLS | Порты API не опубликованы | `docker compose ps` / отсутствие publish | Доступ из docker-сети у процессов с сетью lab | +| Подключение к API с client cert другой роли того же CA | Только LB может быть клиентом API | Allowlist identity `lb` в `tlsutil` | `tests/student_mtls.py` api-client → отказ | Зависит от корректности handshake hook | +| Подбор пароля / login DoS | Доступность и стоимость Argon2 | Redis rate limit 10/30s → 429 Retry-After; бюджет Argon2id 19MiB t=2 p=1 | Ручной/скрипт повторных login | Распределённый IP не учитывается отдельно от username | +| Чужой пользователь читает чужой чат/вложение | ACL membership | Проверки membership в store + smoke ACL | `tests/smoke.py` access errors | Ошибки не раскрывают чужие тела | +| Worker/API читают чужие secrets в OpenBao | Least privilege secrets | Раздельные AppRole+policies | Попытка read чужого path → deny (операторский curl) | Root token у оператора на диске стенда | +| Утечка root/unseal в образ/git | Компрометация всего KV | Secrets gitignored; app без root | `.gitignore`, отсутствие token в env compose | Локальные файлы `secrets/openbao/` | +| Отзыв/истечение leaf под нагрузкой без plaintext fallback | Нет отключения verify | Короткоживущие leaf, `scripts/pki_renew.py`, без verify-off | Track 2 evidence | Ручной rollout на учебном стенде | ## Lifecycle и инциденты -TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском. +Bootstrap: `scripts/bootstrap.py` → PKI → `docker compose up` → `scripts/openbao_bootstrap.py` (init/unseal) → recreate api/worker. + +CA и unseal keys хранит оператор локально в `certs/private/` и `secrets/openbao/`, не в Git. + +При sealed OpenBao API не становится ready (fail-closed). После restart — повторный unseal оператором. diff --git a/src/bao.py b/src/bao.py new file mode 100644 index 0000000..9ac07a6 --- /dev/null +++ b/src/bao.py @@ -0,0 +1,74 @@ +"""OpenBao AppRole: короткоживущий token и scoped KV. Без root/unseal в приложении.""" +from __future__ import annotations + +import json +import os +import ssl +import urllib.error +import urllib.request +from pathlib import Path + +from tlsutil import ca_file + + +class BaoError(RuntimeError): + pass + + +def _read(path: str) -> str: + return Path(path).read_text(encoding="utf-8").strip() + + +def _context() -> ssl.SSLContext: + ctx = ssl.create_default_context(cafile=ca_file()) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + return ctx + + +def _request(method: str, url: str, body: dict | None = None, token: str | None = None) -> dict: + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + if token: + headers["X-Vault-Token"] = token + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, context=_context(), timeout=5) as response: + raw = response.read().decode("utf-8") + except urllib.error.HTTPError as exc: + raise BaoError(f"openbao http {exc.code}") from exc + except urllib.error.URLError as exc: + raise BaoError("openbao unreachable") from exc + if not raw: + return {} + return json.loads(raw) + + +def enabled() -> bool: + return os.environ.get("BAO_ADDR", "").strip() != "" + + +def login_and_read(path: str) -> dict: + """AppRole login → read KV v2 data at path. Fail-closed.""" + addr = os.environ.get("BAO_ADDR", "").rstrip("/") + role_file = os.environ.get("BAO_ROLE_ID_FILE", "/run/secrets/bao_role_id") + secret_file = os.environ.get("BAO_SECRET_ID_FILE", "/run/secrets/bao_secret_id") + if not addr: + raise BaoError("BAO_ADDR") + role_id = _read(role_file) + secret_id = _read(secret_file) + login = _request( + "POST", + f"{addr}/v1/auth/approle/login", + {"role_id": role_id, "secret_id": secret_id}, + ) + token = login.get("auth", {}).get("client_token") + if not token: + raise BaoError("approle login") + payload = _request("GET", f"{addr}/v1/{path}", token=token) + data = payload.get("data", {}).get("data") + if not isinstance(data, dict): + raise BaoError("empty secret") + return data diff --git a/src/blobs.py b/src/blobs.py new file mode 100644 index 0000000..990b256 --- /dev/null +++ b/src/blobs.py @@ -0,0 +1,237 @@ +"""Проверка PNG/JPEG и доступ к приватному bucket. Ключ объекта назначает сервер.""" +import io +import os +from datetime import datetime, timezone + +from minio import Minio +from minio.error import S3Error +from PIL import Image, UnidentifiedImageError +from PIL.Image import DecompressionBombError + +from errors import ApiError, DependencyDown + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + +_client = None + + +def read_secret(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def client(): + global _client + if _client is None: + import urllib3 + + endpoint = os.environ.get("S3_ENDPOINT", "minio:9000").strip() + access = read_secret(os.environ["S3_ACCESS_KEY_FILE"]) + secret = read_secret(os.environ["S3_SECRET_KEY_FILE"]) + secure = os.environ.get("S3_SECURE", "true").strip().lower() == "true" + ca = os.environ.get("TLS_CA_FILE", "/certs/ca.crt") + http_client = urllib3.PoolManager( + timeout=urllib3.Timeout(connect=3, read=30), + cert_reqs="CERT_REQUIRED" if secure else "CERT_NONE", + ca_certs=ca if secure else None, + retries=urllib3.Retry(total=2, backoff_factor=0.2), + ) + _client = Minio(endpoint, access_key=access, secret_key=secret, secure=secure, http_client=http_client) + return _client + + +def bucket_ready(): + try: + return client().bucket_exists(BUCKET) + except Exception: + return False + + +def put_bytes(key, data, content_type, created_at): + stamp = created_at.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + try: + client().put_object( + BUCKET, + key, + io.BytesIO(data), + length=len(data), + content_type=content_type, + metadata={"created-at": stamp}, + ) + except S3Error as exc: + raise DependencyDown() from exc + + +def delete_quiet(key): + try: + client().remove_object(BUCKET, key) + return True + except Exception: + return False + + +def open_object(key): + try: + return client().get_object(BUCKET, key) + except S3Error as exc: + code = getattr(exc, "code", "") + if code in ("NoSuchKey", "NoSuchBucket"): + return None + raise DependencyDown() from exc + + +def object_key(chat_id, attachment_id): + return "chats/{}/attachments/{}/original".format(chat_id, attachment_id) + + +def staging_object_key(attachment_id): + return "staging/{}/original".format(attachment_id) + + +def variant_object_key(attachment_id, pipeline_version, name): + return "ready/{}/v{}/{}".format(attachment_id, int(pipeline_version), name) + + +def header_size(data, declared): + if declared == "image/png": + return _png_size(data) + if declared == "image/jpeg": + return _jpeg_size(data) + return None + + +def read_bytes(key): + body = open_object(key) + if body is None: + return None + try: + return body.read() + finally: + body.close() + body.release_conn() + + +def declared_type(header): + if not header: + return None + media = header.split(";", 1)[0].strip().lower() + if media in ("image/png", "image/jpeg"): + return media + return None + + +def _png_size(data): + if len(data) < 24 or data[:8] != b"\x89PNG\r\n\x1a\n" or data[12:16] != b"IHDR": + return None + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + return width, height + + +def _jpeg_size(data): + if len(data) < 4 or data[:2] != b"\xff\xd8": + return None + index = 2 + limit = len(data) + while index + 8 < limit: + if data[index] != 0xFF: + return None + while index < limit and data[index] == 0xFF: + index += 1 + if index >= limit: + return None + marker = data[index] + index += 1 + if marker in (0xD8, 0xD9): + continue + if index + 2 > limit: + return None + segment = int.from_bytes(data[index:index + 2], "big") + if segment < 2 or index + segment > limit: + return None + if marker in (0xC0, 0xC1, 0xC2): + if segment < 7: + return None + height = int.from_bytes(data[index + 3:index + 5], "big") + width = int.from_bytes(data[index + 5:index + 7], "big") + return width, height + index += segment + return None + + +def _too_many_pixels(width, height): + if width <= 0 or height <= 0: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") + if width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") + + +def inspect_image(data, declared): + if declared == "image/png": + size = _png_size(data) + elif declared == "image/jpeg": + size = _jpeg_size(data) + else: + size = None + detected = None + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + if size is None or detected != declared: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") + _too_many_pixels(size[0], size[1]) + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + except DecompressionBombError as exc: + raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") from exc + return declared + + +def list_objects(prefix=""): + found = [] + prefixes = (prefix,) if prefix else ("chats/", "staging/", "ready/") + try: + for one in prefixes: + found.extend(_list_prefix(one)) + except S3Error as exc: + raise DependencyDown() from exc + return found + + +def _list_prefix(prefix): + found = [] + try: + for item in client().list_objects(BUCKET, prefix=prefix, recursive=True): + if item.is_dir: + continue + stat = client().stat_object(BUCKET, item.object_name) + raw = None + if stat.metadata: + meta = {str(key).lower(): value for key, value in stat.metadata.items()} + raw = meta.get("x-amz-meta-created-at") or meta.get("created-at") + created = _parse_stamp(raw) if raw else None + if created is None and item.last_modified is not None: + created = item.last_modified + found.append({"key": item.object_name, "created_at": created}) + except S3Error as exc: + raise DependencyDown() from exc + return found + + +def _parse_stamp(value): + text = value.strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 0000000..b34f81d --- /dev/null +++ b/src/broker.py @@ -0,0 +1,155 @@ +"""RabbitMQ: publisher confirm при публикации, consumer ack отдельно в worker.""" +import json +import os +from contextlib import contextmanager + +import pika +from opentelemetry.trace import SpanKind +from psycopg.types.json import Json + +from blobs import read_secret +from telemetry import start_span + +PROCESS_QUEUE = "attachments.process" +RETRY_QUEUE = "attachments.retry" +FAILED_QUEUE = "attachments.failed" + + +def broker_url(): + return read_secret(os.environ["BROKER_URL_FILE"]) + + +def _parameters(): + import ssl + + params = pika.URLParameters(broker_url()) + params.heartbeat = 60 + params.blocked_connection_timeout = 10 + params.socket_timeout = 10 + params.connection_attempts = 1 + ca = os.environ.get("TLS_CA_FILE", "/certs/ca.crt") + cert = os.environ.get("BROKER_CLIENT_CERT", "") + key = os.environ.get("BROKER_CLIENT_KEY", "") + if broker_url().startswith("amqps://") or (cert and key): + ctx = ssl.create_default_context(cafile=ca) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + if cert and key: + ctx.load_cert_chain(certfile=cert, keyfile=key) + # URL уже содержит hostname rabbitmq; SSLOptions фиксирует проверку имени. + params.ssl_options = pika.SSLOptions(ctx, "rabbitmq") + return params + + +def ensure_topology(channel): + channel.queue_declare(queue=PROCESS_QUEUE, durable=True) + channel.queue_declare( + queue=RETRY_QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": PROCESS_QUEUE, + }, + ) + channel.queue_declare(queue=FAILED_QUEUE, durable=True) + + +@contextmanager +def broker_channel(): + connection = pika.BlockingConnection(_parameters()) + channel = connection.channel() + channel.confirm_delivery() + ensure_topology(channel) + try: + yield channel + finally: + try: + connection.close() + except Exception: + pass + + +def publish_raw(channel, queue, body, expiration_ms=None): + fields = { + "delivery_mode": 2, + "content_type": "application/json", + } + if expiration_ms is not None: + fields["expiration"] = str(int(expiration_ms)) + with start_span( + "attachments.publish", + kind=SpanKind.PRODUCER, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": queue, + "messaging.operation": "publish", + }, + ): + channel.basic_publish( + exchange="", + routing_key=queue, + body=body, + properties=pika.BasicProperties(**fields), + mandatory=True, + ) + + +def publish_pending(conn, only_job_id=None): + published = 0 + while published < 20: + with conn.transaction(): + conn.execute("SET LOCAL statement_timeout = '20s'") + params = [] + sql = """ + SELECT o.id, o.payload + FROM outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE o.published_at IS NULL + AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= now()) + """ + if only_job_id is not None: + sql += " AND o.job_id = %s" + params.append(only_job_id) + sql += """ + ORDER BY o.created_at + FOR UPDATE OF o SKIP LOCKED + LIMIT 1 + """ + row = conn.execute(sql, params).fetchone() + if row is None: + return published + body = json.dumps( + row["payload"], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + with broker_channel() as channel: + publish_raw(channel, PROCESS_QUEUE, body) + conn.execute( + "UPDATE outbox SET published_at = now() WHERE id = %s AND published_at IS NULL", + (row["id"],), + ) + published += 1 + if only_job_id is not None: + return published + return published + + +def publish_delay(body, delay_ms): + with broker_channel() as channel: + publish_raw(channel, RETRY_QUEUE, body, expiration_ms=max(1000, int(delay_ms))) + + +def publish_failed(body): + with broker_channel() as channel: + publish_raw(channel, FAILED_QUEUE, body) + + +def queue_depth(queue): + with broker_channel() as channel: + declared = channel.queue_declare(queue=queue, passive=True) + return int(declared.method.message_count) + + +def dump_payload(payload): + return Json(payload) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 0000000..341036d --- /dev/null +++ b/src/errors.py @@ -0,0 +1,32 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class DependencyDown(Exception): + """Postgres или Redis не ответили. Наружу это 503 без имени хранилища.""" + + +class IdempotencyConflict(Exception): + """Тот же Idempotency-Key уже зафиксирован с другим телом.""" + + +class AttachmentRejected(Exception): + """Вложение есть, но прикрепить его к сообщению нельзя.""" + + +class AttachmentNotReady(Exception): + """Байты ещё нельзя отдать: queued, processing или failed.""" + + +def error_body(code, message, request_id): + return { + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + } diff --git a/src/faults.py b/src/faults.py new file mode 100644 index 0000000..b18de66 --- /dev/null +++ b/src/faults.py @@ -0,0 +1,49 @@ +"""Точечные сбои для проверки. Без строки в fault_injections код их не включает.""" +import uuid + +from psycopg.pq import TransactionStatus + + +def _idle(conn): + # Чтение выше могло открыть транзакцию. Её откат не теряет запись: + # сама запись начинается следующим transaction() и делает COMMIT. + if conn.info.transaction_status != TransactionStatus.IDLE: + conn.rollback() + + +def take_fault(conn, kind, attachment_id=None): + _idle(conn) + with conn.transaction(): + row = conn.execute( + """ + UPDATE fault_injections AS target + SET armed = false + WHERE target.id = ( + SELECT id + FROM fault_injections + WHERE armed + AND kind = %s + AND (attachment_id IS NULL OR attachment_id = %s) + ORDER BY created_at + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + RETURNING id + """, + (kind, attachment_id), + ).fetchone() + return row is not None + + +def arm_fault(conn, kind, attachment_id=None): + fault_id = uuid.uuid4() + _idle(conn) + with conn.transaction(): + conn.execute( + """ + INSERT INTO fault_injections (id, kind, attachment_id, armed) + VALUES (%s, %s, %s, true) + """, + (fault_id, kind, attachment_id), + ) + return str(fault_id) diff --git a/src/images.py b/src/images.py new file mode 100644 index 0000000..83a705e --- /dev/null +++ b/src/images.py @@ -0,0 +1,129 @@ +"""Варианты изображения. Оригинал не перекодируется.""" +import io + +from PIL import Image, ImageOps, UnidentifiedImageError +from PIL.Image import DecompressionBombError + +from blobs import MAX_PIXELS, header_size + +THUMB_BOX = 256 +CROP_SIZE = 128 +OPT_BOX = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class PermanentJobError(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +class TransientJobError(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +def _rgb(image): + if image.mode == "RGBA" or image.mode == "LA" or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + if image.mode == "RGB": + return image + return image.convert("RGB") + + +def load_rgb(data, declared): + size = header_size(data, declared) + detected = None + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + if size is None or detected != declared: + raise PermanentJobError("invalid_image") + width, height = size + if width <= 0 or height <= 0 or width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS: + raise PermanentJobError("invalid_image") + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + oriented = ImageOps.exif_transpose(image) + rgb = _rgb(oriented) + rgb.load() + return rgb + except DecompressionBombError as exc: + raise PermanentJobError("invalid_image") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise PermanentJobError("invalid_image") from exc + + +def fit_inside(image, box): + width, height = image.size + if width <= box and height <= box: + return image.copy() + if width >= height: + new_w = box + new_h = max(1, (height * box) // width) + else: + new_h = box + new_w = max(1, (width * box) // height) + return image.resize((new_w, new_h), Image.Resampling.LANCZOS) + + +def center_square(image, size): + width, height = image.size + side = min(width, height) + left = (width - side) // 2 + top = (height - side) // 2 + cropped = image.crop((left, top, left + side, top + side)) + if cropped.size != (size, size): + cropped = cropped.resize((size, size), Image.Resampling.LANCZOS) + return cropped + + +def _jpeg(image): + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True) + return { + "bytes": buffer.getvalue(), + "content_type": "image/jpeg", + "width": image.size[0], + "height": image.size[1], + } + + +def _webp(image): + buffer = io.BytesIO() + image.save(buffer, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD) + return { + "bytes": buffer.getvalue(), + "content_type": "image/webp", + "width": image.size[0], + "height": image.size[1], + } + + +def render(data, declared): + rgb = load_rgb(data, declared) + size = header_size(data, declared) + thumb = fit_inside(rgb, THUMB_BOX) + crop = center_square(rgb, CROP_SIZE) + optimized = fit_inside(rgb, OPT_BOX) + return { + "original": { + "bytes": data, + "content_type": declared, + "width": size[0], + "height": size[1], + }, + "thumbnail": _jpeg(thumb), + "crop": _jpeg(crop), + "optimized": _webp(optimized), + } diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 0000000..6186448 --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,896 @@ +"""Состояния задания. Эффект один на (attachment_id, pipeline_version, вариант).""" +import hashlib +import json +import os +import random +import threading +import uuid +from datetime import datetime, timezone + +from psycopg.pq import TransactionStatus +from psycopg.types.json import Json + +from opentelemetry.trace import SpanKind, Status, StatusCode + +from blobs import delete_quiet, put_bytes, read_bytes, variant_object_key +from errors import DependencyDown +from broker import publish_delay, publish_failed, publish_raw, broker_channel, PROCESS_QUEUE +from faults import take_fault +from images import PermanentJobError, TransientJobError, render +from telemetry import ( + extract_context, + now_monotonic, + record_job, + record_time_to_ready, + start_span, + structured_log, +) + +REQUIRED = ("original", "thumbnail", "crop", "optimized") +PUBLIC_CODES = ("invalid_image", "processing_failed", "retry_exhausted", "processing_timeout") + + +class LostLease(Exception): + pass + + +def _idle(conn): + if conn.info.transaction_status != TransactionStatus.IDLE: + conn.rollback() + + +def max_attempts(): + raw = os.environ.get("JOB_MAX_ATTEMPTS", "5") + value = int(raw) + if not 1 <= value <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS") + return value + + +def lease_seconds(): + raw = os.environ.get("JOB_TIMEOUT_SECONDS", "60") + value = int(raw) + if value < 5: + raise RuntimeError("JOB_TIMEOUT_SECONDS") + return value + + +def retention_seconds(): + raw = os.environ.get("STAGING_RETENTION_SECONDS", "86400") + value = int(raw) + if value < 60: + raise RuntimeError("STAGING_RETENTION_SECONDS") + return value + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def backoff_seconds(failed_attempt): + base = min(30.0, 0.5 * (2 ** (failed_attempt - 1))) + return base + random.uniform(0, base * 0.25) + + +def job_message(job_id, attachment_id, pipeline_version, attempt, created_at, carrier=None): + if not isinstance(created_at, str): + created_at = dump_time(created_at) + payload = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created_at, + } + if carrier: + traceparent = carrier.get("traceparent") + tracestate = carrier.get("tracestate") + if isinstance(traceparent, str) and len(traceparent) == 55: + payload["traceparent"] = traceparent + if isinstance(tracestate, str) and 0 < len(tracestate) <= 512: + payload["tracestate"] = tracestate + return payload + + +def _uuid(value): + try: + return str(uuid.UUID(str(value))) + except (ValueError, AttributeError, TypeError): + return None + + +def _stamp(value): + if not isinstance(value, str) or not value.endswith("Z"): + return None + try: + datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError: + return None + return value + + +def parse_job(body): + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeError, ValueError, AttributeError): + return None + if not isinstance(payload, dict): + return None + allowed = { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + "traceparent", + "tracestate", + } + if set(payload) - allowed: + return None + required = allowed - {"traceparent", "tracestate"} + if not required <= set(payload): + return None + if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1": + return None + job_id = _uuid(payload.get("job_id")) + attachment_id = _uuid(payload.get("attachment_id")) + created_at = _stamp(payload.get("created_at")) + version = payload.get("pipeline_version") + attempt = payload.get("attempt") + if job_id is None or attachment_id is None or created_at is None: + return None + if type(version) is not int or type(attempt) is not int: + return None + if version < 1 or attempt < 1: + return None + if "traceparent" in payload: + traceparent = payload["traceparent"] + if not isinstance(traceparent, str) or len(traceparent) != 55: + return None + if "tracestate" in payload: + tracestate = payload["tracestate"] + if not isinstance(tracestate, str) or len(tracestate) > 512: + return None + payload["job_id"] = job_id + payload["attachment_id"] = attachment_id + return payload + + +def _event(conn, job, kind, detail=None): + conn.execute( + """ + INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail) + VALUES (%s, %s, %s, %s, %s) + """, + (job["id"], job["attachment_id"], job["pipeline_version"], kind, detail), + ) + + +def _message_body(job, attempt, created_at): + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], attempt, created_at) + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def fail_job(conn, job, code): + if code not in PUBLIC_CODES and code != "superseded": + code = "processing_failed" + _idle(conn) + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = %s, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status IN ('queued', 'processing') + """, + (code, job["id"]), + ) + if code in PUBLIC_CODES: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND pipeline_version = %s + AND status IN ('queued', 'processing') + """, + (code, job["attachment_id"], job["pipeline_version"]), + ) + _event(conn, job, "failed", code) + try: + publish_failed(_message_body(job, job["attempt"], job["created_at"])) + except Exception: + # Реестр failed уже в базе. Повторная доставка увидит failed и подтвердит сообщение. + return + + +def schedule_retry(conn, job, code, owner): + attempt = int(job["attempt"]) + if attempt >= int(job["max_attempts"]): + fail_job(conn, job, "retry_exhausted") + return + delay = backoff_seconds(attempt) + new_attempt = attempt + 1 + created = utc_now() + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], new_attempt, created) + _idle(conn) + with conn.transaction(): + updated = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, lease_owner = NULL, lease_until = NULL, + next_attempt_at = now() + make_interval(secs => %s), + error_code = NULL, updated_at = now() + WHERE id = %s AND status = 'processing' AND lease_owner = %s + RETURNING id + """, + (new_attempt, delay, job["id"], owner), + ).fetchone() + if updated is None: + return + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (uuid.uuid4(), job["id"], new_attempt, Json(payload), created), + ) + _event(conn, job, "retry", code) + + +def release_expired(conn): + released = 0 + _idle(conn) + with conn.transaction(): + rows = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', lease_owner = NULL, lease_until = NULL, updated_at = now() + WHERE status = 'processing' AND lease_until < now() + RETURNING id, attachment_id, pipeline_version, attempt, created_at + """ + ).fetchall() + for row in rows: + payload = job_message(row["id"], row["attachment_id"], row["pipeline_version"], row["attempt"], utc_now()) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + SELECT %s, %s, %s, %s, now() + WHERE NOT EXISTS ( + SELECT 1 FROM outbox WHERE job_id = %s AND published_at IS NULL + ) + """, + (uuid.uuid4(), row["id"], row["attempt"], Json(payload), row["id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (row["attachment_id"], row["pipeline_version"]), + ) + _event(conn, row, "lease_expired", None) + released += 1 + return released + + +def cleanup_staging(conn): + removed = 0 + _idle(conn) + with conn.transaction(): + rows = conn.execute( + """ + SELECT a.id, a.staging_key + FROM attachments a + WHERE a.staging_key IS NOT NULL + AND a.staging_deleted_at IS NULL + AND a.status IN ('ready', 'failed') + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = a.id + AND j.status IN ('queued', 'processing') + ) + """ + ).fetchall() + for row in rows: + if delete_quiet(row["staging_key"]): + with conn.transaction(): + conn.execute( + """ + UPDATE attachments + SET staging_deleted_at = now() + WHERE id = %s AND staging_deleted_at IS NULL + """, + (row["id"],), + ) + removed += 1 + return removed + + +def _load_job(conn, job_id): + return conn.execute( + """ + SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, + lease_owner, lease_until, next_attempt_at, error_code, created_at + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + + +def _load_attachment(conn, attachment_id): + return conn.execute( + """ + SELECT id, content_type, sha256, object_key, staging_key, staging_deleted_at, + status, pipeline_version + FROM attachments + WHERE id = %s + """, + (attachment_id,), + ).fetchone() + + +def _acquire(conn, job, owner, seconds): + _idle(conn) + with conn.transaction(): + locked = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', lease_owner = %s, + lease_until = now() + make_interval(secs => %s), + updated_at = now() + WHERE id = %s + AND status IN ('queued', 'processing') + AND attempt = %s + AND pipeline_version = %s + AND (lease_owner IS NULL OR lease_owner = %s OR lease_until < now()) + RETURNING id + """, + (owner, seconds, job["id"], job["attempt"], job["pipeline_version"], owner), + ).fetchone() + if locked is None: + return False + conn.execute( + """ + UPDATE attachments + SET status = 'processing', updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing') + """, + (job["attachment_id"], job["pipeline_version"]), + ) + _event(conn, job, "leased", None) + return True + + +def _variants(conn, attachment_id, version): + rows = conn.execute( + """ + SELECT name, object_key, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + """, + (attachment_id, version), + ).fetchall() + return {row["name"]: row for row in rows} + + +def _source(conn, attachment): + if attachment["staging_key"] and attachment["staging_deleted_at"] is None: + data = read_bytes(attachment["staging_key"]) + if data: + return data + data = read_bytes(attachment["object_key"]) + if data: + return data + row = conn.execute( + """ + SELECT object_key + FROM attachment_variants + WHERE attachment_id = %s AND name = 'original' + ORDER BY pipeline_version DESC + LIMIT 1 + """, + (attachment["id"],), + ).fetchone() + if row is None: + raise TransientJobError("processing_failed") + data = read_bytes(row["object_key"]) + if not data: + raise TransientJobError("processing_failed") + return data + + +def _render_limited(data, declared, seconds): + budget = max(5, seconds - 5) + box = [] + errors = [] + + def run(): + try: + box.append(render(data, declared)) + except Exception as exc: + errors.append(exc) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(budget) + if thread.is_alive(): + raise TransientJobError("processing_timeout") + if errors: + exc = errors[0] + if isinstance(exc, (PermanentJobError, TransientJobError)): + raise exc + raise PermanentJobError("invalid_image") + return box[0] + + +def _beat(pool, job_id, owner, seconds, stop): + pause = max(2, seconds // 3) + while not stop.wait(pause): + try: + with pool.connection() as conn: + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE id = %s AND lease_owner = %s AND status = 'processing' + """, + (seconds, job_id, owner), + ) + except Exception: + return + + +def _supersede(conn, job): + _idle(conn) + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'superseded', error_code = 'superseded', lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status IN ('queued', 'processing') + """, + (job["id"],), + ) + _event(conn, job, "superseded", None) + + +def _finalize(conn, job, owner, rendered): + _idle(conn) + with conn.transaction(): + current = conn.execute( + """ + SELECT lease_owner, status, pipeline_version + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job["id"],), + ).fetchone() + if current is None or current["lease_owner"] != owner or current["status"] != "processing": + raise LostLease() + attachment = conn.execute( + """ + SELECT pipeline_version, status + FROM attachments + WHERE id = %s + FOR UPDATE + """, + (job["attachment_id"],), + ).fetchone() + if attachment is None or int(attachment["pipeline_version"]) > int(job["pipeline_version"]): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'superseded', error_code = 'superseded', lease_owner = NULL, + lease_until = NULL, updated_at = now() + WHERE id = %s + """, + (job["id"],), + ) + _event(conn, job, "superseded", None) + return "superseded" + now = utc_now() + for name in REQUIRED: + item = rendered[name] + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key, created_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + ( + job["attachment_id"], + job["pipeline_version"], + name, + item["content_type"], + len(item["bytes"]), + item["width"], + item["height"], + item["sha256"], + item["object_key"], + now, + ), + ) + stored = conn.execute( + """ + SELECT count(*) AS n + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + AND name IN ('original', 'thumbnail', 'crop', 'optimized') + """, + (job["attachment_id"], job["pipeline_version"]), + ).fetchone() + if int(stored["n"]) < len(REQUIRED): + raise TransientJobError("processing_failed") + original = rendered["original"] + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, pipeline_version = %s, + object_key = %s, updated_at = %s + WHERE id = %s AND pipeline_version <= %s + """, + ( + job["pipeline_version"], + original["object_key"], + now, + job["attachment_id"], + job["pipeline_version"], + ), + ) + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = %s + WHERE id = %s AND lease_owner = %s + """, + (now, job["id"], owner), + ) + _event(conn, job, "ready", None) + return "ready" + + +def _drop_staging(conn, attachment): + _idle(conn) + with conn.transaction(): + active = conn.execute( + """ + SELECT count(*) AS n + FROM attachment_jobs + WHERE attachment_id = %s AND status IN ('queued', 'processing') + """, + (attachment["id"],), + ).fetchone() + if int(active["n"]) > 0 or not attachment["staging_key"]: + return + if delete_quiet(attachment["staging_key"]): + with conn.transaction(): + conn.execute( + """ + UPDATE attachments + SET staging_deleted_at = now() + WHERE id = %s AND staging_deleted_at IS NULL + """, + (attachment["id"],), + ) + + +def process_delivery(pool, body, owner): + payload = parse_job(body) + if payload is None: + return "ack" + carrier = {} + if "traceparent" in payload: + carrier["traceparent"] = payload["traceparent"] + if "tracestate" in payload: + carrier["tracestate"] = payload["tracestate"] + parent = extract_context(carrier) if carrier else None + started = now_monotonic() + with start_span( + "attachments.process consume", + kind=SpanKind.CONSUMER, + context=parent, + attributes={ + "messaging.system": "rabbitmq", + "messaging.destination.name": PROCESS_QUEUE, + "messaging.operation": "process", + "job.type": "attachment.process.v1", + }, + ) as span: + return _process_delivery_body(pool, body, owner, payload, span, started) + + +def _process_delivery_body(pool, body, owner, payload, span, started): + import logging + + logger = logging.getLogger("worker") + with pool.connection() as conn: + job = _load_job(conn, payload["job_id"]) + if job is None: + return "ack" + if job["status"] in ("ready", "failed", "superseded"): + return "ack" + if int(payload["attempt"]) != int(job["attempt"]) or int(payload["pipeline_version"]) != int(job["pipeline_version"]): + return "ack" + if job["next_attempt_at"] is not None and job["next_attempt_at"] > utc_now(): + delay = (job["next_attempt_at"] - utc_now()).total_seconds() + publish_delay(body, int(delay * 1000)) + return "ack" + with pool.connection() as conn: + attachment = _load_attachment(conn, job["attachment_id"]) + if attachment is None: + fail_job(conn, job, "invalid_image") + record_job("failed", now_monotonic() - started) + return "ack" + if int(attachment["pipeline_version"]) > int(job["pipeline_version"]): + _supersede(conn, job) + record_job("superseded", now_monotonic() - started) + return "ack" + if not _acquire(conn, job, owner, lease_seconds()): + publish_delay(body, 2000) + return "ack" + structured_log(logger, "job_leased", job_id=str(job["id"]), attempt=int(job["attempt"])) + stop = threading.Event() + beater = threading.Thread( + target=_beat, + args=(pool, job["id"], owner, lease_seconds(), stop), + daemon=True, + ) + beater.start() + try: + with start_span("s3.read_source", kind=SpanKind.CLIENT): + with pool.connection() as conn: + if take_fault(conn, "crash_before_effect", job["attachment_id"]): + os_exit() + if take_fault(conn, "transient_once", job["attachment_id"]): + raise TransientJobError("processing_failed") + present = _variants(conn, job["attachment_id"], job["pipeline_version"]) + source = None if set(REQUIRED) <= set(present) else _source(conn, attachment) + if source is None: + rendered = _rendered_from_rows(conn, job, present) + if source is not None: + if hashlib.sha256(source).hexdigest() != attachment["sha256"]: + raise PermanentJobError("invalid_image") + with start_span("image.render", kind=SpanKind.INTERNAL): + rendered = _render_limited(source, attachment["content_type"], lease_seconds()) + with start_span("s3.write_variants", kind=SpanKind.CLIENT): + for name in REQUIRED: + item = rendered[name] + item["sha256"] = hashlib.sha256(item["bytes"]).hexdigest() + item["object_key"] = variant_object_key(job["attachment_id"], job["pipeline_version"], name) + if name in present: + continue + put_bytes(item["object_key"], item["bytes"], item["content_type"], utc_now()) + with pool.connection() as conn: + if take_fault(conn, "crash_after_objects", job["attachment_id"]): + os_exit() + with start_span("db.finalize_ready", kind=SpanKind.CLIENT): + with pool.connection() as conn: + _finalize(conn, job, owner, rendered) + with pool.connection() as conn: + if take_fault(conn, "crash_after_commit", job["attachment_id"]): + os_exit() + with pool.connection() as conn: + _drop_staging(conn, attachment) + duration = now_monotonic() - started + record_job("ready", duration) + if job.get("created_at") is not None: + created = job["created_at"] + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + record_time_to_ready((utc_now() - created).total_seconds()) + structured_log(logger, "job_ready", job_id=str(job["id"]), duration_ms=round(duration * 1000, 3)) + span.set_status(Status(StatusCode.OK)) + return "ack" + except PermanentJobError as exc: + with pool.connection() as conn: + fail_job(conn, job, exc.code) + record_job("failed", now_monotonic() - started) + span.set_status(Status(StatusCode.ERROR, exc.code)) + structured_log(logger, "job_failed", level=30, job_id=str(job["id"]), error_code=exc.code) + return "ack" + except LostLease: + return "ack" + except (TransientJobError, DependencyDown) as exc: + code = exc.code if isinstance(exc, TransientJobError) else "processing_failed" + with pool.connection() as conn: + schedule_retry(conn, job, code, owner) + record_job("retry", now_monotonic() - started) + span.set_status(Status(StatusCode.ERROR, code)) + structured_log(logger, "job_retry", level=30, job_id=str(job["id"]), error_code=code) + return "ack" + finally: + stop.set() + + +def os_exit(): + os._exit(75) + + +def _rendered_from_rows(conn, job, present): + # Повтор после уже записанного эффекта не перекодирует файлы. + rendered = {} + for name in REQUIRED: + row = present[name] + data = read_bytes(row["object_key"]) + if data is None: + raise TransientJobError("processing_failed") + meta = conn.execute( + """ + SELECT content_type, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (job["attachment_id"], job["pipeline_version"], name), + ).fetchone() + rendered[name] = { + "bytes": data, + "content_type": meta["content_type"], + "width": int(meta["width"]), + "height": int(meta["height"]), + "sha256": meta["sha256"], + "object_key": meta["object_key"], + } + return rendered + + +def bump_pipeline(conn, attachment_id): + _idle(conn) + with conn.transaction(): + row = conn.execute( + "SELECT id, pipeline_version FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if row is None: + return None + version = int(row["pipeline_version"]) + 1 + job_id = uuid.uuid4() + created = utc_now() + final_key = variant_object_key(attachment_id, version, "original") + conn.execute( + """ + UPDATE attachments + SET pipeline_version = %s, status = 'queued', error_code = NULL, + object_key = %s, updated_at = %s + WHERE id = %s + """, + (version, final_key, created, attachment_id), + ) + attempts = max_attempts() + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at, updated_at + ) VALUES (%s, %s, %s, 'queued', 1, %s, %s, %s) + """, + (job_id, attachment_id, version, attempts, created, created), + ) + payload = job_message(job_id, attachment_id, version, 1, created) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job_id, Json(payload), created), + ) + conn.execute( + """ + INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail) + VALUES (%s, %s, %s, 'version', %s) + """, + (job_id, attachment_id, version, str(version)), + ) + return {"job_id": str(job_id), "pipeline_version": version} + + +def replay_job(conn, job_id): + _idle(conn) + with conn.transaction(): + job = conn.execute( + """ + SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job_id,), + ).fetchone() + if job is None: + return {"result": "missing"} + if job["status"] == "ready": + return {"result": "already_ready"} + names = conn.execute( + """ + SELECT name FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + """, + (job["attachment_id"], job["pipeline_version"]), + ).fetchall() + have = {row["name"] for row in names} + if set(REQUIRED) <= have: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status <> 'ready' + """, + (job["id"],), + ) + return {"result": "already_done"} + if job["status"] != "failed": + return {"result": "not_failed"} + created = utc_now() + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = 1, error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = %s + WHERE id = %s + """, + (created, job["id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = %s + WHERE id = %s AND pipeline_version = %s AND status = 'failed' + """, + (created, job["attachment_id"], job["pipeline_version"]), + ) + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], 1, created) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job["id"], Json(payload), created), + ) + _event(conn, job, "replay", None) + return {"result": "queued"} + + +def republish(conn, job_id, attempt=None): + job = conn.execute( + """ + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + if job is None: + return False + chosen = int(job["attempt"] if attempt is None else attempt) + body = _message_body(job, chosen, utc_now()) + with broker_channel() as channel: + publish_raw(channel, PROCESS_QUEUE, body) + return True diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..2f3feec --- /dev/null +++ b/src/main.py @@ -0,0 +1,593 @@ +import json +import logging +import os +import re +import time +import uuid +from contextlib import asynccontextmanager + +import anyio +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Status, StatusCode +from pydantic import ValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from blobs import MAX_BYTES, declared_type, inspect_image, open_object +from errors import ApiError, AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict, error_body +from schemas import CreateChat, CreateMessage, CreateUser +from store import Store, instance_id +from telemetry import record_http, route_template, setup as setup_telemetry, structured_log + +LOGGER = logging.getLogger("messenger") +store = Store() +INSTANCE_ID = instance_id() + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +MAX_JSON_BYTES = 65536 + + +@asynccontextmanager +async def lifespan(_app): + setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-api")) + if os.environ.get("BAO_ADDR"): + from bao import BaoError, login_and_read + + path = os.environ.get("BAO_SECRET_PATH", "secret/data/messenger/api") + + def _wait_bao(): + last = None + for _ in range(60): + try: + login_and_read(path) + return + except BaoError as exc: + last = exc + time.sleep(2) + raise RuntimeError(f"OpenBao unavailable: {last}") + + await anyio.to_thread.run_sync(_wait_bao) + await anyio.to_thread.run_sync(store.startup) + yield + store.shutting_down = True + await anyio.to_thread.run_sync(store.shutdown) + + +app = FastAPI( + title="mtusi-messenger", + version="lab7", + lifespan=lifespan, + redirect_slashes=False, + docs_url="/docs", + redoc_url=None, + openapi_url="/openapi.json", +) + + +def request_id_of(request: Request): + rid = getattr(request.state, "request_id", None) + if rid: + return rid + return str(uuid.uuid4()) + + +def stamp(request: Request, response): + response.headers["X-Request-Id"] = request_id_of(request) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +def json_ok(request: Request, payload, status_code=200): + return stamp(request, JSONResponse(payload, status_code=status_code)) + + +def json_error(request: Request, status, code, message, extra_headers=None): + headers = {"X-Request-Id": request_id_of(request), "X-Instance-Id": INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse(error_body(code, message, request_id_of(request)), status_code=status, headers=headers) + + +@app.middleware("http") +async def wrap_request(request: Request, call_next): + incoming = request.headers.get("X-Request-Id", "") + if incoming and len(incoming) <= 200 and incoming.isprintable() and incoming.strip(): + request.state.request_id = incoming.strip() + else: + request.state.request_id = str(uuid.uuid4()) + if store.shutting_down and request.url.path != "/health/live": + return json_error(request, 503, "unavailable", "Сервис временно недоступен") + started = time.perf_counter() + route = route_template(request.url.path) + is_health = request.url.path in ("/health/live", "/health/ready") + if is_health: + response = await call_next(request) + return stamp(request, response) + tracer = trace.get_tracer("messenger") + with tracer.start_as_current_span( + f"{request.method} {route}", + kind=SpanKind.SERVER, + attributes={ + "http.request.method": request.method, + "http.route": route, + "service.instance.id": INSTANCE_ID, + }, + ) as span: + span.set_attribute("http.request_id", request.state.request_id) + try: + response = await call_next(request) + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + raise + stamped = stamp(request, response) + span.set_attribute("http.response.status_code", stamped.status_code) + if stamped.status_code >= 500: + span.set_status(Status(StatusCode.ERROR)) + duration = time.perf_counter() - started + record_http(request.method, request.url.path, stamped.status_code, duration) + structured_log( + LOGGER, + "http_request", + method=request.method, + route=route, + status=stamped.status_code, + request_id=request.state.request_id, + duration_ms=round(duration * 1000, 3), + ) + return stamped + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def request_validation_handler(request: Request, _exc: RequestValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(ValidationError) +async def pydantic_handler(request: Request, _exc: ValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return json_error(request, 404, "not_found", "Объект не найден") + return json_error(request, exc.status_code if exc.status_code < 500 else 500, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(Exception) +async def unexpected_handler(request: Request, exc: Exception): + if isinstance(exc, ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + LOGGER.error("unhandled %s", type(exc).__name__) + return json_error(request, 500, "internal_error", "Внутренняя ошибка") + + +def unavailable(): + raise ApiError(503, "unavailable", "Сервис временно недоступен") + + +def bearer_unauthorized(): + raise ApiError(401, "unauthorized", "Требуется авторизация", {"WWW-Authenticate": BEARER_CHALLENGE}) + + +def basic_unauthorized(): + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": BASIC_CHALLENGE}) + + +def invalid(): + raise ApiError(400, "invalid_request", "Некорректный запрос") + + +async def call_store(func, *args): + try: + return await anyio.to_thread.run_sync(func, *args) + except DependencyDown: + unavailable() + except IdempotencyConflict: + raise ApiError(409, "conflict", "Ключ уже использован с другим телом") + except AttachmentRejected: + raise ApiError(409, "conflict", "Вложение нельзя прикрепить") + except AttachmentNotReady: + raise ApiError(409, "conflict", "Вложение недоступно") + + +def parse_basic(request: Request): + header = request.headers.get("Authorization", "") + if not header.lower().startswith("basic "): + basic_unauthorized() + raw = header.split(" ", 1)[1].strip() + try: + import base64 + + decoded = base64.b64decode(raw, validate=True).decode("utf-8") + except (ValueError, UnicodeError): + basic_unauthorized() + if ":" not in decoded: + basic_unauthorized() + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(request: Request): + header = request.headers.get("Authorization", "") + # Basic на бизнес-методе не открывает сессию. X-User-Id не смотрим. + if not header.lower().startswith("bearer "): + bearer_unauthorized() + token = header.split(" ", 1)[1].strip() + if len(token) < 32 or len(token) > 512 or any(ch.isspace() for ch in token): + bearer_unauthorized() + return token + + +async def current_user(request: Request): + token = parse_bearer(request) + user = await call_store(store.user_for_token, token) + if user is None: + bearer_unauthorized() + request.state.user_id = user["id"] + request.state.token = token + return user + + +def parse_chat_id(chat_id): + try: + return str(uuid.UUID(chat_id)) + except ValueError: + invalid() + + +def parse_limit(raw): + if raw is None: + return 50 + if not isinstance(raw, str) or len(raw) > 3 or not raw.isdigit(): + invalid() + value = int(raw) + if value < 1 or value > 100: + invalid() + return value + + +async def read_json(request: Request): + ctype = request.headers.get("content-type", "") + if "application/json" not in ctype.lower(): + raise ApiError(415, "unsupported_media_type", "Нужен application/json") + length = request.headers.get("content-length") + if length is not None and (not length.isdigit() or int(length) > MAX_JSON_BYTES): + raise ApiError(413, "payload_too_large", "Слишком большое тело") + raw = await request.body() + if len(raw) > MAX_JSON_BYTES: + raise ApiError(413, "payload_too_large", "Слишком большое тело") + try: + payload = json.loads(raw) + except (ValueError, UnicodeError): + invalid() + if not isinstance(payload, dict): + invalid() + return payload + + +def parse_idempotency_key(request: Request): + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_RE.fullmatch(key): + invalid() + return key + + +@app.get("/health/live") +async def liveness(request: Request): + return json_ok(request, {"status": "ok"}) + + +@app.get("/health/ready") +async def readiness(request: Request): + if store.shutting_down: + raise ApiError(503, "unavailable", "Сервис временно недоступен") + ok = await call_store(store.ready) + if not ok: + unavailable() + return json_ok(request, {"status": "ok"}) + + +@app.post("/api/v1/users", status_code=201) +async def register_user(request: Request, body: CreateUser): + user = await call_store(store.create_user, body.username, body.display_name, body.password) + if user is None: + raise ApiError(409, "conflict", "Имя пользователя занято") + return json_ok(request, user, 201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + username, password = parse_basic(request) + allowed, retry_after = await call_store(store.login_rate_check, username) + if not allowed: + return json_error( + request, + 429, + "rate_limited", + "Слишком много попыток входа", + {"Retry-After": str(retry_after)}, + ) + session = await call_store(store.login, username, password) + if session is None: + await call_store(store.login_rate_fail, username) + basic_unauthorized() + await call_store(store.login_rate_clear, username) + request.state.user_id = session["user"]["id"] + return json_ok(request, session, 201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session(request: Request): + user = await current_user(request) + request.state.user_id = user["id"] + removed = await call_store(store.revoke, request.state.token) + if not removed: + bearer_unauthorized() + return stamp(request, Response(status_code=204)) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + user = await current_user(request) + return json_ok(request, user) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await current_user(request) + payload = await read_json(request) + try: + body = CreateChat.model_validate(payload) + except ValidationError: + invalid() + try: + chat = await call_store(store.create_chat, body.title, body.member_ids, user["id"]) + except ValueError: + invalid() + return json_ok(request, chat, 201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await current_user(request) + limit = parse_limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + if cursor is not None and (not cursor or len(cursor) > 512): + invalid() + try: + items, next_cursor = await call_store(store.list_chats, user["id"], limit, cursor) + except ValueError: + invalid() + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str): + user = await current_user(request) + chat = await call_store(store.get_chat, parse_chat_id(chat_id), user["id"]) + if chat is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + payload = await read_json(request) + key = parse_idempotency_key(request) + try: + body = CreateMessage.model_validate(payload) + except ValidationError: + invalid() + try: + message = await call_store( + store.add_message, canonical, user["id"], body.text, key, body.attachment_ids + ) + except ValueError: + invalid() + if message is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, message, 201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + limit = parse_limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + if cursor is not None and (not cursor or len(cursor) > 512): + invalid() + try: + result = await call_store(store.list_messages, canonical, user["id"], limit, cursor) + except ValueError: + invalid() + if result is None: + raise ApiError(404, "not_found", "Объект не найден") + items, next_cursor = result + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +def check_filename(value): + if not isinstance(value, str) or not value or len(value) > 255: + invalid() + if "/" in value or "\\" in value or "\x00" in value: + invalid() + if any(ord(char) < 32 or ord(char) == 127 for char in value): + invalid() + return value + + +def parse_attachment_id(value): + try: + return str(uuid.UUID(value)) + except (TypeError, ValueError): + invalid() + + +async def read_upload(request: Request): + header = request.headers.get("content-length") + if header is not None: + try: + announced = int(header) + except ValueError: + invalid() + if announced > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ") + if announced < 1: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ") + chunks.append(chunk) + if total < 1: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + return b"".join(chunks) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + member = await call_store(store.is_member, canonical, user["id"]) + if not member: + raise ApiError(404, "not_found", "Объект не найден") + filename = check_filename(request.query_params.get("filename")) + media = declared_type(request.headers.get("content-type")) + if media is None: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + data = await read_upload(request) + content_type = await anyio.to_thread.run_sync(inspect_image, data, media) + saved = await call_store( + store.save_attachment, canonical, user["id"], filename, content_type, data + ) + if saved is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, saved, 202) + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(request: Request, attachment_id: str): + user = await current_user(request) + canonical = parse_attachment_id(attachment_id) + found = await call_store(store.get_attachment, canonical, user["id"]) + if found is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, found) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(request: Request, attachment_id: str): + user = await current_user(request) + canonical = parse_attachment_id(attachment_id) + variant = request.query_params.get("variant") + if variant is None: + variant = "original" + if variant not in ("original", "thumbnail", "crop", "optimized"): + invalid() + opened = await call_store(store.open_attachment, canonical, user["id"], variant) + if opened is None: + raise ApiError(404, "not_found", "Объект не найден") + + def chunks(): + body = open_object(opened["object_key"]) + if body is None: + return + try: + for piece in body.stream(64 * 1024): + yield piece + finally: + body.close() + body.release_conn() + + probe = await anyio.to_thread.run_sync(open_object, opened["object_key"]) + if probe is None: + raise ApiError(404, "not_found", "Объект не найден") + probe.close() + probe.release_conn() + response = StreamingResponse(chunks(), media_type=opened["content_type"]) + response.headers["Content-Length"] = str(opened["size_bytes"]) + return stamp(request, response) + + +def setup_logging(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + + +def _health_only_server(host: str, port: int): + """Локальный HTTP health без mTLS — только 127.0.0.1, бизнес-API сюда не ходит.""" + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path not in ("/health/live", "/health/ready"): + self.send_response(404) + self.end_headers() + return + if self.path == "/health/ready": + pool = getattr(store, "pool", None) + if pool is None or store.shutting_down or not store.ready(): + self.send_response(503) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"unavailable"}') + return + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + + def log_message(self, format, *args): + return + + server = ThreadingHTTPServer((host, port), Handler) + server.serve_forever() + + +if __name__ == "__main__": + import threading + + import uvicorn + + from tlsutil import server_ssl_context + + setup_logging() + setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-api")) + health_host = os.environ.get("HEALTH_HOST", "127.0.0.1") + health_port = int(os.environ.get("HEALTH_PORT", "8081")) + threading.Thread(target=_health_only_server, args=(health_host, health_port), daemon=True).start() + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8443")) + allowed = {x.strip() for x in os.environ.get("MTLS_ALLOWED_CLIENTS", "lb").split(",") if x.strip()} + cert = os.environ.get("TLS_CERT_FILE", "/certs/api.crt") + key = os.environ.get("TLS_KEY_FILE", "/certs/private/api.key") + config = uvicorn.Config( + app, + host=host, + port=port, + ssl_certfile=cert, + ssl_keyfile=key, + timeout_graceful_shutdown=5, + timeout_keep_alive=2, + access_log=False, + log_config=None, + ) + server = uvicorn.Server(config) + config.load() + config.ssl = server_ssl_context(allowed) + server.run() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 0000000..cc35b87 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,112 @@ +"""Однократные миграции. Запускается отдельным контейнером, не каждой репликой API.""" +import logging +import os +import sys +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +LOGGER = logging.getLogger("migrate") +MIGRATION_LOCK = 421003 +CONNECT_TIMEOUT = 3 + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def database_url(): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + +def split_sql(script): + parts = [] + buf = [] + for line in script.splitlines(): + if line.strip().startswith("--"): + continue + buf.append(line) + if line.strip().endswith(";"): + statement = "\n".join(buf).strip() + if statement.endswith(";"): + statement = statement[:-1].strip() + if statement: + parts.append(statement) + buf = [] + tail = "\n".join(buf).strip().rstrip(";").strip() + if tail: + parts.append(tail) + return parts + + +def wait_postgres(conninfo): + last = None + for _ in range(60): + try: + with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT) as conn: + conn.execute("SELECT 1") + return + except psycopg.OperationalError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("postgres not ready: %s", last) + raise SystemExit(1) + + +def apply(conninfo): + directory = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT, row_factory=dict_row) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK,)) + conn.commit() + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + for path in sorted(directory.glob("*.sql")): + version = path.stem + found = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (version,), + ).fetchone() + conn.rollback() + if found: + LOGGER.info("migration %s already applied", version) + continue + with conn.transaction(): + for statement in split_sql(path.read_text(encoding="utf-8")): + conn.execute(statement) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + LOGGER.info("migration %s applied", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK,)) + conn.commit() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + conninfo = database_url() + wait_postgres(conninfo) + apply(conninfo) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 0000000..e70e803 --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,33 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHash, VerificationError, VerifyMismatchError + +# Параметры ближе к учебному стенду, чем к максимальным: time=2, 19 MiB, Argon2id. +# Хеш всё равно медленный password KDF, не SHA-256 и не открытый текст. +hasher = PasswordHasher( + time_cost=2, + memory_cost=19456, + parallelism=1, + hash_len=32, + salt_len=16, +) + +# Чтобы неизвестный username не отвечал заметно быстрее известного. +DUMMY_HASH = hasher.hash("dummy-password-not-a-user") + + +def hash_password(password): + return hasher.hash(password) + + +def verify_password(password, encoded): + try: + return hasher.verify(encoded, password) + except (VerifyMismatchError, VerificationError, InvalidHash): + return False + + +def needs_rehash(encoded): + try: + return hasher.check_needs_rehash(encoded) + except (InvalidHash, VerificationError): + return True diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 0000000..acb822d --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,174 @@ +"""Сверка строк attachments и объектов bucket. + +Свежий объект моложе grace не удаляется: загрузка могла записать файл +раньше, чем строка в базе. Строка, на которую ссылается сообщение, тоже +не удаляется, даже если файла уже нет. По умолчанию только план, без удалений. +""" +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone + +import psycopg +from psycopg.rows import dict_row + +from blobs import BUCKET, delete_quiet, list_objects + + +def grace_seconds(): + raw = os.environ.get("RECONCILE_GRACE_SECONDS", "120") + value = int(raw) + if value < 1: + raise RuntimeError("RECONCILE_GRACE_SECONDS") + return value + + +def database_url(): + path = os.environ.get("DATABASE_URL_FILE") + if not path: + raise RuntimeError("DATABASE_URL_FILE") + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def aware(value): + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def plan(grace): + started = time.perf_counter() + objects = list_objects() + listed = time.perf_counter() + now = datetime.now(timezone.utc) + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + rows = conn.execute( + """ + SELECT a.id, a.object_key, a.staging_key, a.created_at, a.status, + EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = a.id + ) AS in_use + FROM attachments a + """ + ).fetchall() + variant_rows = conn.execute("SELECT object_key FROM attachment_variants").fetchall() + by_key = {} + for row in rows: + by_key[row["object_key"]] = row + if row["staging_key"]: + by_key[row["staging_key"]] = row + for item in variant_rows: + by_key.setdefault(item["object_key"], {"status": "variant"}) + object_keys = {item["key"] for item in objects} + actions = [] + preserved = [] + for item in objects: + row = by_key.get(item["key"]) + if row is not None: + continue + created = aware(item["created_at"]) or now + age = (now - created).total_seconds() + entry = {"key": item["key"], "age_seconds": round(age, 3)} + if age < grace: + entry["reason"] = "fresh_object" + preserved.append(entry) + continue + entry["action"] = "delete_orphan_object" + actions.append(entry) + for row in rows: + if row["object_key"] in object_keys: + continue + created = aware(row["created_at"]) or now + age = (now - created).total_seconds() + entry = { + "attachment_id": str(row["id"]), + "key": row["object_key"], + "age_seconds": round(age, 3), + } + if row["status"] != "ready": + entry["reason"] = "not_ready" + preserved.append(entry) + continue + if row["in_use"]: + entry["reason"] = "missing_but_in_use" + preserved.append(entry) + continue + if age < grace: + entry["reason"] = "fresh_row" + preserved.append(entry) + continue + entry["action"] = "delete_orphan_row" + actions.append(entry) + return { + "bucket": BUCKET, + "grace_seconds": grace, + "objects": len(objects), + "rows": len(rows), + "list_seconds": round(listed - started, 6), + "plan_seconds": round(time.perf_counter() - started, 6), + "actions": actions, + "preserved": preserved, + } + + +def apply_plan(report): + deleted_objects = 0 + deleted_rows = 0 + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + for action in report["actions"]: + if action["action"] == "delete_orphan_object": + if delete_quiet(action["key"]): + deleted_objects += 1 + elif action["action"] == "delete_orphan_row": + with conn.transaction(): + result = conn.execute( + """ + DELETE FROM attachments + WHERE id = %s + AND status = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = attachments.id + ) + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = attachments.id + ) + AND NOT EXISTS ( + SELECT 1 FROM attachment_variants v + WHERE v.attachment_id = attachments.id + ) + """, + (action["attachment_id"],), + ) + deleted_rows += result.rowcount + report["deleted_objects"] = deleted_objects + report["deleted_rows"] = deleted_rows + return report + + +def main(): + parser = argparse.ArgumentParser(description="Сверить вложения в базе и в bucket") + parser.add_argument("--apply", action="store_true", help="выполнить удаления; без флага только план") + parser.add_argument("--grace", type=int, default=None) + args = parser.parse_args() + grace = args.grace if args.grace is not None else grace_seconds() + if grace < 1: + print("grace должен быть не меньше 1 секунды", file=sys.stderr) + return 2 + report = plan(grace) + report["apply"] = bool(args.apply) + if args.apply: + apply_plan(report) + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 0000000..009f577 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,24 @@ +"""Явный повтор failed-задания. Уже готовый эффект второй раз не исполняется.""" +import json +import sys + +import psycopg +from psycopg.rows import dict_row + +from blobs import read_secret +from jobs import replay_job + + +def main(): + if len(sys.argv) != 2: + print("usage: replay.py JOB_ID", file=sys.stderr) + return 2 + url = read_secret(__import__("os").environ["DATABASE_URL_FILE"]) + with psycopg.connect(url, row_factory=dict_row) as conn: + result = replay_job(conn, sys.argv[1]) + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/src/schemas.py b/src/schemas.py new file mode 100644 index 0000000..e11752d --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,128 @@ +import re +import uuid + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +def _visible(value, limit): + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError("text") + return value + + +class CreateUser(StrictModel): + username: str + display_name: str + password: str + + @field_validator("username") + @classmethod + def username_ok(cls, value): + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + raise ValueError("username") + return value + + @field_validator("display_name") + @classmethod + def display_name_ok(cls, value): + return _visible(value, 100) + + @field_validator("password") + @classmethod + def password_ok(cls, value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("password") + # Длина в code points, не в байтах: bcrypt тут не подходит из-за усечения UTF-8. + if len(value) < 12 or len(value) > 128: + raise ValueError("password") + return value + + +class CreateChat(StrictModel): + title: str + member_ids: list + + @field_validator("title") + @classmethod + def title_ok(cls, value): + return _visible(value, 100) + + @field_validator("member_ids") + @classmethod + def member_ids_ok(cls, value): + if not isinstance(value, list) or not (1 <= len(value) <= 100): + raise ValueError("member_ids") + # Канонический UUID (нижний регистр). Иначе один и тот же id + # в разном регистре выглядит как два участника и упирается в PK. + normalized = [] + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("member_ids") + try: + canonical = str(uuid.UUID(item)) + except ValueError as exc: + raise ValueError("member_ids") from exc + if canonical in seen: + raise ValueError("member_ids") + seen.add(canonical) + normalized.append(canonical) + return normalized + + +class CreateMessage(StrictModel): + text: str + attachment_ids: list = [] + + @model_validator(mode="before") + @classmethod + def fill_attachments(cls, value): + # Нет поля и явный [] — одно и то же тело для Idempotency-Key. + if not isinstance(value, dict): + return value + data = dict(value) + data.setdefault("attachment_ids", []) + return data + + @field_validator("text") + @classmethod + def text_ok(cls, value): + if not isinstance(value, str) or len(value) > 4000: + raise ValueError("text") + if value != "" and not value.strip(): + raise ValueError("text") + return value + + @field_validator("attachment_ids") + @classmethod + def attachments_ok(cls, value): + if not isinstance(value, list) or len(value) > 10: + raise ValueError("attachment_ids") + normalized = [] + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("attachment_ids") + try: + canonical = str(uuid.UUID(item)) + except ValueError as exc: + raise ValueError("attachment_ids") from exc + if canonical in seen: + raise ValueError("attachment_ids") + seen.add(canonical) + normalized.append(canonical) + return normalized + + @model_validator(mode="after") + def text_or_file(self): + if self.text == "" and not self.attachment_ids: + raise ValueError("text") + if self.text and not self.text.strip(): + raise ValueError("text") + return self diff --git a/src/store.py b/src/store.py new file mode 100644 index 0000000..664799a --- /dev/null +++ b/src/store.py @@ -0,0 +1,967 @@ +import base64 +import hashlib +import json +import logging +import os +import secrets +import socket +import time +import uuid +from datetime import datetime, timedelta, timezone + +import psycopg +import redis +from psycopg.errors import CheckViolation, ForeignKeyViolation, UniqueViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from errors import AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict +from passwords import DUMMY_HASH, hash_password, needs_rehash, verify_password + +LOGGER = logging.getLogger("messenger") + +POOL_TIMEOUT_SECONDS = 3 +STATEMENT_TIMEOUT_MS = 5000 +LOCK_TIMEOUT_MS = 3000 +REDIS_MAX_CONNECTIONS = 16 +REDIS_SOCKET_TIMEOUT = 2 +SCHEMA_VERSION = "004_jobs" + +SQL_MESSAGES_FIRST = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGES_AFTER = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s + AND (created_at, id) > (%s, %s) +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGE_BY_ID = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE id = %s +""" + + +def instance_id(): + # Docker без hostname: в контейнере это id контейнера, он не меняется, + # пока живёт этот контейнер. stop-one.sh находит контейнер по этому значению. + return socket.gethostname() + + +def pool_max(): + raw = os.environ.get("POOL_MAX", "10") + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError("POOL_MAX") from exc + if not 1 <= value <= 32: + raise RuntimeError("POOL_MAX") + return value + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def parse_time(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("time") + return datetime.fromisoformat(value[:-1] + "+00:00") + + +def encode_cursor(scope, created_at, item_id): + raw = "{}|{}|{}".format(scope, created_at, item_id).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor, expected_scope): + if not isinstance(cursor, str) or not cursor or len(cursor) > 512: + raise ValueError("cursor") + pad = "=" * (-len(cursor) % 4) + try: + decoded = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise ValueError("cursor") from exc + parts = decoded.split("|") + if len(parts) != 3: + raise ValueError("cursor") + scope, created_at, item_id = parts + if scope != expected_scope: + raise ValueError("cursor") + parse_time(created_at) + try: + uuid.UUID(item_id) + except ValueError as exc: + raise ValueError("cursor") from exc + return created_at, item_id + + +def session_key(token): + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +def message_fingerprint(text, attachment_ids): + # Канонический JSON: порядок ключей не важен, пропущенный attachment_ids равен []. + canonical = json.dumps( + {"attachment_ids": list(attachment_ids), "text": text}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def advisory_key(user_id, path, idem_key): + raw = "\0".join((user_id, "POST", path, idem_key)).encode("utf-8") + digest = hashlib.sha256(raw).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +def chat_path(chat_id): + return "/api/v1/chats/" + chat_id + "/messages" + + +def public_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": dump_time(row["created_at"]), + } + + +def public_message(row, attachment_ids=None): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": list(attachment_ids or []), + "created_at": dump_time(row["created_at"]), + } + + +def public_variant(row): + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "width": int(row["width"]), + "height": int(row["height"]), + "sha256": row["sha256"], + } + + +def public_attachment(row, variants=None): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": [public_variant(item) for item in (variants or [])], + "error_code": row["error_code"], + "created_at": dump_time(row["created_at"]), + "updated_at": dump_time(row["updated_at"]), + } + + +def public_chat(row, member_ids=None): + members = member_ids if member_ids is not None else row["member_ids"] + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item) for item in members], + "created_at": dump_time(row["created_at"]), + } + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def _db_down(exc): + raise DependencyDown() from exc + + +class Store: + def __init__(self): + self.pool = None + self.redis = None + self.ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.idem_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + self.pool_size = pool_max() + self.shutting_down = False + if self.ttl < 1 or self.idem_ttl < 1: + raise RuntimeError("TTL") + kind = os.environ.get("SESSION_STORE", "redis").strip().lower() + if kind != "redis": + raise RuntimeError("SESSION_STORE") + + def database_url(self): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + def redis_url(self): + path = os.environ.get("CACHE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("CACHE_URL", "").strip() + if not url: + raise RuntimeError("CACHE_URL_FILE") + return url + + def startup(self): + conninfo = self.database_url() + self._wait_schema(conninfo) + self.pool = ConnectionPool( + conninfo, + min_size=1, + max_size=self.pool_size, + timeout=POOL_TIMEOUT_SECONDS, + max_waiting=64, + max_lifetime=300, + kwargs={ + "connect_timeout": POOL_TIMEOUT_SECONDS, + "row_factory": dict_row, + "application_name": "mtusi-lab7-" + instance_id(), + "options": "-c statement_timeout={} -c lock_timeout={} -c idle_in_transaction_session_timeout=15000".format( + STATEMENT_TIMEOUT_MS, + LOCK_TIMEOUT_MS, + ), + }, + open=True, + ) + redis_kwargs = { + "max_connections": REDIS_MAX_CONNECTIONS, + "socket_timeout": REDIS_SOCKET_TIMEOUT, + "socket_connect_timeout": REDIS_SOCKET_TIMEOUT, + "health_check_interval": 30, + "decode_responses": True, + } + redis_url = self.redis_url() + if redis_url.startswith("rediss://"): + redis_kwargs["ssl_cert_reqs"] = "required" + redis_kwargs["ssl_ca_certs"] = os.environ.get("TLS_CA_FILE", "/certs/ca.crt") + pool = redis.ConnectionPool.from_url(redis_url, **redis_kwargs) + self.redis = redis.Redis(connection_pool=pool) + self._wait_redis() + self._wait_bucket() + try: + from broker import broker_channel, publish_pending + + with broker_channel(): + pass + with self._connect() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("outbox startup deferred %s", type(exc).__name__) + self._start_queue_metrics() + LOGGER.info("ready instance=%s ttl=%s pool_max=%s", instance_id(), self.ttl, self.pool_size) + + def _start_queue_metrics(self): + # Метрики возраста очереди должны жить и при остановленном worker. + import threading + + def loop(): + from broker import PROCESS_QUEUE, RETRY_QUEUE, queue_depth + from telemetry import observe_process, set_queue_stats + + while not self.shutting_down: + try: + age = 0.0 + with self._connect() as conn: + row = conn.execute( + """ + SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """ + ).fetchone() + if row is not None and row["age"] is not None: + age = float(row["age"]) + try: + depth = queue_depth(PROCESS_QUEUE) + except Exception: + depth = 0 + try: + retry_depth = queue_depth(RETRY_QUEUE) + except Exception: + retry_depth = 0 + set_queue_stats(PROCESS_QUEUE, depth, age) + set_queue_stats(RETRY_QUEUE, retry_depth, 0.0) + self.refresh_pool_stats() + observe_process() + except Exception as exc: + LOGGER.warning("queue metrics %s", type(exc).__name__) + time.sleep(5) + + threading.Thread(target=loop, name="queue-metrics", daemon=True).start() + + def shutdown(self): + self.shutting_down = True + if self.pool is not None: + self.pool.close() + if self.redis is not None: + self.redis.close() + + def ready(self): + if self.shutting_down or self.pool is None or self.redis is None: + return False + try: + with self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is None: + return False + self.redis.ping() + from blobs import bucket_ready + + if not bucket_ready(): + return False + except Exception: + LOGGER.warning("ready check failed") + return False + return True + + def _wait_redis(self): + last = None + for _ in range(60): + try: + self.redis.ping() + return + except redis.RedisError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("redis not ready: %s", last) + raise SystemExit(1) + + def _wait_schema(self, conninfo): + # Схему накатывает сервис migrate. Реплики только ждут её, не применяют сами. + last = None + for _ in range(90): + try: + with psycopg.connect(conninfo, connect_timeout=POOL_TIMEOUT_SECONDS, row_factory=dict_row) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is not None: + return + last = "migration missing" + except psycopg.Error as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("schema not ready: %s", last) + raise SystemExit(1) + + def _wait_bucket(self): + from blobs import bucket_ready + + for _ in range(60): + if bucket_ready(): + return + time.sleep(1) + LOGGER.error("object storage not ready") + raise SystemExit(1) + + def refresh_pool_stats(self): + if self.pool is None: + return + try: + from telemetry import set_pool_stats + + stats = self.pool.get_stats() + used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0)) + waiting = int(stats.get("requests_waiting", 0)) + set_pool_stats(max(0, used), waiting) + except Exception: + pass + + def _connect(self): + self.refresh_pool_stats() + try: + return self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) + except PoolTimeout as exc: + _db_down(exc) + + def create_user(self, username, display_name, password): + password_hash = hash_password(password) + user_id = uuid.uuid4() + created = utc_now() + try: + with self._connect() as conn: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (username) DO NOTHING + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash, created), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def login(self, username, password): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + verify_password(password, DUMMY_HASH) + return None + if not verify_password(password, row["password_hash"]): + return None + if needs_rehash(row["password_hash"]): + new_hash = hash_password(password) + try: + with self._connect() as conn: + conn.execute( + "UPDATE users SET password_hash = %s WHERE id = %s", + (new_hash, row["id"]), + ) + except psycopg.OperationalError as exc: + _db_down(exc) + return self._issue(public_user(row)) + + def login_rate_check(self, username): + """Общий для реплик лимит неудачных входов через Redis. (allowed, retry_after).""" + key = f"login:fail:{username.lower()}" + try: + count = int(self.redis.get(key) or 0) + if count < 10: + return True, 0 + ttl = int(self.redis.ttl(key) or 0) + return False, max(1, ttl) + except redis.RedisError as exc: + raise DependencyDown() from exc + + def login_rate_fail(self, username): + key = f"login:fail:{username.lower()}" + try: + count = self.redis.incr(key) + if count == 1: + self.redis.expire(key, 30) + return int(count) + except redis.RedisError as exc: + raise DependencyDown() from exc + + def login_rate_clear(self, username): + key = f"login:fail:{username.lower()}" + try: + self.redis.delete(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + + def _issue(self, user): + token = secrets.token_urlsafe(32) + expires = utc_now() + timedelta(seconds=self.ttl) + payload = json.dumps( + {"user_id": user["id"], "expires_at": dump_time(expires)}, + separators=(",", ":"), + ) + # pxat — абсолютное время. Рестарт Redis не сдвигает срок. + # В значении тоже лежит expires_at: даже если TTL ключа разъедется, сессия кончится вовремя. + try: + self.redis.set(session_key(token), payload, pxat=int(expires.timestamp() * 1000)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return { + "token": token, + "token_type": "Bearer", + "expires_at": dump_time(expires), + "user": user, + } + + def user_for_token(self, token): + key = session_key(token) + try: + raw = self.redis.get(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + if not raw: + return None + try: + payload = json.loads(raw) + expires = parse_time(payload["expires_at"]) + user_id = payload["user_id"] + uuid.UUID(user_id) + except (ValueError, KeyError, TypeError): + return None + if utc_now() >= expires: + try: + self.redis.delete(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + return None + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (user_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def revoke(self, token): + try: + deleted = self.redis.delete(session_key(token)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return deleted == 1 + + def create_chat(self, title, member_ids, caller_id): + try: + caller_uuid = uuid.UUID(str(caller_id)) + member_uuids = [] + seen = set() + for item in member_ids: + member_uuid = uuid.UUID(str(item)) + if member_uuid in seen: + raise ValueError("members") + seen.add(member_uuid) + member_uuids.append(member_uuid) + except ValueError as exc: + raise ValueError("members") from exc + if caller_uuid not in seen: + raise ValueError("members") + chat_id = uuid.uuid4() + created = utc_now() + canonical_members = [str(item) for item in member_uuids] + try: + with self._connect() as conn: + try: + with conn.transaction(): + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created), + ) + for ordinal, member_id in enumerate(member_uuids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, ordinal) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, ordinal), + ) + row = conn.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ).fetchone() + except ForeignKeyViolation as exc: + raise ValueError("members") from exc + except UniqueViolation as exc: + raise ValueError("members") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + return public_chat(row, canonical_members) + + def _member(self, conn, chat_id, user_id): + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + + def get_chat(self, chat_id, user_id): + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s + GROUP BY c.id + """, + (chat_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_chat(row) + + def list_chats(self, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, "chats:" + user_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + sql = """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members m ON m.chat_id = c.id + """ + params = [user_id] + if after is not None: + sql += " WHERE (c.created_at, c.id) > (%s, %s)" + params.extend(after) + sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s" + params.append(limit + 1) + try: + with self._connect() as conn: + rows = conn.execute(sql, params).fetchall() + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_chat(row) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor("chats:" + user_id, last["created_at"], last["id"]) + return page, None + + def _attachment_ids(self, conn, message_ids): + if not message_ids: + return {} + rows = conn.execute( + """ + SELECT message_id, attachment_id + FROM message_attachments + WHERE message_id = ANY(%s::uuid[]) + ORDER BY ordinal + """, + (list(message_ids),), + ).fetchall() + found = {} + for row in rows: + found.setdefault(str(row["message_id"]), []).append(str(row["attachment_id"])) + return found + + def is_member(self, chat_id, user_id): + try: + with self._connect() as conn: + return self._member(conn, chat_id, user_id) is not None + except psycopg.OperationalError as exc: + _db_down(exc) + + def _variants(self, conn, attachment_id, pipeline_version): + return conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name <> 'original' + ORDER BY CASE name + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END + """, + (attachment_id, pipeline_version), + ).fetchall() + + def save_attachment(self, chat_id, owner_id, filename, content_type, data): + from psycopg.types.json import Json + + from blobs import delete_quiet, put_bytes, staging_object_key, variant_object_key + from broker import publish_pending + from faults import take_fault + from jobs import job_message, max_attempts + from telemetry import inject_trace + + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + created = utc_now() + digest = hashlib.sha256(data).hexdigest() + staging = staging_object_key(attachment_id) + final_key = variant_object_key(attachment_id, 1, "original") + carrier = inject_trace({}) + payload = job_message(job_id, attachment_id, 1, 1, created, carrier=carrier) + attempts = max_attempts() + put_bytes(staging, data, content_type, created) + try: + with self._connect() as conn: + with conn.transaction(): + if self._member(conn, chat_id, owner_id) is None: + raise LookupError("chat") + row = conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key, staging_key, status, + pipeline_version, error_code, created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'queued', 1, NULL, %s, %s) + RETURNING id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, status, error_code, created_at, updated_at + """, + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + len(data), + digest, + final_key, + staging, + created, + created, + ), + ).fetchone() + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, status, attempt, + max_attempts, created_at, updated_at + ) + VALUES (%s, %s, 1, 'queued', 1, %s, %s, %s) + """, + (job_id, attachment_id, attempts, created, created), + ) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job_id, Json(payload), created), + ) + except (LookupError, ForeignKeyViolation): + delete_quiet(staging) + return None + except psycopg.OperationalError as exc: + delete_quiet(staging) + _db_down(exc) + except Exception: + delete_quiet(staging) + raise + try: + with self._connect() as conn: + skipped = take_fault(conn, "skip_inline_publish", attachment_id) + if not skipped: + with self._connect() as conn: + publish_pending(conn, only_job_id=job_id) + except Exception as exc: + LOGGER.warning("publish deferred %s", type(exc).__name__) + return public_attachment(row, []) + + def get_attachment(self, attachment_id, user_id): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, a.pipeline_version, + a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + variants = [] + if row is not None and row["status"] == "ready": + variants = self._variants(conn, row["id"], row["pipeline_version"]) + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_attachment(row, variants) + + def open_attachment(self, attachment_id, user_id, variant): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT a.object_key, a.content_type, a.size_bytes, a.status, a.pipeline_version + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + if row is None: + return None + if row["status"] != "ready": + raise AttachmentNotReady() + if variant == "original": + return { + "object_key": row["object_key"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + } + found = conn.execute( + """ + SELECT object_key, content_type, size_bytes + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if found is None: + return None + return { + "object_key": found["object_key"], + "content_type": found["content_type"], + "size_bytes": int(found["size_bytes"]), + } + + def add_message(self, chat_id, sender_id, text, idem_key, attachment_ids): + # Один ключ сериализуется между репликами через advisory lock в той же + # транзакции, где INSERT сообщения. Откат не оставляет ключ занятым. + path = chat_path(chat_id) + digest = message_fingerprint(text, attachment_ids) + lock_key = advisory_key(sender_id, path, idem_key) + message_id = uuid.uuid4() + try: + with self._connect() as conn: + with conn.transaction(): + # Иначе lock_timeout сессии (3 с) оборвёт очередь одинаковых ключей. + conn.execute("SET LOCAL lock_timeout = '15s'") + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + if self._member(conn, chat_id, sender_id) is None: + return None + existing = conn.execute( + """ + SELECT request_hash, message_id, + created_at >= now() - make_interval(secs => %s) AS fresh + FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' + AND request_path = %s AND idem_key = %s + """, + (self.idem_ttl, sender_id, path, idem_key), + ).fetchone() + if existing is not None and existing["fresh"]: + if existing["request_hash"] != digest: + raise IdempotencyConflict() + row = conn.execute(SQL_MESSAGE_BY_ID, (existing["message_id"],)).fetchone() + if row is None: + raise RuntimeError("idempotency target missing") + linked = self._attachment_ids(conn, [row["id"]]).get(str(row["id"]), []) + return public_message(row, linked) + if existing is not None: + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' + AND request_path = %s AND idem_key = %s + """, + (sender_id, path, idem_key), + ) + if attachment_ids: + found = conn.execute( + """ + SELECT id, chat_id, status + FROM attachments + WHERE id = ANY(%s::uuid[]) + """, + (list(attachment_ids),), + ).fetchall() + by_id = {str(item["id"]): item for item in found} + for attachment_id in attachment_ids: + item = by_id.get(attachment_id) + if item is None or str(item["chat_id"]) != chat_id: + return None + if item["status"] == "failed" or item["status"] not in ("queued", "processing", "ready"): + raise AttachmentRejected() + row = conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, body, created_at) + VALUES (%s, %s, %s, %s, clock_timestamp()) + RETURNING id, chat_id, sender_id, body AS text, created_at + """, + (message_id, chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, ordinal) + VALUES (%s, %s, %s) + """, + (row["id"], attachment_id, ordinal), + ) + conn.execute( + """ + INSERT INTO idempotency_keys ( + user_id, http_method, request_path, idem_key, + request_hash, message_id, created_at + ) + VALUES (%s, 'POST', %s, %s, %s, %s, clock_timestamp()) + """, + (sender_id, path, idem_key, digest, row["id"]), + ) + return public_message(row, attachment_ids) + except CheckViolation as exc: + raise ValueError("message") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + + def list_messages(self, chat_id, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, chat_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + if after is None: + rows = conn.execute(SQL_MESSAGES_FIRST, (chat_id, limit + 1)).fetchall() + else: + rows = conn.execute( + SQL_MESSAGES_AFTER, + (chat_id, after[0], after[1], limit + 1), + ).fetchall() + linked = self._attachment_ids(conn, [row["id"] for row in rows[:limit]]) + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_message(row, linked.get(str(row["id"]), [])) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor(chat_id, last["created_at"], last["id"]) + return page, None diff --git a/src/telemetry.py b/src/telemetry.py new file mode 100644 index 0000000..51827aa --- /dev/null +++ b/src/telemetry.py @@ -0,0 +1,473 @@ +"""OpenTelemetry: traces, metrics, structured logs. Секреты и тела сообщений не экспортируются.""" +from __future__ import annotations + +import json +import logging +import os +import socket +import time +from contextlib import contextmanager + +from opentelemetry import metrics, trace +from opentelemetry._logs import set_logger_provider +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.metrics import CallbackOptions, Observation +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader +from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON, ParentBased, TraceIdRatioBased +from opentelemetry.trace import Link, SpanKind, Status, StatusCode +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator + +SERVICE_VERSION = "lab7" +PROPAGATOR = TraceContextTextMapPropagator() +_CONFIGURED = False +_TRACER = None +_METER = None +_STATE = { + "queue_depth": {}, + "oldest_job_age": {}, + "db_pool_used": 0.0, + "db_pool_waiting": 0.0, + "worker_busy": 0.0, + "worker_capacity": 0.0, + "process_cpu": 0.0, + "process_rss": 0.0, +} + +http_requests = None +http_duration = None +jobs_processed = None +job_duration = None +e2e_ready = None + + +def _endpoint(): + return os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel-gateway:4318").rstrip("/") + + +def _headers(): + raw = os.environ.get("OTEL_EXPORTER_OTLP_HEADERS", "") + if not raw: + return {} + out = {} + for part in raw.split(","): + if "=" not in part: + continue + key, value = part.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def _otlp_session(): + """mTLS к otel-gateway: свой client cert + проверка CA сервера.""" + try: + import requests + except ImportError: + return None + ca = os.environ.get("OTEL_EXPORTER_OTLP_CERTIFICATE") or os.environ.get("TLS_CA_FILE", "/certs/ca.crt") + cert = os.environ.get("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", "") + key = os.environ.get("OTEL_EXPORTER_OTLP_CLIENT_KEY", "") + session = requests.Session() + session.verify = ca + if cert and key: + session.cert = (cert, key) + return session + + +def _exporter_common(): + kwargs = {"headers": _headers(), "timeout": 5} + ca = os.environ.get("OTEL_EXPORTER_OTLP_CERTIFICATE") or os.environ.get("TLS_CA_FILE", "/certs/ca.crt") + if ca and _endpoint().startswith("https://"): + kwargs["certificate_file"] = ca + session = _otlp_session() + if session is not None and _endpoint().startswith("https://"): + kwargs["session"] = session + return kwargs + + +def _sampler(): + ratio = os.environ.get("OTEL_TRACES_SAMPLER_ARG", "1.0") + try: + value = float(ratio) + except ValueError: + value = 1.0 + if value >= 1.0: + return ParentBased(ALWAYS_ON) + if value <= 0.0: + return ParentBased(TraceIdRatioBased(0.0)) + return ParentBased(TraceIdRatioBased(value)) + + +def _resource(service_name): + attrs = { + "service.name": service_name, + "service.version": os.environ.get("SERVICE_VERSION", SERVICE_VERSION), + "service.instance.id": socket.gethostname(), + "deployment.environment.name": os.environ.get("APP_ENV", "lab"), + } + extra = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") + for part in extra.split(","): + if "=" not in part: + continue + key, value = part.split("=", 1) + key = key.strip() + value = value.strip() + if key and value and key not in attrs: + attrs[key] = value + return Resource.create(attrs) + + +def _obs_queue_depth(_options: CallbackOptions): + for name, value in list(_STATE["queue_depth"].items()): + yield Observation(float(value), {"messaging.destination.name": name}) + + +def _obs_oldest(_options: CallbackOptions): + for name, value in list(_STATE["oldest_job_age"].items()): + yield Observation(float(value), {"messaging.destination.name": name}) + + +def _obs_db_used(_options: CallbackOptions): + yield Observation(float(_STATE["db_pool_used"])) + + +def _obs_db_waiting(_options: CallbackOptions): + yield Observation(float(_STATE["db_pool_waiting"])) + + +def _obs_worker_busy(_options: CallbackOptions): + yield Observation(float(_STATE["worker_busy"])) + + +def _obs_worker_capacity(_options: CallbackOptions): + yield Observation(float(_STATE["worker_capacity"])) + + +def _obs_cpu(_options: CallbackOptions): + yield Observation(float(_STATE["process_cpu"])) + + +def _obs_rss(_options: CallbackOptions): + yield Observation(float(_STATE["process_rss"])) + + +def attach_log_handler(logger_provider): + handler = LoggingHandler(level=logging.INFO, logger_provider=logger_provider) + root = logging.getLogger() + root.setLevel(logging.INFO) + if not any(isinstance(item, LoggingHandler) for item in root.handlers): + root.addHandler(handler) + for name in ("messenger", "worker", "uvicorn", "uvicorn.error", "uvicorn.access"): + logging.getLogger(name).setLevel(logging.INFO) + + +def setup(service_name): + global _CONFIGURED, _TRACER, _METER + global http_requests, http_duration, jobs_processed, job_duration, e2e_ready + if _CONFIGURED: + # uvicorn мог переустановить handlers после импорта модуля + provider = getattr(setup, "_logger_provider", None) + if provider is not None: + attach_log_handler(provider) + return + if os.environ.get("OTEL_SDK_DISABLED", "").lower() in ("1", "true", "yes"): + _CONFIGURED = True + return + + resource = _resource(service_name) + base = _endpoint() + provider = TracerProvider(resource=resource, sampler=_sampler()) + provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=f"{base}/v1/traces", **_exporter_common()), + max_queue_size=512, + max_export_batch_size=128, + schedule_delay_millis=2000, + export_timeout_millis=5000, + ) + ) + trace.set_tracer_provider(provider) + _TRACER = trace.get_tracer("messenger", SERVICE_VERSION) + + reader = PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=f"{base}/v1/metrics", **_exporter_common()), + export_interval_millis=10000, + export_timeout_millis=5000, + ) + views = ( + View( + instrument_name="messenger.http.duration", + aggregation=ExplicitBucketHistogramAggregation( + [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0] + ), + ), + View( + instrument_name="messenger.jobs.duration", + aggregation=ExplicitBucketHistogramAggregation( + [0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 15.0, 30.0, 60.0] + ), + ), + View( + instrument_name="messenger.attachment.time_to_ready", + aggregation=ExplicitBucketHistogramAggregation( + [0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0] + ), + ), + ) + meter_provider = MeterProvider(resource=resource, metric_readers=[reader], views=views) + metrics.set_meter_provider(meter_provider) + _METER = metrics.get_meter("messenger", SERVICE_VERSION) + + logger_provider = LoggerProvider(resource=resource) + logger_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=f"{base}/v1/logs", **_exporter_common()), + max_queue_size=512, + max_export_batch_size=128, + schedule_delay_millis=2000, + export_timeout_millis=5000, + ) + ) + set_logger_provider(logger_provider) + setup._logger_provider = logger_provider + attach_log_handler(logger_provider) + + http_requests = _METER.create_counter( + "messenger.http.requests", + unit="1", + description="HTTP requests by method, route template and status class", + ) + http_duration = _METER.create_histogram( + "messenger.http.duration", + unit="s", + description="HTTP request duration in seconds", + ) + jobs_processed = _METER.create_counter( + "messenger.jobs.processed", + unit="1", + description="Jobs finished by type and outcome", + ) + job_duration = _METER.create_histogram( + "messenger.jobs.duration", + unit="s", + description="Job processing duration in seconds", + ) + e2e_ready = _METER.create_histogram( + "messenger.attachment.time_to_ready", + unit="s", + description="Seconds from durable acceptance to ready", + ) + _METER.create_observable_up_down_counter( + "messenger.queue.depth", + callbacks=[_obs_queue_depth], + unit="1", + description="Approximate broker queue depth", + ) + _METER.create_observable_gauge( + "messenger.queue.oldest_job_age", + callbacks=[_obs_oldest], + unit="s", + description="Age of the oldest unfinished job", + ) + _METER.create_observable_gauge( + "messenger.db.pool.used", + callbacks=[_obs_db_used], + unit="1", + description="Checked-out DB pool connections", + ) + _METER.create_observable_gauge( + "messenger.db.pool.waiting", + callbacks=[_obs_db_waiting], + unit="1", + description="Waiters for a DB pool connection", + ) + _METER.create_observable_gauge( + "messenger.worker.busy", + callbacks=[_obs_worker_busy], + unit="1", + description="Busy worker slots", + ) + _METER.create_observable_gauge( + "messenger.worker.capacity", + callbacks=[_obs_worker_capacity], + unit="1", + description="Configured worker concurrency", + ) + _METER.create_observable_gauge( + "messenger.process.cpu_seconds", + callbacks=[_obs_cpu], + unit="s", + description="Process user+system CPU seconds", + ) + _METER.create_observable_gauge( + "messenger.process.rss_bytes", + callbacks=[_obs_rss], + unit="By", + description="Process RSS bytes", + ) + _CONFIGURED = True + + +def tracer(): + return _TRACER or trace.get_tracer("messenger", SERVICE_VERSION) + + +def status_class(code): + return f"{int(code) // 100}xx" + + +def route_template(path): + if path in ("/health/live", "/health/ready"): + return path + parts = [p for p in path.strip("/").split("/") if p] + if len(parts) >= 3 and parts[0] == "api" and parts[1] == "v1": + if parts[2] == "users" and len(parts) == 3: + return "/api/v1/users" + if parts[2] == "users" and len(parts) == 4 and parts[3] == "me": + return "/api/v1/users/me" + if parts[2] == "auth" and len(parts) >= 4 and parts[3] == "sessions": + if len(parts) == 4: + return "/api/v1/auth/sessions" + return "/api/v1/auth/sessions/current" + if parts[2] == "chats" and len(parts) == 3: + return "/api/v1/chats" + if parts[2] == "chats" and len(parts) == 4: + return "/api/v1/chats/{chat_id}" + if parts[2] == "chats" and len(parts) == 5 and parts[4] == "messages": + return "/api/v1/chats/{chat_id}/messages" + if parts[2] == "chats" and len(parts) == 5 and parts[4] == "attachments": + return "/api/v1/chats/{chat_id}/attachments" + if parts[2] == "attachments" and len(parts) == 4: + return "/api/v1/attachments/{attachment_id}" + if parts[2] == "attachments" and len(parts) == 5 and parts[4] == "content": + return "/api/v1/attachments/{attachment_id}/content" + return "other" + + +def record_http(method, path, status, duration_s): + if http_requests is None: + return + if path in ("/health/live", "/health/ready"): + return + route = route_template(path) + attrs = { + "http.request.method": method, + "http.route": route, + "http.response.status_class": status_class(status), + } + http_requests.add(1, attrs) + http_duration.record(duration_s, attrs) + + +def inject_trace(carrier): + PROPAGATOR.inject(carrier) + return carrier + + +def extract_context(carrier): + return PROPAGATOR.extract(carrier) + + +def current_trace_fields(): + span = trace.get_current_span() + ctx = span.get_span_context() + if not ctx.is_valid: + return {}, None, None + carrier = {} + inject_trace(carrier) + return carrier, format(ctx.trace_id, "032x"), format(ctx.span_id, "016x") + + +def structured_log(logger, event, level=logging.INFO, **fields): + _carrier, trace_id, span_id = current_trace_fields() + payload = { + "event": event, + "service.name": os.environ.get("OTEL_SERVICE_NAME", "messenger"), + } + if trace_id: + payload["trace_id"] = trace_id + if span_id: + payload["span_id"] = span_id + for key, value in fields.items(): + if value is None: + continue + if key in ("password", "token", "authorization", "body", "text", "bytes"): + continue + payload[key] = value + logger.log(level, json.dumps(payload, ensure_ascii=False, sort_keys=True)) + + +@contextmanager +def start_span(name, kind=SpanKind.INTERNAL, attributes=None, links=None, context=None): + attrs = dict(attributes or {}) + span = tracer().start_span(name, context=context, kind=kind, attributes=attrs, links=links or []) + token = trace.use_span(span, end_on_exit=True) + token.__enter__() + try: + yield span + except Exception as exc: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR, type(exc).__name__)) + raise + finally: + token.__exit__(None, None, None) + + +def link_from_carrier(carrier): + ctx = extract_context(carrier) + span_ctx = trace.get_current_span(ctx).get_span_context() + if span_ctx.is_valid: + return [Link(span_ctx)] + return [] + + +def observe_process(): + try: + import resource + + usage = resource.getrusage(resource.RUSAGE_SELF) + _STATE["process_rss"] = float(usage.ru_maxrss) * 1024.0 + _STATE["process_cpu"] = float(usage.ru_utime + usage.ru_stime) + except Exception: + pass + + +def set_pool_stats(used, waiting): + _STATE["db_pool_used"] = float(used) + _STATE["db_pool_waiting"] = float(waiting) + + +def set_worker_stats(busy, capacity): + _STATE["worker_busy"] = float(busy) + _STATE["worker_capacity"] = float(capacity) + + +def set_queue_stats(name, depth, oldest_age_s): + _STATE["queue_depth"][name] = int(depth) + _STATE["oldest_job_age"][name] = float(oldest_age_s) + + +def record_job(outcome, duration_s, job_type="attachment.process.v1"): + if jobs_processed is None: + return + attrs = {"job.type": job_type, "job.outcome": outcome} + jobs_processed.add(1, attrs) + job_duration.record(duration_s, attrs) + + +def record_time_to_ready(seconds): + if e2e_ready is None: + return + e2e_ready.record(float(seconds), {"job.type": "attachment.process.v1"}) + + +def now_monotonic(): + return time.perf_counter() diff --git a/src/tlsutil.py b/src/tlsutil.py new file mode 100644 index 0000000..1d86c7e --- /dev/null +++ b/src/tlsutil.py @@ -0,0 +1,95 @@ +"""Общие TLS-контексты стенда: проверка CA и (для mTLS) клиентской identity.""" +from __future__ import annotations + +import os +import ssl +from pathlib import Path + + +def _env_path(name: str, default: str | None = None) -> str | None: + value = os.environ.get(name, default) + if value is None or not str(value).strip(): + return None + return str(value).strip() + + +def ca_file() -> str: + path = _env_path("TLS_CA_FILE", "/certs/ca.crt") + if not path or not Path(path).is_file(): + raise RuntimeError("TLS_CA_FILE") + return path + + +def client_identity_ok(peercert, allowed: set[str]) -> bool: + if not peercert: + return False + names = set() + subject = peercert.get("subject") or () + for rdns in subject: + for key, value in rdns: + if key == "commonName": + names.add(value) + for kind, value in peercert.get("subjectAltName") or (): + if kind.lower() in {"dns", "dnsname"}: + names.add(value) + return bool(names.intersection(allowed)) + + +def _guard_handshake(sslobj: ssl.SSLObject, allowed: set[str]) -> None: + peer = sslobj.getpeercert() + if not client_identity_ok(peer, allowed): + raise ssl.SSLError("client identity is not allowed") + + +def server_ssl_context(allowed_clients: set[str]) -> ssl.SSLContext: + cert = _env_path("TLS_CERT_FILE", "/certs/api.crt") + key = _env_path("TLS_KEY_FILE", "/certs/private/api.key") + if not cert or not key: + raise RuntimeError("TLS_CERT_FILE/TLS_KEY_FILE") + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + ctx.load_cert_chain(certfile=cert, keyfile=key) + ctx.load_verify_locations(cafile=ca_file()) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.check_hostname = False + + # asyncio/uvicorn использует wrap_bio, не wrap_socket — вешаем проверку на handshake. + original_bio = ctx.wrap_bio + + def wrap_bio(incoming, outgoing, server_side=False, server_hostname=None, session=None): + sslobj = original_bio( + incoming, + outgoing, + server_side=server_side, + server_hostname=server_hostname, + session=session, + ) + original_hs = sslobj.do_handshake + + def do_handshake(*args, **kwargs): + result = original_hs(*args, **kwargs) + _guard_handshake(sslobj, allowed_clients) + return result + + sslobj.do_handshake = do_handshake # type: ignore[method-assign] + return sslobj + + ctx.wrap_bio = wrap_bio # type: ignore[method-assign] + + original_wrap = ctx.wrap_socket + + def wrap_socket(sock, *args, **kwargs): + ssock = original_wrap(sock, *args, **kwargs) + _guard_handshake(ssock, allowed_clients) + return ssock + + ctx.wrap_socket = wrap_socket # type: ignore[method-assign] + return ctx + + +def client_ssl_context(cert_file: str | None = None, key_file: str | None = None) -> ssl.SSLContext: + ctx = ssl.create_default_context(cafile=ca_file()) + ctx.minimum_version = ssl.TLSVersion.TLSv1_2 + if cert_file and key_file: + ctx.load_cert_chain(certfile=cert_file, keyfile=key_file) + return ctx diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..28af550 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,261 @@ +"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен.""" +import logging +import os +import socket +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone + +import pika +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from blobs import bucket_ready, read_secret +from broker import PROCESS_QUEUE, RETRY_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending, queue_depth +from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired +from telemetry import observe_process, set_queue_stats, set_worker_stats, setup as setup_telemetry, structured_log + +LOGGER = logging.getLogger("worker") +SCHEMA_VERSION = "004_jobs" +BUSY = 0 +BUSY_LOCK = threading.Lock() + + +def database_url(): + return read_secret(os.environ["DATABASE_URL_FILE"]) + + +def concurrency(): + value = int(os.environ.get("WORKER_CONCURRENCY", "2")) + if not 1 <= value <= 8: + raise RuntimeError("WORKER_CONCURRENCY") + return value + + +def wait_schema(conninfo): + last = "error" + for _ in range(90): + try: + with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is not None: + return + last = "migration missing" + except psycopg.Error as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("schema not ready: %s", last) + raise SystemExit(1) + + +def wait_bucket(): + for _ in range(60): + if bucket_ready(): + return + time.sleep(1) + LOGGER.error("object storage not ready") + raise SystemExit(1) + + +def wait_broker(): + last = "error" + for _ in range(60): + try: + with broker_channel(): + return + except Exception as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("broker not ready: %s", last) + raise SystemExit(1) + + +def make_pool(conninfo, slots): + return ConnectionPool( + conninfo, + min_size=1, + max_size=slots * 2 + 2, + timeout=3, + kwargs={ + "connect_timeout": 3, + "row_factory": dict_row, + "application_name": "mtusi-lab6-worker", + "options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000", + }, + open=True, + ) + + +def oldest_job_age_seconds(pool): + try: + with pool.connection() as conn: + row = conn.execute( + """ + SELECT EXTRACT(EPOCH FROM (now() - MIN(created_at))) AS age + FROM attachment_jobs + WHERE status IN ('queued', 'processing') + """ + ).fetchone() + if row is None or row["age"] is None: + return 0.0 + return float(row["age"]) + except Exception: + return 0.0 + + +def refresh_runtime_stats(pool, slots): + global BUSY + try: + depth = queue_depth(PROCESS_QUEUE) + except Exception: + depth = 0 + try: + retry_depth = queue_depth(RETRY_QUEUE) + except Exception: + retry_depth = 0 + age = oldest_job_age_seconds(pool) + set_queue_stats(PROCESS_QUEUE, depth, age) + set_queue_stats(RETRY_QUEUE, retry_depth, 0.0) + with BUSY_LOCK: + busy = BUSY + set_worker_stats(busy, slots) + observe_process() + try: + stats = pool.get_stats() + from telemetry import set_pool_stats + + used = int(stats.get("pool_size", 0)) - int(stats.get("pool_available", 0)) + waiting = int(stats.get("requests_waiting", 0)) + set_pool_stats(max(0, used), waiting) + except Exception: + pass + + +def consume(pool, owner, slots): + executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job") + while True: + connection = None + try: + connection = pika.BlockingConnection(_parameters()) + channel = connection.channel() + ensure_topology(channel) + channel.basic_qos(prefetch_count=slots) + + def on_message(ch, method, _props, body): + executor.submit(run_one, pool, owner, ch, method.delivery_tag, body) + + channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False) + channel.start_consuming() + except Exception as exc: + LOGGER.warning("consumer %s", type(exc).__name__) + time.sleep(1) + finally: + if connection is not None: + try: + connection.close() + except Exception: + pass + + +def run_one(pool, owner, channel, tag, body): + global BUSY + with BUSY_LOCK: + BUSY += 1 + try: + process_delivery(pool, body, owner) + action = "ack" + except Exception as exc: + LOGGER.warning("delivery %s", type(exc).__name__) + action = "nack" + finally: + with BUSY_LOCK: + BUSY = max(0, BUSY - 1) + + def finish(): + try: + if action == "ack": + channel.basic_ack(tag) + else: + channel.basic_nack(tag, requeue=True) + except Exception: + pass + + try: + channel.connection.add_callback_threadsafe(finish) + except Exception: + pass + + +def wait_bao(): + """Fail-closed: worker не стартует без scoped secret из OpenBao.""" + if not os.environ.get("BAO_ADDR"): + return + from bao import BaoError, login_and_read + + path = os.environ.get("BAO_SECRET_PATH", "secret/data/messenger/worker") + last = None + for _ in range(60): + try: + login_and_read(path) + return + except BaoError as exc: + last = exc + time.sleep(2) + LOGGER.error("openbao unavailable: %s", last) + raise SystemExit(1) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + setup_telemetry(os.environ.get("OTEL_SERVICE_NAME", "messenger-worker")) + wait_bao() + conninfo = database_url() + wait_schema(conninfo) + wait_bucket() + wait_broker() + slots = concurrency() + pool = make_pool(conninfo, slots) + owner = socket.gethostname() + stop = threading.Event() + set_worker_stats(0, slots) + + def drain(): + while not stop.wait(1): + try: + with pool.connection() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("drain %s", type(exc).__name__) + + def sweep(): + ticks = 0 + while not stop.wait(5): + ticks += 1 + try: + with pool.connection() as conn: + release_expired(conn) + if ticks % 12 == 0: + cleanup_staging(conn) + refresh_runtime_stats(pool, slots) + except Exception as exc: + LOGGER.warning("sweep %s", type(exc).__name__) + + threading.Thread(target=drain, name="drain", daemon=True).start() + threading.Thread(target=sweep, name="sweep", daemon=True).start() + try: + with pool.connection() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("startup drain %s", type(exc).__name__) + structured_log(LOGGER, "worker_ready", owner=owner, concurrency=slots, lease=lease_seconds()) + LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds()) + consume(pool, owner, slots) + + +if __name__ == "__main__": + main() diff --git a/student-tests/probe_loki.py b/student-tests/probe_loki.py new file mode 100644 index 0000000..2586f2f --- /dev/null +++ b/student-tests/probe_loki.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import json +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + ) + + +def main(): + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read() + except urllib.error.HTTPError: + pass + time.sleep(10) + end = int(time.time()) + start = end - 1800 + for query in ( + '{service_name="messenger-api"}', + '{service_name="messenger-worker"}', + '{service_name=~".+"}', + ): + url = "http://127.0.0.1:3100/loki/api/v1/query_range?" + urllib.parse.urlencode( + { + "query": query, + "start": str(start) + "000000000", + "end": str(end) + "000000000", + "limit": "20", + } + ) + out = compose("exec", "-T", "loki", "wget", "-qO-", url) + try: + data = json.loads(out.stdout) + streams = data.get("data", {}).get("result", []) + lines = sum(len(item.get("values", [])) for item in streams) + print(query, "streams", len(streams), "lines", lines) + if streams and streams[0].get("values"): + print("sample", streams[0]["values"][0][1][:240]) + except Exception as exc: + print(query, "err", exc, out.stdout[:200], out.stderr[:200]) + print("collector tail:") + print(compose("logs", "otel-collector", "--since", "2m").stdout[-2000:]) + q = '{service_name="messenger-api"} |= "http_request"' + url = "http://127.0.0.1:3100/loki/api/v1/query_range?" + urllib.parse.urlencode( + { + "query": q, + "start": str(start) + "000000000", + "end": str(end) + "000000000", + "limit": "10", + } + ) + out = compose("exec", "-T", "loki", "wget", "-qO-", url) + data = json.loads(out.stdout) + streams = data.get("data", {}).get("result", []) + print("http_request streams", len(streams), "lines", sum(len(s.get("values", [])) for s in streams)) + for stream in streams[:3]: + for value in stream.get("values", [])[:2]: + print(value[1][:400]) + + +if __name__ == "__main__": + main() diff --git a/student-tests/probe_traces.py b/student-tests/probe_traces.py new file mode 100644 index 0000000..083d150 --- /dev/null +++ b/student-tests/probe_traces.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +import base64 +import json +import subprocess +import time +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = "http://127.0.0.1:8080" + + +def compose(*args): + return subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + ) + + +def http(method, path, body=None, headers=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(BASE + path, data=data, method=method, headers=headers or {}) + if data is not None: + req.add_header("Content-Type", "application/json") + with urllib.request.urlopen(req, timeout=30) as resp: + raw = resp.read() + payload = json.loads(raw.decode()) if raw else None + return resp.status, dict(resp.headers), payload + + +def main(): + stamp = uuid.uuid4().hex[:10] + users = [] + for idx in range(2): + username = f"tr_{stamp}_{idx}" + http("POST", "/api/v1/users", {"username": username, "display_name": username, "password": "Password123!"}) + auth = "Basic " + base64.b64encode(f"{username}:Password123!".encode()).decode() + _status, _headers, session = http("POST", "/api/v1/auth/sessions", headers={"Authorization": auth}) + users.append(session) + _status, _headers, chat = http( + "POST", + "/api/v1/chats", + {"title": "t", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]}, + headers={"Authorization": "Bearer " + users[0]["token"]}, + ) + png = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + req = urllib.request.Request( + BASE + f"/api/v1/chats/{chat['id']}/attachments?filename=t.png", + data=png, + method="POST", + headers={"Authorization": "Bearer " + users[0]["token"], "Content-Type": "image/png"}, + ) + with urllib.request.urlopen(req) as resp: + attachment = json.loads(resp.read().decode()) + request_id = resp.headers.get("X-Request-Id") + print("upload", resp.status, attachment["id"], request_id) + for _ in range(60): + _status, _headers, payload = http( + "GET", + f"/api/v1/attachments/{attachment['id']}", + headers={"Authorization": "Bearer " + users[0]["token"]}, + ) + if payload.get("status") == "ready": + print("ready") + break + time.sleep(0.5) + time.sleep(6) + for service in ("messenger-api", "messenger-worker"): + q = urllib.parse.urlencode({"tags": f"service.name={service}", "limit": "20"}) + out = compose("exec", "-T", "tempo", "wget", "-qO-", f"http://127.0.0.1:3200/api/search?{q}") + print(service, out.stdout[:800]) + q = urllib.parse.urlencode({"query": '{service_name="messenger-api"} |= "http_request"', "limit": "5"}) + out = compose("exec", "-T", "loki", "wget", "-qO-", "http://127.0.0.1:3100/loki/api/v1/query?" + q) + print("loki", out.stdout[:600]) + # worker stop age probe + compose("stop", "worker") + req = urllib.request.Request( + BASE + f"/api/v1/chats/{chat['id']}/attachments?filename=lag.png", + data=png, + method="POST", + headers={"Authorization": "Bearer " + users[0]["token"], "Content-Type": "image/png"}, + ) + with urllib.request.urlopen(req) as resp: + queued = json.loads(resp.read().decode()) + print("queued", resp.status, queued["id"]) + ages = [] + fired = None + started = time.time() + while time.time() - started < 150: + q = urllib.parse.urlencode( + { + "query": 'max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})' + } + ) + out = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q) + age = None + try: + payload = json.loads(out.stdout) + rows = payload.get("data", {}).get("result", []) + if rows: + age = float(rows[0]["value"][1]) + except Exception: + age = None + ages.append({"t": round(time.time() - started, 1), "age": age}) + alerts = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts") + try: + alert_payload = json.loads(alerts.stdout) + for item in alert_payload.get("data", {}).get("alerts", []): + if item.get("labels", {}).get("alertname") == "GrowingQueueLag" and item.get("state") == "firing": + fired = round(time.time() - started, 1) + break + except Exception: + pass + if fired is not None: + break + time.sleep(5) + compose("start", "worker") + print(json.dumps({"ages": ages[-8:], "fired_after_seconds": fired}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_negative.py b/student-tests/run_negative.py new file mode 100644 index 0000000..da74c89 --- /dev/null +++ b/student-tests/run_negative.py @@ -0,0 +1,136 @@ +"""Негативные проверки API. Токены и пароли не печатаются.""" +import json +import struct +import sys +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png(width, height, payload=b""): + def chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + payload + chunk(b"IEND", b"") + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8080" + client = Client(base) + alice = client.user("n1") + bob = client.user("n2") + eve = client.user("n3") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Негативные", "member_ids": [alice["id"].upper(), bob["id"]]}, + schema="Chat", + )["json"] + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + upper = client.call("GET", "/api/v1/chats/" + chat["id"].upper(), actor=bob, schema="Chat")["json"] + results = [] + + def record(name, status): + results.append({"name": name, "status": status}) + + record("chat_uuid_case", 200 if upper["id"] == chat["id"] else 0) + path = "/api/v1/chats/" + chat["id"] + "/messages" + record("blank_text", client.call("POST", path, 400, actor=alice, body={"text": " "}, headers={"Idempotency-Key": "blank-1"})["status"]) + record("bad_json", client.call("POST", path, 400, actor=alice, raw=b"{", headers={"Content-Type": "application/json", "Idempotency-Key": "bad-json"})["status"]) + record("short_password", client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short"})["status"]) + key = uuid.uuid4().hex + first = client.call("POST", path, 201, actor=alice, body={"text": "одно"}, headers={"Idempotency-Key": key}, schema="Message")["json"] + second = client.call( + "POST", + path, + 201, + actor=alice, + body={"text": "одно", "attachment_ids": []}, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + record("omitted_equals_empty", 201 if first == second else 0) + record("same_key_other_body", client.call("POST", path, 409, actor=alice, body={"text": "другое"}, headers={"Idempotency-Key": key})["status"]) + history = client.call("GET", path.replace("/messages", "/messages?limit=100"), actor=bob, schema="MessagePage")["json"] + record("no_second_row", 200 if sum(1 for item in history["items"] if item["text"] in ("одно", "другое")) == 1 else 0) + other_path = "/api/v1/chats/" + other["id"] + "/messages" + other_message = client.call( + "POST", + other_path, + 201, + actor=alice, + body={"text": "одно"}, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + record("same_key_other_chat", 201 if other_message["id"] != first["id"] else 0) + record("foreign_chat", client.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)["status"]) + record("bad_cursor", client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?cursor=not-a-cursor", 400, actor=alice)["status"]) + upload = "/api/v1/chats/" + chat["id"] + "/attachments" + record("slash_name", client.call("POST", upload + "?filename=a/b.png", 400, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/png"})["status"]) + record("huge_pixels", client.call("POST", upload + "?filename=big.png", 413, actor=alice, raw=png(8000, 8000), headers={"Content-Type": "image/png"})["status"]) + record("bad_png", client.call("POST", upload + "?filename=bad.png", 415, actor=alice, raw=b"\x89PNG\r\n\x1a\nbroken", headers={"Content-Type": "image/png"})["status"]) + record("type_mismatch", client.call("POST", upload + "?filename=no.jpg", 415, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/jpeg"})["status"]) + record("oversize", client.call("POST", upload + "?filename=over.png", 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"})["status"]) + missing = str(uuid.uuid4()) + record("unknown_attachment", client.call("GET", "/api/v1/attachments/" + missing.upper(), 404, actor=alice)["status"]) + record("no_token", client.call("GET", "/api/v1/users/me", 401)["status"]) + payload = { + "command": "python -X utf8 student-tests/run_negative.py", + "result": "PASS negative", + "results": results, + } + body = json.dumps(payload, ensure_ascii=False, indent=2) + print(body) + failed = [item for item in results if item["status"] not in (200, 201, 400, 401, 404, 409, 413, 415)] + if failed: + return 1 + expected = { + "chat_uuid_case": 200, + "blank_text": 400, + "bad_json": 400, + "short_password": 400, + "omitted_equals_empty": 201, + "same_key_other_body": 409, + "no_second_row": 200, + "same_key_other_chat": 201, + "foreign_chat": 404, + "bad_cursor": 400, + "slash_name": 400, + "huge_pixels": 413, + "bad_png": 415, + "type_mismatch": 415, + "oversize": 413, + "unknown_attachment": 404, + "no_token": 401, + } + for name, status in expected.items(): + got = next(item["status"] for item in results if item["name"] == name) + if got != status: + print("mismatch " + name, file=sys.stderr) + return 1 + (ROOT / "evidence" / "negative.json").write_text(body + "\n", encoding="utf-8") + print("PASS negative") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as exc: + print("FAIL " + str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/student-tests/run_observability.py b/student-tests/run_observability.py new file mode 100644 index 0000000..b609587 --- /dev/null +++ b/student-tests/run_observability.py @@ -0,0 +1,474 @@ +#!/usr/bin/env python3 +"""Учебные проверки наблюдаемости и инцидентов. Пароли не печатает.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +OUT = ROOT / "evidence" / "observability.json" + + +def http_json(method, path, body=None, headers=None, timeout=30): + data = None if body is None else json.dumps(body).encode("utf-8") + req = urllib.request.Request( + BASE + path, + data=data, + method=method, + headers=headers or {}, + ) + if data is not None: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read() + payload = json.loads(raw.decode("utf-8")) if raw else None + return resp.status, dict(resp.headers), payload + except urllib.error.HTTPError as exc: + raw = exc.read() + payload = None + if raw: + try: + payload = json.loads(raw.decode("utf-8")) + except ValueError: + payload = {"raw": raw.decode("utf-8", errors="replace")[:200]} + return exc.code, dict(exc.headers), payload + + +def compose(*args): + cmd = ["docker", "compose", "--env-file", ".env", *args] + return subprocess.run(cmd, cwd=str(ROOT), capture_output=True, text=True, check=False) + + +def prom_query(expr): + q = urllib.parse.urlencode({"query": expr}) + url = "http://127.0.0.1:9090/api/v1/query?" + q + # Prometheus не опубликован наружу. Читаем через docker exec. + result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/query?" + q) + if result.returncode != 0: + return None + try: + payload = json.loads(result.stdout) + except ValueError: + return None + rows = payload.get("data", {}).get("result", []) + if not rows: + return 0.0 + return float(rows[0]["value"][1]) + + +def prom_alerts(): + result = compose("exec", "-T", "prometheus", "wget", "-qO-", "http://localhost:9090/api/v1/alerts") + if result.returncode != 0: + return [] + try: + payload = json.loads(result.stdout) + except ValueError: + return [] + return payload.get("data", {}).get("alerts", []) + + +def tempo_search(service, name=None, limit=10): + # Tag search + TraceQL fallback. + tags = "service.name=" + service + q = urllib.parse.urlencode({"tags": tags, "limit": str(limit)}) + result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q) + traces = [] + if result.returncode == 0: + try: + traces = json.loads(result.stdout).get("traces", []) + except ValueError: + traces = [] + if name: + traces = [t for t in traces if name in (t.get("rootTraceName") or "")] + if traces: + return traces + query = f'{{resource.service.name="{service}"}}' + if name: + query = f'{{resource.service.name="{service}" && name="{name}"}}' + q = urllib.parse.urlencode({"q": query, "limit": str(limit)}) + result = compose("exec", "-T", "tempo", "wget", "-qO-", "http://127.0.0.1:3200/api/search?" + q) + if result.returncode != 0: + return [] + try: + return json.loads(result.stdout).get("traces", []) + except ValueError: + return [] + + +def register_and_chat(): + stamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S%f") + users = [] + for idx in range(2): + username = f"obs_{stamp}_{idx}" + status, _, user = http_json( + "POST", + "/api/v1/users", + {"username": username, "display_name": username, "password": "Password123!"}, + ) + assert status == 201, status + status, _, session = http_json( + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + __import__("base64").b64encode(f"{username}:Password123!".encode()).decode()}, + ) + assert status == 201, status + users.append({"user": user, "token": session["token"]}) + status, _, chat = http_json( + "POST", + "/api/v1/chats", + {"title": "obs", "member_ids": [users[0]["user"]["id"], users[1]["user"]["id"]]}, + headers={"Authorization": "Bearer " + users[0]["token"]}, + ) + assert status == 201, status + return users, chat + + +def upload_png(token, chat_id, name="probe.png"): + data = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + req = urllib.request.Request( + f"{BASE}/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(name)}", + data=data, + method="POST", + headers={ + "Authorization": "Bearer " + token, + "Content-Type": "image/png", + "Content-Length": str(len(data)), + }, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + payload = json.loads(resp.read().decode("utf-8")) + return resp.status, dict(resp.headers), payload + + +def api_direct(method, path, token=None, timeout=3): + """Запрос внутрь контейнера API, минуя HAProxy. Нужен, когда ready уже красный.""" + headers = ["-H", "Accept: application/json"] + if token: + headers.extend(["-H", f"Authorization: Bearer {token}"]) + cmd = [ + "exec", + "-T", + "api", + "python", + "-c", + ( + "import os,urllib.request,sys\n" + f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n" + ), + ] + # Проще через wget/python one-liner in api container. + py = ( + "import urllib.request,urllib.error\n" + f"req=urllib.request.Request('http://127.0.0.1:8080{path}', method='{method}')\n" + ) + if token: + py += f"req.add_header('Authorization', 'Bearer {token}')\n" + py += ( + "try:\n" + f" r=urllib.request.urlopen(req, timeout={timeout})\n" + " print(r.status)\n" + "except urllib.error.HTTPError as e:\n" + " print(e.code)\n" + "except Exception as e:\n" + " print(type(e).__name__)\n" + ) + result = compose("exec", "-T", "api", "python", "-c", py) + text = (result.stdout or "").strip().splitlines() + return text[-1] if text else "empty" + + +def wait_ready_attachment(token, attachment_id, timeout=60): + deadline = time.time() + timeout + while time.time() < deadline: + status, _, payload = http_json( + "GET", + f"/api/v1/attachments/{attachment_id}", + headers={"Authorization": "Bearer " + token}, + ) + if status == 200 and payload.get("status") == "ready": + return payload + time.sleep(0.5) + raise TimeoutError(attachment_id) + + +def scenario_trace_link(): + users, chat = register_and_chat() + token = users[0]["token"] + status, headers, attachment = upload_png(token, chat["id"]) + assert status == 202, status + ready = wait_ready_attachment(token, attachment["id"]) + time.sleep(3) + api_traces = tempo_search("messenger-api", "POST /api/v1/chats/{chat_id}/attachments") + worker_traces = tempo_search("messenger-worker", "attachments.process consume") + return { + "ok": True, + "attachment_id": attachment["id"], + "attachment_status": ready["status"], + "request_id": headers.get("X-Request-Id") or headers.get("x-request-id"), + "api_upload_traces": len(api_traces), + "worker_consume_traces": len(worker_traces), + "sample_api_trace": api_traces[0]["traceID"] if api_traces else None, + "sample_worker_trace": worker_traces[0]["traceID"] if worker_traces else None, + } + + +def scenario_worker_stop(): + users, chat = register_and_chat() + token = users[0]["token"] + compose("stop", "worker") + time.sleep(2) + status, _, attachment = upload_png(token, chat["id"], "queued.png") + assert status == 202, status + started = time.time() + fired_at = None + age_samples = [] + while time.time() - started < 90: + age = prom_query('max(messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"})') + age_samples.append({"t": round(time.time() - started, 1), "age": age}) + alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"] + if alerts and fired_at is None: + fired_at = round(time.time() - started, 1) + break + time.sleep(5) + compose("start", "worker") + ready = wait_ready_attachment(token, attachment["id"], timeout=120) + resolved_at = None + recover_started = time.time() + while time.time() - recover_started < 120: + alerts = [a for a in prom_alerts() if a.get("labels", {}).get("alertname") == "GrowingQueueLag" and a.get("state") == "firing"] + if not alerts and fired_at is not None: + resolved_at = round(time.time() - recover_started, 1) + break + time.sleep(5) + return { + "ok": ready["status"] == "ready", + "attachment_id": attachment["id"], + "fired_after_seconds": fired_at, + "resolved_after_seconds": resolved_at, + "age_samples": age_samples[-6:], + "final_age": prom_query('messenger_queue_oldest_job_age_seconds{messaging_destination_name="attachments.process"}'), + } + + +def scenario_db_stop(): + # Отказ Postgres. Через HAProxy ready красный, поэтому бизнес-5xx снимаю прямым + # запросом в контейнер API, чтобы метрики и traces остались от приложения. + users, chat = register_and_chat() + token = users[0]["token"] + before_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)') + compose("stop", "postgres") + started = time.time() + business_codes = [] + ready_via_lb = [] + while time.time() - started < 90: + business_codes.append(api_direct("GET", "/api/v1/users/me", token=token)) + try: + ready_status, _, _ = http_json("GET", "/health/ready", timeout=3) + except Exception as exc: + ready_status = type(exc).__name__ + ready_via_lb.append(ready_status) + time.sleep(1) + high_fired = None + for sample_t in range(0, 90, 5): + pass + compose("start", "postgres") + deadline = time.time() + 90 + while time.time() < deadline: + try: + status, _, payload = http_json("GET", "/health/ready", timeout=3) + if status == 200 and payload and payload.get("status") == "ok": + break + except Exception: + pass + time.sleep(2) + # ждём for: 1m у HighHttpErrorRate и запись метрик + watch_started = time.time() + alerts_timeline = [] + while time.time() - watch_started < 150: + alerts = [ + a + for a in prom_alerts() + if a.get("labels", {}).get("alertname") + in ("HighHttpErrorRate", "ErrorBudgetBurnFast", "ErrorBudgetBurnSlow") + ] + alerts_timeline.append( + { + "t": round(time.time() - watch_started, 1), + "alerts": [ + {"alertname": a.get("labels", {}).get("alertname"), "state": a.get("state")} + for a in alerts + ], + } + ) + if any(a.get("state") == "firing" for a in alerts) and high_fired is None: + high_fired = round(time.time() - watch_started, 1) + if high_fired is not None and not any(a.get("state") == "firing" for a in alerts): + break + time.sleep(5) + after_5xx = prom_query('sum(messenger_http_requests_total{http_response_status_class="5xx"}) or vector(0)') + return { + "ok": True, + "business_statuses_direct_api": business_codes[:8] + ["..."] + business_codes[-4:], + "ready_via_lb": ready_via_lb[:4] + ["..."] + ready_via_lb[-3:], + "http_5xx_before": before_5xx, + "http_5xx_after": after_5xx, + "high_error_seen_after_seconds": high_fired, + "alerts_timeline_tail": alerts_timeline[-6:], + "note": "Прямые запросы в api-контейнер, потому что HAProxy при красном ready отдаёт свою 503-страницу", + } + + +def scenario_telemetry_backend_down(): + users, chat = register_and_chat() + token = users[0]["token"] + compose("stop", "tempo") + compose("stop", "loki") + compose("stop", "prometheus") + time.sleep(2) + mem_before = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip() + status, _, attachment = upload_png(token, chat["id"], "tel.png") + assert status == 202, status + # API должен жить без telemetry backends + ready_status, _, ready_body = http_json("GET", "/health/ready") + text_status, _, message = http_json( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + {"text": "telemetry backends down"}, + headers={ + "Authorization": "Bearer " + token, + "Idempotency-Key": "obs-tel-1", + }, + ) + time.sleep(8) + mem_after = compose("stats", "--no-stream", "--format", "{{.MemUsage}}", "otel-collector").stdout.strip() + compose("start", "prometheus") + compose("start", "loki") + compose("start", "tempo") + time.sleep(5) + compose("restart", "otel-collector") + wait_ready_attachment(token, attachment["id"], timeout=120) + return { + "ok": ready_status == 200 and text_status == 201, + "upload_status": status, + "ready_status": ready_status, + "message_status": text_status, + "message_id": None if not message else message.get("id"), + "otel_mem_before": mem_before, + "otel_mem_after": mem_after, + } + + +def scenario_burn_rate_spike(): + # Короткий всплеск и затем более длинная деградация для multi-window burn-rate. + users, _chat = register_and_chat() + token = users[0]["token"] + + def outage(seconds): + compose("stop", "postgres") + end = time.time() + seconds + while time.time() < end: + api_direct("GET", "/api/v1/users/me", token=token) + time.sleep(0.5) + compose("start", "postgres") + deadline = time.time() + 90 + while time.time() < deadline: + try: + status, _, payload = http_json("GET", "/health/ready", timeout=3) + if status == 200 and payload and payload.get("status") == "ok": + return + except Exception: + pass + time.sleep(2) + + # краткий всплеск ~25 с + brief_started = time.time() + outage(25) + brief_high = None + brief_burn = None + watch = time.time() + while time.time() - watch < 120: + alerts = prom_alerts() + if brief_high is None and any( + a.get("labels", {}).get("alertname") == "HighHttpErrorRate" and a.get("state") == "firing" for a in alerts + ): + brief_high = round(time.time() - brief_started, 1) + if brief_burn is None and any( + a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts + ): + brief_burn = round(time.time() - brief_started, 1) + if brief_high is not None: + break + time.sleep(5) + + # длительная деградация ~6 минут + long_started = time.time() + outage(360) + long_slow = None + long_fast = None + watch = time.time() + while time.time() - watch < 240: + alerts = prom_alerts() + if long_slow is None and any( + a.get("labels", {}).get("alertname") == "ErrorBudgetBurnSlow" and a.get("state") == "firing" for a in alerts + ): + long_slow = round(time.time() - long_started, 1) + if long_fast is None and any( + a.get("labels", {}).get("alertname") == "ErrorBudgetBurnFast" and a.get("state") == "firing" for a in alerts + ): + long_fast = round(time.time() - long_started, 1) + if long_slow is not None: + break + time.sleep(5) + return { + "ok": True, + "brief_spike_seconds": 25, + "brief_high_error_fired_after_seconds": brief_high, + "brief_burn_fast_fired_after_seconds": brief_burn, + "long_degradation_seconds": 360, + "long_burn_slow_fired_after_seconds": long_slow, + "long_burn_fast_fired_after_seconds": long_fast, + "error_ratio_5m": prom_query("messenger:http_error_ratio:5m"), + "error_ratio_30m": prom_query("messenger:http_error_ratio:30m"), + "note": "Короткий всплеск может не зажечь BurnFast (нет ложного page). Длинная деградация должна зажечь BurnSlow.", + } + + +def main(): + report = { + "started_at": datetime.now(timezone.utc).isoformat(), + "base_url": BASE, + "scenarios": {}, + } + for name, func in [ + ("trace_link", scenario_trace_link), + ("worker_stop", scenario_worker_stop), + ("db_stop", scenario_db_stop), + ("telemetry_backend_down", scenario_telemetry_backend_down), + ("burn_rate_spike", scenario_burn_rate_spike), + ]: + print("RUN", name, flush=True) + try: + report["scenarios"][name] = func() + print("OK", name, flush=True) + except Exception as exc: + report["scenarios"][name] = {"ok": False, "error": type(exc).__name__, "detail": str(exc)[:300]} + print("FAIL", name, type(exc).__name__, flush=True) + report["finished_at"] = datetime.now(timezone.utc).isoformat() + OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("WROTE", OUT) + failed = [k for k, v in report["scenarios"].items() if not v.get("ok")] + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/run_pipeline.py b/student-tests/run_pipeline.py new file mode 100644 index 0000000..aab7688 --- /dev/null +++ b/student-tests/run_pipeline.py @@ -0,0 +1,653 @@ +"""Проверки очереди и трека 2. Токены и пароли не печатаются.""" +import hashlib +import json +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +BASE = "http://127.0.0.1:8080" +EVIDENCE = ROOT / "evidence" / "pipeline.json" +INSIDE = r""" +import json, logging, os, hashlib +from datetime import datetime, timezone +logging.disable(logging.CRITICAL) +req = json.loads(REQUEST_JSON) +from blobs import read_secret, read_bytes, put_bytes, delete_quiet +from jobs import bump_pipeline, replay_job, republish +import psycopg +from psycopg.rows import dict_row +url = read_secret(os.environ["DATABASE_URL_FILE"]) +action = req["action"] +out = {"action": action} +with psycopg.connect(url, row_factory=dict_row) as conn: + if action == "republish": + out["ok"] = bool(republish(conn, req["job_id"], req.get("attempt"))) + elif action == "bump": + out["result"] = bump_pipeline(conn, req["attachment_id"]) + elif action == "replay": + out["result"] = replay_job(conn, req["job_id"]) + elif action == "corrupt": + row = conn.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (req["attachment_id"],), + ).fetchone() + key = row["staging_key"] + data = read_bytes(key) + stamp = datetime.now(timezone.utc) + put_bytes(key + ".held", data, "application/octet-stream", stamp) + put_bytes(key, b"not-an-image", "image/png", stamp) + out["original_sha256"] = hashlib.sha256(data).hexdigest() + out["bytes"] = len(data) + elif action == "restore": + row = conn.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (req["attachment_id"],), + ).fetchone() + key = row["staging_key"] + data = read_bytes(key + ".held") + put_bytes(key, data, "image/png", datetime.now(timezone.utc)) + delete_quiet(key + ".held") + out["restored_sha256"] = hashlib.sha256(data).hexdigest() + out["bytes"] = len(data) + else: + raise SystemExit("unknown") +print(json.dumps(out)) +""" + + +def run(args, timeout=180, check=True): + proc = subprocess.run( + args, + cwd=str(ROOT), + capture_output=True, + timeout=timeout, + ) + if check and proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace")[-1200:] + raise RuntimeError(args[0] + " failed: " + err) + return proc + + +def compose(*args, timeout=180): + run(["docker", "compose", "--env-file", ".env", *args], timeout=timeout) + + +def inspect(name, template): + proc = run(["docker", "inspect", "-f", template, name], timeout=30) + return proc.stdout.decode().strip() + + +def wait_health(name, seconds=180): + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + if last.startswith("running healthy"): + return last + time.sleep(1) + raise RuntimeError(name + " not healthy: " + last) + + +def restart_service(service, containers, seconds=180): + if isinstance(containers, str): + containers = [containers] + before = {name: inspect(name, "{{.State.StartedAt}}") for name in containers} + compose("restart", service) + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + ready = True + for name in containers: + started = inspect(name, "{{.State.StartedAt}}") + last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + if started == before[name] or not last.startswith("running healthy"): + ready = False + if ready: + return + time.sleep(1) + raise RuntimeError(service + " did not restart: " + last) + + +def psql(sql): + proc = run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-v", "ON_ERROR_STOP=1", "-q", "-t", "-A", "-c", sql, + ], + timeout=30, + ) + return proc.stdout.decode("utf-8").strip() + + +def state(attachment_id): + sql = """ + SELECT json_build_object( + 'status', a.status, + 'error_code', a.error_code, + 'pipeline_version', a.pipeline_version, + 'jobs', ( + SELECT coalesce(json_agg(json_build_object( + 'id', j.id, 'status', j.status, 'attempt', j.attempt, + 'pipeline_version', j.pipeline_version, 'error_code', j.error_code + ) ORDER BY j.pipeline_version), '[]'::json) + FROM attachment_jobs j WHERE j.attachment_id = a.id + ), + 'variants', ( + SELECT coalesce(json_agg(json_build_object( + 'name', v.name, 'pipeline_version', v.pipeline_version, + 'width', v.width, 'height', v.height, 'content_type', v.content_type + ) ORDER BY v.pipeline_version, v.name), '[]'::json) + FROM attachment_variants v WHERE v.attachment_id = a.id + ), + 'unpublished', ( + SELECT count(*) FROM outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE j.attachment_id = a.id AND o.published_at IS NULL + ), + 'events', ( + SELECT coalesce(json_agg(e.kind ORDER BY e.created_at), '[]'::json) + FROM job_events e WHERE e.attachment_id = a.id + ) + )::text + FROM attachments a WHERE a.id = '{id}'::uuid + """.format(id=attachment_id) + return json.loads(psql(sql)) + + +def arm(kind, attachment_id): + target = "NULL" if attachment_id is None else "'" + attachment_id + "'::uuid" + psql( + "INSERT INTO fault_injections (id, kind, attachment_id, armed) " + "VALUES (gen_random_uuid(), '{kind}', {target}, true)".format(kind=kind, target=target) + ) + + +def queues(): + proc = run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", + "rabbitmqctl", "list_queues", "name", "messages", "--quiet", + ], + timeout=30, + ) + found = {} + for line in proc.stdout.decode().splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + found[parts[0]] = int(parts[1]) + return found + + +def depth(): + last = "" + for _ in range(20): + try: + found = queues() + return found.get("attachments.process", -1) + except RuntimeError as exc: + last = str(exc) + time.sleep(1) + raise RuntimeError(last) + + +def inside(req, timeout=90): + code = INSIDE.replace("REQUEST_JSON", json.dumps(json.dumps(req))) + proc = run( + [ + "docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "-T", + "--entrypoint", "python", "worker", "-c", code, + ], + timeout=timeout, + ) + lines = [line for line in proc.stdout.decode("utf-8").splitlines() if line.startswith("{")] + if not lines: + raise RuntimeError("no json from worker helper") + return json.loads(lines[-1]) + + +def stop_worker(): + compose("stop", "worker") + + +def start_worker(): + compose("start", "worker") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if inspect("mtusi-lab5-worker-1", "{{.State.Status}}") == "running": + return + time.sleep(0.5) + raise RuntimeError("worker did not start") + + +def restart_count(): + return int(inspect("mtusi-lab5-worker-1", "{{.RestartCount}}")) + + +def wait_status(actor, attachment_id, want, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + body = client.call( + "GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment" + )["json"] + last = body["status"] + if last == want: + return body + if want != "failed" and last == "failed": + raise Failure("attachment failed " + attachment_id) + time.sleep(0.4) + raise Failure("timeout status=" + str(last) + " want=" + want) + + +def check_picture(actor, ready, original): + download = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor) + if download["bytes"] != original: + raise Failure("original bytes differ") + names = {item["name"]: item for item in ready["variants"]} + thumb = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=thumbnail", actor=actor + ) + crop = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=crop", actor=actor + ) + optimized = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=optimized", actor=actor + ) + if jpeg_size(thumb["bytes"]) != (256, 160): + raise Failure("thumbnail size") + if jpeg_size(crop["bytes"]) != (128, 128): + raise Failure("crop size") + if optimized["bytes"][:4] != b"RIFF" or optimized["bytes"][8:12] != b"WEBP": + raise Failure("optimized is not webp") + if names["thumbnail"]["width"] != 256 or names["thumbnail"]["height"] != 160: + raise Failure("thumbnail metadata") + if names["crop"]["width"] != 128 or names["crop"]["height"] != 128: + raise Failure("crop metadata") + if names["optimized"]["width"] != 320 or names["optimized"]["height"] != 200: + raise Failure("optimized metadata") + if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"]: + raise Failure("thumbnail sha") + rows = state(ready["id"]) + version_rows = [row for row in rows["variants"] if row["pipeline_version"] == rows["pipeline_version"]] + if len(version_rows) != 4: + raise Failure("variant rows " + str(len(version_rows))) + return { + "pipeline_version": rows["pipeline_version"], + "variant_rows": len(version_rows), + "thumbnail": [names["thumbnail"]["width"], names["thumbnail"]["height"]], + "crop": [names["crop"]["width"], names["crop"]["height"]], + "optimized": [names["optimized"]["width"], names["optimized"]["height"]], + "original_sha256": hashlib.sha256(original).hexdigest(), + } + + +def upload(actor, chat): + att, content = client.attachment(actor, chat) + return att, content + + +results = [] + + +def scenario(name, fn): + started = time.monotonic() + try: + detail = fn() + results.append({"name": name, "ok": True, "seconds": round(time.monotonic() - started, 3), "detail": detail}) + except Exception as exc: + results.append({"name": name, "ok": False, "seconds": round(time.monotonic() - started, 3), "error": type(exc).__name__ + ": " + str(exc)}) + raise + + +client = Client(BASE, timeout=20) + + +def durability(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + blocked = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob) + text = client.message(alice, chat, "текст пока файл в очереди")["json"] + linked = client.message(alice, chat, "", attachments=[att["id"]])["json"] + before_api = depth() + if before_api < 1: + raise Failure("job was not in the durable queue") + restart_service("api", ["mtusi-lab5-api-1", "mtusi-lab5-api-2"]) + after_api = depth() + restart_service("rabbitmq", "mtusi-lab5-rabbitmq-1", 180) + after_rabbit = depth() + if after_api < 1 or after_rabbit < 1: + raise Failure("queue lost the job") + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return { + "upload_status": att["status"], + "content_while_queued": blocked["status"], + "text_status": 201, + "text_id": text["id"], + "linked_id": linked["id"], + "queue_before_api_restart": before_api, + "queue_after_api_restart": after_api, + "queue_after_rabbit_restart": after_rabbit, + "picture": picture, + } + + +def outbox_gap(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + arm("skip_inline_publish", None) + before = depth() + att, content = upload(alice, chat) + info = state(att["id"]) + queued = depth() + if info["unpublished"] != 1 or queued != before: + raise Failure("outbox was published early") + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return {"unpublished_before_worker": info["unpublished"], "queue_depth": queued, "picture": picture} + + +def duplicate(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + job_id = info["jobs"][0]["id"] + published = inside({"action": "republish", "job_id": job_id}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return {"republish": published["ok"], "picture": picture} + + +def reorder(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + job_id = info["jobs"][0]["id"] + inside({"action": "republish", "job_id": job_id, "attempt": 2}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + after = state(att["id"]) + if after["jobs"][0]["attempt"] != 1 or after["status"] != "ready": + raise Failure("reorder changed the job") + return {"attempt": after["jobs"][0]["attempt"], "picture": picture} + + +def stale_version(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + old_job = info["jobs"][0]["id"] + bumped = inside({"action": "bump", "attachment_id": att["id"]}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + before = check_picture(bob, ready, content) + thumb_before = client.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob + )["bytes"] + inside({"action": "republish", "job_id": old_job, "attempt": 1}) + time.sleep(2) + ready2 = client.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + thumb_after = client.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob + )["bytes"] + after = state(att["id"]) + old = [job for job in after["jobs"] if job["id"] == old_job][0] + if ready2["status"] != "ready" or after["pipeline_version"] != 2: + raise Failure("new version was overwritten") + if old["status"] != "superseded": + raise Failure("old job was not superseded") + if thumb_before != thumb_after: + raise Failure("old job changed thumbnail") + return { + "bump": bumped["result"], + "old_job_status": old["status"], + "old_error_code": old["error_code"], + "attachment_error_code": after["error_code"], + "picture": before, + "thumbnail_unchanged": True, + } + + +def crash(kind): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + arm(kind, att["id"]) + start_worker() + born = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}") + ready = wait_status(bob, att["id"], "ready", 120) + picture = check_picture(bob, ready, content) + restarted = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}") + if born == restarted: + raise Failure("worker did not exit") + return {"started_at": born, "restarted_at": restarted, "picture": picture} + + +def transient(): + alice, bob, _eve, chat = client.bootstrap() + stop_worker() + att, _content = upload(alice, chat) + arm("transient_once", att["id"]) + start_worker() + wait_status(bob, att["id"], "ready", 90) + info = state(att["id"]) + version_rows = [row for row in info["variants"] if row["pipeline_version"] == info["pipeline_version"]] + if info["jobs"][0]["attempt"] != 2 or "retry" not in info["events"] or len(version_rows) != 4: + raise Failure("transient retry did not keep one result") + return { + "attempt": info["jobs"][0]["attempt"], + "events": info["events"], + "variant_rows": len(version_rows), + "status": info["status"], + } + + +def permanent_and_replay(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + corrupted = inside({"action": "corrupt", "attachment_id": att["id"]}) + start_worker() + failed = wait_status(bob, att["id"], "failed", 90) + info = state(att["id"]) + restored = inside({"action": "restore", "attachment_id": att["id"]}) + replayed = inside({"action": "replay", "job_id": info["jobs"][0]["id"]}) + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + again = inside({"action": "replay", "job_id": info["jobs"][0]["id"]}) + after = state(att["id"]) + version_rows = [row for row in after["variants"] if row["pipeline_version"] == after["pipeline_version"]] + if failed["error_code"] != "invalid_image" or info["jobs"][0]["attempt"] != 1: + raise Failure("permanent error was retried") + if replayed["result"].get("result") != "queued" or again["result"].get("result") != "already_ready": + raise Failure("replay result") + if len(version_rows) != 4 or restored["restored_sha256"] != corrupted["original_sha256"]: + raise Failure("replay did not keep one result") + return { + "corrupt_sha256": corrupted["original_sha256"], + "failed_code": failed["error_code"], + "attempt_at_fail": info["jobs"][0]["attempt"], + "restored_sha256": restored["restored_sha256"], + "replay": replayed["result"], + "second_replay": again["result"], + "variant_rows": len(version_rows), + "picture": picture, + } + + +def lease(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + arm("skip_inline_publish", None) + before = depth() + att, content = upload(alice, chat) + info = state(att["id"]) + if info["unpublished"] != 1 or depth() != before: + raise Failure("lease setup published the job") + job_id = info["jobs"][0]["id"] + psql( + "UPDATE attachment_jobs SET status = 'processing', lease_owner = 'dead', " + "lease_until = now() - interval '2 minutes', updated_at = now() " + "WHERE id = '{id}'::uuid".format(id=job_id) + ) + psql( + "UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '{id}'::uuid".format( + id=att["id"] + ) + ) + psql("DELETE FROM outbox WHERE job_id = '{id}'::uuid AND published_at IS NULL".format(id=job_id)) + started = time.monotonic() + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + elapsed = round(time.monotonic() - started, 3) + picture = check_picture(bob, ready, content) + info = state(att["id"]) + if "lease_expired" not in info["events"] or info["jobs"][0]["attempt"] != 1: + raise Failure("expired lease was not resumed") + return { + "seconds_until_ready": elapsed, + "events": info["events"], + "attempt": info["jobs"][0]["attempt"], + "picture": picture, + } + + +def http_status(url): + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=10) as response: + return response.status, response.read(4096).decode("utf-8", "replace") + except urllib.error.HTTPError as error: + return error.code, error.read(4096).decode("utf-8", "replace") + + +def leaked(text): + lowered = text.lower() + return [ + word for word in ( + "redis", "postgres", "minio", "password", "select", "rabbit", "amqp", + "secret", "database", "bucket", "messenger-attachments", + ) + if word in lowered + ] + + +def ready_503(): + compose("stop", "redis") + time.sleep(2) + lb_status, lb_body = http_status(BASE + "/health/ready") + proc = run( + [ + "docker", "exec", "mtusi-lab5-api-1", "python", "-c", + "import urllib.request,urllib.error\n" + "req=urllib.request.Request('http://127.0.0.1:8080/health/ready')\n" + "try:\n" + " r=urllib.request.urlopen(req, timeout=5)\n" + " print(r.status)\n" + " print(r.read(4096).decode())\n" + "except urllib.error.HTTPError as e:\n" + " print(e.code)\n" + " print(e.read(4096).decode())\n", + ], + timeout=20, + ) + lines = proc.stdout.decode("utf-8", "replace").splitlines() + api_status = int(lines[0]) + api_body = "\n".join(lines[1:]) + compose("start", "redis") + wait_health("mtusi-lab5-redis-1") + started = time.monotonic() + deadline = started + 40 + recovered = None + while time.monotonic() < deadline: + try: + recovered, _body = http_status(BASE + "/health/ready") + except (urllib.error.URLError, TimeoutError, OSError): + recovered = 0 + if recovered == 200: + break + time.sleep(0.5) + recovery_seconds = round(time.monotonic() - started, 3) + if lb_status != 503 or api_status != 503 or leaked(lb_body) or leaked(api_body): + raise Failure("ready 503 text") + if "Сервис временно недоступен" not in api_body or recovered != 200: + raise Failure("ready 503 text") + return { + "lb_status": lb_status, + "lb_body": lb_body, + "api_status": api_status, + "api_message": "Сервис временно недоступен", + "forbidden": [], + "recovered": recovered, + "recovery_seconds": recovery_seconds, + } + + +def main(): + wait_health("mtusi-lab5-api-1") + wait_health("mtusi-lab5-api-2") + psql("UPDATE fault_injections SET armed = false WHERE armed") + start_worker() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if depth() == 0 and queues().get("attachments.retry", 0) == 0: + break + time.sleep(0.5) + steps = [ + ("durability", durability), + ("outbox_gap", outbox_gap), + ("duplicate", duplicate), + ("reorder", reorder), + ("stale_version", stale_version), + ("crash_before_effect", lambda: crash("crash_before_effect")), + ("crash_after_objects", lambda: crash("crash_after_objects")), + ("crash_after_commit", lambda: crash("crash_after_commit")), + ("transient_once", transient), + ("permanent_replay", permanent_and_replay), + ("lease", lease), + ("ready_503", ready_503), + ] + if len(sys.argv) > 1 and sys.argv[1] == "resume" and EVIDENCE.is_file(): + done = [ + item for item in json.loads(EVIDENCE.read_text(encoding="utf-8"))["results"] if item.get("ok") + ] + results.extend(done) + steps = [item for item in steps if item[0] not in {row["name"] for row in done}] + failed = None + for name, fn in steps: + try: + scenario(name, fn) + except Exception as exc: + failed = exc + break + EVIDENCE.write_text(json.dumps({"results": results}, ensure_ascii=False, indent=2), encoding="utf-8") + if failed is not None: + print("FAIL " + type(failed).__name__ + ": " + str(failed), file=sys.stderr) + return 1 + print("PASS pipeline") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as exc: + print("FAIL " + str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/tests/student_mtls.py b/tests/student_mtls.py new file mode 100644 index 0000000..03a6f12 --- /dev/null +++ b/tests/student_mtls.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Отрицательные проверки внутреннего mTLS к API (docker network).""" +from __future__ import annotations + +import ssl +import subprocess +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def check(cond, msg): + if not cond: + raise SystemExit("FAIL: " + msg) + + +def probe(client_cert: str | None, client_key: str | None, ca: str, expect_ok: bool) -> None: + """curl из контейнера в сети lab к api:8443 с заданным client cert.""" + cmd = [ + "docker", + "run", + "--rm", + "--network", + "mtusi-lab7-net", + "-v", + f"{ROOT / 'certs'}:/certs:ro", + "curlimages/curl:8.10.1", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--cacert", + "/certs/ca.crt", + "--connect-timeout", + "3", + "--max-time", + "5", + ] + if client_cert and client_key: + cmd.extend(["--cert", client_cert, "--key", client_key]) + cmd.append("https://api:8443/health/ready") + # Map host paths already under /certs; pass container paths. + if client_cert: + cmd = [ + "docker", + "run", + "--rm", + "--network", + "mtusi-lab7-net", + "-v", + f"{ROOT / 'certs'}:/certs:ro", + "curlimages/curl:8.10.1", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--cacert", + "/certs/ca.crt", + "--cert", + client_cert, + "--key", + client_key, + "--connect-timeout", + "3", + "--max-time", + "5", + "https://api:8443/health/ready", + ] + else: + cmd = [ + "docker", + "run", + "--rm", + "--network", + "mtusi-lab7-net", + "-v", + f"{ROOT / 'certs'}:/certs:ro", + "curlimages/curl:8.10.1", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--cacert", + "/certs/ca.crt", + "--connect-timeout", + "3", + "--max-time", + "5", + "https://api:8443/health/ready", + ] + proc = subprocess.run(cmd, capture_output=True, text=True) + ok = proc.returncode == 0 and proc.stdout.strip() == "200" + check(ok == expect_ok, f"mtls probe cert={client_cert} ok={ok} expected={expect_ok} out={proc.stdout!r} err={proc.stderr[-200:]!r}") + + +def main(): + # Без client cert → отказ + probe(None, None, "/certs/ca.crt", expect_ok=False) + # Правильная identity lb → успех + probe("/certs/lb.crt", "/certs/private/lb.key", "/certs/ca.crt", expect_ok=True) + # Чужой leaf того же CA (api-client) → отказ identity + probe("/certs/api-client.crt", "/certs/private/api-client.key", "/certs/ca.crt", expect_ok=False) + print("PASS: mTLS rejects missing/wrong identity; lb identity accepted") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/student_openbao.py b/tests/student_openbao.py new file mode 100644 index 0000000..2e8d73d --- /dev/null +++ b/tests/student_openbao.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Allow/deny OpenBao policy для API AppRole (без печати token).""" +from __future__ import annotations + +import json +import ssl +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CA = ROOT / "certs" / "ca.crt" +ADDR = "https://127.0.0.1:8200" + + +def ctx(): + c = ssl.create_default_context(cafile=str(CA)) + c.check_hostname = False + return c + + +def call(method, path, body=None, token=None): + data = None + headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + if token: + headers["X-Vault-Token"] = token + req = urllib.request.Request(ADDR + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, context=ctx(), timeout=5) as response: + raw = response.read().decode("utf-8") + return response.status, json.loads(raw) if raw else {} + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8") + return exc.code, json.loads(raw) if raw else {} + + +def main(): + role = (ROOT / "secrets/bao_api_role_id").read_text(encoding="utf-8").strip() + secret = (ROOT / "secrets/bao_api_secret_id").read_text(encoding="utf-8").strip() + code, payload = call("POST", "/v1/auth/approle/login", {"role_id": role, "secret_id": secret}) + if code >= 400 or "auth" not in payload: + raise SystemExit("FAIL: approle login") + token = payload["auth"]["client_token"] + ok_code, _ = call("GET", "/v1/secret/data/messenger/api", token=token) + deny_code, _ = call("GET", "/v1/secret/data/messenger/worker", token=token) + if ok_code != 200: + raise SystemExit(f"FAIL: allow path status {ok_code}") + if deny_code not in (403, 404): + raise SystemExit(f"FAIL: deny path expected 403/404 got {deny_code}") + print("PASS: AppRole allow api secret; deny worker secret") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- 2.54.0