forked from mtusi2026/mtusi2026lab7
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7353ab97a6 |
@@ -18,3 +18,9 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
operator
|
||||
pki
|
||||
secrets
|
||||
evidence
|
||||
build
|
||||
|
||||
+4
-40
@@ -1,41 +1,5 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
APP_PORT=18443
|
||||
GRAFANA_PORT=13407
|
||||
BAO_PORT=18207
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=https://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=https://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
# TLS and OpenBao: actual paths/ports depend on your implemented trust matrix.
|
||||
PUBLIC_BASE_URL=https://localhost:8443
|
||||
TLS_CA_FILE=/run/secrets/ca.crt
|
||||
TLS_CERT_FILE=/run/secrets/service.crt
|
||||
TLS_KEY_FILE=/run/secrets/service.key
|
||||
BAO_ADDR=https://openbao:8200
|
||||
BAO_CACERT=/run/secrets/bao-ca.crt
|
||||
# No root token here. Bootstrap operator -> short-lived service auth -> scoped token.
|
||||
ARGON2_MEMORY_KIB=19456
|
||||
ARGON2_ITERATIONS=2
|
||||
ARGON2_PARALLELISM=1
|
||||
TRACE_SAMPLE_RATIO=0.25
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
BASE_URL: https://localhost:18443
|
||||
CA_FILE: ${{ github.workspace }}/pki/lb/ca.crt
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: python3 scripts/start.py
|
||||
- run: make check test test-security test-student BASE_URL=$BASE_URL CA_FILE=$CA_FILE
|
||||
- run: make test-internal-security
|
||||
- run: docker compose --profile tools run --rm --no-deps audit /app/student-tests/auth_and_signals.py
|
||||
- run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=$BASE_URL CA_FILE=$CA_FILE
|
||||
- if: always()
|
||||
run: docker compose down -v
|
||||
@@ -30,3 +30,6 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
operator/
|
||||
pki/
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
FROM python:3.12-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 APP_HOST=0.0.0.0 APP_PORT=8080 MALLOC_ARENA_MAX=2
|
||||
WORKDIR /app
|
||||
RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt && rm -rf /usr/local/lib/python3.12/site-packages/pip /usr/local/lib/python3.12/site-packages/pip-*.dist-info /usr/local/bin/pip*
|
||||
COPY src/ ./src/
|
||||
COPY migrations/ ./migrations/
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK NONE
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "src/server.py"]
|
||||
@@ -31,3 +31,17 @@ test-failover:
|
||||
|
||||
test-security:
|
||||
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: test-student
|
||||
test-student:
|
||||
BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" $(PYTHON) -m unittest discover -s student-tests -p 'test_*.py' -v
|
||||
|
||||
.PHONY: up test-internal-security research scan
|
||||
up:
|
||||
python3 scripts/start.py
|
||||
test-internal-security:
|
||||
docker compose --profile tools run --rm --no-deps audit /app/student-tests/tls_audit.py
|
||||
research:
|
||||
BASE_URL="$(BASE_URL)" CA_FILE="$(CA_FILE)" python3 student-tests/research.py
|
||||
scan:
|
||||
python3 student-tests/scan_images.py
|
||||
|
||||
@@ -1,37 +1,222 @@
|
||||
# Отчёт — лабораторная 7
|
||||
# Лабораторная 7: безопасность, mTLS и OpenBao
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: **Майорова Софья, БВТ2301**. Целевая оценка: **5, трек1 — компрометация worker**.
|
||||
Дата локальных проверок:20.11.2026. Ветка: `solution/BVT2301-Mayorova`; основание — оригинальный lab7 skeleton20afc2c790d3e72f6b7666376c9834c1ced1a405.
|
||||
Публикация: [публичная ветка решения](https://gitea.privcloud.tech/mayorovasonya/mtusi2026lab7/src/branch/solution/BVT2301-Mayorova); МР направляется в upstream main с заголовком «ЛР 7 — БВТ2301 — Майорова — на 5». Tested commit записан в clean-clone.json; окончательный commit виден в истории ветки. Hosted CI отдельно не заявляется прошедшим.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Что реализовано
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Собственный Python-мессенджер из lab6 сохранён: пользователи/сессии, чаты, cursor history, идемпотентная отправка сообщений, приватные вложения с фоновым versioned pipeline, PG outbox, повторные доставки/retry/replay/fencing/cleanup, метрики/логи/traces и dashboard.
|
||||
Публичное API lab7 и tests/ не изменены. Заполнены предусмотренные заданием trust-matrix и threat model. README/COURSE/CONTRIBUTING новой работы сохранены.
|
||||
|
||||
## Архитектура и решения
|
||||
В lab7 добавлены проверяемый внешний HTTPS, обязательный native mTLS LB→обе API, TLS/mTLS всех межконтейнерных связей, отдельные identities/ACL, Argon2id, общий login limiter, persistent non-dev OpenBao, bounded secret cache, отзыв/ротация, hardening и отрицательные тесты. Исследование выполняет probe в настоящем worker и восстанавливает те же задания после отзыва.
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
## Чистый clone: операторская инструкция
|
||||
|
||||
Требуются Git, Python3.9+ на host, OpenSSL CLI и Docker Engine/Compose v2. Проверено на macOS/Colima, Linux aarch64 VM4CPU/4GiB; рядом работала собственная lab6. Это не выделенный benchmark host.
|
||||
Все runtime images закреплены digest в compose.yaml/Dockerfile. Фактически проверены Python3.12.16, HAProxy3.0.31, PostgreSQL17.12, Redis7.4.12, SeaweedFS4.51, OpenBao2.4.4, Collector0.132.0, Prometheus3.5.0, Loki3.5.3, Tempo2.8.2, Grafana12.1.1; зависимости Python frozen в requirements.txt (OTel1.45.2, Argon2-cffi25.1.0). Полный scan — security/SCAN_TRIAGE.md.
|
||||
|
||||
```sh
|
||||
git clone --branch solution/BVT2301-Mayorova https://gitea.privcloud.tech/mayorovasonya/mtusi2026lab7.git lab7
|
||||
cd lab7
|
||||
|
||||
# Только на этом Mac при использовании уже настроенного профиля:
|
||||
colima start --profile mtusi-labs
|
||||
export DOCKER_CONTEXT=colima-mtusi-labs
|
||||
|
||||
python3 scripts/start.py
|
||||
export BASE_URL=https://localhost:18443
|
||||
export CA_FILE="$PWD/pki/lb/ca.crt"
|
||||
curl --cacert "$CA_FILE" "$BASE_URL/health/ready"
|
||||
make check test test-security test-student BASE_URL="$BASE_URL" CA_FILE="$CA_FILE"
|
||||
make test-internal-security
|
||||
docker compose --profile tools run --rm --no-deps audit /app/student-tests/auth_and_signals.py
|
||||
```
|
||||
|
||||
`start.py` — явное действие доверенного host-оператора: создаёт свежие CA/leaf/secrets, собирает приложение, запускает stores/Bao, делает init/unseal, policies/KV, bootstrap bucket, SQL migrations, две API/worker/dispatcher/LB и ждёт verified HTTPS readiness. На повторном старте сохраняет keys/данные и не переинициализирует Bao.
|
||||
Первый start может занять минуты (скачивание images). Сам `docker compose up` не заменяет operator initialization.
|
||||
|
||||
Адреса только на loopback host: API **https://localhost:18443**, Grafana **https://localhost:13407**, Bao **https://localhost:18207**.
|
||||
CA публичный `pki/lb/ca.crt`; private keys не распространять. TLS verification не отключать. Для браузера отдельно требуется осознанное доверие локальному CA пользователем; скрипты не меняют системный trust store.
|
||||
Пароль Grafana operator находится в ignored local `operator/settings.json`; в чат/скриншоты/Git не переносить.
|
||||
|
||||
`.env.example` содержит только nonsecret настройки: APP_PORT/GRAFANA_PORT/BAO_PORT — host mappings, SESSION_TTL_SECONDS — user session expiry3600s, TRACE_SAMPLE_RATIO — parent-based25% traces. Внутренний APP_PORT8080 задаётся compose отдельно.
|
||||
Все DSN/пароли/ключи создаёт bootstrap и кладёт в scoped Bao KV; приложение получает их оттуда, а не из `.env`. TLS files монтируются по ролям. Прочие настройки зафиксированы compose/infra.
|
||||
|
||||
Дополнительные воспроизводимые проверки (меняют только текущий compose project):
|
||||
|
||||
```sh
|
||||
# 13 детерминированных pipeline групп требуют остановленных обычных consumers:
|
||||
docker compose stop worker dispatcher
|
||||
docker compose --profile tools run --rm --no-deps audit /app/student-tests/pipeline_checks.py
|
||||
docker compose up -d --no-deps worker dispatcher
|
||||
|
||||
# Три повтора исследования с настоящим отзывом worker:
|
||||
make research BASE_URL="$BASE_URL" CA_FILE="$CA_FILE"
|
||||
|
||||
# Fail-closed при sealed/TTL и восстановление нового job:
|
||||
python3 student-tests/sealed_bao.py
|
||||
|
||||
# Полный restart с сохранением business data и старой telemetry:
|
||||
python3 student-tests/persistence_tls.py
|
||||
|
||||
# Реальный stop ровно выбранного публичным тестом API:
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL="$BASE_URL" CA_FILE="$CA_FILE"
|
||||
docker compose up -d --no-deps api
|
||||
|
||||
python3 student-tests/hardening.py
|
||||
python3 student-tests/grafana_tls.py
|
||||
python3 student-tests/scan_images.py
|
||||
```
|
||||
|
||||
Для независимого повторения чистого развёртывания: `python3 student-tests/clean_clone.py` (нужны свободные18444/13408/18208; создаёт и удаляет только свой уникальный disposable project).
|
||||
Проверенный commit 7d7bd69d8af959bda540b1bc203c9d62e56c3566, длительность 72.354s; после него изменены только операторская уборка временного clone, сохранение denylist при renewal и документация/evidence. API/worker runtime в чистом clone соответствует итоговому решению.
|
||||
|
||||
Сначала `auth_and_signals.py` создаёт actual trace для persistence_tls; если запускаете вручную, сохраните его JSON stdout в `evidence/auth-signals.json` (Compose progress идёт в stderr).
|
||||
Тестовые пользователи/сообщения/файлы синтетические. Исследование отзывает старый leaf/password worker и оставляет новый рабочий набор.
|
||||
|
||||
Безопасная остановка: `docker compose stop`; возобновление с явным unseal: `python3 scripts/start.py`.
|
||||
`docker compose down` удаляет контейнеры/сеть, **сохраняет volumes**.
|
||||
Только для намеренного полного удаления учебных данных собственного текущего проекта: `docker compose down -v --remove-orphans`; после этого для новой установки нужны также новые local generated secrets/PKI. Эта команда не используется recovery/persistence hook.
|
||||
|
||||
## Потоки и границы доверия
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Client -->|HTTPS CA+SAN| LB
|
||||
LB -->|mTLS exact URI lb| API[API x2]
|
||||
API -->|TLS separate role| PG[(PostgreSQL)]
|
||||
API -->|mTLS prefix ACL| S3[(S3 gateway)]
|
||||
API -->|mTLS optional cache| Redis[(Redis profiles)]
|
||||
PG --> Outbox[durable outbox]
|
||||
Dispatcher -->|TLS outbox| PG
|
||||
Dispatcher -->|mTLS producer ACL| Broker[(Redis Streams)]
|
||||
Broker --> Worker
|
||||
Worker -->|TLS column grants| PG
|
||||
Worker -->|mTLS staging read/results create| S3
|
||||
API --> Bao[OpenBao cert-auth / own KV]
|
||||
Worker --> Bao
|
||||
Dispatcher --> Bao
|
||||
API --> Collector[OTel mTLS]
|
||||
Worker --> Collector
|
||||
Dispatcher --> Collector
|
||||
Collector -->|mTLS| Backends[Tempo/Loki]
|
||||
Prometheus -->|mTLS scrape| Collector
|
||||
Grafana -->|mTLS| Backends
|
||||
Grafana -->|mTLS| Prometheus
|
||||
```
|
||||
|
||||
PG — источник истины для users/chats/members/messages, SHA256 session hashes, attachment/job states, outbox/idempotency и failure registry. S3 — bytes staging/results; Redis Streams — durable at-least-once delivery, профильный cache необязателен.
|
||||
PG COMMIT принятого job предшествует publish; worker claim/lease/version защищают UPDATE состояния, результаты создаются условным immutable PUT. Повторная доставка не создаёт второй бизнес-эффект. Сессии и история не в Redis.
|
||||
|
||||
Полная [trust matrix](contracts/trust-matrix.md) содержит endpoints, identities, TLS и права. HAProxy проверяет CA и CN role allowlist+serial denylist; native API дополнительно exact URI SAN `spiffe://mtusi.lab/lb`.
|
||||
CA trust сам по себе не даёт права роли. Test wrong-lb использует trusted cert с разрешённым CNlb, но неправильным URI, и отклоняется API.
|
||||
API→broker отсутствует по архитектуре: API пишет PG outbox, broker имеет только dispatcher/worker. Сертификат API на broker явно отвергается.
|
||||
|
||||
Где продукт без native TLS, proxy и raw process разделяют только собственный network namespace; raw ports bound127.0.0.1. Это проверено из другого контейнера и по реальным listening sockets, включая S3 HTTP/admin/gRPC. Это честно ограниченный локальный hop, не plaintext в общей backend сети.
|
||||
|
||||
## Пароли, сессии и login budget
|
||||
|
||||
Argon2id: memory32768KiB, t2, p1, random salt16bytes, hash32bytes. Encoded string содержит параметры/salt. Успешный вход со старым собственным scrypt или прежними Argon параметрами выполняет rehash.
|
||||
Bearer — случайные256bit; PG хранит только SHA256 token_hash+expiry. Logout удаляет сессию; проверка всегда в PG, даже при profile cache hit.
|
||||
Общий для двух API atomic PG fixed-window limiter:20 login/s, отдельные20 registration/s;429 с Retry-After1. По2 KDF slots/API; при нехватке bounded wait0.5s→503. Максимум KDF RAM64MiB на API при container limit256MiB.
|
||||
|
||||
KDF benchmark:10 повторов hash+verify на каждую memory19/32/64MiB в audit container0.5CPU (API имеет1CPU). Actual timings и 100 параллельных bad logins на обе реплики — [auth-signals.json](evidence/auth-signals.json). Это измерение KDF, не прогноз максимального API RPS.
|
||||
Выбраны32MiB как запас выше минимального профиля задания при bounded concurrency;64MiB при2 slots оставляли бы меньше RAM под Python/pool/TLS. Rate limit fixed-window допускает boundary burst до40/s; полной защиты от распределённого DoS/TLS/DB saturation не заявляем.
|
||||
|
||||
## OpenBao bootstrap и lifecycle
|
||||
|
||||
Non-dev persistent file backend. Operator init3shares/threshold2, затем explicit unseal. Материалы CA/root/shares — только host operator0700/0600, не mounted в apps и даже audit container. Все shares у одного учебного оператора, split custody/HSM нет.
|
||||
Generated service leaf на1сутки — первый bootstrap credential; CA30суток. Exact leaf cert-auth закреплён в Bao; policies только read собственного `secret/data/api|worker|dispatcher`. Запрещены чужой KV, policy read/write и secret write.
|
||||
Token TTL/maxTTL20s, приложение при использовании re-auth каждые5s; bounded cache≤20s. Root token и бессрочный общий bootstrap token приложению не нужны.
|
||||
|
||||
**KV passwords/keys статические**: TTL Bao token не равен автоматическому expiry украденного PG/S3 password. Для инцидента оператор отдельно меняет пароли/ключи, отзывает tokens+serial, убивает старые PG sessions и old proxy connections.
|
||||
`scripts/rotate_worker.py` делает именно это без rebuild. Proxy graceful reload имеет hard-stop≤5s; negative old-leaf проверки идут на новых соединениях через6s.
|
||||
Другим ролям renewal leaf требует operator issuance/update pins/reload в окне обслуживания. Автоматический CA rollover/весь трек2 не заявлены.
|
||||
|
||||
Реальное sealed испытание: bounded cache заканчивается, business endpoint503; после unseal прежний Bearer действует, worker/dispatcher остаются теми же живыми контейнерами и новый job становится ready. Старый service token после20s даёт403, повторный cert-auth работает. Raw auth values не сохраняются.
|
||||
Сохраняется HMAC audit OpenBao в volume; log_raw=false.
|
||||
|
||||
## Hardening и права worker
|
||||
|
||||
Apps/worker/dispatcher UID10001, readonly rootfs, cap_drop ALL, no-new-privileges, tmpfs128MiB, no privileged/host network/Docker socket. Worker1CPU/768MiB, renderer20s/40MP, upload≤10MiB. API×2 по1CPU/256MiB.
|
||||
PG нуждается в entrypoint root caps CHOWN/SETUID/SETGID/DAC_OVERRIDE/FOWNER и writable state, затем работает postgres user. One-shot volume-init имеет CHOWN и network none. Это явные исключения; не выдаются worker.
|
||||
Operator-only audit profile user0 с DAC_OVERRIDE читает выделенные credential fixtures для тестов; не является production worker и не монтирует CAprivate/root/unseal. Другие stateful данные — named volumes.
|
||||
Фактические limits/mounts/ports/listeners и wrong-hostname reject: [hardening.json](evidence/hardening.json).
|
||||
|
||||
Worker может читать attachment/job metadata и staging/results; создавать новые immutable result keys; изменять только разрешённые state/lease/retry columns и писать job_events. Не может читать users/sessions/messages, менять chat_id/source_key, создавать persistent/temporary DDL, делать broker CONFIG/чужие keys, LIST/DELETE bucket или PUTstaging.
|
||||
Он всё равно может испортить разрешённые состояния/свою очередь и прочитать все processing files; tenant isolation и защита от произвольного RCE в PG/kernel не заявлены. Эти границы отражены в [threat model](security/THREAT_MODEL.md).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Реальная команда/наблюдение | Evidence |
|
||||
| --- | --- | --- |
|
||||
| Предыдущий business API через HTTPS, CA/SAN | make check/test/test-security;18 собственных API/image tests PASS | [final-api-tests.txt](evidence/final-api-tests.txt) |
|
||||
| Все прежние durable pipeline свойства | stop worker/dispatcher; audit pipeline_checks.py;13 групп, реальные exit77 до/после side effects/commit | [pipeline-regression.json](evidence/pipeline-regression.json) |
|
||||
| LB→каждая API обязательный mTLS+identity | tls_audit.py: оба DNS IP, no cert/foreign CA/trusted outsider/wrong URI/expired deny | [tls-audit.json](evidence/tls-audit.json) |
|
||||
| Полная матрица TLS/mTLS, raw bypass denied |14 role/endpoint путей,15 plaintext ports недоступны из чужого namespace; PG verify-full | [tls-audit.json](evidence/tls-audit.json), [матрица](contracts/trust-matrix.md) |
|
||||
| Broker/Collector отдельные identities | valid outsider denied, API broker denied; реальный publisher→consumer trace | [tls-audit.json](evidence/tls-audit.json), [auth-signals.json](evidence/auth-signals.json) |
|
||||
| Argon2id, session hashes, expiry/logout, rehash | DB stored hashes, реальная expiry401; legacy/params rehash;9+image/business scenarios | [auth-signals.json](evidence/auth-signals.json),18tests |
|
||||
| Общий limiter и KDF budget |100 запросов,2 instance IDs,429Retry-After1;3 memory profiles×10 | [auth-signals.json](evidence/auth-signals.json) |
|
||||
| Bao non-dev/scoped auth/persistent/operator trust | own KV allowed, чужой KV/policy/write denied; explicit unseal same store | [tls-audit.json](evidence/tls-audit.json),[tls-persistence.json](evidence/tls-persistence.json) |
|
||||
| Secret/leaf rotate+revoke без rebuild | старый реально валидный token403 до TTL, old cert/password/key denied; image IDs неизменны | [worker-compromise-runs.json](evidence/worker-compromise-runs.json) |
|
||||
| Expiry/sealed fail-closed |≤20s cache,503,тот же user Bearer, живые processes/new ready job,expired token403 | [sealed-and-expiry.json](evidence/sealed-and-expiry.json) |
|
||||
| Least privilege/negative ACL/upload | actual non-root worker probe; PG/Redis/S3 deny;10MiB/40MP/20s limits | [worker-compromise-runs.json](evidence/worker-compromise-runs.json),[auth-signals.json](evidence/auth-signals.json) |
|
||||
| Hardening/no secret leaks | inspect effective settings, real sockets, telemetry canaries и repo actual-value scan | [hardening.json](evidence/hardening.json),[runtime-surface.json](evidence/runtime-surface.json) |
|
||||
| Scan + применимость | все11 unique images, реальные fixes, residual risks без blanket ignore | [SCAN_TRIAGE.md](security/SCAN_TRIAGE.md),[image-scans.json](evidence/image-scans.json) |
|
||||
| HTTPS restart с business и telemetry persistence | PG/Redis/S3/Bao/всеtelemetry/Grafana реально restart; старые session/files/trace/logs/fixed metric сохранены | [persistence.txt](evidence/persistence.txt),[tls-persistence.json](evidence/tls-persistence.json),restart snapshots |
|
||||
| Отказ последовательно обеих физических API | unchanged public failover + exact TARGET_INSTANCE_ID hook, exit0 stopped, вторая живая | [failover-1.txt](evidence/failover-1.txt),[failover-other.txt](evidence/failover-other.txt),[stopped-instances.jsonl](evidence/stopped-instances.jsonl) |
|
||||
| Минимум lab6 observability |21 panel dashboard,3 mTLS datasource,2 scrape up,20-span real job trace + logs | [grafana-tls.json](evidence/grafana-tls.json),[auth-signals.json](evidence/auth-signals.json),[RUNBOOK](infra/RUNBOOK.md) |
|
||||
| Трек1 на5,3 повтора | реальные worker права → отзыв/смена leaf+3secrets → те же6 jobs ready/bytes и все подтверждённые сообщения | [worker-compromise-runs.json](evidence/worker-compromise-runs.json) |
|
||||
| Чистый clone с новыми volumes/CA/secrets | независимый compose project/ports, запуск и повтор API/TLS/persistence | PASS: [clean-clone.json](evidence/clean-clone.json),[вывод](evidence/clean-clone.txt) |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Hosted CI не подменяется локальными evidence: GitHub-compatible application workflow добавлен; запуск/approval в Gitea до публикации не подтверждён.
|
||||
|
||||
## Эксперименты и отказы
|
||||
## Исследование на5: компрометация worker
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза: service-specific least privilege уменьшает scope украденных worker credentials, а явный отзыв/ротация прекращает старый доступ без потери уже принятых durable jobs.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Методика `student-tests/research.py`:3 независимых синтетических fixture runs на том же стенде; каждый создаёт новых users/chat/6PNG jobs и новый revoked/current key set. Это не3 разных VM и не статистически мощный performance benchmark.
|
||||
Сначала broad baseline — **явно помеченная operator-controlled модель lab6-style shared credentials**, а не атака живой lab6: видны counts users/sessions/messages, bucket-wide write/delete временного fixture и brokerCONFIG. Значения hashes/contents не выводятся.
|
||||
Затем probe выполняется через docker exec **в настоящем worker UID/mounts**, подтверждает own KV, отсутствие operator files и границы DB/broker.
|
||||
Worker останавливается; API принимает6 вложений в durable state. Один HTTP writer отправляет сообщение и ждёт100ms после ответа (closed loop, nominal ceiling10/s). Оператор отзывает старый Bao cert role/tokens+serial, ротирует PG/Redis/S3 secret и leaf; жёстко завершает old connections и запускает worker тем же образом. Старые credentials отдельно проверяются fresh connections; все те же6IDs ready, исходные bytes и confirmed messages находятся в истории.
|
||||
CPU/RAM — отдельные docker stats snapshots до/после, **не пики**; record содержит caps/units. Latency percentiles относятся только к успешным message requests, не к job-ready и не к времени отзыва.
|
||||
Результаты финальных3 повторов и измерения приведены ниже в автоматически подставленной таблице.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
<!-- ACTUAL_RESULTS_START -->
|
||||
| Повтор | Сообщения / ошибки | Успешные RPS | p50 / p95 / p99, ms | Revoke token, ms | Ротация, s | Jobs ready |
|
||||
| --- | --- | ---: | --- | ---: | ---: | --- |
|
||||
| 1 | 100 / 0 | 8.332 | 13.07 / 24.29 / 54.43 | 130 | 3.107 | 6/6 |
|
||||
| 2 | 101 / 0 | 8.317 | 13.33 / 25.91 / 51.25 | 150 | 2.828 | 6/6 |
|
||||
| 3 | 100 / 0 | 8.383 | 12.39 / 25.63 / 38.78 | 195 | 2.786 | 6/6 |
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Все 301 подтверждённое сообщение найдено, все 18 принятых вложений стали ready с исходными bytes, old token/cert/PG password/Redis password/S3 key отвергнуты; образ worker не менялся. Нулевая наблюдаемая потеря в этих3 коротких прогонах не доказывает устойчивость при произвольной нагрузке.
|
||||
|
||||
Docker stats before: CPU 0.08–5.81% одного ядра по отдельным процессам; memory по каждому контейнеру и точные units — в JSON. Это отдельные snapshots, не max CPU/RAM за эксперимент.
|
||||
|
||||
Docker stats after: CPU 0.60–3.41% одного ядра по отдельным процессам; memory по каждому контейнеру и точные units — в JSON. Это отдельные snapshots, не max CPU/RAM за эксперимент.
|
||||
|
||||
KDF hash+verify median,19/32/64MiB: 28.54ms / 111.36ms / 275.98ms.
|
||||
100 параллельных bad logins: 30×401, 70×429, обе API; Retry-After=1.
|
||||
Sealed: первый503 через19.362s, прежний Bearer снова200 через25.489s от seal, после явного unseal.
|
||||
<!-- ACTUAL_RESULTS_END -->
|
||||
|
||||
PG/S3/Broker остаются доступны и durable; отзыв учётных данных не удаляет jobs. Относительно baseline worker больше не читает accounts/sessions/messages и не управляет чужими keyspaces/входными файлами. Он по-прежнему может портить разрешённый processing scope: malicious-worker result correctness отдельно не доказана.
|
||||
|
||||
## Реальные отказы и границы измерения
|
||||
|
||||
- Полный restart: scopes/actual container IDs/start times до/после в restart-before/after; proxy namespace anchors сохранялись, raw бизнес/telemetry processes и Bao перезапущены. Bao unseal — operator action, без сброса store.
|
||||
- Отказ API:6 успешных публичных сценариев; round-robin initial prefix часто выбирал одну и ту же физическую реплику. Дополнительный обычный GET сменил фазу, после чего неизменённый public test остановил вторую. TARGET_INSTANCE_ID не подменялся. Выбранные2 physical IDs/alias записаны hook; confirmed data/sessions сохранились. В отдельных контрольных сценариях112/110 writes, по1 transient failure, первое восстановление2.682/2.331s.
|
||||
- Чистый first-start выявил гонку S3: head_bucket отвечал раньше готовности data volumes. Bootstrap исправлен на authenticated PUT→GET→DELETE уникального временного объекта; затем чистый start проверен заново.
|
||||
- Один cold bootstrap непосредственно после полного restart превысил2s timeout публичного клиента ещё до failover hook. Прогретые неизменённые сценарии прошли. Поэтому гарантии «все cold requests≤2s» нет.
|
||||
- Sealed Bao обнаружил и помог исправить exit dispatcher из queue observer: APIError теперь обрабатывается как временная недоступность. Повтор проверяет не только200API, но те же живые worker/dispatcher и новый ready job.
|
||||
- При outage Collector bounded SDK queues могут потерять telemetry; бизнес commit от них не зависит. Хранение history24h/limits и SLO определения — RUNBOOK. Системный availability за пределами API (LB/network) не полностью покрывается API request metric.
|
||||
|
||||
## Ограничения и источники
|
||||
|
||||
Стенд single-host: LB/PG/S3/broker/Bao/telemetry — SPOF; нет HA/database PITR/auto-unseal/HSM/independent key custodians. Leaf renewal и CA rollout для всех ролей не автоматизированы. Трек2 не заявляется.
|
||||
Auth TTL не динамическая DB lease. Отзыв проверяет новые соединения; persistent connections ограничены5s, уже начатый renderer20s. IP/tenant quotas и malware scanning нет. Полный scope RCE в image/DB/kernel исключён из эксперимента и остаётся риском.
|
||||
Scan сохранил известные Critical/High findings в сторонних образах; изоляция не объявляется исправлением. До production нужны совместимые обновления/миграция Bao, исправленная PG/libxml, quotas, backups/restore и повторный security review.
|
||||
|
||||
Реализация продолжает только собственные lab1–6. Использован Codex для написания кода/конфигураций/проверок и оформления; результаты не выдуманы, evidence получены реальными командами. Чужие студенческие решения не использовались.
|
||||
Первичные источники: [задание](README.md), [контракт](contracts/README.md), [OWASP Password Storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), [Argon2-cffi API/rehash](https://argon2-cffi.readthedocs.io/en/stable/api.html), [OpenBao cert-auth](https://openbao.org/docs/auth/cert/), [OpenBao policies](https://openbao.org/docs/concepts/policies/), [HAProxy3.0 configuration](https://docs.haproxy.org/3.0/configuration.html), [Docker security](https://docs.docker.com/engine/security/).
|
||||
|
||||
+951
@@ -0,0 +1,951 @@
|
||||
{
|
||||
"name": "mtusi-lab7",
|
||||
"services": {
|
||||
"s3": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/s3.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/s3:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"broker": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/broker.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/broker:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"redis": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/redis.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/redis:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"otel-collector": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/otel-collector.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/otel-collector:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"tempo": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/tempo.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/tempo:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"loki": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/loki.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/loki:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"prometheus": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tls/prometheus.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/prometheus:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
]
|
||||
},
|
||||
"postgres": {
|
||||
"image": "postgres:17-bookworm@sha256:837bed18782820fe967067cb5328b02b0d9992c1be9a37ee0c9c492cfc03af3b",
|
||||
"mem_limit": "512m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": false,
|
||||
"cap_add": [
|
||||
"CHOWN",
|
||||
"SETUID",
|
||||
"SETGID",
|
||||
"DAC_OVERRIDE",
|
||||
"FOWNER"
|
||||
],
|
||||
"environment": {
|
||||
"POSTGRES_USER": "messenger",
|
||||
"POSTGRES_DB": "messenger",
|
||||
"POSTGRES_PASSWORD_FILE": "/run/password"
|
||||
},
|
||||
"volumes": [
|
||||
"postgres_data:/var/lib/postgresql/data",
|
||||
"./pki/postgres:/tls:ro",
|
||||
"./secrets/postgres_password:/run/password:ro",
|
||||
"./secrets/pg-init.sql:/docker-entrypoint-initdb.d/01.sql:ro",
|
||||
"./infra/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro",
|
||||
"./infra/postgres-entrypoint.sh:/start.sh:ro"
|
||||
],
|
||||
"entrypoint": [
|
||||
"sh",
|
||||
"/start.sh"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"healthcheck": {
|
||||
"test": [
|
||||
"CMD",
|
||||
"pg_isready",
|
||||
"-h",
|
||||
"127.0.0.1",
|
||||
"-U",
|
||||
"messenger",
|
||||
"-d",
|
||||
"messenger"
|
||||
],
|
||||
"interval": "2s",
|
||||
"timeout": "1s",
|
||||
"retries": 60
|
||||
}
|
||||
},
|
||||
"broker-raw": {
|
||||
"image": "redis:7.4-bookworm@sha256:2e3eb11462c4e3759e6972678dd051d150ff54b1e60714b3c33b701260172a90",
|
||||
"mem_limit": "192m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "999:999",
|
||||
"command": [
|
||||
"redis-server",
|
||||
"/etc/redis.conf"
|
||||
],
|
||||
"network_mode": "service:broker",
|
||||
"depends_on": [
|
||||
"broker"
|
||||
],
|
||||
"volumes": [
|
||||
"./secrets/broker.conf:/etc/redis.conf:ro",
|
||||
"broker_data:/data"
|
||||
]
|
||||
},
|
||||
"redis-raw": {
|
||||
"image": "redis:7.4-bookworm@sha256:2e3eb11462c4e3759e6972678dd051d150ff54b1e60714b3c33b701260172a90",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "999:999",
|
||||
"command": [
|
||||
"redis-server",
|
||||
"/etc/redis.conf"
|
||||
],
|
||||
"network_mode": "service:redis",
|
||||
"depends_on": [
|
||||
"redis"
|
||||
],
|
||||
"volumes": [
|
||||
"./secrets/redis.conf:/etc/redis.conf:ro"
|
||||
]
|
||||
},
|
||||
"s3-store": {
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"mem_limit": "512m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=64m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"mini",
|
||||
"-dir=/data",
|
||||
"-ip=127.0.0.1",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-s3=false",
|
||||
"-admin.ui=false",
|
||||
"-webdav=false",
|
||||
"-master.telemetry=false",
|
||||
"-master.volumeSizeLimitMB=128",
|
||||
"-volume.max=16",
|
||||
"-volume.fileSizeLimitMB=11",
|
||||
"-volume.concurrentUploadLimitMB=64",
|
||||
"-volume.concurrentDownloadLimitMB=64"
|
||||
],
|
||||
"network_mode": "service:s3",
|
||||
"depends_on": [
|
||||
"s3",
|
||||
"volume-init"
|
||||
],
|
||||
"volumes": [
|
||||
"s3_data:/data"
|
||||
]
|
||||
},
|
||||
"s3-raw": {
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"mem_limit": "384m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"s3",
|
||||
"-filer=127.0.0.1:8888",
|
||||
"-ip.bind=127.0.0.1",
|
||||
"-port=8333",
|
||||
"-config=/run/s3_config",
|
||||
"-iam=false",
|
||||
"-port.iceberg=0",
|
||||
"-port.lance=0",
|
||||
"-concurrentFileUploadLimit=16",
|
||||
"-concurrentUploadLimitMB=32",
|
||||
"-readerCacheSizeMB=16"
|
||||
],
|
||||
"network_mode": "service:s3",
|
||||
"depends_on": [
|
||||
"s3-store"
|
||||
],
|
||||
"volumes": [
|
||||
"./secrets/s3_config:/run/s3_config:ro"
|
||||
]
|
||||
},
|
||||
"bao": {
|
||||
"image": "openbao/openbao:2.4.4@sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "100:1000",
|
||||
"entrypoint": [
|
||||
"bao"
|
||||
],
|
||||
"command": [
|
||||
"server",
|
||||
"-config=/etc/bao.hcl"
|
||||
],
|
||||
"depends_on": [
|
||||
"volume-init"
|
||||
],
|
||||
"networks": [
|
||||
"backend",
|
||||
"edge"
|
||||
],
|
||||
"ports": [
|
||||
"127.0.0.1:${BAO_PORT:-18207}:8200"
|
||||
],
|
||||
"volumes": [
|
||||
"bao_data:/bao/data",
|
||||
"./infra/openbao/config.hcl:/etc/bao.hcl:ro",
|
||||
"./pki/bao:/tls:ro"
|
||||
]
|
||||
},
|
||||
"volume-init": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "0:0",
|
||||
"cap_add": [
|
||||
"CHOWN"
|
||||
],
|
||||
"network_mode": "none",
|
||||
"entrypoint": [
|
||||
"python",
|
||||
"-c",
|
||||
"import os\nfor base,uid,gid in [('/s3',10001,10001),('/bao',100,1000)]:\n os.chown(base,uid,gid)\n for root,dirs,files in os.walk(base):\n for name in dirs+files:os.chown(os.path.join(root,name),uid,gid)\n"
|
||||
],
|
||||
"volumes": [
|
||||
"s3_data:/s3",
|
||||
"bao_data:/bao"
|
||||
]
|
||||
},
|
||||
"api": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 1,
|
||||
"tmpfs": [
|
||||
"/tmp:size=128m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"python",
|
||||
"src/server.py"
|
||||
],
|
||||
"environment": {
|
||||
"SESSION_TTL_SECONDS": "${SESSION_TTL_SECONDS:-3600}",
|
||||
"HTTP_WORKERS": "16",
|
||||
"DB_POOL_MAX": "8",
|
||||
"DB_ACCESS_MODE": "optimized",
|
||||
"S3_ENDPOINT": "https://s3:8443",
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:8443",
|
||||
"TRACE_SAMPLE_RATIO": "${TRACE_SAMPLE_RATIO:-0.25}",
|
||||
"JOB_LEASE_SECONDS": "10",
|
||||
"JOB_TIMEOUT_SECONDS": "20",
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"LOG_LEVEL": "WARNING",
|
||||
"BAO_ROLE": "api"
|
||||
},
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"volumes": [
|
||||
"./pki/api:/tls:ro"
|
||||
],
|
||||
"depends_on": {
|
||||
"migrate": {
|
||||
"condition": "service_completed_successfully"
|
||||
},
|
||||
"bucket-bootstrap": {
|
||||
"condition": "service_completed_successfully"
|
||||
}
|
||||
},
|
||||
"build": ".",
|
||||
"deploy": {
|
||||
"replicas": 2
|
||||
}
|
||||
},
|
||||
"worker": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "768m",
|
||||
"cpus": 1,
|
||||
"tmpfs": [
|
||||
"/tmp:size=128m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"python",
|
||||
"src/pipeline.py",
|
||||
"work"
|
||||
],
|
||||
"environment": {
|
||||
"SESSION_TTL_SECONDS": "${SESSION_TTL_SECONDS:-3600}",
|
||||
"HTTP_WORKERS": "16",
|
||||
"DB_POOL_MAX": "8",
|
||||
"DB_ACCESS_MODE": "optimized",
|
||||
"S3_ENDPOINT": "https://s3:8443",
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:8443",
|
||||
"TRACE_SAMPLE_RATIO": "${TRACE_SAMPLE_RATIO:-0.25}",
|
||||
"JOB_LEASE_SECONDS": "10",
|
||||
"JOB_TIMEOUT_SECONDS": "20",
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"LOG_LEVEL": "WARNING",
|
||||
"BAO_ROLE": "worker"
|
||||
},
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"volumes": [
|
||||
"./pki/worker:/tls:ro"
|
||||
],
|
||||
"depends_on": {
|
||||
"migrate": {
|
||||
"condition": "service_completed_successfully"
|
||||
},
|
||||
"bucket-bootstrap": {
|
||||
"condition": "service_completed_successfully"
|
||||
}
|
||||
}
|
||||
},
|
||||
"dispatcher": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 1,
|
||||
"tmpfs": [
|
||||
"/tmp:size=128m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"python",
|
||||
"src/pipeline.py",
|
||||
"dispatch"
|
||||
],
|
||||
"environment": {
|
||||
"SESSION_TTL_SECONDS": "${SESSION_TTL_SECONDS:-3600}",
|
||||
"HTTP_WORKERS": "16",
|
||||
"DB_POOL_MAX": "8",
|
||||
"DB_ACCESS_MODE": "optimized",
|
||||
"S3_ENDPOINT": "https://s3:8443",
|
||||
"S3_BUCKET": "messenger-attachments",
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT": "https://otel-collector:8443",
|
||||
"TRACE_SAMPLE_RATIO": "${TRACE_SAMPLE_RATIO:-0.25}",
|
||||
"JOB_LEASE_SECONDS": "10",
|
||||
"JOB_TIMEOUT_SECONDS": "20",
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"LOG_LEVEL": "WARNING",
|
||||
"BAO_ROLE": "dispatcher"
|
||||
},
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"volumes": [
|
||||
"./pki/dispatcher:/tls:ro"
|
||||
],
|
||||
"depends_on": {
|
||||
"migrate": {
|
||||
"condition": "service_completed_successfully"
|
||||
},
|
||||
"bucket-bootstrap": {
|
||||
"condition": "service_completed_successfully"
|
||||
}
|
||||
}
|
||||
},
|
||||
"migrate": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "128m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"python",
|
||||
"src/migrate.py"
|
||||
],
|
||||
"environment": {
|
||||
"DATABASE_URL_FILE": "/run/dsn"
|
||||
},
|
||||
"volumes": [
|
||||
"./secrets/migration_database_url:/run/dsn:ro",
|
||||
"./pki/operator:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"depends_on": {
|
||||
"postgres": {
|
||||
"condition": "service_healthy"
|
||||
}
|
||||
}
|
||||
},
|
||||
"bucket-bootstrap": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "128m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"python",
|
||||
"src/bucket_bootstrap.py"
|
||||
],
|
||||
"environment": {
|
||||
"S3_ENDPOINT": "https://s3:8443",
|
||||
"S3_ACCESS_KEY_ID_FILE": "/run/s3_key",
|
||||
"S3_SECRET_ACCESS_KEY_FILE": "/run/s3_secret"
|
||||
},
|
||||
"volumes": [
|
||||
"./pki/operator:/tls:ro",
|
||||
"./secrets/s3_admin_key:/run/s3_key:ro",
|
||||
"./secrets/s3_admin_secret:/run/s3_secret:ro"
|
||||
],
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"depends_on": [
|
||||
"s3-raw"
|
||||
]
|
||||
},
|
||||
"lb": {
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"mem_limit": "64m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "10001:10001",
|
||||
"command": [
|
||||
"haproxy",
|
||||
"-W",
|
||||
"-db",
|
||||
"-f",
|
||||
"/etc/haproxy.cfg"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/haproxy.cfg:/etc/haproxy.cfg:ro",
|
||||
"./pki/lb:/tls:ro"
|
||||
],
|
||||
"networks": [
|
||||
"edge",
|
||||
"backend"
|
||||
],
|
||||
"ports": [
|
||||
"127.0.0.1:${APP_PORT:-18443}:8443"
|
||||
],
|
||||
"depends_on": [
|
||||
"api"
|
||||
]
|
||||
},
|
||||
"otel-collector-raw": {
|
||||
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"command": [
|
||||
"--config=/etc/otel.yaml"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/otel-collector.yaml:/etc/otel.yaml:ro",
|
||||
"./pki/collector:/client:ro"
|
||||
],
|
||||
"network_mode": "service:otel-collector",
|
||||
"depends_on": [
|
||||
"otel-collector"
|
||||
]
|
||||
},
|
||||
"prometheus-raw": {
|
||||
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"command": [
|
||||
"--config.file=/etc/prometheus.yaml",
|
||||
"--web.listen-address=127.0.0.1:9090",
|
||||
"--storage.tsdb.path=/prometheus",
|
||||
"--storage.tsdb.retention.time=24h",
|
||||
"--storage.tsdb.retention.size=256MB"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/prometheus.yaml:/etc/prometheus.yaml:ro",
|
||||
"./infra/alerts.yaml:/etc/prometheus/alerts.yaml:ro",
|
||||
"./pki/prometheus:/tls:ro",
|
||||
"metrics_data:/prometheus"
|
||||
],
|
||||
"network_mode": "service:prometheus",
|
||||
"depends_on": [
|
||||
"prometheus"
|
||||
]
|
||||
},
|
||||
"loki-raw": {
|
||||
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"command": [
|
||||
"-config.file=/etc/loki.yaml"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/loki.yaml:/etc/loki.yaml:ro",
|
||||
"logs_data:/loki"
|
||||
],
|
||||
"network_mode": "service:loki",
|
||||
"depends_on": [
|
||||
"loki"
|
||||
]
|
||||
},
|
||||
"tempo-raw": {
|
||||
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
|
||||
"mem_limit": "384m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"command": [
|
||||
"-config.file=/etc/tempo.yaml"
|
||||
],
|
||||
"volumes": [
|
||||
"./infra/tempo.yaml:/etc/tempo.yaml:ro",
|
||||
"traces_data:/var/tempo"
|
||||
],
|
||||
"network_mode": "service:tempo",
|
||||
"depends_on": [
|
||||
"tempo"
|
||||
]
|
||||
},
|
||||
"grafana": {
|
||||
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
|
||||
"mem_limit": "256m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=32m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"environment": {
|
||||
"GF_SERVER_PROTOCOL": "https",
|
||||
"GF_SERVER_CERT_FILE": "/tls/tls.crt",
|
||||
"GF_SERVER_CERT_KEY": "/tls/tls.key",
|
||||
"GF_SECURITY_ADMIN_USER": "operator",
|
||||
"GF_SECURITY_ADMIN_PASSWORD__FILE": "/run/password",
|
||||
"GF_AUTH_ANONYMOUS_ENABLED": "false",
|
||||
"GF_USERS_ALLOW_SIGN_UP": "false",
|
||||
"GF_ANALYTICS_REPORTING_ENABLED": "false",
|
||||
"GF_ANALYTICS_CHECK_FOR_UPDATES": "false",
|
||||
"GF_SERVER_ROOT_URL": "https://localhost:${GRAFANA_PORT:-13407}"
|
||||
},
|
||||
"networks": [
|
||||
"backend",
|
||||
"edge"
|
||||
],
|
||||
"ports": [
|
||||
"127.0.0.1:${GRAFANA_PORT:-13407}:3000"
|
||||
],
|
||||
"volumes": [
|
||||
"./pki/grafana:/tls:ro",
|
||||
"./secrets/grafana_admin:/run/password:ro",
|
||||
"./secrets/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/lab7.yaml:ro",
|
||||
"./infra/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro",
|
||||
"./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro",
|
||||
"grafana_data:/var/lib/grafana"
|
||||
]
|
||||
},
|
||||
"audit": {
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"mem_limit": "768m",
|
||||
"cpus": 0.5,
|
||||
"tmpfs": [
|
||||
"/tmp:size=128m,mode=1777"
|
||||
],
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"security_opt": [
|
||||
"no-new-privileges:true"
|
||||
],
|
||||
"read_only": true,
|
||||
"user": "0:0",
|
||||
"profiles": [
|
||||
"tools"
|
||||
],
|
||||
"entrypoint": [
|
||||
"python"
|
||||
],
|
||||
"environment": {
|
||||
"BASE_URL": "https://lb:8443",
|
||||
"CA_FILE": "/tls/ca.crt",
|
||||
"DATABASE_URL_FILE": "/operator/database_url",
|
||||
"S3_ENDPOINT": "https://s3:8443",
|
||||
"S3_ACCESS_KEY_ID_FILE": "/run/s3_key",
|
||||
"S3_SECRET_ACCESS_KEY_FILE": "/run/s3_secret",
|
||||
"BROKER_URL_FILE": "/operator/broker_url",
|
||||
"TELEMETRY_ENABLED": "0",
|
||||
"JOB_LEASE_SECONDS": "5"
|
||||
},
|
||||
"networks": [
|
||||
"backend"
|
||||
],
|
||||
"volumes": [
|
||||
"./pki/operator:/tls:ro",
|
||||
"./tests:/app/tests:ro",
|
||||
"./student-tests:/app/student-tests:ro",
|
||||
"./contracts:/app/contracts:ro",
|
||||
"./lab.json:/app/lab.json:ro",
|
||||
"./pki:/identities:ro",
|
||||
"./secrets/s3_admin_key:/run/s3_key:ro",
|
||||
"./secrets/s3_admin_secret:/run/s3_secret:ro",
|
||||
"./operator/settings.json:/operator/settings.json:ro",
|
||||
"./operator/database_url:/operator/database_url:ro",
|
||||
"./operator/broker_url:/operator/broker_url:ro",
|
||||
"./operator/revoked-worker-settings.json:/operator/revoked-worker-settings.json:ro"
|
||||
],
|
||||
"cap_add": [
|
||||
"DAC_OVERRIDE"
|
||||
]
|
||||
}
|
||||
},
|
||||
"networks": {
|
||||
"backend": {
|
||||
"internal": true
|
||||
},
|
||||
"edge": {}
|
||||
},
|
||||
"volumes": {
|
||||
"postgres_data": {},
|
||||
"broker_data": {},
|
||||
"s3_data": {},
|
||||
"bao_data": {},
|
||||
"metrics_data": {},
|
||||
"logs_data": {},
|
||||
"traces_data": {},
|
||||
"grafana_data": {}
|
||||
}
|
||||
}
|
||||
+43
-28
@@ -1,37 +1,52 @@
|
||||
# Матрица доверия — заполнить в решении lab7
|
||||
# Матрица доверия lab7 — Майорова Софья, БВТ2301
|
||||
|
||||
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
|
||||
Все межконтейнерные бизнес- и telemetry-соединения ниже шифруются. Root CA RSA3072 живёт только у оператора в ignored operator/ (0700, файлы0600).
|
||||
Leaf RSA2048: CN=роль, URI SAN=spiffe://mtusi.lab/роль, DNS SAN=имя сервиса/localhost, IP SAN=127.0.0.1; срок1сутки, CA30суток.
|
||||
Python/Go/botocore/Grafana/Prometheus клиенты проверяют CA и hostname/SAN. Проверка TLS нигде не отключается.
|
||||
|
||||
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
|
||||
HAProxy boundaries требуют клиентский сертификат с доверенной цепочкой **и** роль CN из allowlist, затем проверяют serial denylist.
|
||||
Native API дополнительно требует ровно URI SAN spiffe://mtusi.lab/lb. OpenBao cert-auth закрепляет **конкретный leaf**, не любой сертификат общего CA.
|
||||
Чужой valid leaf от нашего CA не равен разрешённой роли. Для PKI с недоверенным issuer-администратором это не защита: оператор CA входит в trust base.
|
||||
|
||||
| Связь | Реальные endpoint и identity | Авторизация / lifecycle | Проверка |
|
||||
| --- | --- | --- | --- |
|
||||
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
|
||||
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
|
||||
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
|
||||
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
|
||||
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
|
||||
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
|
||||
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
|
||||
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
|
||||
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
|
||||
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
|
||||
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
|
||||
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
|
||||
| Клиент → LB | HTTPS localhost:18443 → lb:8443; server SAN localhost/127.0.0.1 | Публичный TLS, затем Basic login/Bearer ACL; клиентский cert снаружи не нужен | make test/test-security; hardening.py wrong-hostname;18 API tests |
|
||||
| LB → API×2 | mTLS api:8080; client lb, server api | API URI allowlist; без cert/чужой CA/outsider/expired/worker — нет HTTP-доступа | tls_audit.py проверяет оба DNS IP, не только одну LB-выдачу |
|
||||
| API/worker/dispatcher → PG | PostgreSQL TLS postgres:5432, sslmode=verify-full | db_api/db_worker/db_dispatcher; отдельные пароли KV; worker только job tables/разрешённые UPDATE columns | tls_audit.py + worker_access.py; old_worker_denied.py после ротации |
|
||||
| API/worker → S3 | HTTPS+mTLS s3:8443 → loopback8333 | CN api/worker/operator; native разные S3 keys, gateway ограничивает prefix+verb; worker PUT results только If-None-Match:* | auth_and_signals.py: staging/results allow/deny, delete/list/admin/traversal denied |
|
||||
| Publisher/worker → broker | rediss broker:8443 → loopback6379; dispatcher / worker | Native ACL producer: attachment-jobs XADD/XGROUP; consumer: jobs/rejected-jobs и нужные ACK/claim; CONFIG/чужие keys запрещены | tls_audit.py, worker_access.py, old_worker_denied.py |
|
||||
| API → broker | N/A: API пишет durable outbox PostgreSQL; публикует отдельный dispatcher | API не получает BROKER_URL и его cert отвергается broker gateway | tls_audit.py api→broker denied |
|
||||
| API/worker/dispatcher → Collector | HTTPS+mTLS otel-collector:8443 → loopback4318 | Разные CN identities; произвольный outsider ingest запрещён; технический resource.name задаёт доверенный код producer | tls_audit.py + actual trace в auth-signals.json |
|
||||
| Collector → Tempo | HTTPS+mTLS tempo:8444 → loopback4318; client collector | Другие app identities не являются backend writer | tls_audit.py; реальный trace |
|
||||
| Collector → Loki | HTTPS+mTLS loki:8443/otlp; client collector | Только collector/grafana/operator в gateway; grafana дополнительно GET-only | tls_audit.py; реальные correlated logs |
|
||||
| Prometheus → Collector metrics | HTTPS+mTLS otel-collector:8444→8889 /8445→8888; client prometheus | Идентичность prometheus/operator; scrape не публикуется на host | tls_audit.py, реальные up=1 |
|
||||
| Grafana → datasources | HTTPS+mTLS prometheus:8443, loki:8443, tempo:8443 | CN grafana; Prometheus GET-only; Loki GET-only; client cert/key в generated ignored provisioning | grafana_tls.py проверяет запрос datasource→Prometheus и secureJsonFields |
|
||||
| API/worker/dispatcher → Bao | Native HTTPS+mTLS bao:8200; cert-auth exact leaf | Policies secret/data/api|worker|dispatcher read-only; token20s, re-auth каждые5s при использовании, cache≤20s | tls_audit.py own/foreign KV/policy denies, sealed_bao.py, rotation |
|
||||
| Оператор → Bao | HTTPS localhost:18207, operator cert + root token из operator/bao-init.json | Init/unseal3shares/threshold2 — явный host-operator action; сервер не имеет shares/root token | bao_operator.py; persistence повторяет unseal существующего store |
|
||||
| Оператор → Grafana | Native HTTPS localhost:13407; operator Basic/login с локальным случайным паролем | Анонимный datasource/admin API401; password только ignored local file | grafana_tls.py |
|
||||
| Оператор → S3 admin | mTLS operator + отдельная S3 admin key; только внутри operator tool/bucket bootstrap | Приложения admin key не получают | bootstrap.py; worker_access.py mounts; S3 negative tests |
|
||||
| API → Redis profile cache | rediss redis:8443 → loopback6379; client api | ACL username api, key user:*, GET/SETEX; cache необязателен, сессия всегда проверяется в PG | TLS audit; API smoke; cache config |
|
||||
| Кластеры/replication | N/A: каждый store одиночный | OpenBao file backend — non-dev persistent, не HA; auto TLS cluster listener8201 не используется для replication | hardening.json, single-host limitation |
|
||||
|
||||
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
|
||||
## Локальные hops и служебные каналы
|
||||
|
||||
## Обязательные отрицательные проверки
|
||||
Для Redis/S3/Collector/Prometheus/Loki/Tempo TLS gateway и backend разделяют **только свой** network namespace через network_mode:service:... .
|
||||
Их plaintext listeners привязаны к127.0.0.1; другие контейнеры backend-сети не могут их открыть. Mount Docker socket, host network и privileged отсутствуют.
|
||||
S3 mini/master/filer/volume/gateway общаются в этом же namespace по loopback, включая внутренние gRPC. Это локальная transport boundary, не заявление о native end-to-end TLS внутри процесса S3.
|
||||
hardening.py читает реальные listening sockets; tls_audit.py проверяет недоступность raw ports из отдельного контейнера.
|
||||
|
||||
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
|
||||
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
|
||||
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
|
||||
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
|
||||
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
|
||||
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
|
||||
LB readiness — реальный GET /health/ready по mTLS с lb cert. Docker application HEALTHCHECK удалён: он не получает приватный LB key для обхода identity.
|
||||
PG pg_isready/Unix socket и local process checks — минимальные local-only проверки; они не превращаются в внешние business listeners.
|
||||
Docker DNS127.0.0.11 — локальная инфраструктура разрешения имён; TLS hostname/CA проверяется после DNS.
|
||||
Наружу опубликованы только loopback TLS18443,13407,18207. PostgreSQL/broker/S3/OTLP/backends/API прямого host mapping не имеют.
|
||||
|
||||
## Bootstrap и эксплуатация
|
||||
## Lifecycle и границы обещаний
|
||||
|
||||
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
|
||||
KV содержит **статические** scoped passwords/keys; это не динамические DB leases. TTL20s относится к токену Bao и bounded application cache, не делает украденный DB password автоматически недействительным.
|
||||
Leaf истекает через1сутки; первый credential — операторский краткоживущий leaf private key, не встроенный постоянный shared token.
|
||||
scripts/rotate_worker.py отзывает outstanding tokens, старый serial, меняет PG/Redis/S3 secrets и leaf, разрывает старые proxy-соединения максимум через5s, убивает PG worker sessions и перезапускает worker без rebuild.
|
||||
Проверка старых credentials делается на **новых** соединениях. Уже выполняющаяся render-попытка дополнительно ограничена20s timeout; arbitrary code compromise этим таймаутом полностью не решается.
|
||||
|
||||
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
|
||||
|
||||
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
|
||||
Оператор владеет CA и всеми shares в защищённой host-папке; независимые custodians/HSM не реализованы.
|
||||
Ни app/worker, ни даже audit-контейнер не монтируют private CA/root/unseal files. Audit profile — явный операторский инструмент с отдельными credential files, не production worker.
|
||||
Ротация CA и автоматический PKI lifecycle трека2 не заявляются. При expiry/ошибке доверия нет HTTP или verify-off fallback.
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
BASE_URL="https://localhost:18443" CA_FILE="/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
|
||||
test_idempotency_conflicting_race (test_api.APIContract) ... ok
|
||||
test_idempotency_invalid_keys (test_api.APIContract) ... ok
|
||||
test_idempotency_scope (test_api.APIContract) ... ok
|
||||
test_idempotency_semantic_body (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_protocol_error_headers (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_nul_round_trip (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
test_async_variants_and_replay (test_images.Images) ... ok
|
||||
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
|
||||
test_private_and_cross_chat (test_images.Images) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 18 tests in 7.417s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,967 @@
|
||||
{
|
||||
"checks": {
|
||||
"passwords": {
|
||||
"argon2id_and_session_hashes": true,
|
||||
"legacy_scrypt_rehashed": true,
|
||||
"argon2_parameter_rehash": true,
|
||||
"real_expiry_401": true
|
||||
},
|
||||
"prefixes": {
|
||||
"worker_read_staging_and_results": true,
|
||||
"worker_create_immutable_result": true,
|
||||
"denied": [
|
||||
"worker PUT staging",
|
||||
"worker DELETE staging",
|
||||
"worker DELETE results",
|
||||
"worker unconditional overwrite",
|
||||
"worker bucket list",
|
||||
"worker create bucket",
|
||||
"API PUT results",
|
||||
"traversal key"
|
||||
]
|
||||
},
|
||||
"signals": {
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"spans": 20,
|
||||
"services_and_job_events_correlated": true,
|
||||
"producer_consumer_parent_match": true,
|
||||
"tls_prometheus_scrape_targets": 2,
|
||||
"sensitive_canaries_absent": true,
|
||||
"forbidden_metric_labels_absent": true
|
||||
},
|
||||
"shared_rate_limit": {
|
||||
"requests": 100,
|
||||
"limited_429": 70,
|
||||
"invalid_401": 30,
|
||||
"actual_replicas": 2,
|
||||
"retry_after_seconds": 1,
|
||||
"kdf_slots_per_replica": 2
|
||||
},
|
||||
"kdf_budget": {
|
||||
"CPU_limit": 0.5,
|
||||
"timings": {
|
||||
"19456": {
|
||||
"hash_plus_verify_median_ms": 28.538944999127125,
|
||||
"hash_plus_verify_max_ms": 77.69014000041352,
|
||||
"runs": 10
|
||||
},
|
||||
"32768": {
|
||||
"hash_plus_verify_median_ms": 111.35763649963337,
|
||||
"hash_plus_verify_max_ms": 161.50791800100706,
|
||||
"runs": 10
|
||||
},
|
||||
"65536": {
|
||||
"hash_plus_verify_median_ms": 275.9806710000703,
|
||||
"hash_plus_verify_max_ms": 284.7436590000143,
|
||||
"runs": 10
|
||||
}
|
||||
},
|
||||
"chosen_memory_kib": 32768,
|
||||
"max_kdf_memory_mib_per_api": 64,
|
||||
"api_memory_limit_mib": 256,
|
||||
"note": "Timings include hash+verify at 0.5 CPU; API has 1 CPU and at most two concurrent KDF operations."
|
||||
}
|
||||
},
|
||||
"trace": {
|
||||
"batches": [
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.45.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "5d6e52d19b9a4ece858a9a5e0b6a1177"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-worker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab7.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "rbuAmpXCcCs=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512061054586",
|
||||
"endTimeUnixNano": "1795174512068783633",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "LU0xd1eU+1o=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512071477151",
|
||||
"endTimeUnixNano": "1795174512077552078",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "7psBevI+/rs=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.get_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512077597704",
|
||||
"endTimeUnixNano": "1795174512079416786",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "get_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.45.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "f0a98e1d929941938612c531e7d276b5"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-dispatcher"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab7.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "Ln03ux3qQpk=",
|
||||
"parentSpanId": "Yfim34Icbgg=",
|
||||
"flags": 768,
|
||||
"name": "attachment publish",
|
||||
"kind": "SPAN_KIND_PRODUCER",
|
||||
"startTimeUnixNano": "1795174512059813142",
|
||||
"endTimeUnixNano": "1795174512060982043",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.45.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "b6da20d0d6004ddf90745f3f40603eba"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab7.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "XM3jomFakXE=",
|
||||
"parentSpanId": "tL9IgJdDqCk=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512014313172",
|
||||
"endTimeUnixNano": "1795174512015572367",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "YKjGBN9wGjk=",
|
||||
"parentSpanId": "tL9IgJdDqCk=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512015615659",
|
||||
"endTimeUnixNano": "1795174512016429177",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "31CfNaog4oU=",
|
||||
"parentSpanId": "tL9IgJdDqCk=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512016722142",
|
||||
"endTimeUnixNano": "1795174512019110279",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "y3lfdg+7fwo=",
|
||||
"parentSpanId": "Yfim34Icbgg=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512019415744",
|
||||
"endTimeUnixNano": "1795174512023220287",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "Yfim34Icbgg=",
|
||||
"parentSpanId": "tL9IgJdDqCk=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512019135029",
|
||||
"endTimeUnixNano": "1795174512024153724",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "tL9IgJdDqCk=",
|
||||
"parentSpanId": "ESIzRFVmd4g=",
|
||||
"flags": 768,
|
||||
"name": "POST /api/v1/chats/{chat_id}/attachments",
|
||||
"kind": "SPAN_KIND_SERVER",
|
||||
"startTimeUnixNano": "1795174512014274088",
|
||||
"endTimeUnixNano": "1795174512024470314",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "http.request.method",
|
||||
"value": {
|
||||
"stringValue": "POST"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.route",
|
||||
"value": {
|
||||
"stringValue": "/api/v1/chats/{chat_id}/attachments"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "http.response.status_code",
|
||||
"value": {
|
||||
"intValue": "202"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{
|
||||
"key": "telemetry.sdk.language",
|
||||
"value": {
|
||||
"stringValue": "python"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.name",
|
||||
"value": {
|
||||
"stringValue": "opentelemetry"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "telemetry.sdk.version",
|
||||
"value": {
|
||||
"stringValue": "1.45.2"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.instance.id",
|
||||
"value": {
|
||||
"stringValue": "5d6e52d19b9a4ece858a9a5e0b6a1177"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.name",
|
||||
"value": {
|
||||
"stringValue": "messenger-worker"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "service.version",
|
||||
"value": {
|
||||
"stringValue": "lab7.1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "deployment.environment.name",
|
||||
"value": {
|
||||
"stringValue": "lab"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"scopeSpans": [
|
||||
{
|
||||
"scope": {
|
||||
"name": "messenger"
|
||||
},
|
||||
"spans": [
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "zaIXAyYKnIU=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512222386880",
|
||||
"endTimeUnixNano": "1795174512224841518",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "de9j3dpp2To=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512224907353",
|
||||
"endTimeUnixNano": "1795174512228579726",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "GqvwpDKrw+Q=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512228643353",
|
||||
"endTimeUnixNano": "1795174512229941007",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "CzXT4UQsfrs=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512230012050",
|
||||
"endTimeUnixNano": "1795174512232093179",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "JRipvb7DZPc=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512232146431",
|
||||
"endTimeUnixNano": "1795174512233290873",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "v0Kc5iBBNfY=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512233349082",
|
||||
"endTimeUnixNano": "1795174512235073829",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "MEoig23QnRk=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.head_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512235112663",
|
||||
"endTimeUnixNano": "1795174512236113352",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "head_object"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "error.type",
|
||||
"value": {
|
||||
"stringValue": "ClientError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {
|
||||
"code": "STATUS_CODE_ERROR"
|
||||
}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "gP7+VNlH0OE=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "s3.put_object",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512236164187",
|
||||
"endTimeUnixNano": "1795174512237930351",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "rpc.system",
|
||||
"value": {
|
||||
"stringValue": "aws-api"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.service",
|
||||
"value": {
|
||||
"stringValue": "S3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "rpc.method",
|
||||
"value": {
|
||||
"stringValue": "put_object"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "EzgTXnrHECs=",
|
||||
"parentSpanId": "HwfARAv1i6o=",
|
||||
"flags": 256,
|
||||
"name": "db.transaction",
|
||||
"kind": "SPAN_KIND_CLIENT",
|
||||
"startTimeUnixNano": "1795174512237964018",
|
||||
"endTimeUnixNano": "1795174512245307265",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "db.system",
|
||||
"value": {
|
||||
"stringValue": "postgresql"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
},
|
||||
{
|
||||
"traceId": "SUVQC/AITRmh9wxltsj+xg==",
|
||||
"spanId": "HwfARAv1i6o=",
|
||||
"parentSpanId": "Ln03ux3qQpk=",
|
||||
"flags": 768,
|
||||
"name": "attachment process",
|
||||
"kind": "SPAN_KIND_CONSUMER",
|
||||
"startTimeUnixNano": "1795174512061016585",
|
||||
"endTimeUnixNano": "1795174512245819026",
|
||||
"attributes": [
|
||||
{
|
||||
"key": "messaging.system",
|
||||
"value": {
|
||||
"stringValue": "redis"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "messaging.destination.name",
|
||||
"value": {
|
||||
"stringValue": "attachment-jobs"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.id",
|
||||
"value": {
|
||||
"stringValue": "8f6fc767-dabb-464b-84b4-01cb9b30cdb9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "job.attempt",
|
||||
"value": {
|
||||
"intValue": "1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"status": {}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"logs": [
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.024169+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-api",
|
||||
"event": "attachment_accepted",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "b4bf48809743a829",
|
||||
"attachment_id": "3c55030c-760d-4c45-a372-d1d656f6d78a",
|
||||
"job_id": "8f6fc767-dabb-464b-84b4-01cb9b30cdb9"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.245714+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "job_processed",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "1f07c0440bf58baa",
|
||||
"job_id": "8f6fc767-dabb-464b-84b4-01cb9b30cdb9",
|
||||
"attempt": 1,
|
||||
"outcome": "ready"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.060886+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-dispatcher",
|
||||
"event": "job_published",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "2e7d37bb1dea4299",
|
||||
"job_id": "8f6fc767-dabb-464b-84b4-01cb9b30cdb9",
|
||||
"attempt": 1
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.024424+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-api",
|
||||
"event": "http_request",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "b4bf48809743a829",
|
||||
"request_id": "2aebfcf9a8424947997e6e8032b192a5",
|
||||
"method": "POST",
|
||||
"route": "/api/v1/chats/{chat_id}/attachments",
|
||||
"status": 202,
|
||||
"duration_seconds": 0.010226
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.224736+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "cda21703260a9c85",
|
||||
"error_type": "ClientError"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.229864+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "1aabf0a432abc3e4",
|
||||
"error_type": "ClientError"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.233223+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "2518a9bdbec364f7",
|
||||
"error_type": "ClientError"
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-11-20T11:35:12.236058+00:00",
|
||||
"severity": "INFO",
|
||||
"service.name": "messenger-worker",
|
||||
"event": "operation_failed",
|
||||
"sampled": true,
|
||||
"trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"span_id": "304a22836dd09d19",
|
||||
"error_type": "ClientError"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,343 @@
|
||||
|
||||
$ fresh bootstrap, build, migrations and verified HTTPS ready
|
||||
Image mtusi-messenger:lab7 Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 729B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 688B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/python:3.12-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [internal] load .dockerignore
|
||||
#4 transferring context: 233B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [1/7] FROM docker.io/library/python:3.12-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df
|
||||
#5 resolve docker.io/library/python:3.12-slim-bookworm@sha256:935c3c5b12357a7e10caa9c1e6ca5692344d97cf229832ef584be2cec9fc03df done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [internal] load build context
|
||||
#6 transferring context: 96.09kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [4/7] COPY requirements.txt ./
|
||||
#7 CACHED
|
||||
|
||||
#8 [6/7] COPY src/ ./src/
|
||||
#8 CACHED
|
||||
|
||||
#9 [2/7] WORKDIR /app
|
||||
#9 CACHED
|
||||
|
||||
#10 [3/7] RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app
|
||||
#10 CACHED
|
||||
|
||||
#11 [5/7] RUN pip install --no-cache-dir -r requirements.txt && rm -rf /usr/local/lib/python3.12/site-packages/pip /usr/local/lib/python3.12/site-packages/pip-*.dist-info /usr/local/bin/pip*
|
||||
#11 CACHED
|
||||
|
||||
#12 [7/7] COPY migrations/ ./migrations/
|
||||
#12 CACHED
|
||||
|
||||
#13 exporting to image
|
||||
#13 exporting layers done
|
||||
#13 exporting manifest sha256:2dfe4693be2797bf22428255af59c4492829ceb4e7be66cb9d9074376e767736 done
|
||||
#13 exporting config sha256:e2273fee14f165f916715b32d394ec8d91638a76a1a1a3030ae95d6d468b14a8 done
|
||||
#13 exporting attestation manifest sha256:1d1ee90f65958c8033c98134a2cf68506d3a7f1c36288e287967d31058dcbe35 done
|
||||
#13 exporting manifest list sha256:81bed584afea3db4977e61298433fac0f040c77280f1aa9a4ae5b3bf9e579524 done
|
||||
#13 naming to docker.io/library/mtusi-messenger:lab7 done
|
||||
#13 unpacking to docker.io/library/mtusi-messenger:lab7 done
|
||||
#13 DONE 0.0s
|
||||
|
||||
#14 resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
Image mtusi-messenger:lab7 Built
|
||||
Network mtusi-lab7-clean-63ee1835_backend Creating
|
||||
Network mtusi-lab7-clean-63ee1835_backend Created
|
||||
Network mtusi-lab7-clean-63ee1835_edge Creating
|
||||
Network mtusi-lab7-clean-63ee1835_edge Created
|
||||
Volume mtusi-lab7-clean-63ee1835_broker_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_broker_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_metrics_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_metrics_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_logs_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_logs_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_postgres_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_postgres_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_s3_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_s3_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_bao_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_bao_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_grafana_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_grafana_data Created
|
||||
Volume mtusi-lab7-clean-63ee1835_traces_data Creating
|
||||
Volume mtusi-lab7-clean-63ee1835_traces_data Created
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Running
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Healthy
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Running
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Running
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Running
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Running
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Running
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Created
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Creating
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Created
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Healthy
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Waiting
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Exited
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Exited
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Exited
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Exited
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Exited
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Exited
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Started
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Started
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Starting
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Started
|
||||
Local PKI ready; CA keys remain in operator/ and are not mounted into services.
|
||||
Local service configuration generated; no credentials printed.
|
||||
OpenBao initialized/unsealed by explicit operator action; scoped service policies and 20s certificate-auth tokens configured.
|
||||
Lab7 processes started; waiting for verified HTTPS readiness.
|
||||
Lab7 ready over trusted HTTPS; operator secrets stay local.
|
||||
|
||||
$ check + public HTTPS smoke + untrusted CA +18 API/image tests
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "https://localhost:18444" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/build/mtusi-lab7-clean-63ee1835/pki/lb/ca.crt"
|
||||
FAIL: POST /api/v1/chats/338ce157-7e05-497d-9468-1b9f9ab4c0db/attachments: expected [202], received 503
|
||||
make: *** [test] Error 1
|
||||
|
||||
$ cleanup ONLY disposable project mtusi-lab7-clean-63ee1835
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-redis-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-broker-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-lb-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-dispatcher-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-grafana-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-broker-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-prometheus-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-redis-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-tempo-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-bao-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-api-2 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-loki-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-loki-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-worker-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-api-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-bucket-bootstrap-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-migrate-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-postgres-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-s3-raw-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-otel-collector-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-s3-store-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Stopping
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-volume-init-1 Removed
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Stopped
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Removing
|
||||
Container mtusi-lab7-clean-63ee1835-s3-1 Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_bao_data Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_metrics_data Removing
|
||||
Network mtusi-lab7-clean-63ee1835_edge Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_traces_data Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_grafana_data Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_postgres_data Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_logs_data Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_broker_data Removing
|
||||
Network mtusi-lab7-clean-63ee1835_backend Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_s3_data Removing
|
||||
Volume mtusi-lab7-clean-63ee1835_bao_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_metrics_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_traces_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_grafana_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_broker_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_s3_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_logs_data Removed
|
||||
Volume mtusi-lab7-clean-63ee1835_postgres_data Removed
|
||||
Network mtusi-lab7-clean-63ee1835_edge Removed
|
||||
Network mtusi-lab7-clean-63ee1835_backend Removed
|
||||
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"pass": true,
|
||||
"commit_tested": "7d7bd69d8af959bda540b1bc203c9d62e56c3566",
|
||||
"project": "mtusi-lab7-clean-e6ba6863",
|
||||
"fresh_secrets_pki_and_volumes": true,
|
||||
"independent_ports": [
|
||||
18444,
|
||||
13408,
|
||||
18208
|
||||
],
|
||||
"results": {
|
||||
"fresh bootstrap, build, migrations and verified HTTPS ready": {
|
||||
"pass": true,
|
||||
"seconds": 30.145
|
||||
},
|
||||
"check + public HTTPS smoke + untrusted CA +18 API/image tests": {
|
||||
"pass": true,
|
||||
"seconds": 9.349
|
||||
},
|
||||
"internal TLS/identity/raw-port/ACL audit": {
|
||||
"pass": true,
|
||||
"seconds": 3.167
|
||||
},
|
||||
"Argon/rehash/expiry/rate limit/S3 ACL/real trace canaries": {
|
||||
"pass": true,
|
||||
"seconds": 9.288
|
||||
},
|
||||
"full business + historical telemetry + existing Bao store restart": {
|
||||
"pass": true,
|
||||
"seconds": 18.679
|
||||
},
|
||||
"Grafana HTTPS/mTLS datasources and21 panels": {
|
||||
"pass": true,
|
||||
"seconds": 0.259
|
||||
},
|
||||
"effective hardening and actual listeners": {
|
||||
"pass": true,
|
||||
"seconds": 1.465
|
||||
}
|
||||
},
|
||||
"seconds": 72.354,
|
||||
"main_lab7_data_reset": false,
|
||||
"only_disposable_project_volumes_deleted": true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 2.682}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 2.1}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.1}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 1.982}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 2.095}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.331}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,30 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 tests/security.py --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
BASE_URL="https://localhost:18443" CA_FILE="/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt" python3 -m unittest discover -s student-tests -p 'test_*.py' -v
|
||||
test_concurrent_messages_and_pagination (test_api.APIContract) ... ok
|
||||
test_cursor_scope_and_validation (test_api.APIContract) ... ok
|
||||
test_foreign_chat (test_api.APIContract) ... ok
|
||||
test_idempotency_concurrent_replicas (test_api.APIContract) ... ok
|
||||
test_idempotency_conflicting_race (test_api.APIContract) ... ok
|
||||
test_idempotency_invalid_keys (test_api.APIContract) ... ok
|
||||
test_idempotency_scope (test_api.APIContract) ... ok
|
||||
test_idempotency_semantic_body (test_api.APIContract) ... ok
|
||||
test_invalid_chat_creation (test_api.APIContract) ... ok
|
||||
test_malformed_json_and_extra_fields (test_api.APIContract) ... ok
|
||||
test_protocol_error_headers (test_api.APIContract) ... ok
|
||||
test_registration_race (test_api.APIContract) ... ok
|
||||
test_unicode_limits (test_api.APIContract) ... ok
|
||||
test_unicode_nul_round_trip (test_api.APIContract) ... ok
|
||||
test_unicode_password_basic_and_logout (test_api.APIContract) ... ok
|
||||
test_async_variants_and_replay (test_images.Images) ... ok
|
||||
test_mime_bytes_pixels_and_filename (test_images.Images) ... ok
|
||||
test_private_and_cross_chat (test_images.Images) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 18 tests in 7.369s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1,23 @@
|
||||
api running
|
||||
api running
|
||||
bao running
|
||||
broker running
|
||||
broker-raw running
|
||||
dispatcher running
|
||||
grafana running
|
||||
lb running
|
||||
loki running
|
||||
loki-raw running
|
||||
otel-collector running
|
||||
otel-collector-raw running
|
||||
postgres running
|
||||
prometheus running
|
||||
prometheus-raw running
|
||||
redis running
|
||||
redis-raw running
|
||||
s3 running
|
||||
s3-raw running
|
||||
s3-store running
|
||||
tempo running
|
||||
tempo-raw running
|
||||
worker running
|
||||
@@ -0,0 +1 @@
|
||||
PASS: native Grafana HTTPS/operator auth, all TLS client datasources, real Prometheus data and trace link configuration.
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"native_https_verified_ca_hostname": true,
|
||||
"anonymous_datasource_api_401": true,
|
||||
"authenticated_operator_api": true,
|
||||
"tls_mtls_datasources": [
|
||||
"Loki",
|
||||
"Prometheus",
|
||||
"Tempo"
|
||||
],
|
||||
"dashboard_uid": "messenger-lab7",
|
||||
"panels": 21,
|
||||
"actual_scrape_targets_up": 2,
|
||||
"loki_trace_link_provisioned": true
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS effective limits, non-root apps, drop ALL/no-new-privileges/read-only; published loopback TLS only, local raw listeners, wrong hostname rejected.
|
||||
@@ -0,0 +1,595 @@
|
||||
{
|
||||
"containers": [
|
||||
{
|
||||
"service": "api",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "bao",
|
||||
"user": "100:1000",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {
|
||||
"8200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18207"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "broker-raw",
|
||||
"user": "999:999",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "bucket-bootstrap",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "dispatcher",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "grafana",
|
||||
"user": "472",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13407"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {
|
||||
"8443/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18443"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "loki",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "loki-raw",
|
||||
"user": "10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "otel-collector",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "otel-collector-raw",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"user": "",
|
||||
"read_only": false,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [
|
||||
"CAP_CHOWN",
|
||||
"CAP_DAC_OVERRIDE",
|
||||
"CAP_FOWNER",
|
||||
"CAP_SETGID",
|
||||
"CAP_SETUID"
|
||||
],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "prometheus",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "prometheus-raw",
|
||||
"user": "nobody",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "redis-raw",
|
||||
"user": "999:999",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "s3",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "s3-raw",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "s3-store",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "tempo",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "tempo-raw",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "volume-init",
|
||||
"user": "0:0",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [
|
||||
"CAP_CHOWN"
|
||||
],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"host_ports": {}
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"user": "10001:10001",
|
||||
"read_only": true,
|
||||
"cap_drop": [
|
||||
"ALL"
|
||||
],
|
||||
"cap_add": [],
|
||||
"no_new_privileges": true,
|
||||
"memory_bytes": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"host_ports": {}
|
||||
}
|
||||
],
|
||||
"ipv4_listeners": {
|
||||
"api-1": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 40503
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8080
|
||||
}
|
||||
],
|
||||
"api-2": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 44643
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8080
|
||||
}
|
||||
],
|
||||
"bao": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 37185
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8200
|
||||
}
|
||||
],
|
||||
"broker": [
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 6379
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 35853
|
||||
}
|
||||
],
|
||||
"grafana": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 44223
|
||||
}
|
||||
],
|
||||
"lb": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 45307
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
}
|
||||
],
|
||||
"loki": [
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 44273
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 3100
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 9095
|
||||
}
|
||||
],
|
||||
"otel-collector": [
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 8888
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 8889
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8444
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8445
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 4318
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 39595
|
||||
}
|
||||
],
|
||||
"prometheus": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 34753
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 9090
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
}
|
||||
],
|
||||
"redis": [
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 34905
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 6379
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
}
|
||||
],
|
||||
"s3": [
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 9333
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 9340
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 23646
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 18333
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 8333
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 12535
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 22535
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 18888
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 8888
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 45173
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 19333
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 19340
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 33646
|
||||
}
|
||||
],
|
||||
"tempo": [
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 9095
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 4318
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8444
|
||||
},
|
||||
{
|
||||
"address": "0.0.0.0",
|
||||
"port": 8443
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.11",
|
||||
"port": 45653
|
||||
},
|
||||
{
|
||||
"address": "127.0.0.1",
|
||||
"port": 3200
|
||||
}
|
||||
]
|
||||
},
|
||||
"wrong_external_hostname_rejected": true,
|
||||
"note": "Raw local hops share only their proxy namespace; PostgreSQL is native TLS. DNS on127.0.0.11 is platform-local, not a business listener."
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,5 @@
|
||||
Container mtusi-lab7-audit-run-9470f081436c Creating
|
||||
Container mtusi-lab7-audit-run-9470f081436c Created
|
||||
Container mtusi-lab7-audit-run-3cef46528198 Creating
|
||||
Container mtusi-lab7-audit-run-3cef46528198 Created
|
||||
PASS: original HTTPS persistence and historical Tempo/Loki/Prometheus; Bao existing store survives.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,119 @@
|
||||
{
|
||||
"invariants": {
|
||||
"commit_before_publish_recovered": true,
|
||||
"publish_before_marker_duplicate_safe": true,
|
||||
"crashes_before_after_side_effect_and_commit": true,
|
||||
"invalid_envelope_observable_dlq": true,
|
||||
"duplicate_and_reorder_no_second_effect": true,
|
||||
"concurrent_duplicate_single_claim": true,
|
||||
"stale_running_version_fenced": true,
|
||||
"old_delivery_after_new_ready_no_overwrite": true,
|
||||
"failed_registry_explicit_replay_and_ready_refusal": true,
|
||||
"transient_backoff_exhaustion_then_replay": true,
|
||||
"invalid_decode_permanent_failed": true,
|
||||
"cleanup_preserves_current_and_fresh_failed": true,
|
||||
"ready_staging_cleanup_idempotent": true
|
||||
},
|
||||
"faults": [
|
||||
{
|
||||
"point": "before_side_effect",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "processing",
|
||||
"recovered": "ready",
|
||||
"attempts": 2,
|
||||
"result_objects": 4
|
||||
},
|
||||
{
|
||||
"point": "after_original",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "processing",
|
||||
"recovered": "ready",
|
||||
"attempts": 2,
|
||||
"result_objects": 4
|
||||
},
|
||||
{
|
||||
"point": "after_objects",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "processing",
|
||||
"recovered": "ready",
|
||||
"attempts": 2,
|
||||
"result_objects": 4
|
||||
},
|
||||
{
|
||||
"point": "after_commit",
|
||||
"exit_code": 77,
|
||||
"state_at_crash": "ready",
|
||||
"recovered": "ready",
|
||||
"attempts": 1,
|
||||
"result_objects": 4
|
||||
}
|
||||
],
|
||||
"retry": {
|
||||
"transitions": [
|
||||
{
|
||||
"state": "queued",
|
||||
"attempt": 1,
|
||||
"error_code": "storage_unavailable"
|
||||
},
|
||||
{
|
||||
"state": "failed",
|
||||
"attempt": 2,
|
||||
"error_code": "retry_exhausted"
|
||||
}
|
||||
],
|
||||
"seconds": 3.427,
|
||||
"replayed_attempt": 3
|
||||
},
|
||||
"transforms": [
|
||||
{
|
||||
"kind": "alpha",
|
||||
"size": [
|
||||
20,
|
||||
10
|
||||
],
|
||||
"exif_removed": true
|
||||
},
|
||||
{
|
||||
"kind": "exif",
|
||||
"size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"exif_removed": true
|
||||
},
|
||||
{
|
||||
"kind": "small",
|
||||
"size": [
|
||||
5,
|
||||
3
|
||||
],
|
||||
"exif_removed": true
|
||||
}
|
||||
],
|
||||
"cleanup": [
|
||||
{
|
||||
"apply": false,
|
||||
"retention_seconds": 86400,
|
||||
"counts": {
|
||||
"ready_staging": 46,
|
||||
"expired_staging": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"apply": true,
|
||||
"retention_seconds": 86400,
|
||||
"counts": {
|
||||
"ready_staging": 46,
|
||||
"expired_staging": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"apply": true,
|
||||
"retention_seconds": 86400,
|
||||
"counts": {
|
||||
"ready_staging": 0,
|
||||
"expired_staging": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"unprivileged_fixture_leaf_renewal_preserved_existing_serial_denylist": true,
|
||||
"production_roles_not_renewed_in_this_probe": true
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
|
||||
python3 tests/security.py --base-url "https://localhost:18443" --ca-file "/Users/ssmayorova/Documents/учеба/Высоконагруженные приложения/lab7/pki/lb/ca.crt"
|
||||
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"candidate_files": 155,
|
||||
"actual_local_secret_values_checked": 72,
|
||||
"leaks": 0,
|
||||
"private_pem_headers_absent": true,
|
||||
"original_public_tests_and_api_contract_unchanged": true,
|
||||
"scope": "Cached + untracked nonignored files; current/revoked operator/settings/Bao values and private PEMs. Not a claim about arbitrary historical leaks."
|
||||
}
|
||||
@@ -0,0 +1,754 @@
|
||||
[
|
||||
{
|
||||
"container": "8398cde71e6e44fad630caf86f2427445f80da9312bf2315e9be48a9297dcbed",
|
||||
"name": "/mtusi-lab7-api-1",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:38:03.995613374Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "b738c046db77a7182d4bfd44dcc936a1699901ff439328000a96fef054ea90e9",
|
||||
"name": "/mtusi-lab7-api-2",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:38:04.092722887Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "d8195b206f487fb062ee5ba958e1c0a650a7967e011329be8b9aa1e536f2424e",
|
||||
"name": "/mtusi-lab7-bao-1",
|
||||
"service": "bao",
|
||||
"image": "openbao/openbao:2.4.4@sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd",
|
||||
"user": "100:1000",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.875511259Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"8200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18207"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/openbao/logs",
|
||||
"name": "189642054cc115ee03eb2a3d0f1c3972476252a500d4613486d70b2c7f370e53"
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/bao/data",
|
||||
"name": "mtusi-lab7_bao_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/bao.hcl",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/openbao/file",
|
||||
"name": "c067c057f8ccfcfe31609e25dc1c2b1ad20bc1eeb2e098c144ab05fd587749ac"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "81d91e475ba5626277feec77efb2a5a26df10d7e8aa3e12ae7af6be6f1c25fbf",
|
||||
"name": "/mtusi-lab7-broker-1",
|
||||
"service": "broker",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.552212846Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "9d72c074c2116de3ccc068c25a0dde44c217136d380697cd988dd03ae5a36acb",
|
||||
"name": "/mtusi-lab7-broker-raw-1",
|
||||
"service": "broker-raw",
|
||||
"image": "redis:7.4-bookworm@sha256:2e3eb11462c4e3759e6972678dd051d150ff54b1e60714b3c33b701260172a90",
|
||||
"user": "999:999",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.577456456Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 201326592,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab7_broker_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/redis.conf",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "38f051bc9c69c0595d51ede1cfa28b1063dae33cb9062b9415e24b2308c7ecce",
|
||||
"name": "/mtusi-lab7-bucket-bootstrap-1",
|
||||
"service": "bucket-bootstrap",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T11:34:50.480058685Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/s3_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/s3_secret",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "2972a7bfb300130c3f881a379301a5a5595dfcbf1a89c53a836c278a573bcd79",
|
||||
"name": "/mtusi-lab7-dispatcher-1",
|
||||
"service": "dispatcher",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:38:03.995433162Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "b76b358d564111a931e28516e460a8b459a00a85ab8db5d3e3a2a3cbdbc003af",
|
||||
"name": "/mtusi-lab7-grafana-1",
|
||||
"service": "grafana",
|
||||
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
|
||||
"user": "472",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.713599928Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13407"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/grafana/provisioning/datasources/lab7.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/grafana/provisioning/dashboards",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/var/lib/grafana/dashboards",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/grafana",
|
||||
"name": "mtusi-lab7_grafana_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/password",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "8d42cad4af6e70571ec2edfc122225155db9fdc1e6038cbedbf0a06f7802f4e9",
|
||||
"name": "/mtusi-lab7-lb-1",
|
||||
"service": "lb",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:38:04.227385416Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"8443/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18443"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "66196ab12ad9ac9d9921864f2d19d887e1f8d3c30a706a475175e1ee2ccaaa25",
|
||||
"name": "/mtusi-lab7-loki-1",
|
||||
"service": "loki",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.613150089Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "021ea088a7ca504744cf6f435205cd6166919162ce891b656c6ba04da60725db",
|
||||
"name": "/mtusi-lab7-loki-raw-1",
|
||||
"service": "loki-raw",
|
||||
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
|
||||
"user": "10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:54.087268854Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/loki.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/loki",
|
||||
"name": "mtusi-lab7_logs_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "bfbeb139a45b238681e3f2480d6fa5798d42f0e0cbb5472a5e486841e55803f5",
|
||||
"name": "/mtusi-lab7-migrate-1",
|
||||
"service": "migrate",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T11:34:49.989494125Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/dsn",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "32b7f5dcee8a7ed60d7a9d2cef69ba404f7d88a3d6688f67be934dfbc0d4fdba",
|
||||
"name": "/mtusi-lab7-otel-collector-1",
|
||||
"service": "otel-collector",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.614424382Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "a954daeaa7dd0178ce6de6f7185fcb7066d97c62019d722c661d119209039333",
|
||||
"name": "/mtusi-lab7-otel-collector-raw-1",
|
||||
"service": "otel-collector-raw",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.531841752Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/otel.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/client",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "afa8c68605a886d1cd3a667a977b84d6d54b9408da4d13d8e0601adace2653f8",
|
||||
"name": "/mtusi-lab7-postgres-1",
|
||||
"service": "postgres",
|
||||
"image": "postgres:17-bookworm@sha256:837bed18782820fe967067cb5328b02b0d9992c1be9a37ee0c9c492cfc03af3b",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.699815519Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/password",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/docker-entrypoint-initdb.d/01.sql",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/postgresql/pg_hba.conf",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/start.sh",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/postgresql/data",
|
||||
"name": "mtusi-lab7_postgres_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "124c12409d9b91fc0913710d43c8250d072bee80a5185ba408f585a66c62b5fb",
|
||||
"name": "/mtusi-lab7-prometheus-1",
|
||||
"service": "prometheus",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.600551168Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "85592df310c72df9aded719cb2d19ff40d98cc60afafa22755a56f66fc9885b1",
|
||||
"name": "/mtusi-lab7-prometheus-raw-1",
|
||||
"service": "prometheus-raw",
|
||||
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
|
||||
"user": "nobody",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.57420888Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/prometheus",
|
||||
"name": "mtusi-lab7_metrics_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus/alerts.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "eb85bf0820c724a8a4186c1ec5668ba901c018ad0238c03f6dff7e31621d9267",
|
||||
"name": "/mtusi-lab7-redis-1",
|
||||
"service": "redis",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.614025868Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "08253582bedb65c1f84cfa303b657aac4d0e36a0229aa0e32185558011733a13",
|
||||
"name": "/mtusi-lab7-redis-raw-1",
|
||||
"service": "redis-raw",
|
||||
"image": "redis:7.4-bookworm@sha256:2e3eb11462c4e3759e6972678dd051d150ff54b1e60714b3c33b701260172a90",
|
||||
"user": "999:999",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.631735949Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/redis.conf",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "9510cdb849e1ba7c98ba0c11dcfb37133e596c632ab519aea925f5ca2bbfb9ad"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "1ea6a697425ff20d9191bdd0702dd5eafa38765dcac47ef2e6f050aeff3972ce",
|
||||
"name": "/mtusi-lab7-s3-1",
|
||||
"service": "s3",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.613419056Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "84ac101caa852787d0a10351c6c299cb0782cd02021f893d72b59ef5def4ded7",
|
||||
"name": "/mtusi-lab7-s3-raw-1",
|
||||
"service": "s3-raw",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:53.532344139Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "b6bfbebcc9aa22f1c6048f93fa402dc86857033870f388824eef3f1c5fd1b452"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/s3_config",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "fe86cdb1e0f8aa638db4ae7fe5b45bed203d4119bc85567c38a4ee64a2a556e6",
|
||||
"name": "/mtusi-lab7-s3-store-1",
|
||||
"service": "s3-store",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:54.974898031Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab7_s3_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "3372304a6659837c814f34039a19948749043f81948b29b28292072fa2f8487e",
|
||||
"name": "/mtusi-lab7-tempo-1",
|
||||
"service": "tempo",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.53813471Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "18644460953d161f9e50c23a5c959ea6a43914237fb91618396f6ebd4a51aa10",
|
||||
"name": "/mtusi-lab7-tempo-raw-1",
|
||||
"service": "tempo-raw",
|
||||
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:38:03.526532036Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/tempo",
|
||||
"name": "mtusi-lab7_traces_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/tempo.yaml",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "0beab7561aa9fb214602d05a55ec79154ba56bf31ee24966f94bb44288bae894",
|
||||
"name": "/mtusi-lab7-volume-init-1",
|
||||
"service": "volume-init",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "0:0",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T11:34:50.435951758Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/s3",
|
||||
"name": "mtusi-lab7_s3_data"
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/bao",
|
||||
"name": "mtusi-lab7_bao_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "05ba596798705c32ad6c581f7009d8966517763d4b06d18a523ec79095291df2",
|
||||
"name": "/mtusi-lab7-worker-1",
|
||||
"service": "worker",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:38:03.994906899Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,754 @@
|
||||
[
|
||||
{
|
||||
"container": "8398cde71e6e44fad630caf86f2427445f80da9312bf2315e9be48a9297dcbed",
|
||||
"name": "/mtusi-lab7-api-1",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:34:51.954723568Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "b738c046db77a7182d4bfd44dcc936a1699901ff439328000a96fef054ea90e9",
|
||||
"name": "/mtusi-lab7-api-2",
|
||||
"service": "api",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:34:52.054845352Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "d8195b206f487fb062ee5ba958e1c0a650a7967e011329be8b9aa1e536f2424e",
|
||||
"name": "/mtusi-lab7-bao-1",
|
||||
"service": "bao",
|
||||
"image": "openbao/openbao:2.4.4@sha256:595c83b42614a4d2b044608e4593c05b019c5db25bc9c185d8fff3ac96c03ddd",
|
||||
"user": "100:1000",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:10:52.120976188Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"8200/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18207"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/openbao/logs",
|
||||
"name": "189642054cc115ee03eb2a3d0f1c3972476252a500d4613486d70b2c7f370e53"
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/bao/data",
|
||||
"name": "mtusi-lab7_bao_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/bao.hcl",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/openbao/file",
|
||||
"name": "c067c057f8ccfcfe31609e25dc1c2b1ad20bc1eeb2e098c144ab05fd587749ac"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "81d91e475ba5626277feec77efb2a5a26df10d7e8aa3e12ae7af6be6f1c25fbf",
|
||||
"name": "/mtusi-lab7-broker-1",
|
||||
"service": "broker",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.552212846Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "9d72c074c2116de3ccc068c25a0dde44c217136d380697cd988dd03ae5a36acb",
|
||||
"name": "/mtusi-lab7-broker-raw-1",
|
||||
"service": "broker-raw",
|
||||
"image": "redis:7.4-bookworm@sha256:2e3eb11462c4e3759e6972678dd051d150ff54b1e60714b3c33b701260172a90",
|
||||
"user": "999:999",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:36.991601786Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 201326592,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab7_broker_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/redis.conf",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "38f051bc9c69c0595d51ede1cfa28b1063dae33cb9062b9415e24b2308c7ecce",
|
||||
"name": "/mtusi-lab7-bucket-bootstrap-1",
|
||||
"service": "bucket-bootstrap",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T11:34:50.480058685Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/s3_key",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/s3_secret",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "2972a7bfb300130c3f881a379301a5a5595dfcbf1a89c53a836c278a573bcd79",
|
||||
"name": "/mtusi-lab7-dispatcher-1",
|
||||
"service": "dispatcher",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:36:49.351580667Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "b76b358d564111a931e28516e460a8b459a00a85ab8db5d3e3a2a3cbdbc003af",
|
||||
"name": "/mtusi-lab7-grafana-1",
|
||||
"service": "grafana",
|
||||
"image": "grafana/grafana:12.1.1@sha256:a1701c2180249361737a99a01bc770db39381640e4d631825d38ff4535efa47d",
|
||||
"user": "472",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:10:51.874179034Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"3000/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "13407"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/password",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/grafana/provisioning/datasources/lab7.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/grafana/provisioning/dashboards",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/var/lib/grafana/dashboards",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/grafana",
|
||||
"name": "mtusi-lab7_grafana_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "8d42cad4af6e70571ec2edfc122225155db9fdc1e6038cbedbf0a06f7802f4e9",
|
||||
"name": "/mtusi-lab7-lb-1",
|
||||
"service": "lb",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:11:02.442998783Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"8443/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "18443"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "66196ab12ad9ac9d9921864f2d19d887e1f8d3c30a706a475175e1ee2ccaaa25",
|
||||
"name": "/mtusi-lab7-loki-1",
|
||||
"service": "loki",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.613150089Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "021ea088a7ca504744cf6f435205cd6166919162ce891b656c6ba04da60725db",
|
||||
"name": "/mtusi-lab7-loki-raw-1",
|
||||
"service": "loki-raw",
|
||||
"image": "grafana/loki:3.5.3@sha256:3165cecce301ce5b9b6e3530284b080934a05cd5cafac3d3d82edcb887b45ecd",
|
||||
"user": "10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:10:52.308693958Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/loki.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/loki",
|
||||
"name": "mtusi-lab7_logs_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "bfbeb139a45b238681e3f2480d6fa5798d42f0e0cbb5472a5e486841e55803f5",
|
||||
"name": "/mtusi-lab7-migrate-1",
|
||||
"service": "migrate",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T11:34:49.989494125Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/dsn",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "32b7f5dcee8a7ed60d7a9d2cef69ba404f7d88a3d6688f67be934dfbc0d4fdba",
|
||||
"name": "/mtusi-lab7-otel-collector-1",
|
||||
"service": "otel-collector",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.614424382Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "a954daeaa7dd0178ce6de6f7185fcb7066d97c62019d722c661d119209039333",
|
||||
"name": "/mtusi-lab7-otel-collector-raw-1",
|
||||
"service": "otel-collector-raw",
|
||||
"image": "otel/opentelemetry-collector-contrib:0.132.0@sha256:e0f565815b2b8e78eb9fdbb80f6190c921ab323aaa8ccceab3255c6d4225f4af",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:10:51.796429311Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/otel.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/client",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "afa8c68605a886d1cd3a667a977b84d6d54b9408da4d13d8e0601adace2653f8",
|
||||
"name": "/mtusi-lab7-postgres-1",
|
||||
"service": "postgres",
|
||||
"image": "postgres:17-bookworm@sha256:837bed18782820fe967067cb5328b02b0d9992c1be9a37ee0c9c492cfc03af3b",
|
||||
"user": "",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:34:46.339789373Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/postgresql/pg_hba.conf",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/start.sh",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/lib/postgresql/data",
|
||||
"name": "mtusi-lab7_postgres_data"
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/password",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/docker-entrypoint-initdb.d/01.sql",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "124c12409d9b91fc0913710d43c8250d072bee80a5185ba408f585a66c62b5fb",
|
||||
"name": "/mtusi-lab7-prometheus-1",
|
||||
"service": "prometheus",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.600551168Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "85592df310c72df9aded719cb2d19ff40d98cc60afafa22755a56f66fc9885b1",
|
||||
"name": "/mtusi-lab7-prometheus-raw-1",
|
||||
"service": "prometheus-raw",
|
||||
"image": "prom/prometheus:v3.5.0@sha256:63805ebb8d2b3920190daf1cb14a60871b16fd38bed42b857a3182bc621f4996",
|
||||
"user": "nobody",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:10:51.819580796Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/prometheus/alerts.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/prometheus",
|
||||
"name": "mtusi-lab7_metrics_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "eb85bf0820c724a8a4186c1ec5668ba901c018ad0238c03f6dff7e31621d9267",
|
||||
"name": "/mtusi-lab7-redis-1",
|
||||
"service": "redis",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.614025868Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "08253582bedb65c1f84cfa303b657aac4d0e36a0229aa0e32185558011733a13",
|
||||
"name": "/mtusi-lab7-redis-raw-1",
|
||||
"service": "redis-raw",
|
||||
"image": "redis:7.4-bookworm@sha256:2e3eb11462c4e3759e6972678dd051d150ff54b1e60714b3c33b701260172a90",
|
||||
"user": "999:999",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:34:46.339244777Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/redis.conf",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "9510cdb849e1ba7c98ba0c11dcfb37133e596c632ab519aea925f5ca2bbfb9ad"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "1ea6a697425ff20d9191bdd0702dd5eafa38765dcac47ef2e6f050aeff3972ce",
|
||||
"name": "/mtusi-lab7-s3-1",
|
||||
"service": "s3",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.613419056Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "84ac101caa852787d0a10351c6c299cb0782cd02021f893d72b59ef5def4ded7",
|
||||
"name": "/mtusi-lab7-s3-raw-1",
|
||||
"service": "s3-raw",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:36.998623618Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/run/s3_config",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "b6bfbebcc9aa22f1c6048f93fa402dc86857033870f388824eef3f1c5fd1b452"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "fe86cdb1e0f8aa638db4ae7fe5b45bed203d4119bc85567c38a4ee64a2a556e6",
|
||||
"name": "/mtusi-lab7-s3-store-1",
|
||||
"service": "s3-store",
|
||||
"image": "chrislusf/seaweedfs:4.51@sha256:f7a49e3f3972ebed5be2ed4befa4a0c68eb7bdddba6e692d42729ceb545134b8",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:10:52.810974582Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/data",
|
||||
"name": "mtusi-lab7_s3_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "3372304a6659837c814f34039a19948749043f81948b29b28292072fa2f8487e",
|
||||
"name": "/mtusi-lab7-tempo-1",
|
||||
"service": "tempo",
|
||||
"image": "haproxy:3.0-alpine@sha256:7430e63d3052198b18a40665b59a5826bc428e8bceb0c561d82258149ef855ee",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T10:57:12.53813471Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/haproxy.cfg",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "18644460953d161f9e50c23a5c959ea6a43914237fb91618396f6ebd4a51aa10",
|
||||
"name": "/mtusi-lab7-tempo-raw-1",
|
||||
"service": "tempo-raw",
|
||||
"image": "grafana/tempo:2.8.2@sha256:0ef775495967cd5d7a6b2e146b6ea695d624803c8db8349fb8ce4164f719f9b7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:11:01.774149059Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 402653184,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/etc/tempo.yaml",
|
||||
"name": null
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/var/tempo",
|
||||
"name": "mtusi-lab7_traces_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "0beab7561aa9fb214602d05a55ec79154ba56bf31ee24966f94bb44288bae894",
|
||||
"name": "/mtusi-lab7-volume-init-1",
|
||||
"service": "volume-init",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "0:0",
|
||||
"running": false,
|
||||
"started_at": "2026-11-20T11:34:50.435951758Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 67108864,
|
||||
"nano_cpus": 500000000,
|
||||
"ports": {},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/s3",
|
||||
"name": "mtusi-lab7_s3_data"
|
||||
},
|
||||
{
|
||||
"type": "volume",
|
||||
"destination": "/bao",
|
||||
"name": "mtusi-lab7_bao_data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"container": "05ba596798705c32ad6c581f7009d8966517763d4b06d18a523ec79095291df2",
|
||||
"name": "/mtusi-lab7-worker-1",
|
||||
"service": "worker",
|
||||
"image": "mtusi-messenger:lab7",
|
||||
"user": "10001:10001",
|
||||
"running": true,
|
||||
"started_at": "2026-11-20T11:37:37.358903064Z",
|
||||
"exit_code": 0,
|
||||
"oom_killed": false,
|
||||
"memory_limit": 805306368,
|
||||
"nano_cpus": 1000000000,
|
||||
"ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"type": "bind",
|
||||
"destination": "/tls",
|
||||
"name": null
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"python": "3.12.16",
|
||||
"architecture": "aarch64",
|
||||
"pointer_bits": 64,
|
||||
"openssl": "OpenSSL 3.0.22 25 Aug 2026",
|
||||
"minizip_zipOpenNewFileInZip4_64_exported_by_libz": false,
|
||||
"otel_sdk": "1.45.2",
|
||||
"argon2": "25.1.0",
|
||||
"distributions": 33
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"sealed_http_observations": [
|
||||
{
|
||||
"elapsed_seconds": 0.062,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 1.078,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 2.114,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 3.129,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 4.144,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 5.16,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 6.175,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 7.189,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 8.203,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 9.214,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 10.226,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 11.246,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 12.262,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 13.275,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 14.287,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 15.302,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 16.32,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 17.333,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 18.344,
|
||||
"status": 200
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 19.362,
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 20.371,
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 21.378,
|
||||
"status": 503
|
||||
},
|
||||
{
|
||||
"elapsed_seconds": 22.386,
|
||||
"status": 503
|
||||
}
|
||||
],
|
||||
"max_secret_cache_seconds": 20,
|
||||
"old_bearer_after_unseal": true,
|
||||
"first_failure_seconds": 19.362,
|
||||
"recovered_seconds": 25.489,
|
||||
"token_expiry": {
|
||||
"ttl": 20,
|
||||
"expired_token_403": true,
|
||||
"new_certificate_auth_succeeds": true
|
||||
},
|
||||
"no_insecure_or_plaintext_fallback": true,
|
||||
"worker_dispatcher_remained_alive": true,
|
||||
"new_job_ready_after_unseal": true
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
{"instance": "be257cab076b4646946f67e69e56b2d3", "container": "05293b655d3b43819f31201b4f4e14c59c3392277697782602b0bc169a3fe6a9", "running": false, "exit_code": 0, "seconds": 0.595}
|
||||
{"instance": "ecd40e87d0244421a8723c48ac7f6fbc", "container": "05293b655d3b43819f31201b4f4e14c59c3392277697782602b0bc169a3fe6a9", "running": false, "exit_code": 0, "seconds": 0.452}
|
||||
{"instance": "eacb0c2c5350491a873fd5d020e00102", "container": "05293b655d3b43819f31201b4f4e14c59c3392277697782602b0bc169a3fe6a9", "running": false, "exit_code": 0, "seconds": 0.368}
|
||||
{"instance": "ae90247b5d154d67a8afaa1de60ff829", "container": "05293b655d3b43819f31201b4f4e14c59c3392277697782602b0bc169a3fe6a9", "running": false, "exit_code": 0, "seconds": 0.351}
|
||||
{"instance": "6ea680203f004496bddb1b034d51f348", "container": "05293b655d3b43819f31201b4f4e14c59c3392277697782602b0bc169a3fe6a9", "running": false, "exit_code": 0, "seconds": 0.375}
|
||||
{"instance": "f3a3b90cc1694551a9be17f9fd964c4e", "container": "92cf19df1a120d28944021521f87c19f1f9658c2b5b7611617d7f455c686d1d5", "running": false, "exit_code": 0, "seconds": 0.617}
|
||||
@@ -0,0 +1,229 @@
|
||||
{
|
||||
"matrix": [
|
||||
{
|
||||
"endpoint": "s3:8443",
|
||||
"allowed_identity": "api",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-api": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "redis:8443",
|
||||
"allowed_identity": "api",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-api": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "broker:8443",
|
||||
"allowed_identity": "worker",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-worker": "SSLError",
|
||||
"api": "identity connection closed"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "otel-collector:8443",
|
||||
"allowed_identity": "api",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-api": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "otel-collector:8443",
|
||||
"allowed_identity": "worker",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-worker": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "otel-collector:8443",
|
||||
"allowed_identity": "dispatcher",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-dispatcher": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "otel-collector:8444",
|
||||
"allowed_identity": "prometheus",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-prometheus": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "otel-collector:8445",
|
||||
"allowed_identity": "prometheus",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-prometheus": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "loki:8443",
|
||||
"allowed_identity": "collector",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-collector": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "loki:8443",
|
||||
"allowed_identity": "grafana",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-grafana": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "tempo:8443",
|
||||
"allowed_identity": "grafana",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-grafana": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "tempo:8444",
|
||||
"allowed_identity": "collector",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-collector": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "prometheus:8443",
|
||||
"allowed_identity": "grafana",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "identity connection closed",
|
||||
"expired-grafana": "SSLError"
|
||||
}
|
||||
},
|
||||
{
|
||||
"endpoint": "bao:8200",
|
||||
"allowed_identity": "api",
|
||||
"positive_connection": true,
|
||||
"negative_cases": {
|
||||
"None": "SSLError",
|
||||
"foreign-client": "SSLError",
|
||||
"outsider": "TLS CA accepted; service-auth denied separately",
|
||||
"expired-api": "SSLError"
|
||||
}
|
||||
}
|
||||
],
|
||||
"api_replicas": [
|
||||
{
|
||||
"address_alias": "replica1",
|
||||
"allowed_lb": true,
|
||||
"negative_cases": [
|
||||
"None",
|
||||
"foreign-client",
|
||||
"outsider",
|
||||
"expired-lb",
|
||||
"wrong-lb",
|
||||
"worker"
|
||||
]
|
||||
},
|
||||
{
|
||||
"address_alias": "replica2",
|
||||
"allowed_lb": true,
|
||||
"negative_cases": [
|
||||
"None",
|
||||
"foreign-client",
|
||||
"outsider",
|
||||
"expired-lb",
|
||||
"wrong-lb",
|
||||
"worker"
|
||||
]
|
||||
}
|
||||
],
|
||||
"plaintext_rejected": [
|
||||
"s3:8333",
|
||||
"s3:8888",
|
||||
"s3:9333",
|
||||
"s3:9340",
|
||||
"broker:6379",
|
||||
"redis:6379",
|
||||
"otel-collector:4318",
|
||||
"otel-collector:8888",
|
||||
"otel-collector:8889",
|
||||
"loki:3100",
|
||||
"loki:9095",
|
||||
"tempo:3200",
|
||||
"tempo:4318",
|
||||
"tempo:9095",
|
||||
"prometheus:9090"
|
||||
],
|
||||
"policies": {
|
||||
"api": {
|
||||
"own_kv_read": true,
|
||||
"foreign_kv_and_policy_read_write_403": true,
|
||||
"token_ttl_seconds": 20
|
||||
},
|
||||
"worker": {
|
||||
"own_kv_read": true,
|
||||
"foreign_kv_and_policy_read_write_403": true,
|
||||
"token_ttl_seconds": 20
|
||||
},
|
||||
"dispatcher": {
|
||||
"own_kv_read": true,
|
||||
"foreign_kv_and_policy_read_write_403": true,
|
||||
"token_ttl_seconds": 20
|
||||
}
|
||||
},
|
||||
"db": {
|
||||
"verify_full_tls": true,
|
||||
"worker_jobs_read_allowed": true,
|
||||
"users_sessions_messages_ddl_denied": true
|
||||
},
|
||||
"broker_acl": {
|
||||
"worker_ping": true,
|
||||
"foreign_key_and_config_denied": true
|
||||
},
|
||||
"plaintext_configuration_rejected": true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"public_business_persistence": true,
|
||||
"old_bearer_and_original_attachment_preserved": true,
|
||||
"same_trace_id": "4945500bf0084d19a1f70c65b6c8fec6",
|
||||
"old_spans_preserved": 20,
|
||||
"old_logs_preserved": 8,
|
||||
"fixed_time_prometheus_metric_preserved": true,
|
||||
"at_epoch": 1795174663.604336,
|
||||
"bao_restarted_and_operator_unsealed_without_reinit": true
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
{
|
||||
"runs": [
|
||||
{
|
||||
"hypothesis": "Service-specific least privilege reduces stolen worker credential scope. Revocation and rollover stop old access while already accepted jobs remain durable.",
|
||||
"baseline_model": {
|
||||
"model": "lab6-style shared full database role and bucket-wide worker S3 key; emulated by explicit operator fixture, NOT live lab6",
|
||||
"business_table_row_counts_visible": {
|
||||
"users": 365,
|
||||
"sessions": 357,
|
||||
"messages": 1850
|
||||
},
|
||||
"bucket_wide_worker_key_can_write_and_delete_staging_without_identity_prefix_gate": true,
|
||||
"fixture_deleted": true,
|
||||
"modeled_broad_broker_CONFIG_allowed": true
|
||||
},
|
||||
"actual_worker_before": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 98,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"resource_snapshots_before": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.83%",
|
||||
"MemUsage": "67.37MiB / 256MiB",
|
||||
"MemPerc": "26.32%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.83%",
|
||||
"MemUsage": "66.71MiB / 256MiB",
|
||||
"MemPerc": "26.06%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "3.79%",
|
||||
"MemUsage": "48.11MiB / 256MiB",
|
||||
"MemPerc": "18.79%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "0.29%",
|
||||
"MemUsage": "63.68MiB / 768MiB",
|
||||
"MemPerc": "8.29%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"rotation": {
|
||||
"revoked_certificate_serial": "870DDEA40BB8D43173A74922991E11E0",
|
||||
"worker_tokens_revoked": 3,
|
||||
"old_valid_token_denied_before_natural_expiry": true,
|
||||
"token_revoke_latency_seconds": 0.13,
|
||||
"passwords_rotated": [
|
||||
"PostgreSQL worker",
|
||||
"Redis worker",
|
||||
"S3 worker"
|
||||
],
|
||||
"new_leaf_certificate": true,
|
||||
"proxy_hard_stop_seconds": 5,
|
||||
"images_rebuilt": false,
|
||||
"seconds": 3.107
|
||||
},
|
||||
"old_access": {
|
||||
"broker": "revoked leaf denied on fresh connection",
|
||||
"s3": "revoked leaf denied on fresh connection",
|
||||
"otel-collector": "revoked leaf denied on fresh connection",
|
||||
"postgres": "old password denied",
|
||||
"broker_old_password_with_new_certificate": "AUTH rejected",
|
||||
"bao": "old exact leaf denied",
|
||||
"s3_old_key_with_new_certificate": "403"
|
||||
},
|
||||
"new_worker_after": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 104,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"workload": {
|
||||
"messages_confirmed": 100,
|
||||
"errors": [],
|
||||
"all_confirmed_messages_found": true,
|
||||
"queued_images_before_revoke": 6,
|
||||
"same_images_ready_after_rollover": 6,
|
||||
"original_bytes_match": true,
|
||||
"elapsed_seconds": 12.002,
|
||||
"success_rps": 8.332,
|
||||
"p50_success_seconds": 0.013067833000000029,
|
||||
"p95_success_seconds": 0.02429433300000028,
|
||||
"p99_success_seconds": 0.05442791699999994,
|
||||
"load_profile": "One HTTP writer, 100ms wait after completion (closed loop, nominal ceiling10/s); six queued PNGs, no saturation search.",
|
||||
"image_unchanged": true,
|
||||
"no_plaintext_or_skip_verify_fallback": true
|
||||
},
|
||||
"resource_snapshots_after": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.67%",
|
||||
"MemUsage": "67.43MiB / 256MiB",
|
||||
"MemPerc": "26.34%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.66%",
|
||||
"MemUsage": "66.89MiB / 256MiB",
|
||||
"MemPerc": "26.13%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "3.41%",
|
||||
"MemUsage": "48.13MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "1.02%",
|
||||
"MemUsage": "65.03MiB / 768MiB",
|
||||
"MemPerc": "8.47%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"repeat": 1
|
||||
},
|
||||
{
|
||||
"hypothesis": "Service-specific least privilege reduces stolen worker credential scope. Revocation and rollover stop old access while already accepted jobs remain durable.",
|
||||
"baseline_model": {
|
||||
"model": "lab6-style shared full database role and bucket-wide worker S3 key; emulated by explicit operator fixture, NOT live lab6",
|
||||
"business_table_row_counts_visible": {
|
||||
"users": 368,
|
||||
"sessions": 360,
|
||||
"messages": 1950
|
||||
},
|
||||
"bucket_wide_worker_key_can_write_and_delete_staging_without_identity_prefix_gate": true,
|
||||
"fixture_deleted": true,
|
||||
"modeled_broad_broker_CONFIG_allowed": true
|
||||
},
|
||||
"actual_worker_before": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 104,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"resource_snapshots_before": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.52%",
|
||||
"MemUsage": "67.44MiB / 256MiB",
|
||||
"MemPerc": "26.34%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.59%",
|
||||
"MemUsage": "66.66MiB / 256MiB",
|
||||
"MemPerc": "26.04%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "3.54%",
|
||||
"MemUsage": "48.13MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "0.08%",
|
||||
"MemUsage": "65.03MiB / 768MiB",
|
||||
"MemPerc": "8.47%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"rotation": {
|
||||
"revoked_certificate_serial": "B2BDA769E83A37B06D649AFA42413D41",
|
||||
"worker_tokens_revoked": 6,
|
||||
"old_valid_token_denied_before_natural_expiry": true,
|
||||
"token_revoke_latency_seconds": 0.15,
|
||||
"passwords_rotated": [
|
||||
"PostgreSQL worker",
|
||||
"Redis worker",
|
||||
"S3 worker"
|
||||
],
|
||||
"new_leaf_certificate": true,
|
||||
"proxy_hard_stop_seconds": 5,
|
||||
"images_rebuilt": false,
|
||||
"seconds": 2.828
|
||||
},
|
||||
"old_access": {
|
||||
"broker": "revoked leaf denied on fresh connection",
|
||||
"s3": "revoked leaf denied on fresh connection",
|
||||
"otel-collector": "revoked leaf denied on fresh connection",
|
||||
"postgres": "old password denied",
|
||||
"broker_old_password_with_new_certificate": "AUTH rejected",
|
||||
"bao": "old exact leaf denied",
|
||||
"s3_old_key_with_new_certificate": "403"
|
||||
},
|
||||
"new_worker_after": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 110,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"workload": {
|
||||
"messages_confirmed": 101,
|
||||
"errors": [],
|
||||
"all_confirmed_messages_found": true,
|
||||
"queued_images_before_revoke": 6,
|
||||
"same_images_ready_after_rollover": 6,
|
||||
"original_bytes_match": true,
|
||||
"elapsed_seconds": 12.144,
|
||||
"success_rps": 8.317,
|
||||
"p50_success_seconds": 0.01332700000000031,
|
||||
"p95_success_seconds": 0.025906332999998227,
|
||||
"p99_success_seconds": 0.05125450000000065,
|
||||
"load_profile": "One HTTP writer, 100ms wait after completion (closed loop, nominal ceiling10/s); six queued PNGs, no saturation search.",
|
||||
"image_unchanged": true,
|
||||
"no_plaintext_or_skip_verify_fallback": true
|
||||
},
|
||||
"resource_snapshots_after": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.66%",
|
||||
"MemUsage": "67.55MiB / 256MiB",
|
||||
"MemPerc": "26.39%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.60%",
|
||||
"MemUsage": "66.86MiB / 256MiB",
|
||||
"MemPerc": "26.12%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "3.34%",
|
||||
"MemUsage": "48.14MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "0.81%",
|
||||
"MemUsage": "64.88MiB / 768MiB",
|
||||
"MemPerc": "8.45%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"repeat": 2
|
||||
},
|
||||
{
|
||||
"hypothesis": "Service-specific least privilege reduces stolen worker credential scope. Revocation and rollover stop old access while already accepted jobs remain durable.",
|
||||
"baseline_model": {
|
||||
"model": "lab6-style shared full database role and bucket-wide worker S3 key; emulated by explicit operator fixture, NOT live lab6",
|
||||
"business_table_row_counts_visible": {
|
||||
"users": 371,
|
||||
"sessions": 363,
|
||||
"messages": 2051
|
||||
},
|
||||
"bucket_wide_worker_key_can_write_and_delete_staging_without_identity_prefix_gate": true,
|
||||
"fixture_deleted": true,
|
||||
"modeled_broad_broker_CONFIG_allowed": true
|
||||
},
|
||||
"actual_worker_before": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 110,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"resource_snapshots_before": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.57%",
|
||||
"MemUsage": "67.61MiB / 256MiB",
|
||||
"MemPerc": "26.41%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.35%",
|
||||
"MemUsage": "66.87MiB / 256MiB",
|
||||
"MemPerc": "26.12%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "5.81%",
|
||||
"MemUsage": "48.13MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "2.06%",
|
||||
"MemUsage": "64.88MiB / 768MiB",
|
||||
"MemPerc": "8.45%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"rotation": {
|
||||
"revoked_certificate_serial": "D8C5BAB015148D0B982EEB5455810BC7",
|
||||
"worker_tokens_revoked": 7,
|
||||
"old_valid_token_denied_before_natural_expiry": true,
|
||||
"token_revoke_latency_seconds": 0.195,
|
||||
"passwords_rotated": [
|
||||
"PostgreSQL worker",
|
||||
"Redis worker",
|
||||
"S3 worker"
|
||||
],
|
||||
"new_leaf_certificate": true,
|
||||
"proxy_hard_stop_seconds": 5,
|
||||
"images_rebuilt": false,
|
||||
"seconds": 2.786
|
||||
},
|
||||
"old_access": {
|
||||
"broker": "revoked leaf denied on fresh connection",
|
||||
"s3": "revoked leaf denied on fresh connection",
|
||||
"otel-collector": "revoked leaf denied on fresh connection",
|
||||
"postgres": "old password denied",
|
||||
"broker_old_password_with_new_certificate": "AUTH rejected",
|
||||
"bao": "old exact leaf denied",
|
||||
"s3_old_key_with_new_certificate": "403"
|
||||
},
|
||||
"new_worker_after": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 116,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"workload": {
|
||||
"messages_confirmed": 100,
|
||||
"errors": [],
|
||||
"all_confirmed_messages_found": true,
|
||||
"queued_images_before_revoke": 6,
|
||||
"same_images_ready_after_rollover": 6,
|
||||
"original_bytes_match": true,
|
||||
"elapsed_seconds": 11.929,
|
||||
"success_rps": 8.383,
|
||||
"p50_success_seconds": 0.012386499999999856,
|
||||
"p95_success_seconds": 0.02562958399999893,
|
||||
"p99_success_seconds": 0.03877966700000002,
|
||||
"load_profile": "One HTTP writer, 100ms wait after completion (closed loop, nominal ceiling10/s); six queued PNGs, no saturation search.",
|
||||
"image_unchanged": true,
|
||||
"no_plaintext_or_skip_verify_fallback": true
|
||||
},
|
||||
"resource_snapshots_after": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.69%",
|
||||
"MemUsage": "67.64MiB / 256MiB",
|
||||
"MemPerc": "26.42%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.76%",
|
||||
"MemUsage": "66.92MiB / 256MiB",
|
||||
"MemPerc": "26.14%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "3.40%",
|
||||
"MemUsage": "48.14MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "0.94%",
|
||||
"MemUsage": "65.04MiB / 768MiB",
|
||||
"MemPerc": "8.47%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"repeat": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
{
|
||||
"hypothesis": "Service-specific least privilege reduces stolen worker credential scope. Revocation and rollover stop old access while already accepted jobs remain durable.",
|
||||
"baseline_model": {
|
||||
"model": "lab6-style shared full database role and bucket-wide worker S3 key; emulated by explicit operator fixture, NOT live lab6",
|
||||
"business_table_row_counts_visible": {
|
||||
"users": 371,
|
||||
"sessions": 363,
|
||||
"messages": 2051
|
||||
},
|
||||
"bucket_wide_worker_key_can_write_and_delete_staging_without_identity_prefix_gate": true,
|
||||
"fixture_deleted": true,
|
||||
"modeled_broad_broker_CONFIG_allowed": true
|
||||
},
|
||||
"actual_worker_before": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 110,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"resource_snapshots_before": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.57%",
|
||||
"MemUsage": "67.61MiB / 256MiB",
|
||||
"MemPerc": "26.41%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.35%",
|
||||
"MemUsage": "66.87MiB / 256MiB",
|
||||
"MemPerc": "26.12%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "5.81%",
|
||||
"MemUsage": "48.13MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "2.06%",
|
||||
"MemUsage": "64.88MiB / 768MiB",
|
||||
"MemPerc": "8.45%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
],
|
||||
"rotation": {
|
||||
"revoked_certificate_serial": "D8C5BAB015148D0B982EEB5455810BC7",
|
||||
"worker_tokens_revoked": 7,
|
||||
"old_valid_token_denied_before_natural_expiry": true,
|
||||
"token_revoke_latency_seconds": 0.195,
|
||||
"passwords_rotated": [
|
||||
"PostgreSQL worker",
|
||||
"Redis worker",
|
||||
"S3 worker"
|
||||
],
|
||||
"new_leaf_certificate": true,
|
||||
"proxy_hard_stop_seconds": 5,
|
||||
"images_rebuilt": false,
|
||||
"seconds": 2.786
|
||||
},
|
||||
"old_access": {
|
||||
"broker": "revoked leaf denied on fresh connection",
|
||||
"s3": "revoked leaf denied on fresh connection",
|
||||
"otel-collector": "revoked leaf denied on fresh connection",
|
||||
"postgres": "old password denied",
|
||||
"broker_old_password_with_new_certificate": "AUTH rejected",
|
||||
"bao": "old exact leaf denied",
|
||||
"s3_old_key_with_new_certificate": "403"
|
||||
},
|
||||
"new_worker_after": {
|
||||
"uid": 10001,
|
||||
"own_kv": true,
|
||||
"root_or_unseal_files_present": false,
|
||||
"job_rows_visible": 116,
|
||||
"allowed_db_operations": [
|
||||
"SELECT job and attachment metadata",
|
||||
"UPDATE allowed job state columns",
|
||||
"pipeline INSERT job_events (covered by successful jobs)"
|
||||
],
|
||||
"denied_db_tables": [
|
||||
"users",
|
||||
"sessions",
|
||||
"messages"
|
||||
],
|
||||
"protected_chat_source_columns_and_ddl_denied": true,
|
||||
"broker_config_denied": true
|
||||
},
|
||||
"workload": {
|
||||
"messages_confirmed": 100,
|
||||
"errors": [],
|
||||
"all_confirmed_messages_found": true,
|
||||
"queued_images_before_revoke": 6,
|
||||
"same_images_ready_after_rollover": 6,
|
||||
"original_bytes_match": true,
|
||||
"elapsed_seconds": 11.929,
|
||||
"success_rps": 8.383,
|
||||
"p50_success_seconds": 0.012386499999999856,
|
||||
"p95_success_seconds": 0.02562958399999893,
|
||||
"p99_success_seconds": 0.03877966700000002,
|
||||
"load_profile": "One HTTP writer, 100ms wait after completion (closed loop, nominal ceiling10/s); six queued PNGs, no saturation search.",
|
||||
"image_unchanged": true,
|
||||
"no_plaintext_or_skip_verify_fallback": true
|
||||
},
|
||||
"resource_snapshots_after": [
|
||||
{
|
||||
"Name": "mtusi-lab7-api-1",
|
||||
"CPUPerc": "0.69%",
|
||||
"MemUsage": "67.64MiB / 256MiB",
|
||||
"MemPerc": "26.42%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-api-2",
|
||||
"CPUPerc": "0.76%",
|
||||
"MemUsage": "66.92MiB / 256MiB",
|
||||
"MemPerc": "26.14%",
|
||||
"PIDs": "21"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-dispatcher-1",
|
||||
"CPUPerc": "3.40%",
|
||||
"MemUsage": "48.14MiB / 256MiB",
|
||||
"MemPerc": "18.80%",
|
||||
"PIDs": "4"
|
||||
},
|
||||
{
|
||||
"Name": "mtusi-lab7-worker-1",
|
||||
"CPUPerc": "0.94%",
|
||||
"MemUsage": "65.04MiB / 768MiB",
|
||||
"MemPerc": "8.47%",
|
||||
"PIDs": "5"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"revoked_certificate_serial": "D8C5BAB015148D0B982EEB5455810BC7",
|
||||
"worker_tokens_revoked": 7,
|
||||
"old_valid_token_denied_before_natural_expiry": true,
|
||||
"token_revoke_latency_seconds": 0.195,
|
||||
"passwords_rotated": [
|
||||
"PostgreSQL worker",
|
||||
"Redis worker",
|
||||
"S3 worker"
|
||||
],
|
||||
"new_leaf_certificate": true,
|
||||
"proxy_hard_stop_seconds": 5,
|
||||
"images_rebuilt": false,
|
||||
"seconds": 2.786
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
# Локальный runbook lab7
|
||||
|
||||
Операторские команды выполнять из корня lab7 с DOCKER_CONTEXT=colima-mtusi-labs на этом Mac.
|
||||
Не удалять volumes при restart/failover. Не передавать secrets/token/private keys в чат, Git или параметры команд.
|
||||
|
||||
## Запуск и доверие
|
||||
|
||||
python3 scripts/start.py выполняет local PKI/credentials generation, build, stores, explicit init/unseal и scoped policies, затем миграции/API/worker/LB.
|
||||
Внешний URL https://localhost:18443; доверенный публичный CA — pki/lb/ca.crt.
|
||||
curl --cacert pki/lb/ca.crt https://localhost:18443/health/ready.
|
||||
Для браузера нужен самостоятельный осознанный trust этого собственного CA; обход browser security warning в автоматизацию не входит.
|
||||
|
||||
После restart Bao возвращается sealed: python3 scripts/bao_operator.py --unseal-only.
|
||||
Операторские материалы только в operator/; серверу они не mounted. Generated datasources содержат Grafana client key и остаются в secrets/, вне Git.
|
||||
|
||||
## Сигналы и SLO
|
||||
|
||||
Resource service.name=messenger-api/worker/dispatcher, service.version=lab7.1, deployment.environment.name=lab, instance UUID.
|
||||
Запросы health исключены из business telemetry; IDs/filename/raw URLs/body не metric labels.
|
||||
Маппинг метрик и histograms сохранён из lab6: http_server_requests_total, http_server_duration_seconds_bucket,
|
||||
jobs_processed_total/job_processing_duration_seconds_bucket/job_ready_duration_seconds_bucket,
|
||||
queue_depth/queue_oldest_age_seconds/queue_observation_ok, db_pool_used/waiting/capacity, worker_busy/capacity, process_cpu_seconds_total/process_resident_memory_bytes/container_memory_bytes.
|
||||
SDK1.45.2 → HTTPS OTLP → Collector → TLS backends. Grafana UID messenger-lab7.
|
||||
|
||||
SLI availability: 1−5xx/все обработанные business requests за5мин; цель≥99%.4xx считаются обработанными, health не входят; LB/network отказы до API этим SLI не покрываются.
|
||||
Latency SLO:≥95% завершённых ready jobs за5мин имеют acceptance→ready≤5сек. Это условная latency, не pipeline availability.
|
||||
Для общего времени агрегировать histogram buckets всех replicas, затем histogram_quantile; не усреднять p95 реплик.
|
||||
Queued/failed показываются отдельно: backlog — не processing latency.
|
||||
|
||||
Prometheus rules/evaluation2s:
|
||||
- APIHighErrorRate: доля5xx за30s>5%, for6s.
|
||||
- QueueLag: oldest>10s и depth>0, for6s.
|
||||
- QueueObservationUnavailable: observer0, for6s.
|
||||
Это учебные operational alerts; burn-rate track и внешняя доставка не заявлены.
|
||||
Prometheus/Loki/Tempo history24h, named volumes; Grafana state/provisioning сохраняется.
|
||||
Лимиты Collector/SDK конечны как в lab6; telemetry не участвует в бизнес-COMMIT, при outage возможны потери.
|
||||
|
||||
## Диагностика
|
||||
|
||||
503 после20s sealed Bao: проверить /v1/sys/seal-status через operator cert, затем explicit unseal.
|
||||
Ошибка не решается отключением TLS или вечным cache. После unseal проверить dispatcher/worker и новый job; sealed_bao.py делает именно это.
|
||||
Worker stop: растут queue_depth/oldest; восстановление worker должно уменьшить backlog и вернуть ready.
|
||||
SQL lock: HTTP503 сопровождается длинным/error db.transaction span, отличает PG от renderer.
|
||||
TLS/identity error: positive CA+hostname probe должен работать, затем no-cert/foreign CA/outsider тесты обязаны отказать.
|
||||
Проверить notAfter и server SAN, CN role/URI SAN, exact leaf pin в Bao, serial denylist. Не считать network timeout доказательством правильного TLS отказа.
|
||||
Остановленный worker gauge может оставаться до30s expiration Collector; pool statistics относятся только к API pool, не всем PG connections.
|
||||
|
||||
## Отзыв и renewal
|
||||
|
||||
python3 scripts/rotate_worker.py — explicit operator incident action: revoke token accessors, serial, PG/Redis/S3 keys и leaf; поддерживает durable backlog, без rebuild.
|
||||
python3 student-tests/compromise_research.py — тот же action с controlled workload и allow/deny evidence.
|
||||
Leaf key — bootstrap identity на1сутки. Для планового renewal других ролей оператор вызывает scripts/pki.py issue через Python, затем обновляет Bao pins и перезапускает/перезагружает соответствующие компоненты в окне обслуживания.
|
||||
API читает leaf при новом TLS accept; proxy reload через SIGUSR2, старые connections hard-stop≤5s; PG server key копируется при старте и требует restart PG.
|
||||
Автоматическое CA rollover/безотказное renewal всех ролей не реализовано (это был бы трек2). CA private key никогда не передаётся сервису для самостоятельного unlimited issuance.
|
||||
|
||||
|
||||
### Плановое продление leaf (с окном недоступности)
|
||||
|
||||
Если CA ещё действителен, оператор останавливает только этот проект, перевыпускает leaf и выполняет обычный start (он обновит exact pins и generated Grafana provisioning). Пароли/volumes не сбрасываются; serial denylist сохраняется при перевыпуске.
|
||||
|
||||
```sh
|
||||
docker compose stop
|
||||
python3 - <<'PY'
|
||||
import sys
|
||||
sys.path.insert(0,'scripts')
|
||||
from pki import issue
|
||||
for role in ['lb','api','worker','dispatcher','operator','s3','broker','redis','otel-collector','collector','prometheus','loki','tempo','grafana','bao','postgres','outsider']:
|
||||
issue(role)
|
||||
PY
|
||||
python3 scripts/start.py
|
||||
```
|
||||
|
||||
Проверить `openssl x509 -in operator/ca.crt -noout -enddate` заранее. Эта процедура не является CA rollover и не решает expiry самого CA. Перед его заменой нужен отдельный план миграции доверия/backup, без автоматического удаления данных.
|
||||
@@ -0,0 +1,27 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
rules:
|
||||
- alert: APIHighErrorRate
|
||||
expr: sum(rate(http_server_requests_total{service_name="messenger-api",status_class="5xx"}[30s])) / clamp_min(sum(rate(http_server_requests_total{service_name="messenger-api"}[30s])), 0.001) > 0.05
|
||||
for: 6s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: API 5xx share exceeds 5 percent in the last 30 seconds
|
||||
runbook: infra/RUNBOOK.md#api-errors
|
||||
- alert: QueueLag
|
||||
expr: max(queue_oldest_age_seconds{service_name="messenger-dispatcher"}) > 10 and max(queue_depth{service_name="messenger-dispatcher"}) > 0
|
||||
for: 6s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Accepted unfinished image work older than 10 seconds
|
||||
runbook: infra/RUNBOOK.md#queue-lag
|
||||
- alert: QueueObservationUnavailable
|
||||
expr: min(queue_observation_ok{service_name="messenger-dispatcher"}) == 0
|
||||
for: 6s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Dispatcher cannot observe PostgreSQL backlog
|
||||
runbook: infra/RUNBOOK.md#api-errors
|
||||
@@ -0,0 +1,713 @@
|
||||
{
|
||||
"uid": "messenger-lab7",
|
||||
"title": "MTUSI lab7 \u2014 TLS messenger security and observability",
|
||||
"schemaVersion": 41,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"tags": [
|
||||
"mtusi",
|
||||
"lab6"
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API throughput (health excluded)",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API 5xx ratio",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(http_server_requests_total{service_name=\"messenger-api\",status_class=\"5xx\"}[1m])) / clamp_min(sum(rate(http_server_requests_total{service_name=\"messenger-api\"}[1m])),0.001)",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API p50 from histogram",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "API p95 from histogram",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "API p99 from histogram",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_duration_seconds_bucket{service_name=\"messenger-api\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker jobs by outcome",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (outcome) (rate(jobs_processed_total{service_name=\"messenger-worker\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Worker processing p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_processing_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Accepted to ready p95",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(job_ready_duration_seconds_bucket{service_name=\"messenger-worker\"}[1m])))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Accepted unfinished jobs",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_depth{service_name=\"messenger-dispatcher\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Oldest unfinished job age",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_oldest_age_seconds{service_name=\"messenger-dispatcher\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "DB pool used",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name) (db_pool_used{service_name=\"messenger-api\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "DB pool waiting",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name) (db_pool_waiting{service_name=\"messenger-api\"})",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Worker busy / capacity",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(worker_busy{service_name=\"messenger-worker\"}) / clamp_min(sum(worker_capacity{service_name=\"messenger-worker\"}),1)",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Process resident memory",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "process_resident_memory_bytes{service_name=~\"messenger-.*\"}",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Process CPU seconds per second",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(service_name)(rate(process_cpu_seconds_total{service_name=~\"messenger-.*\"}[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Container memory incl renderer",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "container_memory_bytes{service_name=~\"messenger-.*\"}",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Local alerts",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "ALERTS{alertstate=\"firing\"}",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"title": "Collector failed span exports",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans_total[1m]))",
|
||||
"legendFormat": "{{service_name}} {{status_class}} {{outcome}}"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 100,
|
||||
"title": "Correlated structured logs: open TraceID for API / publish / worker chain",
|
||||
"type": "logs",
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 72,
|
||||
"w": 24,
|
||||
"h": 10
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-.*\"} | json",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": false,
|
||||
"sortOrder": "Descending"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 101,
|
||||
"title": "Terminal failed jobs",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 82,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"failed\"}[1m]))"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 102,
|
||||
"title": "Retry attempts",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 82,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(job_retries_total{service_name=\"messenger-worker\",outcome=\"retry\"}[1m]))"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: Messenger
|
||||
folder: MTUSI
|
||||
type: file
|
||||
updateIntervalSeconds: 10
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,46 @@
|
||||
global
|
||||
maxconn 1024
|
||||
nbthread 1
|
||||
hard-stop-after 10s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
timeout connect 500ms
|
||||
timeout check 500ms
|
||||
timeout queue 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 3s
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option http-server-close
|
||||
|
||||
resolvers docker
|
||||
nameserver docker_dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold nx 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
hold other 1s
|
||||
hold obsolete 1s
|
||||
|
||||
frontend public
|
||||
bind :8443 ssl crt /tls/tls.pem ssl-min-ver TLSv1.2
|
||||
http-request set-var(txn.request_id) uuid()
|
||||
acl oversized req.hdr_val(Content-Length) gt 10485760
|
||||
http-request return status 413 content-type application/json lf-string '{"error":{"code":"payload_too_large","message":"Maximum 10 MiB","request_id":"%[var(txn.request_id)]"}}' hdr X-Request-Id %[var(txn.request_id)] hdr X-Instance-Id tls-ingress if oversized
|
||||
acl unavailable nbsrv(replicas) lt 1
|
||||
http-request return status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No healthy API","request_id":"%[var(txn.request_id)]"}}' hdr X-Request-Id %[var(txn.request_id)] hdr X-Instance-Id tls-ingress if unavailable
|
||||
default_backend replicas
|
||||
|
||||
backend replicas
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fall 2 rise 2
|
||||
server-template api 1-4 api:8080 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required verifyhost api sni str(api) check check-sni api resolvers docker init-addr none resolve-prefer ipv4
|
||||
@@ -0,0 +1,36 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
http_listen_address: 127.0.0.1
|
||||
grpc_listen_address: 127.0.0.1
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 24h
|
||||
ingestion_rate_mb: 4
|
||||
ingestion_burst_size_mb: 8
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,21 @@
|
||||
ui = false
|
||||
api_addr = "https://bao:8200"
|
||||
cluster_addr = "https://bao:8201"
|
||||
storage "file" {
|
||||
path = "/bao/data"
|
||||
}
|
||||
listener "tcp" {
|
||||
address = "0.0.0.0:8200"
|
||||
tls_cert_file = "/tls/tls.crt"
|
||||
tls_key_file = "/tls/tls.key"
|
||||
tls_client_ca_file = "/tls/ca.crt"
|
||||
tls_require_and_verify_client_cert = true
|
||||
tls_min_version = "tls12"
|
||||
}
|
||||
|
||||
audit "file" "file" {
|
||||
options = {
|
||||
file_path = "/bao/data/audit.json"
|
||||
log_raw = "false"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
path "secret/data/api" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
path "secret/data/dispatcher" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
path "secret/data/worker" {
|
||||
capabilities = ["read"]
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 128
|
||||
send_batch_max_size: 256
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 127.0.0.1:8889
|
||||
metric_expiration: 30s
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: https://tempo:8444
|
||||
tls:
|
||||
ca_file: /client/ca.crt
|
||||
cert_file: /client/tls.crt
|
||||
key_file: /client/tls.key
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 128
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 3s
|
||||
max_elapsed_time: 10s
|
||||
otlphttp/loki:
|
||||
endpoint: https://loki:8443/otlp
|
||||
tls:
|
||||
ca_file: /client/ca.crt
|
||||
cert_file: /client/tls.crt
|
||||
key_file: /client/tls.key
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 128
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 3s
|
||||
max_elapsed_time: 10s
|
||||
service:
|
||||
telemetry:
|
||||
metrics:
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 127.0.0.1
|
||||
port: 8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,3 @@
|
||||
local all all trust
|
||||
hostnossl all all 0.0.0.0/0 reject
|
||||
hostssl all all 0.0.0.0/0 scram-sha-256
|
||||
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cp /tls/tls.key /tmp/postgres.key
|
||||
chmod 600 /tmp/postgres.key
|
||||
chown postgres:postgres /tmp/postgres.key
|
||||
exec docker-entrypoint.sh postgres -c ssl=on -c ssl_cert_file=/tls/tls.crt -c ssl_key_file=/tmp/postgres.key -c hba_file=/etc/postgresql/pg_hba.conf
|
||||
@@ -0,0 +1,21 @@
|
||||
global:
|
||||
scrape_interval: 2s
|
||||
evaluation_interval: 2s
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /tls/ca.crt
|
||||
cert_file: /tls/tls.crt
|
||||
key_file: /tls/tls.key
|
||||
static_configs:
|
||||
- targets: [otel-collector:8444]
|
||||
- job_name: collector
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /tls/ca.crt
|
||||
cert_file: /tls/tls.crt
|
||||
key_file: /tls/tls.key
|
||||
static_configs:
|
||||
- targets: [otel-collector:8445]
|
||||
@@ -0,0 +1,26 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
http_listen_address: 127.0.0.1
|
||||
grpc_listen_address: 127.0.0.1
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 127.0.0.1:4318
|
||||
ingester:
|
||||
trace_idle_period: 5s
|
||||
max_block_duration: 1m
|
||||
max_block_bytes: 1048576
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 24h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,19 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode tcp
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str worker dispatcher operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
server local 127.0.0.1:6379
|
||||
@@ -0,0 +1,23 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode http
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str collector grafana operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
acl reader ssl_c_s_dn(CN) -m str grafana
|
||||
acl read method GET
|
||||
http-request deny deny_status 403 if reader !read
|
||||
server local 127.0.0.1:3100
|
||||
@@ -0,0 +1,40 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode http
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str api worker dispatcher operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
server local 127.0.0.1:4318
|
||||
|
||||
listen tls1
|
||||
mode http
|
||||
bind :8444 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str prometheus operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
server local 127.0.0.1:8889
|
||||
|
||||
listen tls2
|
||||
mode http
|
||||
bind :8445 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str prometheus operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
server local 127.0.0.1:8888
|
||||
@@ -0,0 +1,22 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode http
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str grafana operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
acl read method GET
|
||||
http-request deny deny_status 403 if !read
|
||||
server local 127.0.0.1:9090
|
||||
@@ -0,0 +1,19 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode tcp
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str api operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
server local 127.0.0.1:6379
|
||||
@@ -0,0 +1,45 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode http
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str api worker operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
acl encoded path_sub %
|
||||
acl traversal path_sub ..
|
||||
http-request deny deny_status 403 if encoded
|
||||
http-request deny deny_status 403 if traversal
|
||||
acl immutable req.hdr(If-None-Match) -m str *
|
||||
acl operator ssl_c_s_dn(CN) -m str operator
|
||||
acl api ssl_c_s_dn(CN) -m str api
|
||||
acl worker ssl_c_s_dn(CN) -m str worker
|
||||
acl bucket path -m str /messenger-attachments /messenger-attachments/
|
||||
acl staging path_beg /messenger-attachments/staging/
|
||||
acl results path_beg /messenger-attachments/results/
|
||||
acl safe_read method GET HEAD
|
||||
acl head method HEAD
|
||||
acl put method PUT
|
||||
acl delete method DELETE
|
||||
http-request allow if operator
|
||||
http-request allow if head bucket
|
||||
http-request allow if api safe_read staging
|
||||
http-request allow if api safe_read results
|
||||
http-request allow if api put staging
|
||||
http-request allow if api delete staging
|
||||
http-request allow if worker safe_read staging
|
||||
http-request allow if worker safe_read results
|
||||
http-request allow if worker put results immutable
|
||||
http-request deny deny_status 403
|
||||
server local 127.0.0.1:8333
|
||||
@@ -0,0 +1,30 @@
|
||||
global
|
||||
maxconn 128
|
||||
nbthread 1
|
||||
hard-stop-after 5s
|
||||
|
||||
defaults
|
||||
timeout connect 2s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
timeout http-request 5s
|
||||
|
||||
listen tls0
|
||||
mode http
|
||||
bind :8443 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str grafana operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
server local 127.0.0.1:3200
|
||||
|
||||
listen tls1
|
||||
mode http
|
||||
bind :8444 ssl crt /tls/tls.pem ca-file /tls/ca.crt verify required ssl-min-ver TLSv1.2
|
||||
acl allowed ssl_c_s_dn(CN) -m str collector operator
|
||||
acl revoked ssl_c_serial,hex -i -f /tls/revoked.list
|
||||
tcp-request content reject if !allowed
|
||||
tcp-request content reject if revoked
|
||||
option http-server-close
|
||||
server local 127.0.0.1:4318
|
||||
@@ -0,0 +1,42 @@
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY
|
||||
, username TEXT NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$')
|
||||
, display_name TEXT NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
, password_hash TEXT NOT NULL
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
;
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY
|
||||
, title TEXT NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, last_message_at TIMESTAMPTZ
|
||||
)
|
||||
;
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, user_id UUID NOT NULL REFERENCES users(id)
|
||||
, PRIMARY KEY (chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX chat_members_user ON chat_members (user_id, chat_id)
|
||||
;
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, sender_id UUID NOT NULL REFERENCES users(id)
|
||||
, text TEXT NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
, created_at TIMESTAMPTZ NOT NULL
|
||||
, FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id)
|
||||
;
|
||||
CREATE TABLE sessions (
|
||||
token_hash TEXT PRIMARY KEY
|
||||
, user_id UUID NOT NULL REFERENCES users(id)
|
||||
, expires_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
;
|
||||
CREATE INDEX sessions_expiry ON sessions (expires_at)
|
||||
;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- PostgreSQL TEXT cannot contain U+0000. Store validated UTF-8 losslessly in BYTEA.
|
||||
-- API validation keeps the contract's code-point limits; these bounds cap bytes.
|
||||
ALTER TABLE users
|
||||
DROP CONSTRAINT users_display_name_check
|
||||
, ALTER COLUMN display_name TYPE BYTEA USING convert_to(display_name, 'UTF8')
|
||||
, ADD CONSTRAINT users_display_name_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400)
|
||||
;
|
||||
ALTER TABLE chats
|
||||
DROP CONSTRAINT chats_title_check
|
||||
, ALTER COLUMN title TYPE BYTEA USING convert_to(title, 'UTF8')
|
||||
, ADD CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400)
|
||||
;
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_check
|
||||
, ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8')
|
||||
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000)
|
||||
;
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id)
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, key VARCHAR(128) NOT NULL
|
||||
, request_hash CHAR(64) NOT NULL
|
||||
, message_id UUID REFERENCES messages(id)
|
||||
, expires_at TIMESTAMPTZ NOT NULL
|
||||
, PRIMARY KEY (user_id, chat_id, key)
|
||||
)
|
||||
;
|
||||
CREATE INDEX idempotency_expiry ON idempotency_keys (expires_at)
|
||||
;
|
||||
@@ -0,0 +1,30 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_bytes
|
||||
, ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000)
|
||||
, ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}'
|
||||
;
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY
|
||||
, chat_id UUID NOT NULL REFERENCES chats(id)
|
||||
, owner_id UUID NOT NULL REFERENCES users(id)
|
||||
, filename TEXT NOT NULL
|
||||
, object_key TEXT NOT NULL UNIQUE
|
||||
, content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg'))
|
||||
, size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760)
|
||||
, sha256 CHAR(64) NOT NULL
|
||||
, state TEXT NOT NULL CHECK (state IN ('uploading', 'ready', 'storage_missing'))
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX attachments_chat ON attachments (chat_id, id)
|
||||
;
|
||||
CREATE TABLE message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id)
|
||||
, attachment_id UUID NOT NULL REFERENCES attachments(id)
|
||||
, PRIMARY KEY (message_id, attachment_id)
|
||||
)
|
||||
;
|
||||
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id)
|
||||
;
|
||||
@@ -0,0 +1,40 @@
|
||||
ALTER TABLE attachments
|
||||
DROP CONSTRAINT attachments_state_check
|
||||
, ADD CONSTRAINT attachments_state_check CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed', 'storage_missing'))
|
||||
, ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1
|
||||
, ADD COLUMN staging_key TEXT
|
||||
, ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'
|
||||
, ADD COLUMN error_code TEXT
|
||||
, ADD COLUMN current_job_id UUID
|
||||
;
|
||||
CREATE TABLE attachment_jobs (
|
||||
id UUID PRIMARY KEY
|
||||
, attachment_id UUID NOT NULL REFERENCES attachments(id)
|
||||
, pipeline_version INTEGER NOT NULL
|
||||
, source_key TEXT NOT NULL
|
||||
, state TEXT NOT NULL DEFAULT 'queued' CHECK (state IN ('queued', 'processing', 'ready', 'failed', 'superseded'))
|
||||
, attempt INTEGER NOT NULL DEFAULT 0
|
||||
, max_attempts INTEGER NOT NULL DEFAULT 5
|
||||
, lease_token UUID
|
||||
, lease_until TIMESTAMPTZ
|
||||
, next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, published_at TIMESTAMPTZ
|
||||
, error_code TEXT
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
, UNIQUE (attachment_id, pipeline_version)
|
||||
)
|
||||
;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_current_job FOREIGN KEY (current_job_id) REFERENCES attachment_jobs(id)
|
||||
;
|
||||
CREATE INDEX jobs_due ON attachment_jobs (state, next_attempt_at)
|
||||
;
|
||||
CREATE TABLE job_events (
|
||||
id BIGSERIAL PRIMARY KEY
|
||||
, job_id UUID NOT NULL REFERENCES attachment_jobs(id)
|
||||
, event TEXT NOT NULL
|
||||
, attempt INTEGER NOT NULL
|
||||
, created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
;
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN trace_context JSONB NOT NULL DEFAULT '{}'
|
||||
;
|
||||
@@ -0,0 +1,22 @@
|
||||
CREATE TABLE auth_rate_limits (
|
||||
action TEXT PRIMARY KEY
|
||||
, bucket_start BIGINT NOT NULL
|
||||
, attempts INTEGER NOT NULL
|
||||
)
|
||||
;
|
||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC
|
||||
;
|
||||
REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC
|
||||
;
|
||||
GRANT USAGE ON SCHEMA public TO api_role, worker_role, dispatcher_role
|
||||
;
|
||||
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO api_role
|
||||
;
|
||||
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO api_role
|
||||
;
|
||||
GRANT SELECT, UPDATE ON attachments, attachment_jobs TO worker_role, dispatcher_role
|
||||
;
|
||||
GRANT INSERT, SELECT ON job_events TO worker_role, dispatcher_role
|
||||
;
|
||||
GRANT USAGE, SELECT ON SEQUENCE job_events_id_seq TO worker_role, dispatcher_role
|
||||
;
|
||||
@@ -0,0 +1,2 @@
|
||||
DROP ROLE IF EXISTS db_baseline
|
||||
;
|
||||
@@ -0,0 +1,13 @@
|
||||
REVOKE UPDATE ON attachments, attachment_jobs FROM worker_role, dispatcher_role
|
||||
;
|
||||
GRANT UPDATE (state, object_key, variants, error_code, updated_at) ON attachments TO worker_role
|
||||
;
|
||||
GRANT UPDATE (state, error_code, updated_at) ON attachments TO dispatcher_role
|
||||
;
|
||||
GRANT UPDATE (state, attempt, lease_token, lease_until, next_attempt_at, published_at, error_code, updated_at)
|
||||
ON attachment_jobs TO worker_role, dispatcher_role
|
||||
;
|
||||
REVOKE ALL ON schema_migrations FROM api_role
|
||||
;
|
||||
GRANT SELECT ON schema_migrations TO api_role
|
||||
;
|
||||
@@ -0,0 +1,2 @@
|
||||
REVOKE TEMPORARY ON DATABASE messenger FROM PUBLIC
|
||||
;
|
||||
@@ -0,0 +1,33 @@
|
||||
argon2-cffi-bindings==25.1.0
|
||||
argon2-cffi==25.1.0
|
||||
boto3==1.40.61
|
||||
botocore==1.40.76
|
||||
certifi==2026.10.23
|
||||
cffi==2.0.0
|
||||
charset-normalizer==3.5.2
|
||||
googleapis-common-protos==1.76.0
|
||||
idna==3.20
|
||||
importlib-metadata==8.7.1
|
||||
jmespath==1.1.0
|
||||
opentelemetry-api==1.45.2
|
||||
opentelemetry-exporter-http-transport==0.66b2
|
||||
opentelemetry-exporter-otlp-common==0.66b2
|
||||
opentelemetry-exporter-otlp-proto-common==1.45.2
|
||||
opentelemetry-exporter-otlp-proto-http==1.45.2
|
||||
opentelemetry-proto==1.45.2
|
||||
opentelemetry-sdk==1.45.2
|
||||
opentelemetry-semantic-conventions==0.66b2
|
||||
pillow==13.0.0
|
||||
protobuf==6.33.6
|
||||
psycopg-binary==3.2.3
|
||||
psycopg-pool==3.2.4
|
||||
psycopg==3.2.3
|
||||
pycparser==2.23
|
||||
python-dateutil==2.9.0.post0
|
||||
redis==5.2.1
|
||||
requests==2.34.2
|
||||
s3transfer==0.14.0
|
||||
six==1.17.0
|
||||
typing-extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
zipp==4.1.2
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Explicit operator init/unseal/configure. Root and shares only on protected host."""
|
||||
import json
|
||||
import ssl
|
||||
from pathlib import Path
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import os
|
||||
from pki import ROOT
|
||||
|
||||
URL=os.environ.get('BAO_OPERATOR_URL','https://localhost:18207')
|
||||
def call(path,data=None,token=None,method=None):
|
||||
ctx=ssl.create_default_context(cafile=str(ROOT/'operator/ca.crt'))
|
||||
ctx.load_cert_chain(str(ROOT/'pki/operator/tls.crt'),str(ROOT/'pki/operator/tls.key'))
|
||||
headers={'Content-Type':'application/json'}
|
||||
if token:headers['X-Vault-Token']=token
|
||||
req=urllib.request.Request(URL+'/v1/'+path,data=None if data is None else json.dumps(data).encode(),headers=headers,method=method)
|
||||
op=urllib.request.build_opener(urllib.request.ProxyHandler({}),urllib.request.HTTPSHandler(context=ctx))
|
||||
with op.open(req,timeout=60 if path in ["sys/init","sys/unseal"] else 5) as r:return json.load(r) if r.status!=204 else {}
|
||||
|
||||
def unseal():
|
||||
init=ROOT/'operator/bao-init.json'
|
||||
if not call('sys/init')['initialized']:
|
||||
if init.exists():
|
||||
import shutil
|
||||
shutil.copyfile(init,ROOT/'operator/bao-init.previous.json')
|
||||
(ROOT/'operator/bao-init.previous.json').chmod(0o600)
|
||||
info=call('sys/init',{'secret_shares':3,'secret_threshold':2})
|
||||
init.write_text(json.dumps(info));init.chmod(0o600)
|
||||
data=json.loads(init.read_text())
|
||||
if call('sys/seal-status')['sealed']:
|
||||
for key in data['keys'][:2]:call('sys/unseal',{'key':key})
|
||||
return data['root_token']
|
||||
|
||||
def configure():
|
||||
root=unseal()
|
||||
mounts=call('sys/mounts',token=root)
|
||||
if 'secret/' not in mounts:call('sys/mounts/secret',{'type':'kv','options':{'version':'2'}},root)
|
||||
auth=call('sys/auth',token=root)
|
||||
if 'cert/' not in auth:call('sys/auth/cert',{'type':'cert'},root)
|
||||
audits=call('sys/audit',token=root)
|
||||
assert 'file/' in audits, 'Declarative audit device must be enabled'
|
||||
settings=json.loads((ROOT/'operator/settings.json').read_text())
|
||||
for role in ['api','worker','dispatcher']:
|
||||
policy=(ROOT/'infra/openbao/policies'/(role+'.hcl')).read_text()
|
||||
call('sys/policies/acl/'+role,{'policy':policy},root)
|
||||
# Exact leaf pinning: another valid certificate from the same CA cannot impersonate this service.
|
||||
call('auth/cert/certs/'+role,{'certificate':(ROOT/'pki'/role/'tls.crt').read_text(),'token_policies':[role],'token_ttl':'20s','token_max_ttl':'20s','token_no_default_policy':True},root)
|
||||
db='postgresql://db_'+role+':'+settings[role]+'@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/tls/ca.crt'
|
||||
values={'DATABASE_URL':db}
|
||||
if role=='api':
|
||||
values['CACHE_URL']='rediss://api:'+settings['cache']+'@redis:8443/0'
|
||||
else:
|
||||
values['BROKER_URL']='rediss://'+role+':'+settings['broker_'+role]+'@broker:8443/0'
|
||||
if role in ['api','worker']:
|
||||
values['S3_ACCESS_KEY_ID']=settings['s3_'+role+'_key'];values['S3_SECRET_ACCESS_KEY']=settings['s3_'+role]
|
||||
call('secret/data/'+role,{'data':values},root)
|
||||
print('OpenBao initialized/unsealed by explicit operator action; scoped service policies and 20s certificate-auth tokens configured.')
|
||||
|
||||
if __name__=='__main__':
|
||||
import sys
|
||||
if '--unseal-only' in sys.argv:
|
||||
import time
|
||||
for _ in range(60):
|
||||
try:call('sys/init');break
|
||||
except Exception:time.sleep(.3)
|
||||
unseal();print('OpenBao unsealed by operator.')
|
||||
else:configure()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Local secrets/PKI generation, idempotent. Never prints secret values."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
from pki import ROOT,bootstrap
|
||||
|
||||
def main():
|
||||
bootstrap()
|
||||
path=ROOT/'operator/settings.json'
|
||||
if path.exists():data=json.loads(path.read_text())
|
||||
else:
|
||||
data={role:secrets.token_hex(24) for role in ['postgres','api','worker','dispatcher','cache','broker_worker','broker_dispatcher','broker_operator','grafana','s3_api','s3_worker','s3_admin']}
|
||||
data.update({role+'_key':secrets.token_hex(12) for role in ['s3_api','s3_worker','s3_admin']})
|
||||
path.write_text(json.dumps(data));path.chmod(0o600)
|
||||
sql=[]
|
||||
for role in ['api','worker','dispatcher']:
|
||||
sql.append('CREATE ROLE '+role+'_role NOLOGIN;')
|
||||
sql.append("CREATE ROLE db_"+role+" LOGIN PASSWORD '"+data[role]+"' IN ROLE "+role+"_role;")
|
||||
(ROOT/'secrets/pg-init.sql').write_text('\n'.join(sql)+'\n')
|
||||
if 'broker_operator' not in data:
|
||||
data['broker_operator']=secrets.token_hex(24)
|
||||
path.write_text(json.dumps(data,indent=2)+'\n')
|
||||
path.chmod(0o600)
|
||||
(ROOT/'secrets/postgres_password').write_text(data['postgres'])
|
||||
identities=[]
|
||||
for role in ['api','worker','admin']:
|
||||
identities.append({'name':role,'credentials':[{'accessKey':data['s3_'+role+'_key'],'secretKey':data['s3_'+role]}],
|
||||
'actions':['Admin','Read','Write','List'] if role=='admin' else ['Read:messenger-attachments','Write:messenger-attachments']})
|
||||
# Prefix and verb authorization is additionally enforced at the mTLS S3 gateway.
|
||||
(ROOT/'secrets/s3_config').write_text(json.dumps({'identities':identities}))
|
||||
(ROOT/'secrets/redis.conf').write_text('bind 127.0.0.1\nport 6379\nprotected-mode yes\nsave ""\nappendonly no\nmaxmemory 32mb\nmaxmemory-policy allkeys-lru\nuser default off\nuser api on >'+data['cache']+' ~user:* +get +setex +ping +client|setinfo\n')
|
||||
(ROOT/'secrets/broker.conf').write_text('bind 127.0.0.1\nport 6379\nprotected-mode yes\nsave ""\nappendonly yes\nappendfsync always\nmaxmemory 96mb\nmaxmemory-policy noeviction\nuser default off\nuser dispatcher on >'+data['broker_dispatcher']+' ~attachment-jobs +ping +xadd +xgroup|create +client|setinfo\nuser worker on >'+data['broker_worker']+' ~attachment-jobs ~rejected-jobs +ping +xgroup|create +xautoclaim +xreadgroup +xadd +xack +xdel +eval +client|setinfo\nuser operator on >'+data['broker_operator']+' ~* +@all\n')
|
||||
url='postgresql://messenger:'+data['postgres']+'@postgres:5432/messenger?sslmode=verify-full&sslrootcert=/tls/ca.crt'
|
||||
(ROOT/'operator/database_url').write_text(url)
|
||||
(ROOT/'operator/broker_url').write_text('rediss://operator:'+data['broker_operator']+'@broker:8443/0')
|
||||
(ROOT/'secrets/migration_database_url').write_text(url)
|
||||
(ROOT/'secrets/grafana_admin').write_text(data['grafana'])
|
||||
(ROOT/'secrets/s3_admin_key').write_text(data['s3_admin_key'])
|
||||
(ROOT/'secrets/s3_admin_secret').write_text(data['s3_admin'])
|
||||
for f in (ROOT/'secrets').iterdir():f.chmod(0o644)
|
||||
old=ROOT/'operator/revoked-worker-settings.json'
|
||||
if not old.exists():old.write_text('{}')
|
||||
for f in (ROOT/'operator').iterdir():
|
||||
if f.is_file():f.chmod(0o600)
|
||||
# Native TLS client provisioning; private keys only in ignored generated configuration.
|
||||
sources=[]
|
||||
for name,url in [('Prometheus','https://prometheus:8443'),('Loki','https://loki:8443'),('Tempo','https://tempo:8443')]:
|
||||
options={'tlsAuth':True,'tlsAuthWithCACert':True,'tlsSkipVerify':False}
|
||||
if name=='Loki':
|
||||
options['derivedFields']=[{'name':'TraceID','datasourceUid':'tempo','matcherRegex':'"trace_id":"([a-f0-9]{32})"','url':'$${__value.raw}'}]
|
||||
if name=='Tempo':
|
||||
options['tracesToLogsV2']={'datasourceUid':'loki','spanStartTimeShift':'-1m','spanEndTimeShift':'1m','customQuery':True,'query':'{service_name=~"messenger-.*"} | json | trace_id="$${__trace.traceId}"'}
|
||||
sources.append({'name':name,'uid':name.lower(),'type':name.lower(),'access':'proxy','url':url,'jsonData':options,
|
||||
'secureJsonData':{key:(ROOT/'pki/grafana'/file).read_text() for key,file in [('tlsCACert','ca.crt'),('tlsClientCert','tls.crt'),('tlsClientKey','tls.key')]}})
|
||||
(ROOT/'secrets/grafana-datasources.yaml').write_text(json.dumps({'apiVersion':1,'datasources':sources},indent=2))
|
||||
env=ROOT/'.env'
|
||||
if not env.exists():env.write_text((ROOT/'.env.example').read_text());env.chmod(0o600)
|
||||
print('Local service configuration generated; no credentials printed.')
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Operator-only local PKI. Private CA never mounted into any service."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import subprocess
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
def command(*args):
|
||||
subprocess.run(['openssl',*map(str,args)],check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
|
||||
def issue(role,days=1,foreign=False,identity=None,uri_identity=None):
|
||||
name=identity or role
|
||||
folder=ROOT/'pki'/role
|
||||
folder.mkdir(parents=True,exist_ok=True)
|
||||
ca=ROOT/'operator'/('foreign' if foreign else 'ca')
|
||||
config=folder/'request.cnf'
|
||||
config.write_text('[req]\nprompt=no\ndistinguished_name=dn\n[dn]\nCN='+name+'\n[ext]\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName=DNS:'+name+',DNS:localhost,IP:127.0.0.1,URI:spiffe://mtusi.lab/'+(uri_identity or name)+'\n')
|
||||
command('req','-new','-newkey','rsa:2048','-nodes','-keyout',folder/'tls.key','-out',folder/'request.csr','-config',config)
|
||||
if days:
|
||||
command('x509','-req','-in',folder/'request.csr','-CA',str(ca)+'.crt','-CAkey',str(ca)+'.key','-set_serial','0x'+secrets.token_hex(16),'-out',folder/'tls.crt','-days',days,'-sha256','-extfile',config,'-extensions','ext')
|
||||
else:
|
||||
op=ROOT/'operator'
|
||||
(op/'issued').mkdir(exist_ok=True)
|
||||
(op/'index.txt').touch()
|
||||
(op/'serial').write_text(secrets.token_hex(16)+'\n')
|
||||
cfg=op/'expired-ca.cnf'
|
||||
cfg.write_text('[ca]\ndefault_ca=local\n[local]\ndatabase='+str(op/'index.txt')+'\nserial='+str(op/'serial')+'\nnew_certs_dir='+str(op/'issued')+'\ncertificate='+str(ca)+'.crt\nprivate_key='+str(ca)+'.key\ndefault_md=sha256\npolicy=policy\n[policy]\ncommonName=supplied\n')
|
||||
command('ca','-batch','-notext','-config',cfg,'-in',folder/'request.csr','-out',folder/'tls.crt','-startdate','20000101000000Z','-enddate','20000102000000Z','-extfile',config,'-extensions','ext')
|
||||
|
||||
(folder/'tls.pem').write_bytes((folder/'tls.crt').read_bytes()+(folder/'tls.key').read_bytes())
|
||||
(folder/'ca.crt').write_bytes((ROOT/'operator/ca.crt').read_bytes())
|
||||
if not (folder/'revoked.list').exists():
|
||||
(folder/'revoked.list').write_text('00\n')
|
||||
# Parent pki is 0700 on host; a service mounts only its own folder.
|
||||
for f in folder.iterdir():f.chmod(0o644)
|
||||
return folder
|
||||
|
||||
def bootstrap():
|
||||
for n in ['operator','pki','secrets']:
|
||||
f=ROOT/n;f.mkdir(exist_ok=True);f.chmod(0o700)
|
||||
for role in ['ca','foreign']:
|
||||
base=ROOT/'operator'/role
|
||||
if not base.with_suffix('.key').exists():
|
||||
config=ROOT/'operator'/(role+'.cnf')
|
||||
config.write_text('[req]\nprompt=no\ndistinguished_name=dn\nx509_extensions=ext\n[dn]\nCN=MTUSI-LAB7-'+role+'\n[ext]\nbasicConstraints=critical,CA:TRUE\nkeyUsage=critical,keyCertSign,cRLSign\n')
|
||||
command('req','-x509','-newkey','rsa:3072','-nodes','-sha256','-days','30','-keyout',str(base)+'.key','-out',str(base)+'.crt','-config',config)
|
||||
base.with_suffix('.key').chmod(0o600)
|
||||
for role in ['lb','api','worker','dispatcher','operator','s3','broker','redis','otel-collector','collector','prometheus','loki','tempo','grafana','bao','postgres','outsider']:
|
||||
if not (ROOT/'pki'/role/'tls.key').exists():issue(role)
|
||||
if not (ROOT/'pki/foreign-client/tls.key').exists():issue('foreign-client',foreign=True)
|
||||
for identity in ['lb','api','worker','dispatcher','collector','prometheus','grafana']:
|
||||
if not (ROOT/'pki'/('expired-'+identity)/'tls.crt').exists():issue('expired-'+identity,days=0,identity=identity)
|
||||
if not (ROOT/'pki/wrong-lb/tls.crt').exists():issue('wrong-lb',identity='lb',uri_identity='outsider')
|
||||
print('Local PKI ready; CA keys remain in operator/ and are not mounted into services.')
|
||||
|
||||
if __name__=='__main__':bootstrap()
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
python3 scripts/snapshot.py > evidence/restart-before.json
|
||||
docker compose stop api lb worker dispatcher
|
||||
# Keep proxy namespace anchors: restarting the raw processes preserves the local boundary.
|
||||
docker compose restart postgres broker-raw redis-raw s3-store s3-raw otel-collector-raw prometheus-raw loki-raw tempo-raw grafana bao
|
||||
python3 scripts/bao_operator.py --unseal-only
|
||||
docker compose up -d --no-deps api lb worker dispatcher
|
||||
python3 scripts/snapshot.py > evidence/restart-after.json
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Explicit operator revocation/rollover; no image rebuild and no root credential in apps."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import ssl
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from pki import ROOT,issue
|
||||
from bootstrap import main as bootstrap
|
||||
from bao_operator import URL,call,unseal,configure
|
||||
|
||||
def dc(*a,**kw):return subprocess.run(['docker','compose',*a],cwd=ROOT,check=True,**kw)
|
||||
def run():
|
||||
started=time.monotonic()
|
||||
dc('stop','worker')
|
||||
old=ROOT/'pki/revoked-worker';old.mkdir(exist_ok=True)
|
||||
for file in (ROOT/'pki/worker').iterdir():
|
||||
if file.is_file():shutil.copyfile(file,old/file.name)
|
||||
# Exact cert serial is public correlation data; keys and passwords remain local.
|
||||
serial=subprocess.check_output(['openssl','x509','-in',str(old/'tls.crt'),'-serial','-noout'],text=True).strip().split('=')[1]
|
||||
settings=json.loads((ROOT/'operator/settings.json').read_text())
|
||||
oldfile=ROOT/'operator/revoked-worker-settings.json';oldfile.write_text(json.dumps(settings));oldfile.chmod(0o600)
|
||||
root=unseal()
|
||||
def old_service(path,data=None,token=None):
|
||||
ctx=ssl.create_default_context(cafile=str(ROOT/'pki/worker/ca.crt'))
|
||||
ctx.load_cert_chain(str(ROOT/'pki/worker/tls.crt'),str(ROOT/'pki/worker/tls.key'))
|
||||
headers={'Content-Type':'application/json'}
|
||||
if token:headers['X-Vault-Token']=token
|
||||
req=urllib.request.Request(URL+'/v1/'+path,
|
||||
data=None if data is None else json.dumps(data).encode(),headers=headers)
|
||||
op=urllib.request.build_opener(urllib.request.ProxyHandler({}),urllib.request.HTTPSHandler(context=ctx))
|
||||
with op.open(req,timeout=5) as response:return json.load(response)
|
||||
auth=old_service('auth/cert/login',{'name':'worker'})['auth']
|
||||
old_token=auth['client_token']
|
||||
assert old_service('secret/data/worker',token=old_token)['data']['data']
|
||||
token_created=time.monotonic()
|
||||
call('auth/cert/certs/worker',token=root,method='DELETE')
|
||||
# Revoke all outstanding worker tokens instead of assuming deleting a role revokes tokens.
|
||||
revoked_tokens=0
|
||||
try:accessors=call('auth/token/accessors',token=root,method='LIST')['data']['keys']
|
||||
except Exception:accessors=[]
|
||||
for accessor in accessors:
|
||||
try:
|
||||
info=call('auth/token/lookup-accessor',{'accessor':accessor},root)
|
||||
if 'worker' in info['data']['policies']:
|
||||
call('auth/token/revoke-accessor',{'accessor':accessor},root);revoked_tokens+=1
|
||||
except Exception:pass # An already-expired token is not still usable.
|
||||
try:old_service('secret/data/worker',token=old_token)
|
||||
except urllib.error.HTTPError as error:assert error.code==403
|
||||
else:raise AssertionError('Outstanding worker token still reads secrets after revocation')
|
||||
revoke_latency=time.monotonic()-token_created
|
||||
assert revoke_latency < auth['lease_duration']-1, 'Prove revocation before natural token expiry'
|
||||
for folder in (ROOT/'pki').iterdir():
|
||||
listing=folder/'revoked.list'
|
||||
if listing.exists() and folder.name!='revoked-worker':
|
||||
rows=set(listing.read_text().splitlines());rows.add(serial.lower());listing.write_text('\n'.join(sorted(rows))+'\n')
|
||||
for proxy in ['broker','s3','otel-collector']:
|
||||
dc('kill','-s','SIGUSR2',proxy)
|
||||
settings['worker']=secrets.token_hex(24)
|
||||
settings['broker_worker']=secrets.token_hex(24)
|
||||
settings['s3_worker']=secrets.token_hex(24)
|
||||
settings['s3_worker_key']=secrets.token_hex(12)
|
||||
file=ROOT/'operator/settings.json';file.write_text(json.dumps(settings));file.chmod(0o600)
|
||||
issue('worker')
|
||||
bootstrap()
|
||||
dc('--profile','tools','run','--rm','--no-deps','audit','/app/student-tests/rotation_db.py')
|
||||
# Raw process restart keeps the stable TLS/network boundary and persistent broker/S3 data.
|
||||
dc('restart','broker-raw','s3-raw')
|
||||
configure()
|
||||
dc('up','-d','--no-deps','worker')
|
||||
result={'revoked_certificate_serial':serial,'worker_tokens_revoked':revoked_tokens,'old_valid_token_denied_before_natural_expiry':True,'token_revoke_latency_seconds':round(revoke_latency,3),
|
||||
'passwords_rotated':['PostgreSQL worker','Redis worker','S3 worker'],
|
||||
'new_leaf_certificate':True,'proxy_hard_stop_seconds':5,'images_rebuilt':False,
|
||||
'seconds':round(time.monotonic()-started,3)}
|
||||
(ROOT/'evidence/worker-rotation.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
print('Worker identity revoked, fresh secrets/certificate installed; no secrets printed.')
|
||||
return result
|
||||
if __name__=='__main__':run()
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Sanitized process and resource evidence; never dump container environments."""
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
ids = subprocess.check_output(['docker', 'compose', 'ps', '-aq'], text=True).split()
|
||||
records = []
|
||||
for container in ids:
|
||||
raw = subprocess.check_output(['docker', 'inspect', container], text=True)
|
||||
data = json.loads(raw)[0]
|
||||
records.append({'container': data['Id'], 'name': data['Name'], 'service': data['Config']['Labels']['com.docker.compose.service'], 'image': data['Config']['Image'], 'user': data['Config']['User'], 'running': data['State']['Running'], 'started_at': data['State']['StartedAt'], 'exit_code': data['State']['ExitCode'], 'oom_killed': data['State']['OOMKilled'], 'memory_limit': data['HostConfig']['Memory'], 'nano_cpus': data['HostConfig']['NanoCpus'], 'ports': data['NetworkSettings']['Ports'], 'mounts': [{'type': m['Type'], 'destination': m['Destination'], 'name': m.get('Name')} for m in data['Mounts']]})
|
||||
print(json.dumps(records, indent=2))
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Reproducible explicit operator bring-up; no automatic unseal inside the Bao container."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from bootstrap import main as bootstrap
|
||||
from bao_operator import configure,call
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
def dc(*args):subprocess.run(['docker','compose',*args],cwd=ROOT,check=True)
|
||||
bootstrap()
|
||||
dc('build','api')
|
||||
dc('up','-d','postgres','bao','s3-raw','broker-raw','redis-raw','otel-collector-raw','prometheus-raw','loki-raw','tempo-raw','grafana')
|
||||
for _ in range(90):
|
||||
try:call('sys/init');break
|
||||
except Exception:time.sleep(1)
|
||||
else:raise SystemExit('OpenBao TLS startup timeout')
|
||||
configure()
|
||||
# Filer bootstrap is asynchronous; do not mistake initial socket creation for ready S3.
|
||||
for _ in range(60):
|
||||
result=subprocess.run(['docker','compose','run','--rm','--no-deps','bucket-bootstrap'],cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
if result.returncode==0:break
|
||||
time.sleep(1)
|
||||
else:raise SystemExit('Authenticated S3 bootstrap timeout')
|
||||
dc('up','-d','migrate')
|
||||
dc('up','-d','api','lb','worker','dispatcher')
|
||||
print('Lab7 processes started; waiting for verified HTTPS readiness.')
|
||||
import ssl,urllib.request
|
||||
url=os.environ.get('BASE_URL','https://localhost:'+os.environ.get('APP_PORT','18443'))
|
||||
context=ssl.create_default_context(cafile=str(ROOT/'pki/lb/ca.crt'))
|
||||
for _ in range(120):
|
||||
try:
|
||||
with urllib.request.urlopen(url+'/health/ready',context=context,timeout=2) as r:
|
||||
if r.status==200:break
|
||||
except OSError:pass
|
||||
time.sleep(.5)
|
||||
else:raise SystemExit('HTTPS readiness timeout')
|
||||
print('Lab7 ready over trusted HTTPS; operator secrets stay local.')
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/stop_one.py
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Stop the exact alias requested by the public failover test, using operator exec."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
def dc(*a):return subprocess.check_output(['docker',*a],text=True).strip()
|
||||
target=os.environ['TARGET_INSTANCE_ID'];started=time.monotonic()
|
||||
for container in dc('compose','ps','-q','api').splitlines():
|
||||
alias=dc('exec',container,'cat','/tmp/instance')
|
||||
if alias==target:
|
||||
dc('stop','--time','8',container)
|
||||
state=json.loads(dc('inspect','--format','{{json .State}}',container))
|
||||
assert not state['Running']
|
||||
record={'instance':target,'container':container,'running':False,'exit_code':state['ExitCode'],'seconds':round(time.monotonic()-started,3)}
|
||||
with Path('evidence/stopped-instances.jsonl').open('a') as f:f.write(json.dumps(record)+'\n')
|
||||
print(json.dumps(record));break
|
||||
else:raise SystemExit('Exact alias not found')
|
||||
@@ -0,0 +1,60 @@
|
||||
# Разбор сканирования lab7
|
||||
|
||||
Дата: 20.11.2026. Это inventory/triage локального учебного стенда, не утверждение об отсутствии уязвимостей и не разрешение выставлять сервисы в Интернет.
|
||||
|
||||
## Воспроизведение и охват
|
||||
|
||||
`DOCKER_CONTEXT=colima-mtusi-labs python3 student-tests/scan_images.py` сканирует все 11 уникальных образов из compose.yaml по фактическому image ID.
|
||||
Сканер Trivy0.67.2 закреплён digest в скрипте; база уязвимостей — официальный GHCR. В scanner передаётся docker-save archive, Docker socket туда не mounted.
|
||||
Лимит scanner:512MiB/1CPU. `--resume` использует только совпавшие image IDs; для свежей базы запускать без resume.
|
||||
Полные значения CVE/package/installed/fixed/severity сохранены в [image-scans.json](../evidence/image-scans.json). `findings_columns` задаёт формат массивов. Старые counts сохранены отдельно; прежние образы не входят в текущие11.
|
||||
|
||||
| Образ (точный digest в compose/Dockerfile и JSON) | Critical | High | Medium |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| chrislusf/seaweedfs:4.51 | 0 | 2 | 15 |
|
||||
| grafana/grafana:12.1.1 | 15 | 182 | 272 |
|
||||
| grafana/loki:3.5.3 | 2 | 54 | 58 |
|
||||
| grafana/tempo:2.8.2 | 2 | 59 | 65 |
|
||||
| haproxy:3.0-alpine | 0 | 0 | 10 |
|
||||
| openbao/openbao:2.4.4 | 9 | 91 | 83 |
|
||||
| otel/opentelemetry-collector-contrib:0.132.0 | 7 | 74 | 74 |
|
||||
| prom/prometheus:v3.5.0 | 4 | 108 | 125 |
|
||||
| mtusi-messenger:lab7 | 2 | 99 | 142 |
|
||||
| postgres:17-bookworm | 4 | 139 | 238 |
|
||||
| redis:7.4-bookworm | 1 | 37 | 104 |
|
||||
|
||||
Counts — записи package/CVE, а не число доказанных эксплойтов. Low/Unknown также сохранены в JSON. Отсутствие FixedVersion не означает отсутствие риска.
|
||||
|
||||
## Что исправлено по результатам scan
|
||||
|
||||
- База Python3.12.8 заменена закреплённой Python3.12.16/bookworm; actual runtime — aarch64/64bit/OpenSSL3.0.22. Старый runtime pip удалён после установки frozen dependencies. В финальном application scan нет language-package findings; OS findings остаются.
|
||||
- OTel SDK обновлён до1.45.2: исходные CVE-2026-114603 и CVE-2026-126360 затрагивали обработку trace context. Дополнительно до propagator ограничены размеры traceparent55 и tracestate512; значения headers не пишутся в логи.
|
||||
- TLS ingress/gateways HAProxy3.0.5→3.0.31. У старого образа было2Critical/21High, у выбранного digest —0Critical/0High/10Medium. Это важно для внешнего TLS, где нельзя опираться только на сетевую изоляцию.
|
||||
- PostgreSQL17.2→17.12 в той же major17 с сохранением volumes; counts Critical24→4, High250→139. Redis7.4.1→7.4.12, Critical11→1, High121→37. Обновление особенно существенно для авторизованного worker: он выполняет SQL и разрешённый EVAL, поэтому authenticated attack surface нельзя объявить недоступной.
|
||||
- PG-superuser и Redis operator получили независимые случайные credentials. Worker/API/dispatcher используют другие роли и ключи; operator passwords не передаются приложениям.
|
||||
- После обновления выполнены публичный HTTPS smoke, TLS/ACL audit,18 API-тестов,13 pipeline fault/regression групп, worker revocation и persistence. Это проверка совместимости, не CVE exploit test.
|
||||
|
||||
## Применимость и остаточные риски
|
||||
|
||||
| Findings/группа | Фактическая поверхность | Решение и граница |
|
||||
| --- | --- | --- |
|
||||
| CVE-2025-7458, libsqlite3-0 в application/PG | Наш API использует PostgreSQL; SQLite не используется бизнес-кодом. Наличие библиотеки само по себе не создаёт SQL endpoint | Не включать SQLite в обработку недоверенных данных до patched base. Grafana использует собственную SQLite: это отдельная поверхность, к ней это исключение не относится |
|
||||
| CVE-2023-45853, zlib1g | Advisory относится к contrib minizip zipOpenNewFileInZip4_64. Проверка фактического libz application не нашла этот exported symbol ([runtime-surface.json](../evidence/runtime-surface.json)); Pillow читает PNG/JPEG, не ZIP | Узкое обоснование для этой функции/образа; это не исключение всех zlib и не доказательство безопасности любого archive parser |
|
||||
| CVE-2026-6653, libxml2 в PG | Даже ограниченный SQL user может обращаться к встроенным XML-функциям. ACL таблиц не защищает от RCE внутри DB process | Остающийся риск. Не давать произвольный SQL внешним пользователям; worker DB credential считается ограниченным, но не sandbox от CVE. Для production нужен patched supported PG/base и повторный scan |
|
||||
| CVE-2025-68121, Go TLS session resumption в PG helper/Go сервисах | Это language-package finding; application native Python/OpenSSL TLS не использует Go. Для Go endpoint/клиентов resumption path может существовать | Сопоставлять с конкретным binary/endpoint, не списывать все findings как unused. mTLS+exact pin/revoke не исправляют библиотеку; остающийся риск Go сервисов |
|
||||
| CVE-2026-31789 в OpenBao OpenSSL | Указанная OpenSSL ветка имеет critical X509 path для32bit; реальный стенд64bit. В HAProxy библиотека уже обновлена | Архитектурная неприменимость конкретного32bit случая не оправдывает остальные Bao findings. Сохранён full inventory |
|
||||
| CVE-2026-33815/33816, pgx в Bao | В Bao включены только KVv2 и cert-auth; database secret engine не mounted, PG dynamic leases не заявлены | Уменьшенная поверхность именно DB-engine path; не blanket ignore Bao/pgx во всех сценариях |
|
||||
| CVE-2026-33757/33758, OIDC в Bao | OIDC auth mount отсутствует; UI выключен | Эти OIDC paths не используются текущей конфигурацией. Повторить triage перед включением OIDC/UI |
|
||||
| CVE-2026-63132, recovery mode token timing | Сервер работает в обычном non-dev режиме, recovery mode не включён | Этот режим не доступен приложениям; host/root оператор по-прежнему trust base |
|
||||
| Другие Bao grpc/Go HTTP/TLS findings, включая CVE-2026-33186 | Listener8200 требует trusted client cert, затем policy. Скомпрометированный worker всё равно имеет валидную identity и может атаковать pre-authorization parser | НЕ объявлены устранёнными. Bao2.4.4 оставлен с проверенным persistent file backend. Переход на актуальный backend/version требует отдельной миграции+backup/restore rehearsal; auto-reset данных не применяется |
|
||||
| Grafana15Critical/182High | HTTPS только127.0.0.1:13407, anonymous datasource401; operator login, provisioned dashboard без сторонних plugins. Часть findings — bundled OS/JS/Go | Локальная доступность снижает exposure, не исправляет parser/auth bugs. Перед общим доступом обновить supported image и проверить provisioning/queries. Браузер/host считается доверенным |
|
||||
| Loki/Tempo/Prometheus/Collector Go/HTTP/dependency findings | Нет host mapping, native raw listeners только loopback внутри proxy namespace; mTLS CN allowlist до backend | Это ограничение доступности, не patch. Разрешённый producer/compromised worker может дойти до Collector и злоупотреблять telemetry. Нужны актуальные compatible images+повторные интеграционные тесты |
|
||||
| SeaweedFS2High | S3 только через mTLS gateway; raw8333/admin/gRPC loopback, отдельные native keys и prefix/verb filter | Нет публичного raw/admin входа. Permission tests не доказывают отсутствие S3/parser CVE; обновление и quotas нужны перед production |
|
||||
| Остальные Debian Low/Medium/High и Unknown | Библиотеки остаются в образах, не все входят в реально вызываемый путь. Часть advisory не имеет Debian fixed version | Не применялось массовое ignore/скрытие severity. Точный inventory приложен; остаток явно принят только для локальной лабораторной |
|
||||
|
||||
## Приоритет дальнейших действий
|
||||
|
||||
До публикации стенда в общую сеть: обновить OpenBao с контролируемой миграцией persistent backend, Grafana и Go telemetry backends; проверить patched libxml2/PG. Сделать DB/Bao backup+restore rehearsal, независимые custodians shares, per-tenant quotas и сетевую сегментацию по ролям.
|
||||
Одни TLS/readonly/cap_drop не заменяют обновления. В этой работе исследуется снижение последствий украденной **сервисной identity** и проверяется отзыв; эксплуатация kernel/DB/image RCE не моделируется.
|
||||
|
||||
Первичные сведения по CVE доступны в official advisory ссылках базы Trivy; начальные package records сохраняются в JSON. Пример: [Go vulnerability database](https://vuln.go.dev/), [Debian security tracker](https://security-tracker.debian.org/), [OpenBao security advisories](https://github.com/openbao/openbao/security/advisories).
|
||||
+30
-12
@@ -1,19 +1,37 @@
|
||||
# Модель угроз — заполнить
|
||||
# Модель угроз lab7
|
||||
|
||||
## Активы и акторы
|
||||
Активы: пароли и bearer-сессии пользователей, сообщения/файлы, durable jobs, service secrets, CA/unseal/root material и диагностическая история.
|
||||
Акторы: внешний клиент; участник чата; другой authenticated user; атакующий с кодом/ключами worker; доверенный оператор host/CA.
|
||||
Рассматриваем собственный локальный стенд и синтетические данные. Компрометация macOS/Docker daemon/операторского аккаунта исключена из защищаемой границы: там находятся доверенные bootstrap keys.
|
||||
|
||||
TODO: пользовательские credentials/сессии, сообщения/вложения, service identities, CA/secrets, telemetry. Акторы: участник чата, посторонний пользователь, внешний клиент, скомпрометированный сервис, оператор. Опишите доступ атакующего и границы учебного стенда.
|
||||
Границы и точные ports/identities — [trust matrix](../contracts/trust-matrix.md).
|
||||
Связь «правильный сертификат» не равна разрешению на произвольную операцию: дополнительно работают URI/CN allowlists, Bao policy, PG/Redis ACL, S3 prefix/verb checks и пользовательский ACL.
|
||||
|
||||
## Границы доверия
|
||||
| Конкретная угроза | Контроль | Воспроизводимый тест | Остаток |
|
||||
| --- | --- | --- | --- |
|
||||
| Внешний MITM подменяет LB или hostname | CA+SAN, TLS1.2+, без insecure | public test-security, hardening.py wrong hostname | CA/host operator доверен |
|
||||
| Worker идёт прямо в API и выдаёт себя за LB | Native mTLS, URI SAN ровно lb, API не mapped на host | tls_audit.py оба API, worker/outsider denied | Украденный lb key потребует отдельного отзыва |
|
||||
| Valid outsider cert получает broker/Collector доступ | После CA verify проверяется CN role, отдельные identities | tls_audit.py no-cert/foreign-CA/outsider/expired | Скомпрометированный разрешённый producer может отравлять свою telemetry |
|
||||
| Eve читает/прикрепляет чужое изображение или подменяет X-User-Id | Bearer+PG membership,404 без раскрытия объекта |18 API/image tests и smoke | Worker по назначению читает все staging/results, не tenant sandbox |
|
||||
| Offline утечка password/session DB | Argon2id32MiB,t2,p1; legacy verify→rehash; SHA256 только для случайного256-bit session token | auth_and_signals.py DB hashes/rehash/expiry/logout | Компрометация самого API даёт бизнес-доступ; Argon не скрывает слабый пароль навсегда |
|
||||
| Login flood на две реплики | Atomic PG limiter20/s, Retry-After1;2KDF slots/API, bounded workers/body |100 параллельных bad logins,2 реальных instance IDs; KDF measurements | Fixed-window boundary burst до40, TLS/DB saturation не исчерпываются этим тестом |
|
||||
| Worker крадёт API/root secrets из Bao | Exact leaf auth, only own KV policy, root/shares не mounted | tls_audit.py policy read/write/foreign KV403; actual worker mount probe | Собственные worker credentials и все processing inputs остаются доступны |
|
||||
| Worker меняет владельца/источник job, читает sessions или управляет broker | PG table/column grants, schema CREATE revoked; Redis key/command ACL | worker_access.py, old_worker_denied.py; pipeline regression | Разрешённые state fields/queue processing можно испортить; exactly-once и malicious-worker correctness не обещаются |
|
||||
| Worker переписывает staging/ready files | TLS identity+prefix/verb gate; worker results PUT только conditional create, no DELETE/LIST | auth_and_signals.py + replay/fencing tests | Можно создавать новые results keys/занимать quota; quotas per tenant не реализованы |
|
||||
| Похищенные worker key/token остаются после инцидента | Revoke real tokens до TTL; serial denylist; новые PG/Redis/S3 secrets; kill old connections |3 независимых compromise runs под HTTP-нагрузкой | Host CA compromise требует другого recovery/CA rollover |
|
||||
| Sealed Bao молча оставляет сервис на вечном cache | Cache≤20s,503; dispatcher продолжает ждать, после unseal job ready | sealed_bao.py: старый Bearer и живые worker/dispatcher | Уже начатая попытка ограничена отдельным renderer timeout; украденные static passwords нужны отзывать отдельно |
|
||||
| Upload exhaustion или image bomb |10MiB ingress+API,40MP, isolated renderer1CPU/768MiB/20s,tmpfs128MiB | image tests, bounds из собственного lab5 | Это лимиты локального single-host процесса, не полноценный malware scanner |
|
||||
| Секрет случайно уходит в tracing/logs/Git/image | Allowlisted instrumentation, no raw SQL/body/Authorization/filename; ignored PKI/secret dirs | auth-and-signals canaries, actual-value scan, image build-context exclusions | Это проверка доверенного кода, не DLP для arbitrary compromised worker |
|
||||
|
||||
TODO: схема и ссылка на заполненную contracts/trust-matrix.md; публичные/внутренние интерфейсы, процессы, секреты и их выдача.
|
||||
## Эксплуатация и последствия компрометации
|
||||
|
||||
| Угроза и конкретный путь | Нарушаемый инвариант | Контроль | Отрицательный тест | Остаточное ограничение |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO | TODO |
|
||||
Трек1 выполняет probe **внутри настоящего non-root worker** с его mounts и сервисной identity.
|
||||
До/после least privilege сравнивается с явно помеченной моделью старой доверенной сети/shared DB credentials в operator-controlled fixture; живая lab6 не атакуется.
|
||||
Baseline получает только counts из синтетических таблиц и временный S3 fixture, который удаляется; пароли/хеши/содержимое переписки в evidence не сохраняются.
|
||||
|
||||
На 4 минимум 6 конкретных угроз. Не заменяйте сценарии словами «SQL injection», «безопасный Docker», «mTLS включён» без пути и проверки.
|
||||
После отзыва identity проверены отдельные отказы: старый Bao token до expiry, old leaf при тех же CA/ролях, old PG/Redis password и old S3 key с новым cert.
|
||||
Данные принятых jobs живут в PG/S3/AOF и не зависят от локальной памяти worker; после восстановления проверяются те же ID и исходные bytes.
|
||||
|
||||
## Lifecycle и инциденты
|
||||
|
||||
TODO: bootstrap, expiry/renewal, revoke/rotation, компрометация identity, sealed OpenBao и восстановление. Указать владельца CA/unseal и способ защиты bootstrap credentials. Сопоставить scan findings с реальной применимостью и исправлением/остаточным риском.
|
||||
Root CA и shares защищены host ACL0700/0600 и не доступны контейнерам. На учебном host один оператор владеет всеми shares — split custody не заявляется.
|
||||
Уязвимости images/dependencies не объявляются отсутствующими: [SCAN_TRIAGE.md](SCAN_TRIAGE.md) связывает фактические findings с reachable surface, исправлениями и остаточными рисками.
|
||||
Один host/LB/DB/S3/broker/Bao/telemetry stores остаются SPOF. Стенд предназначен для локальной защиты, не для публикации в Интернет.
|
||||
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""Explicit local operator tools; no administrative HTTP endpoint."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
|
||||
from pipeline import connect, audit
|
||||
from objects import Objects
|
||||
from postgres import setting
|
||||
|
||||
|
||||
def replay(job_id):
|
||||
with connect() as conn, conn.transaction():
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job_id,)).fetchone()
|
||||
if job is None or job['state'] != 'failed':
|
||||
raise ValueError('Replay requires a failed job')
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND current_job_id = %s FOR UPDATE', (job_id,)).fetchone()
|
||||
if attachment is None or attachment['state'] != 'failed' or Objects(setting).head(job['source_key']) is None:
|
||||
raise ValueError('Current failed source is unavailable')
|
||||
conn.execute('''
|
||||
UPDATE attachment_jobs
|
||||
SET state = 'queued'
|
||||
, max_attempts = attempt + %s
|
||||
, published_at = NULL
|
||||
, next_attempt_at = clock_timestamp()
|
||||
, error_code = NULL
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
''', (int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), job_id))
|
||||
conn.execute("UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job_id,))
|
||||
audit(conn, job, 'operator_replay')
|
||||
return {'job_id': str(job_id), 'replayed': True}
|
||||
|
||||
|
||||
def version(attachment_id, number):
|
||||
if number not in (1, 2):
|
||||
raise ValueError('Only implemented pipeline versions 1 and 2 are supported')
|
||||
with connect() as conn, conn.transaction():
|
||||
row = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone()
|
||||
if row is None or number != row['pipeline_version'] + 1 or row['state'] in ('uploading', 'failed', 'storage_missing'):
|
||||
raise ValueError('Version must advance the current process exactly once')
|
||||
old = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (row['current_job_id'],)).fetchone()
|
||||
source = row['object_key'] if row['state'] == 'ready' else old['source_key']
|
||||
if Objects(setting).head(source) is None:
|
||||
raise ValueError('Source unavailable')
|
||||
conn.execute("UPDATE attachment_jobs SET state = 'superseded', lease_token = NULL, lease_until = NULL WHERE 1=1 AND id = %s", (old['id'],))
|
||||
audit(conn, old, 'operator_supersede')
|
||||
job_id = uuid.uuid4()
|
||||
conn.execute('''
|
||||
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts)
|
||||
VALUES (%s, %s, %s, %s, %s)
|
||||
''', (job_id, attachment_id, number, source, int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))))
|
||||
conn.execute('''
|
||||
UPDATE attachments
|
||||
SET pipeline_version = %s
|
||||
, current_job_id = %s
|
||||
, state = 'queued'
|
||||
, variants = '[]'
|
||||
, error_code = NULL
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
''', (number, job_id, attachment_id))
|
||||
return {'job_id': str(job_id), 'version': number}
|
||||
|
||||
|
||||
def cleanup(apply=False, retention=86400):
|
||||
if retention < 86400:
|
||||
raise ValueError('Retention must be >=24 hours')
|
||||
objects = Objects(setting)
|
||||
counts = {'ready_staging': 0, 'expired_staging': 0}
|
||||
with connect() as conn:
|
||||
# Intent rows cover every valid staging PUT. Never collect active jobs.
|
||||
last = '00000000-0000-0000-0000-000000000000'
|
||||
while True:
|
||||
rows = conn.execute('SELECT id FROM attachments WHERE 1=1 AND id > %s ORDER BY id LIMIT 100', (last,)).fetchall()
|
||||
if not rows:
|
||||
break
|
||||
for candidate in rows:
|
||||
last = candidate['id']
|
||||
with conn.transaction():
|
||||
row = conn.execute('''
|
||||
SELECT *
|
||||
FROM attachments
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
AND (state = 'ready' OR (state IN ('uploading', 'failed') AND updated_at < clock_timestamp() - %s * INTERVAL '1 second'))
|
||||
FOR UPDATE SKIP LOCKED
|
||||
''', (last, retention)).fetchone()
|
||||
if row is None or not row['staging_key'] or not row['staging_key'].startswith('staging/'):
|
||||
continue
|
||||
counts['ready_staging' if row['state'] == 'ready' else 'expired_staging'] += 1
|
||||
if apply:
|
||||
objects.delete(row['staging_key'])
|
||||
if row['state'] == 'uploading' and row['current_job_id'] is None:
|
||||
conn.execute('DELETE FROM attachments WHERE 1=1 AND id = %s', (last,))
|
||||
else:
|
||||
conn.execute('UPDATE attachments SET staging_key = NULL WHERE 1=1 AND id = %s', (last,))
|
||||
return dict(apply=apply, retention_seconds=retention, counts=counts)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('action', choices=['failed', 'replay', 'version', 'cleanup'])
|
||||
p.add_argument('identifier', nargs='?')
|
||||
p.add_argument('--version', type=int)
|
||||
p.add_argument('--apply', action='store_true')
|
||||
args = p.parse_args()
|
||||
try:
|
||||
if args.action == 'failed':
|
||||
with connect() as conn:
|
||||
result = conn.execute("SELECT id, attachment_id, pipeline_version, attempt, max_attempts, error_code FROM attachment_jobs WHERE 1=1 AND state = 'failed' ORDER BY created_at LIMIT 100").fetchall()
|
||||
elif args.action == 'replay':
|
||||
result = replay(uuid.UUID(args.identifier))
|
||||
elif args.action == 'version':
|
||||
result = version(uuid.UUID(args.identifier), args.version)
|
||||
else:
|
||||
result = cleanup(args.apply)
|
||||
print(json.dumps(result, default=str, indent=2))
|
||||
except Exception:
|
||||
print(json.dumps({'error': 'Operator action refused or dependency unavailable; no secrets logged.'}))
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Idempotent private bucket bootstrap, executed only with administrative S3 keys."""
|
||||
import uuid
|
||||
from botocore.exceptions import ClientError
|
||||
from objects import Objects
|
||||
from postgres import setting
|
||||
|
||||
objects = Objects(setting)
|
||||
try:
|
||||
objects.client.create_bucket(Bucket=objects.bucket)
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] not in ('BucketAlreadyOwnedByYou', 'BucketAlreadyExists'):
|
||||
raise RuntimeError('Bucket bootstrap failed') from None
|
||||
objects.ready()
|
||||
# Bucket metadata can be ready before a fresh Seaweed volume accepts file writes.
|
||||
key='staging/bootstrap-'+uuid.uuid4().hex
|
||||
payload=b'lab7-readiness-probe'
|
||||
try:
|
||||
objects.client.put_object(Bucket=objects.bucket,Key=key,Body=payload)
|
||||
assert objects.client.get_object(Bucket=objects.bucket,Key=key)['Body'].read()==payload
|
||||
finally:
|
||||
objects.client.delete_object(Bucket=objects.bucket,Key=key)
|
||||
print('Private bucket ready; static authenticated identities loaded from local secret config.')
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Shared HTTP contract; storage is supplied by each laboratory."""
|
||||
import base64
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, code, message):
|
||||
self.status, self.code, self.message = status, code, message
|
||||
|
||||
|
||||
def invalid():
|
||||
raise APIError(400, 'invalid_request', 'Неверные параметры запроса')
|
||||
|
||||
|
||||
def unauthorized():
|
||||
raise APIError(401, 'unauthorized', 'Требуется действительная идентификация')
|
||||
|
||||
|
||||
def missing():
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
|
||||
|
||||
def identifier(value):
|
||||
try:
|
||||
if not isinstance(value, str):
|
||||
invalid()
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
invalid()
|
||||
|
||||
|
||||
def text(value, maximum, minimum=1):
|
||||
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
|
||||
invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def fields(value, required):
|
||||
if not isinstance(value, dict) or set(value) != set(required):
|
||||
invalid()
|
||||
|
||||
|
||||
def username(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
def now():
|
||||
return dt.datetime.now(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def page_args(query, scope):
|
||||
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
|
||||
invalid()
|
||||
limit = query.get('limit', ['50'])[0]
|
||||
if not re.fullmatch(r'[0-9]+', limit):
|
||||
invalid()
|
||||
try:
|
||||
limit = int(limit)
|
||||
except ValueError:
|
||||
invalid()
|
||||
if not 1 <= limit <= 100:
|
||||
invalid()
|
||||
cursor = None
|
||||
if 'cursor' in query:
|
||||
raw = query['cursor'][0]
|
||||
try:
|
||||
if not raw or len(raw) > 1024:
|
||||
invalid()
|
||||
decoded = json.loads(base64.b64decode(raw + '=' * (-len(raw) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(decoded, list) or len(decoded) != 3 or decoded[0] != scope:
|
||||
invalid()
|
||||
stamp = decoded[1]
|
||||
if not isinstance(stamp, str) or not stamp.endswith('Z'):
|
||||
invalid()
|
||||
stamp = dt.datetime.fromisoformat(stamp[:-1] + '+00:00').isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
cursor = (stamp, identifier(decoded[2]))
|
||||
except (ValueError, TypeError, UnicodeError):
|
||||
invalid()
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def page(items, limit, scope):
|
||||
more = len(items) > limit
|
||||
items = items[:limit]
|
||||
cursor = None
|
||||
if more:
|
||||
last = items[-1]
|
||||
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
|
||||
return {'items': items, 'next_cursor': cursor}
|
||||
@@ -0,0 +1,13 @@
|
||||
"""One-shot migration job; replicas never execute DDL."""
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from postgres import setting
|
||||
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=5) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2301003)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT PRIMARY KEY)')
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||
if not conn.execute('SELECT version FROM schema_migrations WHERE 1=1 AND version = %s', (path.name,)).fetchone():
|
||||
conn.execute(path.read_text())
|
||||
conn.execute('INSERT INTO schema_migrations (version) VALUES (%s)', (path.name,))
|
||||
print('Applied', path.name)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Bounded S3 operations. No root credentials or filename-derived object keys."""
|
||||
import contextlib
|
||||
import telemetry as tele
|
||||
import hashlib
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unicodedata
|
||||
import warnings
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from PIL import Image, PngImagePlugin, UnidentifiedImageError
|
||||
|
||||
from common import APIError, invalid
|
||||
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
||||
PngImagePlugin.MAX_TEXT_CHUNK = 1024 * 1024
|
||||
PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
|
||||
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
|
||||
UPLOAD_SLOTS = threading.BoundedSemaphore(8)
|
||||
DECODE_SLOTS = threading.BoundedSemaphore(1)
|
||||
|
||||
|
||||
def unavailable():
|
||||
raise APIError(503, 'unavailable', 'Объектное хранилище временно недоступно')
|
||||
|
||||
|
||||
def filename(value):
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= 255 or not value.strip():
|
||||
invalid()
|
||||
if any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in value):
|
||||
invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
return value
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def bounded_upload():
|
||||
if not UPLOAD_SLOTS.acquire(blocking=False):
|
||||
raise APIError(503, 'unavailable', 'Все слоты загрузки заняты')
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
UPLOAD_SLOTS.release()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def receive(handler):
|
||||
lengths = handler.headers.get_all('Content-Length', [])
|
||||
if len(lengths) != 1 or handler.headers.get('Transfer-Encoding'):
|
||||
invalid()
|
||||
try:
|
||||
length = int(lengths[0])
|
||||
except ValueError:
|
||||
invalid()
|
||||
if length > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Максимум 10 MiB')
|
||||
if length <= 0:
|
||||
invalid()
|
||||
mime = handler.headers.get_content_type()
|
||||
if mime not in ('image/png', 'image/jpeg'):
|
||||
raise APIError(415, 'unsupported_media_type', 'Требуется PNG или JPEG')
|
||||
deadline = time.monotonic() + 15
|
||||
digest = hashlib.sha256()
|
||||
# Ephemeral bounded spool is never a durable source of a confirmed attachment.
|
||||
with tempfile.TemporaryFile(mode='w+b') as source:
|
||||
remaining = length
|
||||
while remaining:
|
||||
handler.connection.settimeout(min(3, max(0.01, deadline - time.monotonic())))
|
||||
if time.monotonic() >= deadline:
|
||||
invalid()
|
||||
try:
|
||||
chunk = handler.rfile.read1(min(64 * 1024, remaining))
|
||||
except (OSError, TimeoutError):
|
||||
invalid()
|
||||
if not chunk:
|
||||
invalid()
|
||||
digest.update(chunk)
|
||||
source.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if not DECODE_SLOTS.acquire(timeout=10):
|
||||
raise APIError(503, 'unavailable', 'Проверка изображений перегружена')
|
||||
try:
|
||||
source.flush()
|
||||
for decode in (False,):
|
||||
source.seek(0)
|
||||
# Pillow.close releases pixel buffers and may close its input file.
|
||||
# A duplicate descriptor preserves the original for the S3 PUT.
|
||||
with os.fdopen(os.dup(source.fileno()), 'rb') as image_file:
|
||||
image = Image.open(image_file, formats=['PNG', 'JPEG'])
|
||||
try:
|
||||
if image.width * image.height > MAX_PIXELS:
|
||||
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей')
|
||||
if Image.MIME[image.format] != mime:
|
||||
raise APIError(415, 'unsupported_media_type', 'MIME не соответствует изображению')
|
||||
if decode:
|
||||
image.load()
|
||||
else:
|
||||
image.verify()
|
||||
finally:
|
||||
image.close() # Free pixels BEFORE releasing the decoder semaphore.
|
||||
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
|
||||
raise APIError(413, 'payload_too_large', 'Максимум 40 миллионов пикселей') from None
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from None
|
||||
finally:
|
||||
DECODE_SLOTS.release()
|
||||
source.seek(0)
|
||||
yield source, length, mime, digest.hexdigest()
|
||||
|
||||
|
||||
class Objects:
|
||||
def __init__(self, setting):
|
||||
self.bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
endpoint=os.environ.get('S3_ENDPOINT','https://s3:8443')
|
||||
if not endpoint.startswith('https://'):
|
||||
raise ValueError('S3_ENDPOINT must use verified HTTPS')
|
||||
self.client = boto3.client('s3', endpoint_url=endpoint, region_name='us-east-1', verify='/tls/ca.crt', aws_access_key_id=setting('S3_ACCESS_KEY_ID'), aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'), config=Config(client_cert=('/tls/tls.crt','/tls/tls.key'), signature_version='s3v4', s3={'addressing_style': 'path'}, connect_timeout=1, read_timeout=3, max_pool_connections=16, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required'))
|
||||
self.client = tele.S3Client(self.client)
|
||||
|
||||
def head(self, key):
|
||||
try:
|
||||
return self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response['Error']['Code'] in ('404', 'NoSuchKey', 'NotFound'):
|
||||
return None
|
||||
unavailable()
|
||||
except BotoCoreError:
|
||||
unavailable()
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
|
||||
def put(self, key, source, length, mime, digest):
|
||||
try:
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=source, ContentLength=length, ContentType=mime, Metadata={'sha256': digest})
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
|
||||
def get(self, key):
|
||||
try:
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
|
||||
def delete(self, key):
|
||||
try:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (ClientError, BotoCoreError):
|
||||
unavailable()
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Argon2id plus bounded one-time verification of our legacy scrypt hashes."""
|
||||
import hashlib
|
||||
import hmac
|
||||
import threading
|
||||
from argon2 import PasswordHasher,Type
|
||||
from argon2.exceptions import VerificationError,InvalidHashError
|
||||
from common import APIError
|
||||
|
||||
HASHER=PasswordHasher(time_cost=2,memory_cost=32768,parallelism=1,hash_len=32,salt_len=16,type=Type.ID)
|
||||
SLOTS=threading.BoundedSemaphore(2)
|
||||
|
||||
def guard():
|
||||
if not SLOTS.acquire(timeout=.5):
|
||||
raise APIError(503,'unavailable','Аутентификация временно перегружена')
|
||||
|
||||
def password_hash(password):
|
||||
guard()
|
||||
try:return HASHER.hash(password)
|
||||
finally:SLOTS.release()
|
||||
|
||||
def verify(password,stored):
|
||||
guard()
|
||||
try:
|
||||
if stored.startswith('scrypt$'):
|
||||
parts=stored.split('$')
|
||||
if parts[1:4]!=['16384','8','1']:return False,False
|
||||
digest=hashlib.scrypt(password.encode(),salt=bytes.fromhex(parts[4]),n=16384,r=8,p=1,dklen=32,maxmem=64*1024*1024)
|
||||
return hmac.compare_digest(digest.hex(),parts[5]),True
|
||||
try:return HASHER.verify(stored,password),HASHER.check_needs_rehash(stored)
|
||||
except (VerificationError,InvalidHashError):return False,False
|
||||
finally:SLOTS.release()
|
||||
+396
@@ -0,0 +1,396 @@
|
||||
"""Durable outbox, Redis Streams delivery, bounded worker and fenced result commits."""
|
||||
import vault
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import telemetry as tele
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
import redis
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from common import APIError
|
||||
from objects import Objects, MAX_BYTES
|
||||
from postgres import setting
|
||||
|
||||
STREAM, GROUP = 'attachment-jobs', 'image-workers'
|
||||
LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '60'))
|
||||
STOP = threading.Event()
|
||||
|
||||
|
||||
def raw_connect():
|
||||
return psycopg.connect(setting('DATABASE_URL'), row_factory=dict_row, autocommit=True, connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1500')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connect():
|
||||
with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
|
||||
with raw_connect() as connection:
|
||||
yield connection
|
||||
|
||||
|
||||
def broker():
|
||||
client = redis.Redis.from_url(setting('BROKER_URL'), decode_responses=True, socket_connect_timeout=1, socket_timeout=3, **vault.redis_tls())
|
||||
try:
|
||||
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
|
||||
except redis.ResponseError as error:
|
||||
if 'BUSYGROUP' not in str(error):
|
||||
raise
|
||||
return client
|
||||
|
||||
|
||||
def event(job):
|
||||
return {'schema_version': 1, 'job_id': str(job['id']), 'type': 'attachment.process.v1', 'attachment_id': str(job['attachment_id']), 'pipeline_version': job['pipeline_version'], 'attempt': job['attempt'] + 1, 'created_at': job['created_at'].isoformat().replace('+00:00', 'Z'), **(job.get('trace_context') or {})}
|
||||
|
||||
|
||||
def validate(message):
|
||||
required = {'schema_version', 'job_id', 'type', 'attachment_id', 'pipeline_version', 'attempt', 'created_at'}
|
||||
if not isinstance(message, dict) or set(message) - required - {'traceparent', 'tracestate'} or not required <= set(message):
|
||||
return False
|
||||
try:
|
||||
uuid.UUID(message['job_id']); uuid.UUID(message['attachment_id'])
|
||||
stamp = dt.datetime.fromisoformat(message['created_at'].replace('Z', '+00:00'))
|
||||
if stamp.tzinfo is None or not message['created_at'].endswith('Z'):
|
||||
return False
|
||||
if 'traceparent' in message and (not isinstance(message['traceparent'], str) or not re.fullmatch(r'00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}', message['traceparent'])):
|
||||
return False
|
||||
if 'tracestate' in message and (not isinstance(message['tracestate'], str) or len(message['tracestate']) > 512):
|
||||
return False
|
||||
return type(message['schema_version']) is int and message['schema_version'] == 1 and message['type'] == 'attachment.process.v1' and all(type(message[k]) is int and message[k] >= 1 for k in ['pipeline_version', 'attempt'])
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
|
||||
def audit(conn, job, name):
|
||||
conn.execute('INSERT INTO job_events (job_id, event, attempt) VALUES (%s, %s, %s)', (job['id'], name, job['attempt']))
|
||||
|
||||
|
||||
def retry(conn, job, code, permanent=False):
|
||||
failed = permanent or job['attempt'] >= job['max_attempts']
|
||||
state = 'failed' if failed else 'queued'
|
||||
code = code if permanent or not failed else 'retry_exhausted'
|
||||
delay = min(8, .5 * 2 ** min(job['attempt'] - 1, 4)) + random.uniform(0, .5)
|
||||
conn.execute('''
|
||||
UPDATE attachment_jobs
|
||||
SET state = %s
|
||||
, error_code = %s
|
||||
, lease_token = NULL
|
||||
, lease_until = NULL
|
||||
, published_at = NULL
|
||||
, next_attempt_at = clock_timestamp() + %s * INTERVAL '1 second'
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
''', (state, code, delay, job['id']))
|
||||
conn.execute('''
|
||||
UPDATE attachments
|
||||
SET state = %s
|
||||
, error_code = %s
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND current_job_id = %s
|
||||
''', (state, code if failed else None, job['id']))
|
||||
audit(conn, job, 'failed:' + code if failed else 'retry:' + code)
|
||||
tele.count('job_retries', {'job_type':'image', 'outcome':'failed' if failed else 'retry'})
|
||||
tele.log('job_failed' if failed else 'job_retry', job_id=str(job['id']), attempt=job['attempt'], error_code=code)
|
||||
|
||||
|
||||
def dispatch_once(client, fault=None):
|
||||
with connect() as conn:
|
||||
with conn.transaction():
|
||||
expired = conn.execute("SELECT * FROM attachment_jobs WHERE 1=1 AND state = 'processing' AND lease_until < clock_timestamp() FOR UPDATE SKIP LOCKED LIMIT 20").fetchall()
|
||||
for job in expired:
|
||||
retry(conn, job, 'lease_expired')
|
||||
with conn.transaction():
|
||||
jobs = conn.execute('''
|
||||
SELECT j.*
|
||||
FROM attachment_jobs j
|
||||
JOIN attachments a ON a.current_job_id = j.id
|
||||
WHERE 1=1
|
||||
AND j.state = 'queued'
|
||||
AND j.next_attempt_at <= clock_timestamp()
|
||||
AND (j.published_at IS NULL OR j.published_at < clock_timestamp() - %s * INTERVAL '1 second')
|
||||
ORDER BY j.created_at
|
||||
LIMIT 10
|
||||
FOR UPDATE OF j SKIP LOCKED
|
||||
''', (LEASE,)).fetchall()
|
||||
for job in jobs:
|
||||
with tele.span('attachment publish', {'messaging.system':'redis', 'messaging.destination.name':STREAM},
|
||||
tele.SpanKind.PRODUCER, job.get('trace_context') or {}):
|
||||
message = event(job)
|
||||
message.update(tele.carrier())
|
||||
client.xadd(STREAM, {'event': json.dumps(message, separators=(',', ':'))})
|
||||
tele.log('job_published', job_id=str(job['id']), attempt=job['attempt']+1)
|
||||
if fault:
|
||||
fault('after_publish', job)
|
||||
conn.execute('UPDATE attachment_jobs SET published_at = clock_timestamp() WHERE 1=1 AND id = %s', (job['id'],))
|
||||
audit(conn, job, 'published')
|
||||
return len(jobs)
|
||||
|
||||
|
||||
def claim(message):
|
||||
with connect() as conn, conn.transaction():
|
||||
job = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (message['job_id'],)).fetchone()
|
||||
if job is None:
|
||||
return None
|
||||
attachment = conn.execute('SELECT * FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (job['attachment_id'],)).fetchone()
|
||||
if str(job['attachment_id']) != message['attachment_id'] or job['pipeline_version'] != message['pipeline_version']:
|
||||
return None
|
||||
if attachment['current_job_id'] != job['id'] or attachment['pipeline_version'] != job['pipeline_version']:
|
||||
audit(conn, job, 'stale_noop')
|
||||
return None
|
||||
if job['state'] in ('ready', 'failed', 'superseded'):
|
||||
audit(conn, job, 'terminal_noop')
|
||||
return None
|
||||
if job['state'] != 'queued' or job['next_attempt_at'] > dt.datetime.now(dt.timezone.utc) or message['attempt'] != job['attempt'] + 1:
|
||||
audit(conn, job, 'duplicate_or_early_noop')
|
||||
return None
|
||||
token = uuid.uuid4()
|
||||
job = conn.execute('''
|
||||
UPDATE attachment_jobs
|
||||
SET state = 'processing'
|
||||
, attempt = attempt + 1
|
||||
, lease_token = %s
|
||||
, lease_until = clock_timestamp() + %s * INTERVAL '1 second'
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING *
|
||||
''', (token, LEASE, job['id'])).fetchone()
|
||||
conn.execute("UPDATE attachments SET state = 'processing', updated_at = clock_timestamp() WHERE 1=1 AND current_job_id = %s", (job['id'],))
|
||||
audit(conn, job, 'claimed')
|
||||
return job, attachment
|
||||
|
||||
|
||||
class Permanent(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def immutable(objects, key, path, mime, sha):
|
||||
head = objects.head(key)
|
||||
if head is not None:
|
||||
if head['ContentLength'] != path.stat().st_size or head.get('Metadata', {}).get('sha256') != sha:
|
||||
raise Permanent('result_conflict')
|
||||
return
|
||||
try:
|
||||
with path.open('rb') as source:
|
||||
objects.client.put_object(Bucket=objects.bucket, Key=key, Body=source, ContentLength=path.stat().st_size, ContentType=mime, Metadata={'sha256': sha}, IfNoneMatch='*')
|
||||
except ClientError as error:
|
||||
if error.response['ResponseMetadata']['HTTPStatusCode'] in (409, 412):
|
||||
head = objects.head(key)
|
||||
if head and head['ContentLength'] == path.stat().st_size and head.get('Metadata', {}).get('sha256') == sha:
|
||||
return
|
||||
raise
|
||||
|
||||
|
||||
def process_core(message, fault=None):
|
||||
claimed = claim(message)
|
||||
if claimed is None:
|
||||
return 'noop'
|
||||
job, attachment = claimed
|
||||
objects = Objects(setting)
|
||||
try:
|
||||
if fault:
|
||||
fault('before_side_effect', job)
|
||||
with tempfile.TemporaryDirectory(prefix='job-') as temporary:
|
||||
directory = Path(temporary)
|
||||
source = directory / 'original'
|
||||
head = objects.head(job['source_key'])
|
||||
if head is None:
|
||||
raise Permanent('source_missing')
|
||||
response = objects.get(job['source_key'])
|
||||
digest, size = hashlib.sha256(), 0
|
||||
deadline = time.monotonic() + 15
|
||||
with response['Body'] as stream, source.open('wb') as out:
|
||||
for chunk in iter(lambda: stream.read(64 * 1024), b''):
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES or time.monotonic() > deadline:
|
||||
raise Permanent('invalid_image')
|
||||
digest.update(chunk); out.write(chunk)
|
||||
if size != attachment['size_bytes'] or digest.hexdigest() != attachment['sha256']:
|
||||
raise Permanent('checksum_mismatch')
|
||||
result = subprocess.run([sys.executable, str(Path(__file__).with_name('render.py')), str(source), str(directory), str(job['pipeline_version'])], capture_output=True, timeout=int(os.environ.get('JOB_TIMEOUT_SECONDS', '20')))
|
||||
if result.returncode == 2:
|
||||
raise Permanent('invalid_image')
|
||||
if result.returncode:
|
||||
raise RuntimeError('render_failed')
|
||||
variants = json.loads(result.stdout)
|
||||
prefix = 'results/' + str(attachment['id']) + '/v' + str(job['pipeline_version']) + '/'
|
||||
original_key = prefix + 'original'
|
||||
immutable(objects, original_key, source, attachment['content_type'], attachment['sha256'])
|
||||
if fault:
|
||||
fault('after_original', job)
|
||||
for variant in variants:
|
||||
variant['object_key'] = prefix + variant['name']
|
||||
immutable(objects, variant['object_key'], directory / variant['name'], variant['content_type'], variant['sha256'])
|
||||
if fault:
|
||||
fault('after_objects', job)
|
||||
with connect() as conn, conn.transaction():
|
||||
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
|
||||
changed = conn.execute('''
|
||||
UPDATE attachments
|
||||
SET state = 'ready'
|
||||
, object_key = %s
|
||||
, variants = %s
|
||||
, error_code = NULL
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND current_job_id = %s
|
||||
AND pipeline_version = %s
|
||||
AND %s
|
||||
RETURNING id
|
||||
''', (original_key, Jsonb(variants), job['id'], job['pipeline_version'], current['state'] == 'processing' and current['lease_token'] == job['lease_token'] and current['lease_until'] > dt.datetime.now(dt.timezone.utc))).fetchone()
|
||||
if changed:
|
||||
conn.execute("UPDATE attachment_jobs SET state = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp() WHERE 1=1 AND id = %s", (job['id'],))
|
||||
audit(conn, job, 'ready')
|
||||
else:
|
||||
audit(conn, job, 'fenced_noop')
|
||||
if fault:
|
||||
fault('after_commit', job)
|
||||
if changed and job['source_key'].startswith('staging/') and os.environ.get('BAO_ROLE') != 'worker':
|
||||
objects.delete(job['source_key'])
|
||||
if changed:
|
||||
tele.duration('job_ready_duration', (dt.datetime.now(dt.timezone.utc)-job['created_at']).total_seconds(), {'job_type':'image'})
|
||||
return 'ready' if changed else 'fenced'
|
||||
except Exception as error:
|
||||
code = str(error) if isinstance(error, Permanent) else 'processing_timeout' if isinstance(error, subprocess.TimeoutExpired) else 'storage_unavailable' if isinstance(error, (APIError, ClientError, BotoCoreError)) else 'processing_failed'
|
||||
with connect() as conn, conn.transaction():
|
||||
current = conn.execute('SELECT * FROM attachment_jobs WHERE 1=1 AND id = %s FOR UPDATE', (job['id'],)).fetchone()
|
||||
if current['state'] == 'processing' and current['lease_token'] == job['lease_token']:
|
||||
retry(conn, current, code, isinstance(error, Permanent))
|
||||
return 'failed_or_retry'
|
||||
finally:
|
||||
objects.client.close()
|
||||
|
||||
|
||||
def process(message, fault=None):
|
||||
started = time.monotonic()
|
||||
with tele.span('attachment process', {'messaging.system':'redis', 'messaging.destination.name':STREAM,
|
||||
'job.id':message['job_id'], 'job.attempt':message['attempt']},
|
||||
tele.SpanKind.CONSUMER, message):
|
||||
with tele.LOCK:
|
||||
tele.STATE['worker_busy'] += 1
|
||||
outcome = 'error'
|
||||
try:
|
||||
outcome = process_core(message, fault)
|
||||
if outcome == 'failed_or_retry':
|
||||
tele.trace.get_current_span().set_status(tele.Status(tele.StatusCode.ERROR))
|
||||
return outcome
|
||||
finally:
|
||||
with tele.LOCK:
|
||||
tele.STATE['worker_busy'] -= 1
|
||||
attributes = {'job_type':'image', 'outcome':outcome}
|
||||
tele.count('jobs_processed', attributes)
|
||||
tele.duration('job_processing_duration', time.monotonic()-started, attributes)
|
||||
tele.log('job_processed', job_id=message['job_id'], attempt=message['attempt'], outcome=outcome)
|
||||
|
||||
|
||||
def observe_queue():
|
||||
try:
|
||||
with raw_connect() as conn:
|
||||
row = conn.execute("""
|
||||
SELECT
|
||||
count(*) AS depth
|
||||
, coalesce(extract(epoch FROM clock_timestamp() - min(created_at)), 0) AS age
|
||||
FROM attachment_jobs
|
||||
WHERE 1=1
|
||||
AND state IN ('queued', 'processing')
|
||||
""").fetchone()
|
||||
tele.STATE.update(queue_depth=row['depth'], queue_oldest_age=float(row['age']), queue_observation_ok=1)
|
||||
except (psycopg.Error, APIError):
|
||||
tele.STATE['queue_observation_ok'] = 0
|
||||
|
||||
|
||||
def ack(client, identifier):
|
||||
# Atomic ack+delete; PostgreSQL is the durable completion/replay registry.
|
||||
client.eval("redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, identifier)
|
||||
|
||||
|
||||
def consume(once=False, fault=None):
|
||||
client = broker()
|
||||
consumer = uuid.uuid4().hex
|
||||
while not STOP.is_set():
|
||||
messages = []
|
||||
try:
|
||||
vault.ensure()
|
||||
pending = client.xautoclaim(STREAM, GROUP, consumer, min_idle_time=LEASE * 1000, start_id='0-0', count=1)[1]
|
||||
if pending:
|
||||
messages = pending
|
||||
else:
|
||||
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=500)
|
||||
messages = batches[0][1] if batches else []
|
||||
for identifier, fields in messages:
|
||||
raw = fields.get('event', '')
|
||||
try:
|
||||
message = json.loads(raw) if len(raw) <= 4096 else None
|
||||
except ValueError:
|
||||
message = None
|
||||
if validate(message):
|
||||
process(message, fault)
|
||||
else:
|
||||
client.xadd('rejected-jobs', {'source_message_id': identifier, 'sha256': hashlib.sha256(raw.encode()).hexdigest(), 'reason': 'invalid_envelope'}, maxlen=1000, approximate=False)
|
||||
ack(client, identifier)
|
||||
if once:
|
||||
return
|
||||
except (redis.RedisError, psycopg.Error, APIError):
|
||||
STOP.wait(.5)
|
||||
if once and not messages:
|
||||
return
|
||||
client.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('mode', choices=['dispatch', 'work'])
|
||||
parser.add_argument('--once', action='store_true')
|
||||
parser.add_argument('--fault-point', choices=['before_side_effect', 'after_original', 'after_objects', 'after_commit', 'after_publish'])
|
||||
args = parser.parse_args()
|
||||
tele.configure('messenger-dispatcher' if args.mode=='dispatch' else 'messenger-worker')
|
||||
# Operator-only opt-in injection; no request header or public endpoint controls it.
|
||||
def fault(point, job):
|
||||
if point == args.fault_point:
|
||||
os._exit(77)
|
||||
signal.signal(signal.SIGTERM, lambda *_: STOP.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: STOP.set())
|
||||
if args.mode == 'dispatch':
|
||||
client = broker()
|
||||
while not STOP.is_set():
|
||||
observe_queue()
|
||||
try:
|
||||
vault.ensure()
|
||||
dispatch_once(client, fault)
|
||||
except (redis.RedisError, psycopg.Error, APIError):
|
||||
pass
|
||||
if args.once:
|
||||
break
|
||||
STOP.wait(.2)
|
||||
else:
|
||||
count = int(os.environ.get('WORKER_CONCURRENCY', '1'))
|
||||
if not 1 <= count <= 2:
|
||||
raise ValueError('WORKER_CONCURRENCY must be 1..2')
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
|
||||
futures = [pool.submit(consume, args.once, fault) for _ in range(count)]
|
||||
for result in futures:
|
||||
result.result()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
tele.shutdown()
|
||||
+427
@@ -0,0 +1,427 @@
|
||||
import base64
|
||||
import contextlib
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
import telemetry as tele
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
import redis
|
||||
|
||||
from objects import Objects, unavailable
|
||||
|
||||
from common import APIError, fields, identifier, invalid, missing, page, text, unauthorized, username
|
||||
|
||||
from passwords import password_hash, verify
|
||||
import vault
|
||||
|
||||
|
||||
def setting(name):
|
||||
if os.environ.get('BAO_ROLE'):
|
||||
return vault.value(name)
|
||||
path = os.environ.get(name + '_FILE')
|
||||
return Path(path).read_text().strip() if path else os.environ[name]
|
||||
|
||||
|
||||
def public(row):
|
||||
return {k: (v.decode('utf-8') if isinstance(v, bytes) else [str(x) for x in v] if isinstance(v, list) else str(v) if isinstance(v, uuid.UUID) else v.isoformat(timespec='microseconds').replace('+00:00', 'Z') if isinstance(v, dt.datetime) else v) for k, v in row.items() if k != 'password_hash'}
|
||||
|
||||
|
||||
|
||||
class Store:
|
||||
persistent = True
|
||||
|
||||
def __init__(self):
|
||||
self.ttl = int(os.environ.get('SESSION_TTL_SECONDS', '3600'))
|
||||
if not 1 <= self.ttl <= 86400:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be 1..86400')
|
||||
self.access_mode = os.environ.get('DB_ACCESS_MODE', 'optimized')
|
||||
if self.access_mode not in ('baseline', 'optimized'):
|
||||
raise ValueError('DB_ACCESS_MODE must be baseline or optimized')
|
||||
pool_size = int(os.environ.get('DB_POOL_MAX', '8'))
|
||||
if not 1 <= pool_size <= 32:
|
||||
raise ValueError('DB_POOL_MAX must be 1..32')
|
||||
self.pool = ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=pool_size, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=2000 -c lock_timeout=1500'}, open=True)
|
||||
self.pool.wait(timeout=10)
|
||||
tele.pool(self.pool)
|
||||
self.cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.15, socket_timeout=0.15, decode_responses=True, **vault.redis_tls())
|
||||
self.objects = Objects(setting)
|
||||
self.dummy_hash = password_hash('dummy-password-for-unknown-users')
|
||||
with self.connection() as conn:
|
||||
if not conn.execute("SELECT version FROM schema_migrations WHERE 1=1 AND version = '010_no_temporary_ddl.sql'").fetchone():
|
||||
raise RuntimeError('Run the separate migration service first')
|
||||
|
||||
@contextlib.contextmanager
|
||||
def connection(self):
|
||||
try:
|
||||
with tele.span('db.transaction', {'db.system':'postgresql'}, tele.SpanKind.CLIENT):
|
||||
with self.pool.connection() as conn:
|
||||
yield conn
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
except (psycopg.Error, PoolTimeout):
|
||||
raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
self.objects.client.close()
|
||||
|
||||
def ready(self):
|
||||
self.objects.ready()
|
||||
with self.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
|
||||
def rate_limit(self, action, budget):
|
||||
with self.connection() as conn:
|
||||
row=conn.execute('''
|
||||
INSERT INTO auth_rate_limits (action, bucket_start, attempts)
|
||||
VALUES (%s, floor(extract(epoch FROM clock_timestamp())), 1)
|
||||
ON CONFLICT (action) DO UPDATE
|
||||
SET bucket_start = EXCLUDED.bucket_start
|
||||
, attempts = CASE WHEN auth_rate_limits.bucket_start = EXCLUDED.bucket_start
|
||||
THEN auth_rate_limits.attempts + 1 ELSE 1 END
|
||||
RETURNING attempts
|
||||
''',(action,)).fetchone()
|
||||
if row['attempts']>budget:
|
||||
raise APIError(429,'rate_limited','Повторите попытку позже')
|
||||
|
||||
def register(self, body):
|
||||
self.rate_limit('register', 20)
|
||||
fields(body, ('username', 'display_name', 'password'))
|
||||
name, display = username(body['username']), text(body['display_name'], 100)
|
||||
# Passwords may contain spaces and colons; only length/UTF-8 are restricted.
|
||||
password = body['password']
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||
invalid()
|
||||
try:
|
||||
password.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
hashed = password_hash(password)
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('''
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
''', (uuid.uuid4(), name, display.encode('utf-8'), hashed)).fetchone()
|
||||
return public(row)
|
||||
|
||||
def login(self, headers):
|
||||
self.rate_limit('login', 20)
|
||||
authorization = headers.get('Authorization', '')
|
||||
try:
|
||||
scheme, encoded = authorization.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
unauthorized()
|
||||
name, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
except (ValueError, UnicodeError):
|
||||
unauthorized()
|
||||
if not re_username(name) or not 12 <= len(password) <= 128:
|
||||
unauthorized()
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('SELECT id, username, display_name, created_at, password_hash FROM users WHERE 1=1 AND username = %s', (name,)).fetchone()
|
||||
hashed = row['password_hash'] if row else self.dummy_hash
|
||||
verified, rehash = verify(password, hashed)
|
||||
if row is None or not verified:
|
||||
unauthorized()
|
||||
if rehash:
|
||||
upgraded = password_hash(password)
|
||||
with self.connection() as conn:
|
||||
conn.execute('UPDATE users SET password_hash = %s WHERE 1=1 AND id = %s AND password_hash = %s', (upgraded,row['id'],hashed))
|
||||
token = secrets.token_urlsafe(32)
|
||||
with self.connection() as conn:
|
||||
expiry = conn.execute('''
|
||||
INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES (%s, %s, clock_timestamp() + %s * INTERVAL '1 second')
|
||||
RETURNING expires_at
|
||||
''', (hashlib.sha256(token.encode()).hexdigest(), row['id'], self.ttl)).fetchone()['expires_at']
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': public({'expires_at': expiry})['expires_at'], 'user': public(row)}
|
||||
|
||||
def token_hash(self, headers):
|
||||
parts = headers.get('Authorization', '').split(' ')
|
||||
if len(parts) != 2 or parts[0].lower() != 'bearer' or len(parts[1]) != 43 or not parts[1].isascii():
|
||||
unauthorized()
|
||||
return hashlib.sha256(parts[1].encode()).hexdigest()
|
||||
|
||||
def authenticate(self, headers):
|
||||
token_hash = self.token_hash(headers)
|
||||
with self.connection() as conn:
|
||||
session = conn.execute('''
|
||||
SELECT user_id
|
||||
FROM sessions
|
||||
WHERE 1=1
|
||||
AND token_hash = %s
|
||||
AND expires_at > clock_timestamp()
|
||||
''', (token_hash,)).fetchone()
|
||||
if session is None:
|
||||
unauthorized()
|
||||
key = 'user:' + str(session['user_id'])
|
||||
try:
|
||||
cached = self.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (redis.RedisError, ValueError):
|
||||
pass
|
||||
row = conn.execute('SELECT id, username, display_name, created_at FROM users WHERE 1=1 AND id = %s', (session['user_id'],)).fetchone()
|
||||
if row is None:
|
||||
unauthorized()
|
||||
user = public(row)
|
||||
try:
|
||||
self.cache.setex(key, 60, json.dumps(user))
|
||||
except redis.RedisError:
|
||||
pass
|
||||
return user
|
||||
|
||||
def logout(self, headers):
|
||||
with self.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE 1=1 AND token_hash = %s', (self.token_hash(headers),))
|
||||
|
||||
def chat_in(self, conn, actor, chat_id):
|
||||
row = conn.execute('''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE 1=1
|
||||
AND c.id = %s
|
||||
AND cm.user_id = %s
|
||||
''', (chat_id, actor['id'])).fetchone()
|
||||
if row is None:
|
||||
missing()
|
||||
members = conn.execute('SELECT user_id FROM chat_members WHERE 1=1 AND chat_id = %s ORDER BY user_id', (chat_id,)).fetchall()
|
||||
return dict(public(row), member_ids=[str(v['user_id']) for v in members])
|
||||
|
||||
def chat(self, actor, chat_id):
|
||||
with self.connection() as conn:
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
|
||||
def create_chat(self, actor, body):
|
||||
fields(body, ('title', 'member_ids'))
|
||||
title = text(body['title'], 100)
|
||||
members = body['member_ids']
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
invalid()
|
||||
members = [identifier(v) for v in members]
|
||||
if len(set(members)) != len(members) or actor['id'] not in members:
|
||||
invalid()
|
||||
with self.connection() as conn:
|
||||
chat_id = uuid.uuid4()
|
||||
conn.execute('INSERT INTO chats (id, title) VALUES (%s, %s)', (chat_id, title.encode('utf-8')))
|
||||
# Checking after INSERT exercises rollback of the whole composite operation.
|
||||
found = conn.execute('SELECT id FROM users WHERE 1=1 AND id = ANY(%s::uuid[])', (members,)).fetchall()
|
||||
if len(found) != len(members):
|
||||
invalid()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', [(chat_id, v) for v in members])
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
|
||||
def list_chats(self, actor, limit, cursor, scope):
|
||||
with self.connection() as conn:
|
||||
where = 'AND (c.created_at, c.id) > (%s::timestamptz, %s::uuid)' if cursor else ''
|
||||
params = [actor['id']] + list(cursor or ()) + [limit + 1]
|
||||
rows = conn.execute('''
|
||||
SELECT c.id
|
||||
FROM chats c
|
||||
JOIN chat_members cm ON cm.chat_id = c.id
|
||||
WHERE 1=1
|
||||
AND cm.user_id = %s
|
||||
''' + where + '''
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
''', params).fetchall()
|
||||
return page([self.chat_in(conn, actor, v['id']) for v in rows], limit, scope)
|
||||
|
||||
def require_member(self, conn, actor, chat_id):
|
||||
if self.access_mode == 'baseline':
|
||||
return self.chat_in(conn, actor, chat_id)
|
||||
if not conn.execute('''
|
||||
SELECT 1
|
||||
FROM chat_members
|
||||
WHERE 1=1
|
||||
AND chat_id = %s
|
||||
AND user_id = %s
|
||||
''', (chat_id, actor['id'])).fetchone():
|
||||
missing()
|
||||
|
||||
def send(self, actor, chat_id, body, key):
|
||||
if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
invalid()
|
||||
if not isinstance(body, dict) or set(body) - {'text', 'attachment_ids'} or 'text' not in body:
|
||||
invalid()
|
||||
attachments = body.get('attachment_ids', [])
|
||||
if not isinstance(attachments, list) or len(attachments) > 10:
|
||||
invalid()
|
||||
attachments = [identifier(v) for v in attachments]
|
||||
if len(set(attachments)) != len(attachments):
|
||||
invalid()
|
||||
content = body['text']
|
||||
if not isinstance(content, str) or len(content) > 4000 or (not attachments and not content.strip()):
|
||||
invalid()
|
||||
try:
|
||||
content.encode('utf-8')
|
||||
except UnicodeError:
|
||||
invalid()
|
||||
canonical = json.dumps({'text': content, 'attachment_ids': attachments}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
digest = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
with self.connection() as conn:
|
||||
self.require_member(conn, actor, chat_id)
|
||||
# The unique key arbitrates concurrent processes; everything commits together.
|
||||
claimed = conn.execute('''
|
||||
INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, expires_at)
|
||||
VALUES (%s, %s, %s, %s, clock_timestamp() + INTERVAL '24 hours')
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash
|
||||
, message_id = NULL
|
||||
, expires_at = EXCLUDED.expires_at
|
||||
WHERE 1=1
|
||||
AND idempotency_keys.expires_at <= clock_timestamp()
|
||||
RETURNING request_hash
|
||||
''', (actor['id'], chat_id, key, digest)).fetchone()
|
||||
if claimed is None:
|
||||
previous = conn.execute('''
|
||||
SELECT ik.request_hash
|
||||
, m.id
|
||||
, m.chat_id
|
||||
, m.sender_id
|
||||
, m.text
|
||||
, m.created_at
|
||||
, m.attachment_ids
|
||||
FROM idempotency_keys ik
|
||||
JOIN messages m ON m.id = ik.message_id
|
||||
WHERE 1=1
|
||||
AND ik.user_id = %s
|
||||
AND ik.chat_id = %s
|
||||
AND ik.key = %s
|
||||
''', (actor['id'], chat_id, key)).fetchone()
|
||||
if previous['request_hash'] != digest:
|
||||
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||
previous.pop('request_hash')
|
||||
return public(previous)
|
||||
if attachments:
|
||||
rows = conn.execute('''
|
||||
SELECT id, state
|
||||
FROM attachments
|
||||
WHERE 1=1
|
||||
AND id = ANY(%s::uuid[])
|
||||
AND chat_id = %s
|
||||
ORDER BY id
|
||||
FOR SHARE
|
||||
''', (attachments, chat_id)).fetchall()
|
||||
if len(rows) != len(attachments):
|
||||
missing()
|
||||
if any(v['state'] not in ('ready', 'queued', 'processing') for v in rows):
|
||||
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||
# Per-chat row lock preserves committed history order across replicas/clocks.
|
||||
stamp = conn.execute('''
|
||||
UPDATE chats
|
||||
SET last_message_at = GREATEST(clock_timestamp(), last_message_at + INTERVAL '1 microsecond')
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING last_message_at
|
||||
''', (chat_id,)).fetchone()['last_message_at']
|
||||
row = conn.execute('''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (%s, %s, %s, %s, %s, %s::uuid[])
|
||||
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
''', (uuid.uuid4(), chat_id, actor['id'], content.encode('utf-8'), stamp, attachments)).fetchone()
|
||||
if attachments:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO message_attachments (message_id, attachment_id) VALUES (%s, %s)', [(row['id'], v) for v in attachments])
|
||||
conn.execute('''
|
||||
UPDATE idempotency_keys
|
||||
SET message_id = %s
|
||||
WHERE 1=1
|
||||
AND user_id = %s
|
||||
AND chat_id = %s
|
||||
AND key = %s
|
||||
''', (row['id'], actor['id'], chat_id, key))
|
||||
return public(row)
|
||||
|
||||
def history(self, actor, chat_id, limit, cursor, scope):
|
||||
with self.connection() as conn:
|
||||
self.require_member(conn, actor, chat_id)
|
||||
where = 'AND (created_at, id) > (%s::timestamptz, %s::uuid)' if cursor else ''
|
||||
params = [chat_id] + list(cursor or ()) + [limit + 1]
|
||||
rows = conn.execute('''
|
||||
SELECT id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages
|
||||
WHERE 1=1
|
||||
AND chat_id = %s
|
||||
''' + where + '''
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
''', params).fetchall()
|
||||
return page([public(v) for v in rows], limit, scope)
|
||||
|
||||
def upload(self, actor, chat_id, name, source, length, mime, digest):
|
||||
attachment_id, job_id = uuid.uuid4(), uuid.uuid4()
|
||||
key = 'staging/' + str(attachment_id)
|
||||
with self.connection() as conn:
|
||||
self.require_member(conn, actor, chat_id)
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2301005)')
|
||||
pending = conn.execute("SELECT count(*) AS n FROM attachments WHERE 1=1 AND state IN ('uploading', 'queued', 'processing')").fetchone()['n']
|
||||
if pending >= int(os.environ.get('MAX_PENDING_JOBS', '100')):
|
||||
raise APIError(503, 'unavailable', 'Очередь принятых работ заполнена')
|
||||
conn.execute('''
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, staging_key, content_type, size_bytes, sha256, state)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')
|
||||
''', (attachment_id, chat_id, actor['id'], name, 'results/' + str(attachment_id) + '/v1/original', key, mime, length, digest))
|
||||
with self.connection() as conn:
|
||||
if conn.execute('SELECT id FROM attachments WHERE 1=1 AND id = %s FOR UPDATE', (attachment_id,)).fetchone() is None:
|
||||
unavailable()
|
||||
self.objects.put(key, source, length, mime, digest)
|
||||
conn.execute('''
|
||||
INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, source_key, max_attempts, trace_context)
|
||||
VALUES (%s, %s, 1, %s, %s, %s)
|
||||
''', (job_id, attachment_id, key, int(os.environ.get('JOB_MAX_ATTEMPTS', '5')), Jsonb(tele.carrier())))
|
||||
row = conn.execute('''
|
||||
UPDATE attachments
|
||||
SET state = 'queued'
|
||||
, current_job_id = %s
|
||||
, updated_at = clock_timestamp()
|
||||
WHERE 1=1
|
||||
AND id = %s
|
||||
RETURNING *
|
||||
''', (job_id, attachment_id)).fetchone()
|
||||
tele.log('attachment_accepted', attachment_id=str(attachment_id), job_id=str(job_id))
|
||||
# Capture the queued snapshot, even if the worker finishes before HTTP reply.
|
||||
return self.attachment_public(row)
|
||||
|
||||
def attachment_public(self, row):
|
||||
keys = ('id', 'chat_id', 'owner_id', 'filename', 'content_type', 'size_bytes', 'sha256', 'created_at', 'updated_at')
|
||||
result = public({k: row[k] for k in keys})
|
||||
variants = [{k: v for k, v in item.items() if k != 'object_key'} for item in row['variants']]
|
||||
return dict(result, status=row['state'], variants=variants, error_code=row['error_code'])
|
||||
|
||||
def attachment(self, actor, attachment_id):
|
||||
with self.connection() as conn:
|
||||
row = conn.execute('''
|
||||
SELECT a.*
|
||||
FROM attachments a
|
||||
JOIN chat_members cm ON cm.chat_id = a.chat_id
|
||||
WHERE 1=1
|
||||
AND a.id = %s
|
||||
AND cm.user_id = %s
|
||||
''', (attachment_id, actor['id'])).fetchone()
|
||||
if row is None:
|
||||
missing()
|
||||
if row['state'] == 'storage_missing':
|
||||
unavailable()
|
||||
if row['state'] == 'uploading':
|
||||
raise APIError(409, 'conflict', 'Приём не завершён')
|
||||
return row
|
||||
|
||||
|
||||
def re_username(value):
|
||||
import re
|
||||
return re.fullmatch(r'[a-z0-9_]{3,32}', value)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Isolated image decoder/resizer; no network, database or credentials."""
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (15, 16))
|
||||
from PIL import Image, ImageOps, PngImagePlugin
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
PngImagePlugin.MAX_TEXT_CHUNK = PngImagePlugin.MAX_TEXT_MEMORY = 1024 * 1024
|
||||
warnings.filterwarnings('error', category=Image.DecompressionBombWarning)
|
||||
|
||||
|
||||
def render(source, directory, version):
|
||||
output = []
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
raise ValueError('pixels')
|
||||
image.load()
|
||||
ImageOps.exif_transpose(image, in_place=True)
|
||||
rgba = None
|
||||
if image.mode in ('RGBA', 'LA') or (image.mode == 'P' and 'transparency' in image.info):
|
||||
rgba = image if image.mode == 'RGBA' else image.convert('RGBA')
|
||||
rgb = Image.new('RGB', rgba.size, 'white')
|
||||
alpha = rgba.getchannel('A')
|
||||
rgb.paste(rgba, mask=alpha)
|
||||
alpha.close()
|
||||
else:
|
||||
rgb = image if image.mode == 'RGB' else image.convert('RGB')
|
||||
try:
|
||||
for name, bound, fmt, quality in [('thumbnail', 256, 'JPEG', 85), ('crop', 128, 'JPEG', 85), ('optimized', 1280, 'WEBP', 82 if version == 1 else 70)]:
|
||||
if name == 'crop':
|
||||
result = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(.5, .5))
|
||||
else:
|
||||
scale = min(1, bound / max(rgb.size))
|
||||
size = tuple(max(1, math.floor(v * scale)) for v in rgb.size)
|
||||
result = rgb.resize(size, Image.Resampling.LANCZOS)
|
||||
try:
|
||||
path = directory / name
|
||||
options = {'quality': quality, 'exif': b'', 'icc_profile': None}
|
||||
if fmt == 'WEBP':
|
||||
options['method'] = 4
|
||||
result.save(path, format=fmt, **options)
|
||||
raw = path.read_bytes()
|
||||
output.append({'name': name, 'width': result.width, 'height': result.height, 'content_type': 'image/jpeg' if fmt == 'JPEG' else 'image/webp', 'size_bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
finally:
|
||||
result.close()
|
||||
finally:
|
||||
if rgb is not image:
|
||||
rgb.close()
|
||||
if rgba is not None and rgba is not image:
|
||||
rgba.close()
|
||||
return output
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
print(json.dumps(render(sys.argv[1], Path(sys.argv[2]), int(sys.argv[3]))))
|
||||
except (OSError, ValueError, SyntaxError, Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise SystemExit(2)
|
||||
+345
@@ -0,0 +1,345 @@
|
||||
"""Bounded threaded API; persistent correctness state is shared across replicas."""
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import http.server
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import vault
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from objects import bounded_upload, receive, filename, unavailable
|
||||
|
||||
import telemetry as tele
|
||||
|
||||
from common import APIError, identifier, invalid, missing, page_args
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
|
||||
|
||||
def unique_object(pairs):
|
||||
result = {}
|
||||
for key, value in pairs:
|
||||
if key in result:
|
||||
invalid()
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.connection.settimeout(3)
|
||||
|
||||
def send_error(self, code, message=None, explain=None):
|
||||
# BaseHTTPRequestHandler otherwise emits HTML without contract headers.
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.route = ''
|
||||
self.close_connection = True
|
||||
status = 413 if code == 413 else 400
|
||||
self.reply(status, {'error': {'code': 'payload_too_large' if status == 413 else 'invalid_request', 'message': 'Некорректный HTTP-запрос', 'request_id': self.request_id}})
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def reply(self, status, body):
|
||||
if status == 200 and isinstance(body, tuple):
|
||||
return self.download(*body)
|
||||
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
|
||||
if status >= 400:
|
||||
self.close_connection = True
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(raw)))
|
||||
if self.close_connection:
|
||||
self.send_header('Connection', 'close')
|
||||
self.send_header('X-Request-Id', self.request_id)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
if status == 401 and self.server.store.persistent:
|
||||
self.send_header('WWW-Authenticate', 'Basic realm="messenger", charset="UTF-8"' if self.route == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
|
||||
if status == 429:
|
||||
self.send_header('Retry-After','1')
|
||||
self.end_headers()
|
||||
if self.command != 'HEAD':
|
||||
self.wfile.write(raw)
|
||||
|
||||
def download(self, row, response, chunk):
|
||||
stream = response['Body']
|
||||
try:
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', row['content_type'])
|
||||
self.send_header('Content-Length', str(row['size_bytes']))
|
||||
self.send_header('X-Request-Id', self.request_id)
|
||||
self.send_header('X-Instance-Id', self.server.instance_id)
|
||||
self.send_header('X-Content-Type-Options', 'nosniff')
|
||||
self.send_header('Cache-Control', 'private, no-store')
|
||||
self.end_headers()
|
||||
deadline = time.monotonic() + 15
|
||||
while chunk:
|
||||
self.wfile.write(chunk)
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError()
|
||||
chunk = stream.read(64 * 1024)
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
def body(self):
|
||||
lengths = self.headers.get_all('Content-Length', [])
|
||||
if len(lengths) != 1 or self.headers.get('Transfer-Encoding'):
|
||||
invalid()
|
||||
try:
|
||||
length = int(lengths[0])
|
||||
except ValueError:
|
||||
invalid()
|
||||
if length > 32768:
|
||||
raise APIError(413, 'payload_too_large', 'Максимальный JSON — 32768 байт')
|
||||
if length <= 0:
|
||||
invalid()
|
||||
if self.headers.get_content_type() != 'application/json':
|
||||
raise APIError(415, 'unsupported_media_type', 'Требуется application/json')
|
||||
try:
|
||||
raw = self.rfile.read(length)
|
||||
if len(raw) != length:
|
||||
invalid()
|
||||
return json.loads(raw.decode('utf-8'), object_pairs_hook=unique_object, parse_constant=lambda _: invalid())
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
invalid()
|
||||
|
||||
def dispatch(self):
|
||||
self.request_id = uuid.uuid4().hex
|
||||
self.route = urlsplit(self.path).path
|
||||
template = tele.route(self.route)
|
||||
method = self.command if self.command in ('GET','POST','DELETE','PUT','PATCH','HEAD','OPTIONS') else 'OTHER'
|
||||
attrs = {'method':method, 'route':template}
|
||||
started = time.monotonic()
|
||||
context = tele.span(method+' '+template, {'http.request.method':method, 'http.route':template},
|
||||
tele.SpanKind.SERVER, {k:self.headers[k] for k in ('traceparent','tracestate') if k in self.headers})
|
||||
# Health probes do not enter business telemetry or create noisy dependency spans.
|
||||
if template.startswith('/health/'):
|
||||
context = tele.suppressed()
|
||||
status = 500
|
||||
with context as current:
|
||||
try:
|
||||
status, result = self.route_request()
|
||||
except APIError as exc:
|
||||
status = exc.status
|
||||
result = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
|
||||
except Exception:
|
||||
result = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
|
||||
try:
|
||||
self.reply(status, result)
|
||||
except (BrokenPipeError, ConnectionResetError, TimeoutError, OSError):
|
||||
self.close_connection = True
|
||||
if not template.startswith('/health/'):
|
||||
attrs['status_class'] = str(status//100)+'xx'
|
||||
tele.count('http_server_requests', attrs)
|
||||
tele.duration('http_server_duration', time.monotonic()-started, attrs)
|
||||
if current is not None:
|
||||
current.set_attribute('http.response.status_code', status)
|
||||
if status >= 500:
|
||||
current.set_status(tele.Status(tele.StatusCode.ERROR))
|
||||
tele.log('http_request', request_id=self.request_id, method=method, route=template, status=status, duration_seconds=round(time.monotonic()-started,6))
|
||||
|
||||
def route_request(self):
|
||||
method, path, store = self.command, self.route, self.server.store
|
||||
if path != '/health/live':
|
||||
vault.ensure()
|
||||
if method == 'GET' and path in ('/health/live', '/health/ready'):
|
||||
if path.endswith('ready'):
|
||||
store.ready()
|
||||
return 200, {'status': 'ok'}
|
||||
if method == 'POST' and path == '/api/v1/users':
|
||||
return 201, store.register(self.body())
|
||||
if store.persistent and method == 'POST' and path == '/api/v1/auth/sessions':
|
||||
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
||||
invalid()
|
||||
return 201, store.login(self.headers)
|
||||
actor = store.authenticate(self.headers)
|
||||
if store.persistent and method == 'DELETE' and path == '/api/v1/auth/sessions/current':
|
||||
store.logout(self.headers)
|
||||
return 204, None
|
||||
if method == 'GET' and path == '/api/v1/users/me':
|
||||
return 200, actor
|
||||
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
|
||||
if path == '/api/v1/chats':
|
||||
if method == 'POST':
|
||||
return 201, store.create_chat(actor, self.body())
|
||||
if method == 'GET':
|
||||
scope = 'chats:' + actor['id']
|
||||
limit, cursor = page_args(query, scope)
|
||||
return 200, store.list_chats(actor, limit, cursor, scope)
|
||||
parts = path.split('/')
|
||||
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'attachments'] and method == 'GET':
|
||||
row = store.attachment(actor, identifier(parts[4]))
|
||||
if len(parts) == 5:
|
||||
if query:
|
||||
invalid()
|
||||
return 200, store.attachment_public(row)
|
||||
if parts[5] == 'content':
|
||||
if set(query) - {'variant'} or any(len(v) != 1 for v in query.values()):
|
||||
invalid()
|
||||
if row['state'] != 'ready':
|
||||
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||
variant = query.get('variant', ['original'])[0]
|
||||
if variant != 'original':
|
||||
item = next((v for v in row['variants'] if v['name'] == variant), None)
|
||||
if item is None:
|
||||
missing()
|
||||
row = dict(row, object_key=item['object_key'], content_type=item['content_type'], size_bytes=item['size_bytes'])
|
||||
response = store.objects.get(row['object_key'])
|
||||
try:
|
||||
chunk = response['Body'].read(64 * 1024)
|
||||
if response['ContentLength'] != row['size_bytes'] or not chunk:
|
||||
unavailable()
|
||||
except Exception:
|
||||
response['Body'].close()
|
||||
unavailable()
|
||||
return 200, (row, response, chunk)
|
||||
if len(parts) in (5, 6) and parts[1:4] == ['api', 'v1', 'chats']:
|
||||
chat_id = identifier(parts[4])
|
||||
if store.access_mode == 'baseline':
|
||||
store.chat(actor, chat_id)
|
||||
if len(parts) == 5 and method == 'GET':
|
||||
return 200, store.chat(actor, chat_id)
|
||||
if len(parts) == 6 and parts[5] == 'attachments' and method == 'POST':
|
||||
store.chat(actor, chat_id)
|
||||
if set(query) != {'filename'} or len(query['filename']) != 1:
|
||||
invalid()
|
||||
name = filename(query['filename'][0])
|
||||
with bounded_upload():
|
||||
with receive(self) as (source, length, mime, digest):
|
||||
return 202, store.upload(actor, chat_id, name, source, length, mime, digest)
|
||||
if len(parts) == 6 and parts[5] == 'messages':
|
||||
if method == 'POST':
|
||||
keys = self.headers.get_all('Idempotency-Key', [])
|
||||
return 201, store.send(actor, chat_id, self.body(), keys[0] if len(keys) == 1 else None)
|
||||
if method == 'GET':
|
||||
scope = 'messages:' + chat_id
|
||||
limit, cursor = page_args(query, scope)
|
||||
return 200, store.history(actor, chat_id, limit, cursor, scope)
|
||||
missing()
|
||||
|
||||
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = do_OPTIONS = dispatch
|
||||
|
||||
|
||||
class Server(http.server.HTTPServer):
|
||||
def get_request(self):
|
||||
raw,address=super().get_request()
|
||||
raw.settimeout(2)
|
||||
context=ssl.create_default_context(ssl.Purpose.CLIENT_AUTH,cafile='/tls/ca.crt')
|
||||
context.minimum_version=ssl.TLSVersion.TLSv1_2
|
||||
context.load_cert_chain('/tls/tls.crt','/tls/tls.key')
|
||||
context.verify_mode=ssl.CERT_REQUIRED
|
||||
try:
|
||||
conn=context.wrap_socket(raw,server_side=True)
|
||||
peer=conn.getpeercert()
|
||||
identities={v for k,v in peer.get('subjectAltName',[]) if k=='URI'}
|
||||
revoked={int(x.strip(),16) for x in Path('/tls/revoked.list').read_text().splitlines() if x.strip()}
|
||||
if identities!={'spiffe://mtusi.lab/lb'} or int(peer['serialNumber'],16) in revoked:
|
||||
conn.close()
|
||||
raise OSError('Client identity denied')
|
||||
return conn,address
|
||||
except Exception:
|
||||
raw.close()
|
||||
raise
|
||||
|
||||
request_queue_size = 64
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, address, store):
|
||||
self.store = store
|
||||
self.instance_id = tele.INSTANCE
|
||||
workers = int(os.environ.get('HTTP_WORKERS', '16'))
|
||||
if not 1 <= workers <= 128:
|
||||
raise ValueError('HTTP_WORKERS must be 1..128')
|
||||
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers)
|
||||
self.slots = threading.BoundedSemaphore(workers * 2)
|
||||
self.connections = set()
|
||||
self.connections_lock = threading.Lock()
|
||||
super().__init__(address, Handler)
|
||||
|
||||
def process_request(self, request, address):
|
||||
if not self.slots.acquire(blocking=False):
|
||||
rid = uuid.uuid4().hex
|
||||
raw = json.dumps({'error': {'code': 'unavailable', 'message': 'Очередь запросов заполнена', 'request_id': rid}}).encode()
|
||||
try:
|
||||
request.settimeout(0.2)
|
||||
request.sendall(('HTTP/1.0 503 Service Unavailable\r\nContent-Type: application/json\r\nX-Instance-Id: ' + self.instance_id + '\r\nX-Request-Id: ' + rid + '\r\nContent-Length: ' + str(len(raw)) + '\r\n\r\n').encode() + raw)
|
||||
except OSError:
|
||||
pass
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
with self.connections_lock:
|
||||
self.connections.add(request)
|
||||
self.executor.submit(self.worker, request, address)
|
||||
|
||||
def worker(self, request, address):
|
||||
try:
|
||||
self.finish_request(request, address)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
with self.connections_lock:
|
||||
self.connections.discard(request)
|
||||
self.slots.release()
|
||||
|
||||
def close_connections(self):
|
||||
with self.connections_lock:
|
||||
connections = list(self.connections)
|
||||
for connection in connections:
|
||||
try:
|
||||
connection.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def server_close(self):
|
||||
super().server_close()
|
||||
# Bound draining even when a slow client keeps an incomplete request alive.
|
||||
deadline = threading.Timer(5, self.close_connections)
|
||||
deadline.start()
|
||||
try:
|
||||
self.executor.shutdown(wait=True)
|
||||
finally:
|
||||
deadline.cancel()
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=os.environ.get('LOG_LEVEL', 'INFO'), format='%(asctime)s %(levelname)s %(message)s')
|
||||
tele.configure('messenger-api')
|
||||
from postgres import Store
|
||||
store = Store()
|
||||
server = Server((os.environ.get('APP_HOST', '0.0.0.0'), int(os.environ.get('APP_PORT', '8080'))), store)
|
||||
Path('/tmp/instance').write_text(server.instance_id)
|
||||
stopping = threading.Event()
|
||||
|
||||
def stop(signum, frame):
|
||||
if not stopping.is_set():
|
||||
stopping.set()
|
||||
LOG.info('shutdown_started signal=%s', signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGUSR1, lambda *_: LOG.warning('pool_stats instance=%s stats=%s', server.instance_id, json.dumps(store.pool.get_stats())))
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
LOG.info('started port=%s', server.server_address[1])
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
store.close()
|
||||
tele.shutdown()
|
||||
LOG.info('shutdown_complete')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Allowlisted manual SDK telemetry; bounded exporters never carry business payloads."""
|
||||
import contextlib
|
||||
import contextvars
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import resource
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from opentelemetry import trace, metrics
|
||||
from opentelemetry.trace import SpanKind, Status, StatusCode
|
||||
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import TracerProvider, SpanProcessor
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SpanExporter, SpanExportResult
|
||||
from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
||||
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
||||
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
||||
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
|
||||
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
||||
|
||||
INSTANCE = uuid.uuid4().hex
|
||||
ACTIVE = False
|
||||
SUPPRESSED = contextvars.ContextVar('telemetry_suppressed', default=False)
|
||||
PROVIDERS = []
|
||||
COUNTERS = {}
|
||||
HISTOGRAMS = {}
|
||||
STATE = {'worker_busy': 0, 'queue_depth': 0, 'queue_oldest_age': 0, 'queue_observation_ok': 1,
|
||||
'spans_finished': 0, 'spans_exported': 0, 'spans_export_failed': 0}
|
||||
LOCK = threading.Lock()
|
||||
PROPAGATOR = TraceContextTextMapPropagator()
|
||||
LOGGER = logging.getLogger('messenger.telemetry')
|
||||
TRACER = trace.get_tracer('messenger')
|
||||
POOL = None
|
||||
SERVICE = ''
|
||||
BUCKETS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 20, 30, 60]
|
||||
|
||||
class FinishedSpans(SpanProcessor):
|
||||
def on_end(self, span):
|
||||
with LOCK:
|
||||
STATE['spans_finished'] += 1
|
||||
|
||||
class CountedExporter(SpanExporter):
|
||||
def __init__(self, delegate):
|
||||
self.delegate = delegate
|
||||
|
||||
def export(self, spans):
|
||||
result = self.delegate.export(spans)
|
||||
with LOCK:
|
||||
STATE['spans_exported' if result == SpanExportResult.SUCCESS else 'spans_export_failed'] += len(spans)
|
||||
return result
|
||||
|
||||
def shutdown(self):
|
||||
self.delegate.shutdown()
|
||||
|
||||
def force_flush(self, timeout_millis=30000):
|
||||
return self.delegate.force_flush(timeout_millis)
|
||||
|
||||
def configure(service):
|
||||
global ACTIVE, TRACER, SERVICE
|
||||
if PROVIDERS or os.environ.get('TELEMETRY_ENABLED', '1') == '0':
|
||||
return
|
||||
SERVICE = service
|
||||
ratio = float(os.environ.get('TRACE_SAMPLE_RATIO', '0.25'))
|
||||
if not 0 <= ratio <= 1:
|
||||
raise ValueError('TRACE_SAMPLE_RATIO must be 0..1')
|
||||
resource_attributes = Resource.create({
|
||||
'service.name': service, 'service.version': 'lab7.1',
|
||||
'service.instance.id': INSTANCE, 'deployment.environment.name': 'lab',
|
||||
})
|
||||
endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'https://otel-collector:8443').rstrip('/')
|
||||
if not endpoint.startswith('https://'):
|
||||
raise ValueError('OTLP endpoint must use verified HTTPS')
|
||||
provider = TracerProvider(resource=resource_attributes, sampler=ParentBased(TraceIdRatioBased(ratio)))
|
||||
provider.add_span_processor(FinishedSpans())
|
||||
provider.add_span_processor(BatchSpanProcessor(
|
||||
CountedExporter(OTLPSpanExporter(endpoint=endpoint+'/v1/traces', timeout=1, certificate_file='/tls/ca.crt', client_certificate_file='/tls/tls.crt', client_key_file='/tls/tls.key')),
|
||||
max_queue_size=512, max_export_batch_size=128, schedule_delay_millis=500,
|
||||
export_timeout_millis=1500))
|
||||
trace.set_tracer_provider(provider)
|
||||
TRACER = trace.get_tracer('messenger')
|
||||
reader = PeriodicExportingMetricReader(
|
||||
OTLPMetricExporter(endpoint=endpoint+'/v1/metrics', timeout=1, certificate_file='/tls/ca.crt', client_certificate_file='/tls/tls.crt', client_key_file='/tls/tls.key'),
|
||||
export_interval_millis=2000, export_timeout_millis=1500)
|
||||
meter_provider = MeterProvider(resource=resource_attributes, metric_readers=[reader],
|
||||
views=[View(instrument_name='*duration', aggregation=ExplicitBucketHistogramAggregation(BUCKETS))])
|
||||
metrics.set_meter_provider(meter_provider)
|
||||
meter = metrics.get_meter('messenger')
|
||||
for name in ['http_server_requests', 'jobs_processed', 'job_retries']:
|
||||
COUNTERS[name] = meter.create_counter(name)
|
||||
for name in ['http_server_duration', 'job_processing_duration', 'job_ready_duration']:
|
||||
HISTOGRAMS[name] = meter.create_histogram(name, unit='s')
|
||||
def gauge(name, getter, unit=''):
|
||||
def observe(options):
|
||||
try:
|
||||
yield metrics.Observation(getter(), {})
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
meter.create_observable_gauge(name, callbacks=[observe], unit=unit)
|
||||
for name in ['queue_depth', 'queue_oldest_age', 'queue_observation_ok', 'worker_busy',
|
||||
'spans_finished', 'spans_exported', 'spans_export_failed']:
|
||||
gauge(name, lambda name=name: STATE[name], 's' if name=='queue_oldest_age' else '')
|
||||
gauge('worker_capacity', lambda: int(os.environ.get('WORKER_CONCURRENCY', '1')) if service=='messenger-worker' else 0)
|
||||
gauge('db_pool_used', lambda: max(0, POOL.get_stats().get('pool_size',0)-POOL.get_stats().get('pool_available',0)) if POOL else 0)
|
||||
gauge('db_pool_waiting', lambda: POOL.get_stats().get('requests_waiting',0) if POOL else 0)
|
||||
gauge('db_pool_capacity', lambda: int(os.environ.get('DB_POOL_MAX','8')) if POOL else 0)
|
||||
gauge('process_resident_memory', lambda: int(Path('/proc/self/statm').read_text().split()[1])*os.sysconf('SC_PAGE_SIZE'), 'By')
|
||||
gauge('container_memory', lambda: int(Path('/sys/fs/cgroup/memory.current').read_text()), 'By')
|
||||
def cpu(options):
|
||||
use = resource.getrusage(resource.RUSAGE_SELF)
|
||||
yield metrics.Observation(use.ru_utime+use.ru_stime, {})
|
||||
meter.create_observable_counter('process_cpu', callbacks=[cpu], unit='s')
|
||||
log_provider = LoggerProvider(resource=resource_attributes)
|
||||
log_provider.add_log_record_processor(BatchLogRecordProcessor(
|
||||
OTLPLogExporter(endpoint=endpoint+'/v1/logs', timeout=1, certificate_file='/tls/ca.crt', client_certificate_file='/tls/tls.crt', client_key_file='/tls/tls.key'),
|
||||
max_queue_size=256, max_export_batch_size=64, schedule_delay_millis=500,
|
||||
export_timeout_millis=1500))
|
||||
LOGGER.setLevel(logging.INFO)
|
||||
LOGGER.propagate = False
|
||||
LOGGER.addHandler(LoggingHandler(level=logging.INFO, logger_provider=log_provider))
|
||||
PROVIDERS.extend([provider, meter_provider, log_provider])
|
||||
ACTIVE = True
|
||||
|
||||
def pool(value):
|
||||
global POOL
|
||||
POOL = value
|
||||
|
||||
def route(path):
|
||||
if path in ['/health/live', '/health/ready', '/api/v1/users', '/api/v1/users/me',
|
||||
'/api/v1/auth/sessions', '/api/v1/auth/sessions/current', '/api/v1/chats']:
|
||||
return path
|
||||
patterns = [
|
||||
(r'/api/v1/chats/[^/]+/messages', '/api/v1/chats/{chat_id}/messages'),
|
||||
(r'/api/v1/chats/[^/]+/attachments', '/api/v1/chats/{chat_id}/attachments'),
|
||||
(r'/api/v1/chats/[^/]+', '/api/v1/chats/{chat_id}'),
|
||||
(r'/api/v1/attachments/[^/]+/content', '/api/v1/attachments/{attachment_id}/content'),
|
||||
(r'/api/v1/attachments/[^/]+', '/api/v1/attachments/{attachment_id}'),
|
||||
]
|
||||
return next((template for pattern,template in patterns if re.fullmatch(pattern,path)), 'unmatched')
|
||||
|
||||
@contextlib.contextmanager
|
||||
def span(name, attributes=None, kind=SpanKind.INTERNAL, carrier=None):
|
||||
if not ACTIVE or SUPPRESSED.get() or (name == 'db.transaction' and not trace.get_current_span().get_span_context().is_valid):
|
||||
yield trace.get_current_span()
|
||||
return
|
||||
if carrier is not None:
|
||||
carrier={k:v for k,v in carrier.items() if
|
||||
(k=='traceparent' and len(v)==55) or (k=='tracestate' and len(v)<=512)}
|
||||
context = PROPAGATOR.extract(carrier) if carrier is not None else None
|
||||
with TRACER.start_as_current_span(name, attributes=attributes or {}, kind=kind,
|
||||
context=context, record_exception=False, set_status_on_exception=False) as current:
|
||||
try:
|
||||
yield current
|
||||
except Exception as error:
|
||||
current.set_attribute('error.type', type(error).__name__)
|
||||
current.set_status(Status(StatusCode.ERROR))
|
||||
log('operation_failed', error_type=type(error).__name__)
|
||||
raise
|
||||
|
||||
@contextlib.contextmanager
|
||||
def suppressed():
|
||||
token = SUPPRESSED.set(True)
|
||||
try:
|
||||
yield None
|
||||
finally:
|
||||
SUPPRESSED.reset(token)
|
||||
|
||||
def carrier():
|
||||
result = {}
|
||||
if ACTIVE:
|
||||
PROPAGATOR.inject(result)
|
||||
return result
|
||||
|
||||
def log(event, **attributes):
|
||||
if not ACTIVE:
|
||||
return
|
||||
# Callers pass only fixed event names, bounded outcomes and technical correlation IDs.
|
||||
context = trace.get_current_span().get_span_context()
|
||||
body = {'timestamp': dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
'severity': 'INFO', 'service.name': SERVICE, 'event': event,
|
||||
'sampled': bool(context.trace_flags.sampled), 'trace_id': format(context.trace_id,'032x'), 'span_id': format(context.span_id,'016x'),
|
||||
**attributes}
|
||||
LOGGER.info(json.dumps(body, ensure_ascii=True, separators=(',',':')))
|
||||
|
||||
def count(name, attributes=None):
|
||||
if ACTIVE:
|
||||
COUNTERS[name].add(1, attributes or {})
|
||||
|
||||
def duration(name, seconds, attributes=None):
|
||||
if ACTIVE:
|
||||
HISTOGRAMS[name].record(max(0, seconds), attributes or {})
|
||||
|
||||
def flush():
|
||||
for provider in PROVIDERS:
|
||||
provider.force_flush(timeout_millis=5000)
|
||||
|
||||
def shutdown():
|
||||
for provider in PROVIDERS:
|
||||
provider.shutdown()
|
||||
|
||||
class S3Client:
|
||||
"""Span each SDK request without serializing args, keys, URLs or SDK exceptions."""
|
||||
def __init__(self, client):
|
||||
self.client = client
|
||||
|
||||
def __getattr__(self, name):
|
||||
method = getattr(self.client,name)
|
||||
if name not in {'head_object','get_object','put_object','delete_object','head_bucket','create_bucket','list_objects_v2'}:
|
||||
return method
|
||||
def call(*args, **kwargs):
|
||||
with span('s3.'+name, {'rpc.system':'aws-api','rpc.service':'S3','rpc.method':name}, SpanKind.CLIENT):
|
||||
return method(*args, **kwargs)
|
||||
return call
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Short service certificate authentication; bounded secret cache, no root/bootstrap token."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from common import APIError
|
||||
|
||||
LOCK=threading.RLock()
|
||||
DATA={}
|
||||
VALID_UNTIL=0
|
||||
NEXT_REFRESH=0
|
||||
|
||||
def request(path, data=None, token=None):
|
||||
context=ssl.create_default_context(cafile='/tls/ca.crt')
|
||||
context.load_cert_chain('/tls/tls.crt','/tls/tls.key')
|
||||
headers={'Content-Type':'application/json'}
|
||||
if token:headers['X-Vault-Token']=token
|
||||
req=urllib.request.Request('https://bao:8200/v1/'+path,
|
||||
data=None if data is None else json.dumps(data).encode(),headers=headers)
|
||||
opener=urllib.request.build_opener(urllib.request.ProxyHandler({}),urllib.request.HTTPSHandler(context=context))
|
||||
with opener.open(req,timeout=2) as response:return json.load(response)
|
||||
|
||||
def ensure():
|
||||
global DATA,VALID_UNTIL,NEXT_REFRESH
|
||||
if not os.environ.get('BAO_ROLE'):return
|
||||
with LOCK:
|
||||
now=time.monotonic()
|
||||
if now>=NEXT_REFRESH:
|
||||
NEXT_REFRESH=now+5
|
||||
try:
|
||||
auth=request('auth/cert/login',{'name':os.environ['BAO_ROLE']})['auth']
|
||||
values=request('secret/data/'+os.environ['BAO_ROLE'],token=auth['client_token'])['data']['data']
|
||||
DATA=values
|
||||
# Re-authentication replaces renewal; cache never outlives auth TTL.
|
||||
VALID_UNTIL=now+min(20,auth['lease_duration'])
|
||||
except Exception:
|
||||
pass # No raw response/exception: it can contain auth material.
|
||||
if now>=VALID_UNTIL:
|
||||
raise APIError(503,'unavailable','Сервисные полномочия временно недоступны')
|
||||
|
||||
def value(name):
|
||||
ensure()
|
||||
return DATA[name]
|
||||
|
||||
def redis_tls():
|
||||
return dict(ssl_ca_certs='/tls/ca.crt',ssl_certfile='/tls/tls.crt',
|
||||
ssl_keyfile='/tls/tls.key',ssl_cert_reqs='required',ssl_check_hostname=True)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user