556 changed files with 98155 additions and 65 deletions
+14
View File
@@ -0,0 +1,14 @@
# Лабораторная 7
Основа: ЛР6 f60297b8e2f252d2b40bf0cdbbcfa6ef5e447496. Цель 5, трек 1 «Компрометация worker».
- Задание README.md, COURSE.md, contracts/ и tests/ сохраняются. contracts/trust-matrix.md заполняется по заданию.
- OpenSpec secure-lab7-messenger завершён: openspec/changes/archive/2026-11-14-secure-lab7-messenger. Новые изменения начинаются с нового change.
- Python 3.14; ruff: 4 пробела, двойные кавычки. Без комментариев в коде.
- Слои: domain/application зависят от портов; инфраструктура и transport подключаются в ioc.
- make check validate lint до коммита; собственные tests и реальные HTTPS сценарии до заявки PASS.
- Ветка solution/bpi2303-kostylev, английские commits. origin — fork ЛР7, upstream — исходная ЛР7.
- Docker project mtusi2026lab7, отдельные volumes; не изменять другие лабораторные.
- Root token, unseal shares, CA key только в operator каталоге вне Git и mounts сервисов.
- Не публиковать secrets/.env/credentials в Git, логи и telemetry. Отрицательные тесты только своего стенда.
- Отчёт содержит фактические команды/результаты/ограничения. Текст видео в ../тексты/ после проверки результатов.
+58
View File
@@ -0,0 +1,58 @@
FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder
COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
ENV UV_COMPILE_BYTECODE=1 \
UV_LINK_MODE=copy \
UV_PYTHON_DOWNLOADS=never \
UV_PROJECT_ENVIRONMENT=/opt/venv
WORKDIR /build
COPY pyproject.toml uv.lock README.md ./
RUN uv sync --locked --no-dev --no-install-project --no-editable
COPY src ./src
RUN uv sync --locked --no-dev --no-editable
FROM python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PATH="/opt/venv/bin:$PATH" \
HOST=0.0.0.0 \
PORT=8080 \
LOG_LEVEL=INFO \
WEB_CONCURRENCY=2 \
REQUEST_TIMEOUT_SECONDS=8 \
DRAIN_SECONDS=2 \
SHUTDOWN_TIMEOUT_SECONDS=6
RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends haproxy && rm -rf /var/lib/apt/lists/*
RUN python -m pip uninstall -y pip setuptools
RUN groupadd --system --gid 10001 messenger \
&& useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
COPY --from=builder --chown=root:root /opt/venv /opt/venv
WORKDIR /app
COPY --chown=root:root alembic.ini ./
COPY --chown=root:root alembic ./alembic
COPY --chown=root:root scripts/secure_runtime.py /app/secure_runtime.py
ENTRYPOINT ["python", "/app/secure_runtime.py"]
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/live',timeout=2).status==200 else 1)"]
CMD ["python", "-m", "gunicorn", \
"--config", "python:messenger.gunicorn_conf", \
"messenger.main:create_app()"]
+285 -4
View File
@@ -1,17 +1,54 @@
PYTHON ?= python3
BASE_URL ?= https://localhost:8443
CA_FILE ?=
BASE_URL ?= https://localhost:18443
CA_FILE ?= $(CURDIR)/secrets/pki/operator/ca.crt
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover test-security
VENV ?= .venv
UV ?= uv
COMPOSE ?= docker compose
RATE ?= 100
FROM_REV ?=
SHIFT ?= 0
SCRIPTS_PY = scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py \
scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py \
scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
SCENARIO ?= failover-upload
PSQL = $(COMPOSE) exec -T postgres psql -U messenger -d messenger -X -q
OBS_CHECK = $(PYTHON) scripts/obs_check.py
APP_METRICS = {job=~"messenger-.*"}
TRACK2_OUTPUT ?= evidence/track2.json
TRACK2_SCENARIOS = smoke_pipeline,duplicate_after_ready,crash_commit_publish,crash_publish_confirm,crash_after_s3_before_commit,crash_after_commit_before_ack,lease_expiry_fencing,crash_before_side_effect,worker_kill_mid_task,duplicate_in_flight,reorder_attempts,reorder_versions,stale_after_new_version
PIPELINE_VERSION ?= 2
PYTEST_ARGS ?=
.PHONY: help check validate test load test-persistence test-failover \
install lint format typecheck student-test \
secrets up down reset-data migrate ps logs restore lb-check failover-evidence stack-evidence \
persistence-evidence storage-evidence cache-evidence test-faults seed explain \
limits detection failover-each rolling-update lab4-level4 \
scenario s3-check attachments-evidence upload-faults upload-memory \
images queues topology worker-stop worker-start broker-restart suite-preflight level3 \
pipeline-evidence level4 replay failed dlq \
obs-validate telemetry-check \
faults-evidence retry-check reconcile reconcile-demo reconcile-cost load-evidence \
clean-clone-evidence overload-leftovers
.NOTPARALLEL:
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "ЛР7: install; security-bootstrap; up; check validate lint; student-test; test test-security"
@echo "security-audit: TLS/ACL и hardening; security-track: три опыта отзыва worker"
@echo "security-lifecycle: sealed/restart/TTL и общий login limit; security-evidence: telemetry/права"
@echo "test-persistence ACTION_SCRIPT=scripts/restart.sh; python3 scripts/security_failover.py"
@echo "security-unseal: явный operator unseal; down: остановка; reset-data: удаление volumes"
@echo "REPORT.md — действующие команды ЛР7. Остальные исследовательские targets перенесены из ЛР3–6."
check:
$(PYTHON) scripts/check_contract.py
validate: PYTHON := $(if $(wildcard $(VENV)/bin/python),$(VENV)/bin/python,$(PYTHON))
validate:
$(PYTHON) scripts/validate_openapi.py
@@ -29,5 +66,249 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
install:
$(UV) sync --group dev
lint:
$(VENV)/bin/ruff check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format --check src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ty check src student-tests alembic
format:
$(VENV)/bin/ruff check --fix src student-tests $(SCRIPTS_PY)
$(VENV)/bin/ruff format src student-tests $(SCRIPTS_PY)
typecheck:
$(VENV)/bin/ty check src student-tests alembic
student-test:
$(VENV)/bin/pytest $(PYTEST_ARGS)
secrets:
sh scripts/secrets.sh
sh scripts/broker-secrets.sh
sh scripts/secrets-lab6.sh
up:
sh scripts/secure-up.sh
down:
$(COMPOSE) --profile bench --profile tools --profile reconcile down
reset-data:
@echo "Удаляются volumes с учебными данными стенда"
$(COMPOSE) --profile bench --profile tools --profile reconcile down -v --remove-orphans
migrate:
$(COMPOSE) run --rm migrate
ps:
$(COMPOSE) ps
logs:
$(COMPOSE) logs --tail 200
restore:
scripts/restore.sh
lb-check:
$(PYTHON) scripts/probe.py check --rounds 20 --out evidence/lb-check.json
retry-check:
$(PYTHON) scripts/probe.py retry --out evidence/retry.json
limits:
$(PYTHON) scripts/limits.py --out evidence/limits.txt
detection:
rm -rf evidence/detection
for run in 1 2 3; do sh scripts/detection.sh evidence/detection/run$$run || exit 1; done
failover-each:
RATE=$(RATE) sh scripts/failover-each.sh evidence/failover-each
rolling-update:
ifneq ($(FROM_REV),)
git archive "$(FROM_REV)" | docker build -q -t "mtusi-messenger:$(FROM_REV)" -
API_IMAGE="mtusi-messenger:$(FROM_REV)" $(COMPOSE) up -d --no-deps --force-recreate --wait api
scripts/lb-wait.sh
endif
$(COMPOSE) build -q api
RATE=$(RATE) $(PYTHON) scripts/probe.py load --expect replace --duration 90 \
--out evidence/rolling$(if $(FROM_REV),-from-$(FROM_REV)) \
-- scripts/rolling-update.sh
lab4-level4: limits lb-check detection failover-each rolling-update
scenario:
$(PYTHON) scripts/lab4_scenarios.py $(SCENARIO) --base-url "$(BASE_URL)" --restore --output evidence/$(SCENARIO).json
scripts/lb-wait.sh
s3-check:
sh scripts/s3-check.sh > evidence/s3-security.txt; status=$$?; cat evidence/s3-security.txt; exit $$status
attachments-evidence:
sh scripts/evidence-attachments.sh
upload-faults:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD) BASE_URL=$(BASE_URL)"; \
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s --no-header --tb=short \
-p no:cacheprovider student-tests/stack/test_attachment_faults.py; } > evidence/upload-faults.txt; \
status=$$?; cat evidence/upload-faults.txt; exit $$status
upload-memory:
$(PYTHON) scripts/upload_memory.py --base-url "$(BASE_URL)" $(ARGS)
load-evidence:
sh scripts/load-evidence.sh
overload-leftovers:
sh scripts/overload-leftovers.sh
clean-clone-evidence:
sh scripts/clean-clone.sh
reconcile:
$(COMPOSE) run --rm --no-deps reconciler $(ARGS)
reconcile-demo:
$(PYTHON) scripts/reconcile_track.py demo --base-url "$(BASE_URL)" $(ARGS)
reconcile-cost:
$(PYTHON) scripts/reconcile_track.py cost $(ARGS)
failover-evidence:
sh scripts/failover-evidence.sh "$(BASE_URL)" "$(SHIFT)"
stack-evidence:
sh scripts/evidence-stack.sh
persistence-evidence:
sh scripts/persistence-evidence.sh "$(BASE_URL)" > evidence/persistence.txt
cat evidence/persistence.txt
storage-evidence:
sh scripts/evidence-storage.sh
cache-evidence:
sh scripts/evidence-cache.sh > evidence/cache.txt
cat evidence/cache.txt
test-faults:
BASE_URL="$(BASE_URL)" $(VENV)/bin/pytest -m stack -v -s student-tests/stack
faults-evidence:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; echo "\$$ make test-faults"; \
$(MAKE) --no-print-directory test-faults 2>&1; } | sed 's#$(CURDIR)#<repo>#g' > evidence/faults.txt
cat evidence/faults.txt
grep -qE '^=+ [0-9]+ passed in' evidence/faults.txt
seed:
$(PSQL) -f - < scripts/sql/seed-history.sql
explain: seed
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \
echo "\$$ make explain # scripts/sql/explain-history.sql"; \
$(PSQL) -f - < scripts/sql/explain-history.sql; } > evidence/explain-history.txt
cat evidence/explain-history.txt
images:
$(PYTHON) scripts/make_images.py
queues:
sh scripts/broker.sh queues
topology:
{ echo "date: $$(date -u +%FT%TZ) commit: $$(git rev-parse --short HEAD)"; \
echo "$$ rabbitmqctl list_queues -p messenger name type durable policy messages"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_queues -p messenger name type durable policy messages; \
echo "$$ rabbitmqctl list_consumers -p messenger"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_consumers -p messenger queue_name ack_required prefetch_count; \
echo "$$ rabbitmqctl list_permissions -p messenger"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_permissions -p messenger; \
echo "$$ rabbitmqctl list_users"; \
$(COMPOSE) exec -T broker rabbitmqctl -q list_users; \
echo "$$ docker compose ps"; \
$(COMPOSE) ps --format '{{.Service}}\t{{.Status}}\t{{.Ports}}'; } > evidence/topology.txt
cat evidence/topology.txt
worker-stop:
sh scripts/worker.sh stop
worker-start:
sh scripts/worker.sh start
broker-restart:
sh scripts/broker.sh restart
suite-preflight:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --preflight
level3:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 3 --output evidence/level3.json
level4:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 4 --output evidence/level4.json
failed:
$(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin failed
replay:
@test -n "$(ID)" || (echo "Set ID=<attachment_id>"; exit 2)
$(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin replay --attachment-id "$(ID)"
dlq:
sh scripts/broker.sh dlq
pipeline-evidence:
$(PYTHON) scripts/pipeline_evidence.py --base-url "$(BASE_URL)" > evidence/pipeline-stand.txt; \
status=$$?; cat evidence/pipeline-stand.txt; exit $$status
obs-validate:
sh scripts/obs-validate.sh
telemetry-check:
$(OBS_CHECK) grafana > evidence/grafana.json
$(OBS_CHECK) dashboards --strict > evidence/dashboards.json
$(OBS_CHECK) request --base-url "$(BASE_URL)" > evidence/trace-request.json
$(OBS_CHECK) job --base-url "$(BASE_URL)" > evidence/trace-job.json
$(OBS_CHECK) leaks --base-url "$(BASE_URL)" > evidence/leaks.json
$(OBS_CHECK) query 'count by (__name__, job) ($(APP_METRICS))' \
'count by (http_route, http_request_method) (http_server_request_duration_seconds_count)' \
'count by (job_type, outcome) (messenger_jobs_processed_total)' > evidence/cardinality.json
@echo "telemetry-check: evidence/grafana.json dashboards.json trace-request.json trace-job.json leaks.json cardinality.json"
.PHONY: track2 reprocess
track2:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/pipeline_suite.py --level 5 --only "$(TRACK2_SCENARIOS)" --output "$(TRACK2_OUTPUT)"
reprocess:
@test -n "$(ID)" || (echo "Set ID=<attachment_id>"; exit 2)
$(COMPOSE) run --rm --no-deps -T worker python -m messenger.admin reprocess --attachment-id "$(ID)" --pipeline-version "$(PIPELINE_VERSION)"
.PHONY: test-security security-bootstrap security-unseal
test-security:
$(PYTHON) tests/security.py --base-url "$(BASE_URL)" $(TLS_ARGS)
security-bootstrap:
$(MAKE) secrets
$(PYTHON) scripts/pki.py bootstrap
$(PYTHON) scripts/security_operator.py passwords
security-unseal:
$(PYTHON) scripts/security_operator.py unseal
.PHONY: security-audit security-track security-lifecycle security-evidence
security-audit:
docker compose run --rm --no-deps security-audit
$(PYTHON) scripts/security_inventory.py
security-track:
$(PYTHON) scripts/worker_compromise.py
security-lifecycle:
$(PYTHON) scripts/security_lifecycle.py
security-evidence:
$(PYTHON) scripts/security_evidence.py
+114 -23
View File
@@ -1,37 +1,128 @@
# Отчёт — лабораторная 7
# Лабораторная 7 — безопасность, mTLS и OpenBao
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
PR: [mtusi2026/mtusi2026lab7#15](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab7/pulls/15), base main, head из настоящего fork. CI Gitea для первого вклада ожидает разрешения сопровождающего; локальные проверки выполнены.
Автор: Костылев Артём, БПИ2303. Целевая оценка: **5**, трек1 **«Компрометация worker»**.
Исходное задание: `mtusi2026/mtusi2026lab7`, skeleton20afc2c790d3e72f6b7666376c9834c1ced1a405. Настоящий fork: https://gitea.privcloud.tech/akostylev2303/mtusi2026lab7, parent — репозиторий преподавателя. Ветка `solution/bpi2303-kostylev`. Основа реализации — моя ЛР6 f60297b8e2f252d2b40bf0cdbbcfa6ef5e447496; [provenance](evidence/forward-provenance.json). README/COURSE/контракт/API и публичные tests новой ЛР7 сохранены. Заполнена предусмотренная заданием trust-matrix.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Git/SSH-доступ к Gitea, Docker Compose v2, uv, Python3.9+ для operator scripts, OpenSSL CLI. `make install` устанавливает Python3.14 и зависимости uv.lock. Проверено на macOS arm64, Docker Desktop28.3.2. Рекомендуется Docker VM≥8GiB; другие лабораторные остановить, но volumes не удалять.
## Архитектура и решения
```sh
git clone --branch solution/bpi2303-kostylev ssh://git@gitea.privcloud.tech:2222/akostylev2303/mtusi2026lab7.git
cd mtusi2026lab7
printf 'COMPOSE_PROJECT_NAME=mtusi2026lab7\n' > .env
make install
make check validate lint
make up
make test test-security
make security-audit
make student-test
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab6, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
`make up` создаёт локальные secrets и CA, собирает image, запускает зависимости, инициализирует/разблокирует **новый** OpenBao, делает миграции, применяет DB/S3 policies и запускает две API-реплики, relay, worker, LB. Первый init автоматизирован, shares/root сохранены только в operator directory. **Уже инициализированный sealed Bao автоматически не разблокируется**: `make security-unseal`, затем `make up`.
## Доказательства критериев
Endpoint: https://localhost:18443; CA: `secrets/pki/operator/ca.crt`. Grafana https://localhost:27300 требует operator client certificate и Basic admin. OpenBao https://localhost:28200. Эти host ports bound127.0.0.1. Не импортировать CA в системное глобальное доверие; передавать `--cacert`/CA_FILE адресно.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
```sh
curl --cacert secrets/pki/operator/ca.crt https://localhost:18443/health/ready
make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 scripts/security_failover.py
make security-lifecycle
make security-track
make security-evidence
```
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Lifecycle/track намеренно останавливают доступ/worker **только этого учебного стенда**. `make down` сохраняет volumes. `make reset-data` явно удаляет данные; для полного нового Bao bootstrap нужен новый operator directory, нельзя смешивать старые shares с новым Raft volume.
## Эксперименты и отказы
`LAB7_OPERATOR_DIR` задаёт operator directory; default `~/.local/share/mtusi2026lab7/operator`. Для второго изолированного стенда нужны свои COMPOSE_PROJECT_NAME, operator directory и LB_HTTPS_PORT/GRAFANA_HTTPS_PORT/BAO_HTTPS_PORT/BAO_OPERATOR_URL. Самостоятельный clean-clone сценарий: `sh scripts/clean-clone.sh`; использует свободные19443/28300/29200 и новый project/CA/volumes, сохраняет volumes для осмотра. [Проверка чистого клона](evidence/security/clean-clone.txt) коммита170a76a завершилась exit_code0: install/check/validate/lint, новый bootstrap, HTTPS smoke, внешний TLS,103внутренние проверки и hardening. Исходный стенд на время проверки был остановлен для освобождения RAM; его volumes сохранены.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Реализация и объявленные переходы
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Сохранены пользователи/Argon2, PostgreSQL sessions и Valkey cache, ACL, optimistic idempotency, история/пагинация, S3 attachments, outbox→RabbitMQ→worker, fencing/lease/retry/DLQ и OTel metrics/logs/traces/Grafana. Новая миграция0008 добавляет общий login budget. На новой базе вся цепочка0001–0008; upgrade legacy проверяется собственными тестами.
## Ограничения и следующие шаги
Изменения ЛР7: HTTPS ingress; mTLS LB→каждая API; mTLS gateways dependencies/telemetry; PostgreSQL native TLS verify-full; persistent non-dev OpenBao2.7.2 Raft; cert auth/scoped KV; supervisor re-auth/expiry; отдельные DB роли и S3/Rabbit ACL; rehash слабого Argon2; общий login limiter; усиленный container runtime. API по-прежнему stateless, вся business/session state вне реплик.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Граница plaintext — **loopback внутри одного namespace** proxy/backend. Внешняя контейнерная сеть переносит TLS. [Матрица](contracts/trust-matrix.md) перечисляет порты, role allowlist, локальные hop и проверки. Все leaf24ч; CA10лет. Автоматический leaf renewal/CA rollover не реализован — выбран трек1. До истечения leaf его перевыпускает оператор; `scripts/pki.py issue <role>` создаёт новый cert/key, сервисную роль надо заново привязать `security_operator.py configure --role <role>`, затем перезапустить соответствующие процессы/gateways. Для OpenBao restart нужен explicit unseal. Не запускать учебный PKI на месяцы без обслуживания.
Рабочие секреты — static KV с политикой read своего path. TokenTTL60с/max300с, re-auth10с. Supervisor хранит конфигурацию в памяти/окружении дочернего процесса и прекращает работу после истечения последнего допуска; секреты не передаются через Docker Compose env. Это **не** dynamic DB lease: скомпрометированный процесс может игнорировать supervisor, поэтому реальный revoke дополнен rotation пароля, CRL и закрытием соединений.
Миграции используют отдельный operator-only owner messenger; API/worker его credentials не получают. Runtime приложения: uid10001, readonly, no-new-privileges, cap_dropALL, tmpfs/tmp. Stateful data volumes остаются writable: PG/S3/Rabbit/Valkey/Bao/telemetry. bao-init — one-shot root только CHOWN, networknone, runtime Bao non-root. Worker2CPU/1GiB, concurrency2, renderer30с/448MiB на процесс, job60с; API1CPU/512MiB на реплику. [Hardening](evidence/security/hardening.json).
Перенесённые скрипты старых нагрузочных исследований остаются исходным историческим материалом: их старые HTTP/host backend URLs и fault overlays не являются командами ЛР7. Действующий маршрут проверки ЛР7 приведён выше. Старые численные нагрузочные результаты не выдаются за новые HTTPS измерения.
## Проверки и результаты
Реальный вывод — [evidence/security](evidence/security/README.md), включая неуспешные первые запуски и исправления. Подтверждённый результат:
| Проверка | Результат |
| --- | --- |
| make check / validate / lint | PASS; Ruff+format+ty |
| Свои тесты | **997passed,2skipped,19deselected**; stack tests вынесены в явные сценарии |
| HTTPS public smoke / external TLS | PASS, правильный CA принят, недоверенный отвергнут |
| Persistence | PASS: данные и уже выданная сессия после перезапуска |
| Public failover | 129 успешных записей,0failed, first recovery3.154с |
| Отдельный отказ каждой API | 30+30 успешных записей,0failed;2.756/2.734с; сессия, сообщение, original сохранены |
| Внутренняя matrix/ACL | **103проверки PASS**: TLS, обе API, plaintext, Bao, DB/S3/Rabbit/Valkey, backend query/ingest |
| Shared login budget | 30 concurrent →20×401+10×429, обе реплики, Retry-After,0.320с |
| Bao sealed+restart | API прекратил обслуживание через56.955с; restart остаётся sealed; unseal восстановил за12.595с; сессия/сообщение сохранены |
| Telemetry | 3084metric series,1000log streams,20traces;26secret values/canaries,0exactmatches |
997 тестов прошли после исправления собственных fixtures: новая migration head0008, отдельный RabbitMQ fixture со своим loopback/port policy, очистка тестовой БД от outbox между тестами. Assertions публичных tests не менялись. Первая ошибка metadata сравнения выявила Integer/BigInteger несоответствие модели login_budget — исправлено.
В IAM логах S3 обнаружены3 **access-key ID** (идентификаторы principal), не secret access keys. Первый слишком широкий поиск учитывал их как секреты; исправленный audit явно считает идентификаторы отдельно. Проверка ограничена текущими Docker logs и конечной выборкой telemetry; она не является доказательством для всех будущих входов.
## Стоимость паролей и login DoS
Argon2id19456KiB=19MiB,t2,p1, random salt. На отдельном контейнере1CPU/256MiB по100verify:
| Concurrent | median ms | p95 ms | CPU ms/verify | peak RSS KiB |
| --- | --- | --- | --- | --- |
| 1 | 10.341 | 11.651 | 10.531 | 41632 |
| 2 | 11.587 | 65.264 | 11.647 | 61088 |
| 4 | 85.386 | 94.708 | 15.126 | 100000 |
[Сырые данные](evidence/security/kdf-budget.json). 20login/10с ограничивают среднюю стоимость KDF примерно21–30CPU-ms/с всего стенда при этом эксперименте; worst rehash добавляет ещё hash. Это оценка по measuredCPU, не гарантия latency под произвольной нагрузкой. SQL UPSERT одного bucket сериализует решение до KDF, DB clock исключает расхождение часов реплик. Fixed-window допускает до40попыток на стыке двух окон. Глобальный лимит ограничивает и добросовестных пользователей; для production нужна более детальная account/IP policy. Registration имеет отдельные concurrency/time bounds, не входит в этот budget.
## Трек1: компрометация worker
Гипотеза: отдельные права ограничивают последствия захвата worker; полноценный отзыв identity+secret и восстановление сохраняют уже принятые задания.
Метод: на изолированном стенде оператор использует реальные worker credentials, проверяет DB/S3/queue allow/deny, сравнивает с broad operator правами. Затем удаляет worker cert-auth; ждёт bounded stop; API принимает3вложения, которые остаются queued. Оператор выдаёт новый leaf, отзывает предыдущий через CRL, меняет DB пароль, завершает PG sessions и перезапускает gateways/worker **без build**. Старые credentials испытываются новыми соединениями. Для каждой задачи проверяются ready, SHA256 оригинала и отсутствие дополнительных publication rows. Три повтора. Скрипт: `make security-track`.
| Операция | Broad operator | Worker |
| --- | --- | --- |
| Читать users / создавать таблицу | allow | deny |
| Писать/удалять private prefix S3 | allow | deny |
| Читать staging/originals, писать originals/variants | allow | allow |
| Публиковать/создавать очередь | allow | deny |
| Читать/ack attachments.process | allow | allow |
| Читать migration KV / менять Bao policy | allow | deny |
Broad опыт создаёт отдельную тестовую очередь, откатывает DDL и удаляет тестовый объект; production права worker не расширяются. [Broad evidence](evidence/security/broad-rights.json), [worker scope](evidence/security/trust-audit-final.json).
Итоговая серия после добавления проверки publication rows:
| Повтор | Stop, с | Recovery, с | Queued→ready | Jobs / variants на вложение |
| --- | --- | --- | --- | --- |
| 1 | 60.285 | 12.923 | 3→3 | 1 / 3, version1 |
| 2 | 56.717 | 20.992 | 3→3 | 1 / 3, version1 |
| 3 | 56.953 | 12.906 | 3→3 | 1 / 3, version1 |
9/9 originals совпадают по SHA256; 15/15 проверок старых credentials дали ожидаемый отказ. [Итоговые измерения](evidence/security/worker-compromise.json). Первая серия сохранена отдельно: recovery12.897/13.133/13.181с. Вторая серия имеет один более долгий restart20.992с; он не исключён из результата.
Ограничение вывода: задачам не причинялось преднамеренное повреждение разрешёнными worker операциями. Worker скомпрометированный действительно может менять доступные ему originals/variants/jobs, ack/удалять доступные сообщения. Least privilege уменьшает область ущерба, но не гарантирует сохранность от злонамеренного worker в собственной области. At-least-once может повторять вычисления; гарантируется единичная публикация DB-state на версию, не «ровно один запуск CPU».
Текст обучающего видео: [docs/lab7-video.md](docs/lab7-video.md). Все четыре вопроса защиты разобраны с фактическими результатами.
## Ограничения и источники
Single-node зависимости и Bao — не HA. TLS proxies добавляют процессы/лимиты/точки отказа; CA/operator trusted. Статические KV пароли требуют явной rotation. После24ч leaf нужно обслужить. Ротация worker вызывает краткий reconnect других клиентов S3/broker/Collector при закрытии старых TLS sessions. Исследования на локальной VM не дают production SLO.
Scan не нулевой: [triage](security/SCAN_TRIAGE.md), 1critical/49high/90medium/65low/5unknown package findings. Уязвимости и условия применимости не скрыты; образ не заявляется безопасным для публичного production. Inherited Unicode-surrogate JSON edge case может приводить к500; контракт публичных корректных запросов сохранён.
Источники: README/COURSE/contracts текущей ЛР7, собственная ЛР6 по указанному SHA; [OpenBao cert auth](https://openbao.org/docs/auth/cert/), [policies](https://openbao.org/docs/concepts/policies/), [TCP listener](https://openbao.org/docs/configuration/listener/tcp/), [OWASP Password Storage](https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html), официальная документация PostgreSQL/HAProxy/Docker и package advisories из scan. Context7 был вызван, но вернул исчерпанную месячную квоту; использованы официальные источники. Версии Python — uv.lock, images — digest в compose/Dockerfile.
Codex использован для планирования OpenSpec, переноса кода, реализации, запуска проверок, исправлений, анализа результатов и подготовки отчёта/сценария видео. Исходные неуспешные прогоны сохранены. Итоговая оценка принадлежит преподавателю, PASS тестов её автоматически не назначает.
+4
View File
@@ -0,0 +1,4 @@
[alembic]
script_location = %(here)s/alembic
prepend_sys_path = %(here)s/src %(here)s/alembic
path_separator = space
+111
View File
@@ -0,0 +1,111 @@
import asyncio
import os
import ssl
from alembic import context
from alembic.script import ScriptDirectory
from legacy import adopt_legacy
from sqlalchemy import Connection, func, make_url, pool, select
from sqlalchemy.ext.asyncio import create_async_engine
from messenger.attachments.infrastructure.postgres import (
AttachmentRow,
JobEventRow,
JobRow,
OutboxRow,
VariantRow,
)
from messenger.identity.infrastructure.login_protection import LoginBudgetRow
from messenger.identity.infrastructure.postgres import SessionRow, UserRow
from messenger.messaging.infrastructure.postgres import (
ChatMemberRow,
ChatRow,
IdempotencyKeyRow,
MessageAttachmentRow,
MessageRow,
)
from messenger.shared.config import PlatformSettings
LOCK_KEY = 20_260_002
MODELS = (
LoginBudgetRow,
UserRow,
SessionRow,
ChatRow,
ChatMemberRow,
MessageRow,
IdempotencyKeyRow,
AttachmentRow,
JobRow,
OutboxRow,
JobEventRow,
VariantRow,
MessageAttachmentRow,
)
target_metadata = UserRow.metadata
config = context.config
def include_object(_obj, name, _type, _reflected, _compare_to):
return name != "schema_migrations"
def run_migrations(connection: Connection) -> None:
with connection.engine.connect() as lock:
lock.execute(select(func.pg_advisory_lock(LOCK_KEY)))
lock.commit()
try:
with connection.begin():
adopt_legacy(connection, ScriptDirectory.from_config(config))
context.configure(
connection=connection,
target_metadata=target_metadata,
transaction_per_migration=True,
compare_type=True,
include_object=include_object,
)
with context.begin_transaction():
context.run_migrations()
finally:
lock.execute(select(func.pg_advisory_unlock(LOCK_KEY)))
lock.commit()
async def run_async_migrations() -> None:
dsn = config.attributes.get("database_url")
if dsn is None:
dsn = PlatformSettings().database_url.get_secret_value()
engine = create_async_engine(
make_url(dsn).set(drivername="postgresql+asyncpg"),
poolclass=pool.NullPool,
hide_parameters=True,
connect_args={
**(
{"ssl": ssl.create_default_context(cafile=os.environ["PGSSLROOTCERT"])}
if os.environ.get("PGSSLROOTCERT")
else {}
),
"timeout": 10,
"server_settings": {"lock_timeout": "10000", "statement_timeout": "60000"},
},
)
try:
async with engine.connect() as connection:
await connection.run_sync(run_migrations)
finally:
await engine.dispose()
if context.is_offline_mode():
context.configure(
dialect_name="postgresql",
target_metadata=target_metadata,
literal_binds=True,
transaction_per_migration=True,
)
with context.begin_transaction():
context.run_migrations()
elif config.attributes.get("connection") is not None:
run_migrations(config.attributes["connection"])
else:
asyncio.run(run_async_migrations())
+94
View File
@@ -0,0 +1,94 @@
from uuid import uuid4
from alembic.migration import MigrationContext
from alembic.operations import Operations
from alembic.script import ScriptDirectory
from sqlalchemy import Connection, MetaData, Table, func, inspect, select, text
from sqlalchemy.schema import CreateSchema
LEGACY = {
"0001_init": "9a13618f078ae9056431391e2e5197a31b981ae1fff047a9b3421ac750d3fd0d",
"0002_idempotency_keys": "d9159a35553249a1e51af5f2e4e7c39b014e97593c49748e13f3799839063dd0",
"0003_attachments": "c4417f560538a083efba1c67f660147dc94532fcbb2fa91f3f3e8258a0f01f2e",
"0004_jobs": "c3887fa7a886ae8153a216aa43fd4c3c6190a2074e8ce20c3e77f29bfb01550e",
"0005_failed_registry": "02b80061a08a6929feb092148c453ad970e3d001901c651b5a73b90b1e40f145",
"0006_attachment_quarantine": (
"541f5399d87fac3584c4102b4e0bf92c23cd372cc20a7d56dad03e771773202c"
),
}
SNAPSHOT = text("""
WITH relations AS (
SELECT c.* FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = :schema AND c.relkind IN ('r', 'p')
AND c.relname NOT IN ('schema_migrations', 'alembic_version')
)
SELECT 'table' AS kind, c.relname AS name,
concat_ws('|', c.relkind, c.relpersistence,
c.relrowsecurity, c.relforcerowsecurity) AS definition
FROM relations c
UNION ALL
SELECT 'column', c.relname || '.' || a.attname,
concat_ws('|', a.attnum, format_type(a.atttypid, a.atttypmod), a.attnotnull,
a.attidentity, a.attgenerated, a.attcollation::regcollation,
pg_get_expr(d.adbin, d.adrelid))
FROM relations c JOIN pg_attribute a ON a.attrelid = c.oid
LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum
WHERE a.attnum > 0 AND NOT a.attisdropped
UNION ALL
SELECT 'constraint', c.relname || '.' || k.conname,
concat_ws('|', pg_get_constraintdef(k.oid), k.convalidated)
FROM relations c JOIN pg_constraint k ON k.conrelid = c.oid
UNION ALL
SELECT 'index', c.relname || '.' || x.relname,
concat_ws('|', pg_get_indexdef(i.indexrelid), i.indisvalid, i.indisready)
FROM relations c JOIN pg_index i ON i.indrelid = c.oid
JOIN pg_class x ON x.oid = i.indexrelid
UNION ALL
SELECT 'trigger', c.relname || '.' || t.tgname,
concat_ws('|', pg_get_triggerdef(t.oid), t.tgenabled)
FROM relations c JOIN pg_trigger t ON t.tgrelid = c.oid WHERE NOT t.tgisinternal
ORDER BY kind, name
""")
def snapshot(connection: Connection, schema: str) -> list[tuple[str, str, str]]:
return [
(kind, name, definition.replace(f"{schema}.", ""))
for kind, name, definition in connection.execute(SNAPSHOT, {"schema": schema})
]
def adopt_legacy(connection: Connection, scripts: ScriptDirectory) -> None:
migration = MigrationContext.configure(connection)
if migration.get_current_heads():
return
tables = set(inspect(connection).get_table_names(schema="public"))
if tables <= {"alembic_version"}:
return
if "schema_migrations" not in tables:
raise RuntimeError("Existing schema has no legacy migration journal")
journal = Table("schema_migrations", MetaData(), schema="public", autoload_with=connection)
applied = dict(connection.execute(select(journal.c.version, journal.c.checksum)).tuples().all())
expected = dict(list(LEGACY.items())[: len(applied)])
if applied != expected:
raise RuntimeError("Unknown or changed legacy migrations")
if not applied:
if tables - {"schema_migrations", "alembic_version"}:
raise RuntimeError("Existing schema has an empty migration journal")
return
revision = list(expected)[-1]
actual = snapshot(connection, "public")
reference = f"alembic_verify_{uuid4().hex}"
savepoint = connection.begin_nested()
try:
connection.execute(CreateSchema(reference))
connection.execute(select(func.set_config("search_path", reference, True)))
with Operations.context(MigrationContext.configure(connection)):
for script in reversed(list(scripts.walk_revisions(base="base", head=revision))):
script.module.upgrade()
wanted = snapshot(connection, reference)
finally:
savepoint.rollback()
if actual != wanted:
raise RuntimeError("Legacy schema differs from the recorded migrations")
migration.stamp(scripts, revision)
+16
View File
@@ -0,0 +1,16 @@
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
+87
View File
@@ -0,0 +1,87 @@
import sqlalchemy as sa
from alembic import op
revision = "0001_init"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"users",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("username", sa.Text(), nullable=False),
sa.Column("display_name", sa.Text(), nullable=False),
sa.Column("password_hash", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("password_hash LIKE '$argon2id$%'", name="users_password_hash_check"),
sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="users_username_format"),
sa.CheckConstraint(
"char_length(display_name) BETWEEN 1 AND 100", name="users_display_name_check"
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("username", name="users_username_key"),
)
op.create_table(
"sessions",
sa.Column("token_hash", sa.LargeBinary(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("expires_at > created_at", name="sessions_check"),
sa.CheckConstraint("octet_length(token_hash) = 32", name="sessions_token_hash_check"),
sa.ForeignKeyConstraint(["user_id"], ["users.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("token_hash"),
)
op.create_index("sessions_user_idx", "sessions", ["user_id"], unique=False)
op.create_table(
"chats",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("title", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("last_message_at", sa.DateTime(timezone=True), nullable=True),
sa.CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="chats_title_check"),
sa.PrimaryKeyConstraint("id"),
)
op.create_table(
"chat_members",
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.SmallInteger(), nullable=False),
sa.Column("chat_created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("position BETWEEN 1 AND 100", name="chat_members_position_check"),
sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["user_id"],
["users.id"],
),
sa.PrimaryKeyConstraint("chat_id", "user_id"),
)
op.create_index(
"chat_members_user_idx",
"chat_members",
["user_id", "chat_created_at", "chat_id"],
unique=False,
)
op.create_table(
"messages",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("sender_id", sa.Uuid(), nullable=False),
sa.Column("text", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("char_length(text) BETWEEN 1 AND 4000", name="messages_text_check"),
sa.ForeignKeyConstraint(
["chat_id", "sender_id"],
["chat_members.chat_id", "chat_members.user_id"],
),
sa.PrimaryKeyConstraint("id"),
)
op.create_index(
"messages_history_idx", "messages", ["chat_id", "created_at", "id"], unique=False
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+33
View File
@@ -0,0 +1,33 @@
import sqlalchemy as sa
from alembic import op
revision = "0002_idempotency_keys"
down_revision = "0001_init"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"idempotency_keys",
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("user_id", sa.Uuid(), nullable=False),
sa.Column("key", sa.Text(), nullable=False),
sa.Column("fingerprint", sa.LargeBinary(), nullable=False),
sa.Column("message_id", sa.Uuid(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="idempotency_keys_key_check"),
sa.CheckConstraint(
"octet_length(fingerprint) = 32", name="idempotency_keys_fingerprint_check"
),
sa.ForeignKeyConstraint(
["chat_id", "user_id"],
["chat_members.chat_id", "chat_members.user_id"],
),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("chat_id", "user_id", "key"),
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+77
View File
@@ -0,0 +1,77 @@
import sqlalchemy as sa
from alembic import op
revision = "0003_attachments"
down_revision = "0002_idempotency_keys"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint("messages_text_check", "messages", type_="check")
op.create_check_constraint("messages_text_check", "messages", "char_length(text) <= 4000")
op.create_table(
"attachments",
sa.Column("id", sa.Uuid(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("owner_id", sa.Uuid(), nullable=False),
sa.Column("filename", sa.Text(), nullable=False),
sa.Column("content_type", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("sha256", sa.Text(), nullable=False),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
sa.Column("status", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("object_key", name="attachments_object_key_key"),
sa.UniqueConstraint("id", "chat_id", name="attachments_chat_key"),
sa.ForeignKeyConstraint(
["chat_id", "owner_id"],
["chat_members.chat_id", "chat_members.user_id"],
name="attachments_owner_fkey",
),
sa.CheckConstraint(
"char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_check"
),
sa.CheckConstraint(
"content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_check"
),
sa.CheckConstraint(
"size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_check"
),
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachments_sha256_check"),
sa.CheckConstraint("width > 0", name="attachments_width_check"),
sa.CheckConstraint("height > 0", name="attachments_height_check"),
sa.CheckConstraint("status IN ('pending', 'ready')", name="attachments_status_check"),
)
op.create_index(
"attachments_pending_idx",
"attachments",
["created_at"],
postgresql_where=sa.text("status = 'pending'"),
)
op.create_table(
"message_attachments",
sa.Column("message_id", sa.Uuid(), nullable=False),
sa.Column("position", sa.SmallInteger(), nullable=False),
sa.Column("chat_id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.PrimaryKeyConstraint("message_id", "position"),
sa.UniqueConstraint("message_id", "attachment_id"),
sa.CheckConstraint("position BETWEEN 0 AND 9", name="message_attachments_position_check"),
sa.ForeignKeyConstraint(["message_id"], ["messages.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(
["attachment_id", "chat_id"],
["attachments.id", "attachments.chat_id"],
name="message_attachments_attachment_fkey",
ondelete="RESTRICT",
),
)
op.create_index("message_attachments_attachment_idx", "message_attachments", ["attachment_id"])
def downgrade() -> None:
raise RuntimeError("Downgrade would delete application data")
+146
View File
@@ -0,0 +1,146 @@
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql
revision = "0004_jobs"
down_revision = "0003_attachments"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.drop_constraint("attachments_status_check", "attachments", type_="check")
op.create_check_constraint(
"attachments_status_check",
"attachments",
"status IN ('pending', 'queued', 'processing', 'ready', 'failed')",
)
for name in ("staging_key", "original_key"):
op.add_column("attachments", sa.Column(name, sa.Text(collation="C"), nullable=True))
op.create_unique_constraint(f"attachments_{name}_key", "attachments", [name])
op.add_column(
"attachments",
sa.Column("published_version", sa.Integer(), nullable=False, server_default="0"),
)
op.create_check_constraint(
"attachments_published_version_check", "attachments", "published_version >= 0"
)
op.add_column("attachments", sa.Column("error_code", sa.Text(), nullable=True))
op.create_check_constraint(
"attachments_error_code_check",
"attachments",
"error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted')",
)
op.alter_column("attachments", "object_key", nullable=True)
attachments = sa.table(
"attachments",
sa.column("status"),
sa.column("object_key"),
sa.column("original_key"),
sa.column("staging_key"),
sa.column("published_version"),
)
op.execute(
attachments.update()
.where(attachments.c.status == "ready")
.values(original_key=attachments.c.object_key, published_version=1)
)
op.execute(
attachments.update()
.where(attachments.c.status == "pending")
.values(staging_key=attachments.c.object_key)
)
op.create_index(
"attachments_staged_idx",
"attachments",
["updated_at"],
postgresql_where=sa.column("staging_key").is_not(None),
)
op.create_table(
"attachment_variants",
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
sa.Column("pipeline_version", sa.Integer(), nullable=False),
sa.Column("name", sa.Text(), nullable=False),
sa.Column("object_key", sa.Text(collation="C"), nullable=False),
sa.Column("content_type", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("sha256", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.PrimaryKeyConstraint("attachment_id", "pipeline_version", "name"),
sa.UniqueConstraint("object_key", name="attachment_variants_object_key_key"),
sa.CheckConstraint(
"pipeline_version >= 1", name="attachment_variants_pipeline_version_check"
),
sa.CheckConstraint(
"name IN ('thumbnail', 'crop', 'optimized')", name="attachment_variants_name_check"
),
sa.CheckConstraint(
"content_type IN ('image/jpeg', 'image/png', 'image/webp')",
name="attachment_variants_content_type_check",
),
sa.CheckConstraint("size_bytes >= 1", name="attachment_variants_size_bytes_check"),
sa.CheckConstraint("width >= 1", name="attachment_variants_width_check"),
sa.CheckConstraint("height >= 1", name="attachment_variants_height_check"),
sa.CheckConstraint("sha256 ~ '^[0-9a-f]{64}$'", name="attachment_variants_sha256_check"),
)
op.create_table(
"jobs",
sa.Column("job_id", sa.Uuid(), primary_key=True),
sa.Column("attachment_id", sa.Uuid(), sa.ForeignKey("attachments.id"), nullable=False),
sa.Column("pipeline_version", sa.Integer(), nullable=False),
sa.Column("status", sa.Text(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=False),
sa.Column("claims", sa.Integer(), nullable=False, server_default="0"),
sa.Column("lease_token", sa.Uuid(), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("attachment_id", "pipeline_version", name="jobs_effect_key"),
sa.CheckConstraint("pipeline_version >= 1", name="jobs_pipeline_version_check"),
sa.CheckConstraint(
"status IN ('queued', 'processing', 'done', 'failed', 'superseded')",
name="jobs_status_check",
),
sa.CheckConstraint("attempt >= 1", name="jobs_attempt_check"),
sa.CheckConstraint("claims >= 0", name="jobs_claims_check"),
)
op.create_index(
"jobs_lease_idx",
"jobs",
["lease_until"],
postgresql_where=sa.column("status") == "processing",
)
op.create_table(
"outbox",
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
sa.Column("job_id", sa.Uuid(), sa.ForeignKey("jobs.job_id"), nullable=False),
sa.Column("payload", postgresql.JSONB(), nullable=False),
sa.Column("available_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index(
"outbox_pending_idx",
"outbox",
["available_at", "id"],
postgresql_where=sa.column("published_at").is_(None),
)
op.create_index("outbox_job_idx", "outbox", ["job_id"])
op.create_table(
"job_events",
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), primary_key=True),
sa.Column("job_id", sa.Uuid(), nullable=False),
sa.Column("attachment_id", sa.Uuid(), nullable=False),
sa.Column("kind", sa.Text(), nullable=False),
sa.Column("attempt", sa.Integer(), nullable=True),
sa.Column("detail", postgresql.JSONB(), nullable=False, server_default="{}"),
sa.Column("at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
op.create_index("job_events_attachment_idx", "job_events", ["attachment_id", "id"])
def downgrade() -> None:
raise RuntimeError("Downgrade would delete pipeline data")
+20
View File
@@ -0,0 +1,20 @@
import sqlalchemy as sa
from alembic import op
revision = "0005_failed_registry"
down_revision = "0004_jobs"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_index(
"jobs_failed_idx",
"jobs",
[sa.column("updated_at").desc()],
postgresql_where=sa.column("status") == "failed",
)
def downgrade() -> None:
op.drop_index("jobs_failed_idx", table_name="jobs")
@@ -0,0 +1,28 @@
import sqlalchemy as sa
from alembic import op
revision = "0006_attachment_quarantine"
down_revision = "0005_failed_registry"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("attachments", sa.Column("quarantine_reason", sa.Text(), nullable=True))
op.create_check_constraint(
"attachments_quarantine_reason_check",
"attachments",
"quarantine_reason IN ('missing', 'size_mismatch', 'sha_mismatch')",
)
op.add_column(
"attachments", sa.Column("quarantined_at", sa.DateTime(timezone=True), nullable=True)
)
op.create_check_constraint(
"attachments_quarantine_check",
"attachments",
"(quarantine_reason IS NULL) = (quarantined_at IS NULL)",
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete quarantine data")
+19
View File
@@ -0,0 +1,19 @@
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects.postgresql import JSONB
revision = "0007_trace_context"
down_revision = "0006_attachment_quarantine"
branch_labels = None
depends_on = None
def upgrade() -> None:
for table in ("jobs", "outbox"):
op.add_column(
table, sa.Column("trace_context", JSONB(), nullable=False, server_default="{}")
)
def downgrade() -> None:
raise RuntimeError("Downgrade would delete trace correlation data")
+21
View File
@@ -0,0 +1,21 @@
import sqlalchemy as sa
from alembic import op
revision = "0008_login_budget"
down_revision = "0007_trace_context"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table(
"login_budget",
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("window", sa.BigInteger(), nullable=False),
sa.Column("attempts", sa.Integer(), nullable=False),
sa.CheckConstraint("id = 1 AND attempts > 0"),
)
def downgrade() -> None:
op.drop_table("login_budget")
+1179
View File
File diff suppressed because it is too large Load Diff
+26 -27
View File
@@ -1,37 +1,36 @@
# Матрица доверия — заполнить в решении lab7
# Матрица доверия ЛР7
Это обязательная матрица покрытия. Для каждой строки укажите реальный порт, endpoint, CA, service identity, проверку SAN/ACL, срок жизни credentials и команду положительного/отрицательного теста. TLS шифрует и подтверждает сервер; mTLS также проверяет клиента. При доверенном CA всё равно нужно ограничивать разрешённую identity. Plaintext hop допустим только внутри закрытого локального transport/proxy boundary, без опубликованного обхода; границу и rationale нужно доказать.
Один локальный CA. Сервер проверяется по CA и DNS SAN конкретного сервиса; клиент — по цепочке и allowlist CN (CN здесь удостоверенная роль, а не произвольный HTTP-заголовок). Leaf: 24 часа. CA key и unseal shares находятся вне репозитория у оператора. `make security-audit` проверяет положительные и отрицательные подключения; JSON: [trust-audit-final](../evidence/security/trust-audit-final.json).
| Связь | Минимум на 3 | Дополнительно на 4 | Доказательство / конфиг |
| Связь | Реальный endpoint и проверка | Идентичность / права | Проверка |
| --- | --- | --- | --- |
| Клиент → ingress/LB | TLS, CA + hostname, Bearer | Проверка expiry/revocation сессии и общего login rate limit | TODO |
| LB → API (все реплики) | mTLS + allowlist LB identity | Ротация leaf без обхода проверки | TODO |
| API/worker → PostgreSQL | TLS verify-full + раздельные ограниченные DB roles | Ротация credentials; mTLS — допустимое усиление | TODO |
| API/worker → S3 | TLS + разные scoped credentials/prefix rights | Проверка denied prefix и ротация | TODO |
| API/worker → broker | TLS + разные producer/consumer identities и ACL | mTLS + ограничение допустимых SAN/ролей | TODO |
| API/worker → Collector | TLS; сеть/аутентификация запрещает произвольный ingest | mTLS с раздельными identities | TODO |
| Collector → telemetry backends | TLS + scoped backend credentials или mTLS | Ротация и проверка least privilege | TODO |
| Grafana → data sources | TLS с проверкой сервера, ограниченная identity | Отрицательные проверки доступа | TODO |
| API/worker/agent → OpenBao | TLS + сервисный auth + scoped policy | Renewal/expiry/revocation | TODO |
| Оператор → OpenBao/Grafana/S3 admin | TLS + authentication, доступ локальный/через защищённый канал | Разделение operator/application roles | TODO |
| API → Redis/Valkey, если используется | TLS + ACL/пароль с минимальными правами | Ротация; mTLS по возможности | TODO |
| Replication/кластерные каналы, если добавлены | TLS/mTLS либо подтверждённый шифрованный transport | Проверка заявленной модели отказа | TODO |
| Клиент → LB | HTTPS localhost:18443, CA+SAN localhost, TLS≥1.2 | Basic только вход; затем Bearer, ACL чата | `make test test-security`, `make security-lifecycle`: logout/чужой доступ, недоверенный CA, общий 429 |
| LB → каждая API | api:8443 mTLS, SAN api; localhost:8080 только в контейнере API | CN lb; operator допущен для диагностики. api/worker/outsider не являются LB | `mtls:api`, `api-replica:<IP>`: правильный/без cert/чужой CA/неверная роль; `hostname:api` |
| API/worker/relay/reconciler → PostgreSQL | postgres:5432 native TLS, Python SSLContext с hostname verification, hostnossl reject | messenger_api, messenger_worker, messenger_relay, messenger_reconciler; миграции отдельным messenger owner | worker DB SELECT jobs разрешён; users/sessions/DDL запрещены; plaintext/чужой CA запрещены; старый пароль после rotation не принимается |
| API/worker/relay/reconciler → S3 | локальный proxy 19000 → s3:9443 mTLS → loopback:9000 | CN allowlist; затем SigV4. API staging/originals, worker читает originals/staging/variants и пишет originals/variants; relay удаляет staging; reconciler отдельная policy | worker PUT/GET variants 200, чужой prefix/DELETE variants 403; `worker-compromise` отзывает cert |
| relay/worker → broker | локальный proxy 5672 → broker:5671 mTLS → loopback:5672 | CN api/relay/worker/operator; AMQP messenger-api producer и messenger-worker consumer; API не публикует напрямую — outbox relay | consumer получает задачу; publish/configure worker запрещены; нет cert/foreignCA/outsider запрещены |
| API/relay/worker → Collector | localhost:4318 → otel-collector:44318 mTLS → loopback:4318 | отдельные CN api, relay, worker; operator для тестов | полная отрицательная серия, отзыв workercert, реальные метрики/логи/trace |
| Collector → Prometheus/Loki/Tempo | prometheus:9443 /api/v1/otlp, loki:9443 /otlp, tempo:44318 /v1/traces; mTLS | CN otel-collector. Prom/Loki proxy разрешает только ingest prefix; Tempo ingest отделён от query | correct role/foreign/no cert/outsider/SAN; Collector query к Prom/Loki 403 |
| Grafana → источники | localhost:19090/13100/13200 → Prom/Loki/Tempo:9443, mTLS | CN grafana; query разрешён, ingest Prom/Loki 403; Tempo ingest порт не допускает grafana | `telemetry-query:*`, `grafana-write-denied:*` |
| Сервисы → OpenBao | openbao:8200 native HTTPS CA+SAN; cert auth дополнительно привязан к конкретному leaf | api/worker/relay/migrate/reconciler читают только secret/data/своя-роль. Token TTL60с max300с; re-auth каждые10с | own200, foreign403, policy-write403; sealed/restart/expiry и новый/старый cert |
| Оператор → OpenBao | localhost:28200 HTTPS; operator cert, root token только для provisioning | init 3 shares / threshold2, unseal только оператор | `security-lifecycle`: перезапуск остаётся sealed; данные/политики сохраняются |
| Оператор → Grafana | localhost:27300 HTTPS mTLS + Basic admin; порт bound127.0.0.1 | operator cert + отдельный Grafana password | транспортная серия; admin пароль не смонтирован в API/worker |
| Оператор → S3 admin | `docker compose run --rm s3-cli ...`, network namespace s3, localhost:9000 | root credentials только в operator job, профиль tools; внешнего plaintext admin endpoint нет | `broad-rights.json`: root операции vs worker403; сетевой :9000 закрыт |
| Оператор → Rabbit admin | broker:15671 mTLS → loopback:15672; не опубликован на host | CN operator + messenger-ops | worker configure/publish deny; operator isolated queue create/publish/consume/delete разрешены |
| API → Valkey | localhost:6379 → valkey:6380 mTLS → loopback:6379 | CN api/operator, ACL user api, только session:*/revoked:* и перечисленные команды | own-prefix разрешён; foreign-prefix и CONFIG запрещены; plaintext закрыт |
| Prometheus → scrape | otel-collector:48888, Loki/Tempo/Grafana:9443, mTLS; self localhost:9090 | CN prometheus | gateway allowlist; внешний plaintext health/metrics закрыт |
| Кластер / replication | N/A: PostgreSQL, Valkey, Rabbit, S3 и OpenBao single-node | Bao Raft служебный cluster transport native TLS; Rabbit distribution/EPMD только loopback | broker:25672/4369 снаружи namespace закрыты; HA зависимостей не заявляется |
Нет выбранного компонента — отметьте N/A с причиной, а не оставляйте пустую строку. Внутренний публичный health listener, если нужен, минимален и изолирован; через него нельзя обращаться к бизнес-API без mTLS. Отдельно задокументируйте scrape/health/bootstrap каналы: они тоже не должны создавать обход.
## Закрытый локальный hop
## Обязательные отрицательные проверки
HAProxy каждого backend использует `network_mode: service:<backend>`. Backend слушает 127.0.0.1, proxy — TLS порт. У API/worker/relay исходящие proxy находятся в том же контейнере. Ни один plaintext hop не пересекает container network namespace. PostgreSQL/OpenBao используют native TLS. Это граница доверия namespace: процесс, уже захвативший этот контейнер, может обращаться к локальному backend. Сетевой сосед — не может. `plaintext-closed:*` проверяет это реально, а не по отсутствию `ports` в YAML.
1. HTTPS с правильным CA/hostname проходит; с пустым/чужим CA и неправильным hostname — ошибка верификации, не «сервер недоступен».
2. LB→API: без client cert, с cert чужого CA, с запрещённой identity от доверенного CA — отказ; с разрешённой LB identity — успех.
3. На 4 повторите набор для broker и Collector; ограничение на network policy не заменяет проверку identity.
4. Пользователь вне чата не читает metadata/content и не прикрепляет чужой файл; поддельный X-User-Id не меняет Bearer identity.
5. OpenBao app policy читает только свой secret/path; запись policy и чтение чужого path запрещены. Worker не получает admin credentials API/БД/S3.
6. На 4 отзовите secret/cert и дождитесь контролируемого эффекта с учётом действующих соединений, TLS resumption и cache/TTL. Покажите новое подключение, иначе тест отзыва может ничего не проверить.
Health: HTTP исключительно loopback внутри соответствующего namespace; healthtools копирует BusyBox в общий read-only tools volume для локальных health команд. Stats LB только 127.0.0.1:8404 внутри LB. Публичный health проходит через HTTPS/mTLS как остальные запросы. Docker health запускается операторским runtime, не публичным отдельным портом.
## Bootstrap и эксплуатация
## Bootstrap, expiry, revoke
В репозитории только открытые конфиги, policy templates и scripts. CA private key, leaf keys, root token, unseal shares и SecretID не коммитятся, не запекаются в образ и не печатаются в отчёте. Для локального стенда оператор создаёт их вне Git и передаёт через files/secrets с ограниченными правами. Опишите, кто имеет доступ к CA/unseal, что нужно после рестарта OpenBao и как восстановить работу.
`make security-bootstrap` создаёт CA/private keys и служебные credentials локально. Operator directory по умолчанию `~/.local/share/mtusi2026lab7/operator`, mode0700; CA key/init.json0600. `secrets/` исключён из Git и Docker context. В сервис монтируется только каталог его leaf; root token и shares отсутствуют в mounts API/worker. Тестовый security-audit имеет все leaf исключительно как явная operator job.
OpenBao не использует dev-mode. Выберите persistent storage и auth метод (например AppRole с короткоживущим SecretID/response wrapping, cert auth или иная обоснованная identity). Не подменяйте root token постоянной переменной BAO_TOKEN в приложении. Рабочие tokens имеют policy, TTL и renewal/re-auth сценарий. Статические KV secrets на 3 допустимы при реальном чтении через сервисную identity; динамические DB credentials — хорошее расширение, но не обязательное название технологии.
Первый credential сервиса — его 24-часовой leaf/key, выданный оператором, плюс доверенный CA. Он не вечен и не запечён в image. OpenBao выдаёт scoped KV; конфигурация находится в памяти/окружении дочернего процесса. Supervisor переаутентифицируется каждые10с и останавливает процесс после срока последнего token (60с, плюс ограниченный shutdown≤8с). Недоступность/sealed не включает insecure fallback.
Разрешённый компромисс учебного стенда: ручной operator init/unseal, если он безопасно документирован и не нужен при каждом обычном рестарте API. Автоматическое хранение всех unseal shares рядом с OpenBao без защиты не считается решённым bootstrap.
KV пароли статические: TTL token не равен TTL DB-пароля. Злоумышленник не обязан подчиняться supervisor. Поэтому полный отзыв в `make security-track` удаляет cert auth, перевыпускает leaf, помещает старый serial в CRL, перезапускает затронутые TLS gateways (закрывает старые соединения), меняет DB-пароль и завершает PG sessions. Это показано новыми подключениями со старыми credentials. Renewal leaf ручной; автоматический rollover CA и HA Bao не заявляются.
+140
View File
@@ -0,0 +1,140 @@
# Лабораторная 7. Безопасность мессенджера: TLS, mTLS и OpenBao
Сценарий обучающего видео, ориентировочно 12–15 минут. Автор: Костылев Артём, БПИ2303. Основа — фактический [REPORT.md](../REPORT.md) и [результаты проверок](../evidence/security/README.md). Цель — объяснить решение и ответить на все четыре вопроса защиты. Курсивом выделены подсказки для демонстрации; остальные абзацы можно читать как текст озвучки.
## 1. Что защищаем
*На экране: схема клиент → LB → две API; рядом PostgreSQL, Valkey, S3, RabbitMQ, worker и OpenTelemetry.*
В предыдущих работах мы сделали мессенджер, который хранит сообщения, распределяет запросы между репликами, обрабатывает вложения через очередь и собирает метрики, логи и трассировки. В седьмой лабораторной сохраняем эти возможности и защищаем границы между компонентами.
У такого приложения несколько разных угроз. Посторонний пользователь может пытаться прочитать чужое вложение. Соседний контейнер — обратиться к API напрямую, минуя балансировщик. А если атакующий получил права worker, он может попробовать добраться до паролей пользователей, чужих секретов или административных операций.
Поэтому недостаточно просто включить HTTPS на внешнем адресе. Нужно разобрать каждую связь: кто подключается, кому он доверяет, как подтверждает свою личность и какие действия ему разрешены. Эти ответы собраны в матрице доверия.
На практике клиент обращается к HTTPS на localhost, порт 18443. Балансировщик общается с API по mTLS. PostgreSQL использует собственный TLS, а для части остальных компонентов работают TLS-прокси. Незашифрованный участок разрешён только на loopback внутри одного контейнерного network namespace. Другой контейнер до такого порта добраться не может — мы проверили это отдельными сетевыми запросами.
## 2. Шифрование, идентичность и разрешение — три разных проверки
*На экране: вопрос защиты №1 и три подписи: «шифрование», «кто подключился», «что разрешено».*
Первый вопрос: какая часть mTLS отвечает за шифрование, какая — за идентичность, а где находится разрешение на действие?
Шифрование обеспечивает TLS-протокол. После согласования ключей данные передаются так, чтобы сетевой наблюдатель не мог прочитать содержимое или незаметно его изменить.
Идентичность сервера проверяется по сертификату. Клиент проверяет доверенный центр сертификации, срок действия и имя сервера в SAN. Например, подключаясь к PostgreSQL, приложение должно получить сертификат именно для postgres. Просто сертификата от знакомого CA недостаточно, если имя сервера другое.
При взаимном TLS сертификат предъявляет ещё и клиент. Сервер проверяет его подлинность. Но следующий шаг — авторизация: имеет ли эта подтверждённая личность право обращаться к конкретному сервису?
В нашем стенде HAProxy проверяет разрешённую роль клиента из удостоверенного CN. Дальше действуют прикладные права. PostgreSQL проверяет роль и права на таблицы. S3 — подпись запроса и policy на префиксы объектов. RabbitMQ — права producer или consumer. OpenBao — policy на путь секрета.
Вот пример результата. Worker смог читать задания, но получил отказ при чтении users и sessions. Он смог записать вариант изображения в разрешённую область S3, но попытка записи в чужой префикс вернула 403. Соединение при этом было защищено. Отказ произошёл на уровне прав на действие.
Это отвечает на первый вопрос: TLS защищает канал, сертификаты подтверждают стороны, а allowlist и прикладные ACL определяют разрешённые действия.
## 3. Почему любой сертификат нашего CA не превращает клиента в балансировщик
*На экране: вопрос №2; результаты `mtls:api` и `api-replica` из trust-audit-final.json.*
Второй вопрос: почему доверенный CA ещё не означает, что любой его клиент может быть LB?
Центр сертификации подтверждает, что сертификат выдан определённой личности. Он не говорит: «этому клиенту разрешено всё».
Представим паспорта сотрудников. Подлинный паспорт подтверждает личность, но сам по себе не даёт доступ в серверную. Для серверной нужен отдельный список допущенных людей. Аналогично API должен проверить не только подпись сертификата, но и роль клиента.
В нашем API разрешены роль lb и оператор для диагностики. Worker не становится балансировщиком только потому, что его сертификат подписан тем же CA. Мы специально создали outsider — сертификат от нашего доверенного CA с другой ролью.
Для обеих API-реплик проверили четыре случая: правильный сертификат LB, отсутствие сертификата, сертификат чужого CA и outsider от нашего CA. Разрешённый клиент получил доступ, три остальных случая были отклонены. Отдельно проверили неправильное имя сервера.
То же правило применили к брокеру и Collector. В сумме внутренний аудит содержит 103 успешные проверки: это и допустимые подключения, и ожидаемые отказы. Здесь «успешная проверка отказа» означает, что запрещённая операция действительно не прошла.
## 4. Где находится первый секрет для OpenBao
*На экране: вопрос №3; схема «оператор → leaf/key сервиса → cert auth → scoped token → свой KV path». Самих ключей и tokens не показывать.*
Третий вопрос: где хранится первый секрет для доступа к OpenBao и кто имеет право unseal?
Хранилище секретов не устраняет необходимость первоначального доверия. Сервису всё равно нужно как-то доказать, что он имеет право получить свой секрет.
У нас первым credential является клиентский сертификат сервиса и его приватный ключ. Оператор создаёт локальный CA и выдаёт отдельные сертификаты API, worker, relay и другим ролям. Приватный ключ CA остаётся на хосте оператора, вне Git и вне mounts приложений. Сам сертификат CA — открытые данные: он нужен для проверки подписей.
В API монтируется только ключ и сертификат API. В worker — только материал worker. Эти ключи не запечены в Docker image. Leaf действует 24 часа, поэтому его нужно обслуживать; автоматическое обновление всех сертификатов в этой работе не заявлено.
OpenBao работает не в dev-mode и хранит состояние в постоянном Raft volume. При первом init создаются три unseal shares, порог — две. В нашем учебном варианте они вместе с root token лежат в защищённом operator directory на хосте. В реальной эксплуатации shares стоило бы разделить между независимыми хранителями. Само наличие трёх строк в одном файле ещё не создаёт независимого контроля.
Сервис аутентифицируется своим сертификатом и получает короткоживущий token. Его policy разрешает только чтение собственного пути, например secret/data/worker. В опыте worker успешно прочитал свой путь, а чтение migration secret и попытка изменить policy получили 403.
Итак, первый секрет выдаёт оператор. Root token и unseal shares нужны оператору, а не приложению. Сертификат сервиса — ограниченный по сроку bootstrap credential, а не вечный общий пароль ко всем секретам.
## 5. Что происходит при expiry, sealed OpenBao и рестарте API
*На экране: вопрос №4; lifecycle.json и шкала времени: seal → 56,955 секунды → остановка обслуживания → unseal → 12,595 секунды → восстановление.*
Четвёртый вопрос: что происходит при истечении credentials, остановке OpenBao и рестарте API?
Наш supervisor повторно аутентифицируется каждые десять секунд. Выданный token действует шестьдесят секунд. Пока последняя авторизация действительна, уже работающий процесс может продолжать обслуживать запросы. Если OpenBao недоступен или sealed, бесконечно продлевать этот допуск нельзя. После истечения срока supervisor завершает приложение, с ограниченным временем на корректную остановку.
В эксперименте мы запечатали OpenBao. API прекратил обслуживание через 56,955 секунды. Затем перезапустили OpenBao и проверили, что он остался sealed. Автоматического получения всех shares из контейнера не было.
После явного операторского unseal обслуживание восстановилось за 12,595 секунды. Уже выданная пользовательская сессия продолжила работать, а созданное сообщение сохранилось. Это показывает, что данные и сессии не жили только в памяти API.
Если перезапустить API при работающем OpenBao, он заново получает scoped конфигурацию. Если перезапустить его при sealed OpenBao, он не должен обходить хранилище и запускаться с постоянным запасным секретом. В реализации такой fallback отсутствует.
Здесь важно различать два срока. TTL OpenBao token ограничивает доступ к хранилищу и работу нашего supervisor. Сам пароль БД хранится в static KV и не становится недействительным автоматически через шестьдесят секунд. Злоумышленник, уже укравший пароль, может не выполнять правила supervisor. Поэтому для настоящего отзыва мы дополнительно меняем пароль, закрываем соединения и отзываем сертификат. Это проверяется в исследовательском треке.
## 6. Как ограничиваем дорогую проверку паролей
*На экране: Argon2id — 19 MiB, t=2, p=1; таблица kdf-budget.json.*
Для паролей используется Argon2id. В хеше сохраняются salt и параметры. Если после успешного входа обнаружены старые параметры, приложение пересчитывает hash. Обновление условное: оно не должно перезаписать результат конкурирующего изменения пароля.
Но хороший password hash намеренно дорогой. Значит, поток неверных паролей способен расходовать CPU. Мы измерили эту стоимость: в отдельном контейнере с одним CPU медиана одиночной проверки составила около 10,3 миллисекунды, p95 — 11,7 миллисекунды. При четырёх параллельных проверках p95 вырос примерно до 94,7 миллисекунды. Эти цифры относятся к нашему стенду, а не ко всем компьютерам.
Перед Argon2 действует общий для реплик бюджет: двадцать попыток за десятисекундное окно. Решение принимается атомарно в PostgreSQL. Если бы каждая API считала попытки только в своей памяти, добавление реплики увеличивало бы общий лимит. Здесь обе реплики используют одну запись бюджета.
Проверка отправила тридцать одновременных неверных входов. Двадцать получили 401 после проверки credentials, десять — 429 с Retry-After до дорогого KDF. Запросы обслуживались обеими репликами.
У решения есть цена: глобальный лимит может задержать и нормальных пользователей. На границе фиксированных окон возможны два близких burst. Мы фиксируем эти ограничения, а не объявляем лабораторный limiter универсальной защитой от любого DoS.
## 7. Исследование: у атакующего права worker
*На экране: таблица broad operator / worker, затем три строки worker-compromise.json.*
Для оценки пять выбран трек «Компрометация worker». Гипотеза: отдельные права уменьшают область ущерба, а полный отзыв и восстановление позволяют обработать сохранённые задания.
Сначала сравнили реальные права. Оператор смог прочитать users, выполнить DDL в откатываемой транзакции, записать объект в private prefix и создать тестовую очередь. Worker эти операции выполнить не смог. При этом его рабочие действия — чтение задания и работа с разрешёнными изображениями — остались доступны.
Затем провели три повтора. Отозвали cert-auth worker и дождались остановки. Пока worker не обрабатывал очередь, API принял три вложения. Мы проверили, что они остались queued.
После этого выдали новый сертификат, внесли serial старого в CRL, изменили DB-пароль и завершили прежние PostgreSQL sessions. Перезапуск TLS gateways закрыл уже установленные соединения: иначе смена правил могла бы повлиять только на новые подключения, а старые продолжили бы работать.
Worker восстановили без пересборки image. Проверили новые подключения со старым сертификатом к брокеру, S3, Collector и OpenBao, а также со старым паролем к БД. Во всех трёх повторах старый доступ был отклонён.
В итоговой серии остановка заняла 60,285, 56,717 и 56,953 секунды. Восстановление — 12,923, 20,992 и 12,906 секунды. Более медленный второй повтор мы оставили в результатах.
Все девять вложений стали ready. SHA256 скачанных originals совпали с отправленными файлами. Для каждого attachment в БД проверены одна job, три variants и published_version, равная единице. Это подтверждает сохранность принятых задач и единичную публикацию состояния. Это не означает, что в системе at-least-once вычисление никогда не может выполниться повторно.
И ещё граница вывода: мы не просили злонамеренный worker удалить разрешённые ему данные. Он всё ещё может портить свои jobs, originals и variants или подтверждать полученные задания. Least privilege ограничивает последствия захвата, но не делает доступные worker данные неуязвимыми.
## 8. Что в итоге доказано
*На экране: 997 tests; 103 TLS/ACL checks; clean-clone; ссылки на evidence. Затем короткая строка с ограничениями.*
Полный набор собственных тестов завершился: 997 passed, два skipped. Девятнадцать stack tests отделены от обычного запуска; реальные HTTPS, persistence и failover сценарии выполнялись явно.
При последовательной остановке каждой из двух API-реплик удалось выполнить по тридцать записей без неуспешных попыток. Сессия, уже подтверждённое сообщение и вложение сохранились. Метрики, логи и traces продолжили поступать через защищённые связи.
Мы также просканировали зависимости и runtime image. Доступные исправления применены, но нулевого количества CVE нет. Остались package findings, включая один critical в неиспользуемом FastCGI-пути HAProxy; применимость и остаточный риск разобраны в отчёте. Это учебный стенд на localhost, а не готовое разрешение выставить сервис в Интернет.
Главный результат работы — проверяемые границы доверия. Мы знаем, какой сертификат принимается, какие операции доступны worker, как действует отзыв и что происходит, когда OpenBao перестаёт выдавать доступ. Каждый из этих ответов связан с конкретным опытом и сохранённым результатом.
## Карта вопросов защиты
| Вопрос задания | Раздел сценария | Доказательство |
| --- | --- | --- |
| Шифрование, идентичность и разрешение в mTLS | 2 | trust-audit-final.json: transport + DB/S3/queue ACL |
| Почему любой доверенный CA client не является LB | 3 | outsider, no-cert, foreign-CA проверки каждого API IP |
| Первый секрет OpenBao и право unseal | 4 | PKI bootstrap, scoped policies, operator directory, restart sealed |
| Expiry, остановка OpenBao и restart API | 5 | lifecycle.json + worker-compromise.json |
Не показывать содержимое secrets/, init.json, root token, unseal shares, private keys или Authorization headers. Для демонстрации использовать конфиги без значений секретов и агрегированные JSON evidence.
+447
View File
@@ -0,0 +1,447 @@
{
"source": "https://gitea.privcloud.tech/akostylev2303/mtusi2026lab6",
"source_sha": "f60297b8e2f252d2b40bf0cdbbcfa6ef5e447496",
"skeleton_sha": "20afc2c790d3e72f6b7666376c9834c1ced1a405",
"copied_files": [
"Dockerfile",
"alembic.ini",
"alembic/env.py",
"alembic/legacy.py",
"alembic/script.py.mako",
"alembic/versions/0001_init.py",
"alembic/versions/0002_idempotency_keys.py",
"alembic/versions/0003_attachments.py",
"alembic/versions/0004_jobs.py",
"alembic/versions/0005_failed_registry.py",
"alembic/versions/0006_attachment_quarantine.py",
"alembic/versions/0007_trace_context.py",
"infra/compose.faults.yaml",
"infra/grafana/dashboards/messenger-api.json",
"infra/grafana/dashboards/messenger-pipeline.json",
"infra/grafana/provisioning/alerting/alerting.yaml",
"infra/grafana/provisioning/dashboards/messenger.yaml",
"infra/grafana/provisioning/datasources/datasources.yaml",
"infra/grafana/provisioning/plugins/plugins.yaml",
"infra/haproxy/haproxy.cfg",
"infra/k6/messages.js",
"infra/loki/loki.yaml",
"infra/otel/collector-balanced.yaml",
"infra/otel/collector-tail.yaml",
"infra/otel/collector.yaml",
"infra/prometheus/prometheus.yml",
"infra/prometheus/rules/messenger.alerts.yml",
"infra/prometheus/rules/messenger.recording.yml",
"infra/prometheus/runbooks.md",
"infra/rabbitmq/definitions.json",
"infra/rabbitmq/enabled_plugins",
"infra/rabbitmq/rabbitmq.conf",
"infra/s3/api-policy.json",
"infra/s3/cli.sh",
"infra/s3/init.sh",
"infra/s3/lifecycle.json",
"infra/s3/reconciler-policy.json",
"infra/s3/worker-policy.json",
"infra/tempo/tempo.yaml",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/.openspec.yaml",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/design.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/proposal.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/specs/lab1-event-publication/spec.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/specs/lab1-inmemory-capacity/spec.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/specs/lab1-messenger-chats/spec.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/specs/lab1-messenger-history/spec.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/specs/lab1-messenger-identity/spec.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/specs/lab1-service-operability/spec.md",
"openspec/changes/archive/2026-09-29-add-lab1-messenger/tasks.md",
"openspec/changes/archive/2026-10-12-typed-boundary-schemas/.openspec.yaml",
"openspec/changes/archive/2026-10-12-typed-boundary-schemas/design.md",
"openspec/changes/archive/2026-10-12-typed-boundary-schemas/proposal.md",
"openspec/changes/archive/2026-10-12-typed-boundary-schemas/tasks.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/.openspec.yaml",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/design.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/proposal.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab1-event-publication/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab1-inmemory-capacity/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab1-messenger-chats/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab1-messenger-history/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab1-messenger-identity/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab1-service-operability/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab2-auth-sessions/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/specs/lab2-persistent-storage/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-postgres-sessions/tasks.md",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/.openspec.yaml",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/design.md",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/proposal.md",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/specs/lab2-dependency-failures/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/specs/lab2-history-index/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/specs/lab2-session-cache/spec.md",
"openspec/changes/archive/2026-10-28-add-lab2-session-cache-and-resilience/tasks.md",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/.openspec.yaml",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/design.md",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/proposal.md",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/specs/lab1-service-operability/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/specs/lab2-dependency-failures/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/specs/lab3-failover-operations/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-failover-operations/tasks.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/.openspec.yaml",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/design.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/proposal.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/specs/lab1-messenger-history/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/specs/lab1-service-operability/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/specs/lab2-persistent-storage/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/specs/lab3-idempotent-messages/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/specs/lab3-load-balancing/spec.md",
"openspec/changes/archive/2026-10-28-add-lab3-load-balanced-replicas/tasks.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/.openspec.yaml",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/design.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/proposal.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/specs/lab1-messenger-history/spec.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/specs/lab2-persistent-storage/spec.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/specs/lab3-idempotent-messages/spec.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/specs/lab3-load-balancing/spec.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/specs/lab4-attachments/spec.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/specs/lab4-object-storage/spec.md",
"openspec/changes/archive/2026-10-28-add-lab4-image-attachments/tasks.md",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/.openspec.yaml",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/design.md",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/proposal.md",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/specs/lab4-attachments/spec.md",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/specs/lab4-object-storage/spec.md",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/specs/lab4-upload-resilience/spec.md",
"openspec/changes/archive/2026-10-29-add-lab4-upload-resilience/tasks.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/.openspec.yaml",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/design.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/proposal.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/specs/lab1-messenger-history/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/specs/lab3-load-balancing/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/specs/lab4-attachments/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/specs/lab4-upload-resilience/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/specs/lab5-attachment-pipeline/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/specs/lab5-job-delivery/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-async-attachments/tasks.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/.openspec.yaml",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/design.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/proposal.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/specs/lab4-attachments/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/specs/lab5-attachment-pipeline/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/specs/lab5-image-variants/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/specs/lab5-job-recovery/spec.md",
"openspec/changes/archive/2026-10-29-add-lab5-variants-and-recovery/tasks.md",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/.openspec.yaml",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/design.md",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/proposal.md",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/specs/lab1-service-operability/spec.md",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/specs/lab6-service-telemetry/spec.md",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/specs/lab6-telemetry-stack/spec.md",
"openspec/changes/archive/2026-11-06-add-lab6-telemetry-pipeline/tasks.md",
"openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/.openspec.yaml",
"openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/design.md",
"openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/proposal.md",
"openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/specs/lab6-observability-resilience/spec.md",
"openspec/changes/archive/2026-11-06-complete-lab6-observability-resilience/tasks.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/.openspec.yaml",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/design.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/findings.json",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/proposal.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/specs/lab2-persistent-storage/spec.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/specs/lab4-storage-reconciler/spec.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/specs/lab5-versioned-pipeline/spec.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/tasks.md",
"openspec/changes/archive/2026-11-06-forward-lab5-sqlalchemy-foundation/verification.md",
"openspec/changes/archive/2026-11-06-measure-lab6-observability-cost/.openspec.yaml",
"openspec/changes/archive/2026-11-06-measure-lab6-observability-cost/design.md",
"openspec/changes/archive/2026-11-06-measure-lab6-observability-cost/proposal.md",
"openspec/changes/archive/2026-11-06-measure-lab6-observability-cost/specs/lab6-observability-cost/spec.md",
"openspec/changes/archive/2026-11-06-measure-lab6-observability-cost/tasks.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/.openspec.yaml",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/design.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/proposal.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab1-messenger-chats/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab1-messenger-history/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab1-messenger-identity/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab2-auth-sessions/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab2-session-cache/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab3-load-balancing/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab4-attachments/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab5-image-variants/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab5-job-delivery/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/specs/lab5-job-recovery/spec.md",
"openspec/changes/archive/2026-11-07-forward-lab5-published-fixes/tasks.md",
"openspec/config.yaml",
"openspec/notes/architecture-patterns-python.md",
"openspec/specs/lab1-event-publication/spec.md",
"openspec/specs/lab1-messenger-chats/spec.md",
"openspec/specs/lab1-messenger-history/spec.md",
"openspec/specs/lab1-messenger-identity/spec.md",
"openspec/specs/lab1-service-operability/spec.md",
"openspec/specs/lab2-auth-sessions/spec.md",
"openspec/specs/lab2-dependency-failures/spec.md",
"openspec/specs/lab2-history-index/spec.md",
"openspec/specs/lab2-persistent-storage/spec.md",
"openspec/specs/lab2-session-cache/spec.md",
"openspec/specs/lab3-failover-operations/spec.md",
"openspec/specs/lab3-idempotent-messages/spec.md",
"openspec/specs/lab3-load-balancing/spec.md",
"openspec/specs/lab4-attachments/spec.md",
"openspec/specs/lab4-object-storage/spec.md",
"openspec/specs/lab4-storage-reconciler/spec.md",
"openspec/specs/lab4-upload-resilience/spec.md",
"openspec/specs/lab5-attachment-pipeline/spec.md",
"openspec/specs/lab5-image-variants/spec.md",
"openspec/specs/lab5-job-delivery/spec.md",
"openspec/specs/lab5-job-recovery/spec.md",
"openspec/specs/lab5-versioned-pipeline/spec.md",
"openspec/specs/lab6-observability-cost/spec.md",
"openspec/specs/lab6-observability-resilience/spec.md",
"openspec/specs/lab6-service-telemetry/spec.md",
"openspec/specs/lab6-telemetry-stack/spec.md",
"pyproject.toml",
"scripts/broker-secrets.sh",
"scripts/broker.sh",
"scripts/capture_evidence.py",
"scripts/clean-clone.sh",
"scripts/detection.sh",
"scripts/docker-stats.sh",
"scripts/evidence-attachments.sh",
"scripts/evidence-cache.sh",
"scripts/evidence-stack.sh",
"scripts/evidence-storage.sh",
"scripts/failover-each.sh",
"scripts/failover-evidence.sh",
"scripts/k6.sh",
"scripts/lab4_scenarios.py",
"scripts/lab4_seed.py",
"scripts/lab6.sh",
"scripts/lb-stats.sh",
"scripts/lb-wait.sh",
"scripts/limits.py",
"scripts/load-evidence.sh",
"scripts/make_images.py",
"scripts/obs-validate.sh",
"scripts/obs_check.py",
"scripts/obs_cost.py",
"scripts/obs_cost_summary.py",
"scripts/obs_incidents.py",
"scripts/obs_recovery.py",
"scripts/overload-leftovers.sh",
"scripts/persistence-evidence.sh",
"scripts/pg-stats.sh",
"scripts/pipeline_evidence.py",
"scripts/pipeline_suite.py",
"scripts/probe.py",
"scripts/reconcile-inject.sh",
"scripts/reconcile_check.py",
"scripts/reconcile_track.py",
"scripts/restart.sh",
"scripts/restore.sh",
"scripts/rolling-update.sh",
"scripts/s3-check.sh",
"scripts/scaling.py",
"scripts/secrets-lab6.sh",
"scripts/secrets.sh",
"scripts/sql/explain-history.sql",
"scripts/sql/seed-history.sql",
"scripts/stand-usage.sh",
"scripts/stop-one.sh",
"scripts/upload_memory.py",
"scripts/worker.sh",
"src/messenger/__init__.py",
"src/messenger/admin.py",
"src/messenger/attachments/__init__.py",
"src/messenger/attachments/application/__init__.py",
"src/messenger/attachments/application/bus.py",
"src/messenger/attachments/application/commands.py",
"src/messenger/attachments/application/errors.py",
"src/messenger/attachments/application/handlers.py",
"src/messenger/attachments/application/pipeline.py",
"src/messenger/attachments/application/ports.py",
"src/messenger/attachments/application/queries.py",
"src/messenger/attachments/application/reconcile.py",
"src/messenger/attachments/config.py",
"src/messenger/attachments/domain/__init__.py",
"src/messenger/attachments/domain/errors.py",
"src/messenger/attachments/domain/events.py",
"src/messenger/attachments/domain/jobs.py",
"src/messenger/attachments/domain/models.py",
"src/messenger/attachments/domain/values.py",
"src/messenger/attachments/domain/variants.py",
"src/messenger/attachments/entrypoints/__init__.py",
"src/messenger/attachments/entrypoints/admin.py",
"src/messenger/attachments/entrypoints/errors.py",
"src/messenger/attachments/entrypoints/heartbeat.py",
"src/messenger/attachments/entrypoints/http.py",
"src/messenger/attachments/entrypoints/ioc.py",
"src/messenger/attachments/entrypoints/journal.py",
"src/messenger/attachments/entrypoints/reconcile.py",
"src/messenger/attachments/entrypoints/relay.py",
"src/messenger/attachments/entrypoints/schemas.py",
"src/messenger/attachments/entrypoints/sweeper.py",
"src/messenger/attachments/entrypoints/worker.py",
"src/messenger/attachments/infrastructure/__init__.py",
"src/messenger/attachments/infrastructure/faults.py",
"src/messenger/attachments/infrastructure/images.py",
"src/messenger/attachments/infrastructure/jitter.py",
"src/messenger/attachments/infrastructure/png.py",
"src/messenger/attachments/infrastructure/postgres.py",
"src/messenger/attachments/infrastructure/queue_metrics.py",
"src/messenger/attachments/infrastructure/rabbitmq.py",
"src/messenger/attachments/infrastructure/s3.py",
"src/messenger/attachments/infrastructure/telemetry.py",
"src/messenger/gunicorn_conf.py",
"src/messenger/identity/__init__.py",
"src/messenger/identity/application/__init__.py",
"src/messenger/identity/application/bus.py",
"src/messenger/identity/application/commands.py",
"src/messenger/identity/application/handlers.py",
"src/messenger/identity/application/ports.py",
"src/messenger/identity/application/queries.py",
"src/messenger/identity/config.py",
"src/messenger/identity/domain/__init__.py",
"src/messenger/identity/domain/errors.py",
"src/messenger/identity/domain/events.py",
"src/messenger/identity/domain/models.py",
"src/messenger/identity/domain/values.py",
"src/messenger/identity/entrypoints/__init__.py",
"src/messenger/identity/entrypoints/authentication.py",
"src/messenger/identity/entrypoints/errors.py",
"src/messenger/identity/entrypoints/http.py",
"src/messenger/identity/entrypoints/ioc.py",
"src/messenger/identity/entrypoints/journal.py",
"src/messenger/identity/entrypoints/schemas.py",
"src/messenger/identity/infrastructure/__init__.py",
"src/messenger/identity/infrastructure/argon2.py",
"src/messenger/identity/infrastructure/postgres.py",
"src/messenger/identity/infrastructure/valkey.py",
"src/messenger/main.py",
"src/messenger/messaging/__init__.py",
"src/messenger/messaging/application/__init__.py",
"src/messenger/messaging/application/bus.py",
"src/messenger/messaging/application/commands.py",
"src/messenger/messaging/application/handlers.py",
"src/messenger/messaging/application/ports.py",
"src/messenger/messaging/application/queries.py",
"src/messenger/messaging/config.py",
"src/messenger/messaging/domain/__init__.py",
"src/messenger/messaging/domain/errors.py",
"src/messenger/messaging/domain/events.py",
"src/messenger/messaging/domain/keys.py",
"src/messenger/messaging/domain/models.py",
"src/messenger/messaging/domain/values.py",
"src/messenger/messaging/entrypoints/__init__.py",
"src/messenger/messaging/entrypoints/errors.py",
"src/messenger/messaging/entrypoints/http.py",
"src/messenger/messaging/entrypoints/ioc.py",
"src/messenger/messaging/entrypoints/journal.py",
"src/messenger/messaging/entrypoints/schemas.py",
"src/messenger/messaging/infrastructure/__init__.py",
"src/messenger/messaging/infrastructure/cursor.py",
"src/messenger/messaging/infrastructure/postgres.py",
"src/messenger/relay.py",
"src/messenger/shared/__init__.py",
"src/messenger/shared/application/__init__.py",
"src/messenger/shared/application/commands.py",
"src/messenger/shared/application/errors.py",
"src/messenger/shared/application/messagebus.py",
"src/messenger/shared/application/ports.py",
"src/messenger/shared/config.py",
"src/messenger/shared/domain/__init__.py",
"src/messenger/shared/domain/errors.py",
"src/messenger/shared/domain/events.py",
"src/messenger/shared/entrypoints/__init__.py",
"src/messenger/shared/entrypoints/actor.py",
"src/messenger/shared/entrypoints/config.py",
"src/messenger/shared/entrypoints/errors.py",
"src/messenger/shared/entrypoints/health.py",
"src/messenger/shared/entrypoints/ioc.py",
"src/messenger/shared/entrypoints/pipeline.py",
"src/messenger/shared/entrypoints/schemas.py",
"src/messenger/shared/entrypoints/telemetry.py",
"src/messenger/shared/infrastructure/__init__.py",
"src/messenger/shared/infrastructure/clock.py",
"src/messenger/shared/infrastructure/ids.py",
"src/messenger/shared/infrastructure/logging.py",
"src/messenger/shared/infrastructure/postgres.py",
"src/messenger/shared/infrastructure/readiness.py",
"src/messenger/shared/infrastructure/runtime_metrics.py",
"src/messenger/shared/infrastructure/telemetry.py",
"src/messenger/shared/infrastructure/trace_context.py",
"src/messenger/worker.py",
"student-tests/adapters/test_rabbitmq.py",
"student-tests/adapters/test_s3_storage.py",
"student-tests/api/test_access.py",
"student-tests/api/test_attachments.py",
"student-tests/api/test_bounded_contexts.py",
"student-tests/api/test_cache.py",
"student-tests/api/test_concurrency.py",
"student-tests/api/test_consistency.py",
"student-tests/api/test_cost_experiment.py",
"student-tests/api/test_drain.py",
"student-tests/api/test_events.py",
"student-tests/api/test_http_telemetry.py",
"student-tests/api/test_idempotency.py",
"student-tests/api/test_instance.py",
"student-tests/api/test_job_returning.py",
"student-tests/api/test_message_attachments.py",
"student-tests/api/test_migrations.py",
"student-tests/api/test_negative.py",
"student-tests/api/test_nul_validation.py",
"student-tests/api/test_operability.py",
"student-tests/api/test_pagination.py",
"student-tests/api/test_pipeline.py",
"student-tests/api/test_pool.py",
"student-tests/api/test_reconciler.py",
"student-tests/api/test_recovery.py",
"student-tests/api/test_sessions.py",
"student-tests/api/test_timeout.py",
"student-tests/api/test_trace_context.py",
"student-tests/api/test_upload_consistency.py",
"student-tests/api/test_versioned_pipeline.py",
"student-tests/api/test_wire_format.py",
"student-tests/conftest.py",
"student-tests/fakes.py",
"student-tests/fixtures/legacy/0001_init.sql",
"student-tests/fixtures/legacy/0002_idempotency_keys.sql",
"student-tests/fixtures/legacy/0003_attachments.sql",
"student-tests/fixtures/legacy/0004_jobs.sql",
"student-tests/fixtures/legacy/0005_failed_registry.sql",
"student-tests/fixtures/legacy/0006_attachment_quarantine.sql",
"student-tests/pictures.py",
"student-tests/pipeline.py",
"student-tests/png_trns_fixtures.py",
"student-tests/stack/test_attachment_faults.py",
"student-tests/stack/test_faults.py",
"student-tests/stand.py",
"student-tests/tools/forward_conditions.py",
"student-tests/tools/forward_network.py",
"student-tests/tools/forward_runtime.py",
"student-tests/tools/lb_errors/client.py",
"student-tests/tools/lb_errors/compose.yaml",
"student-tests/tools/lb_errors/server.py",
"student-tests/tools/seed_storage.py",
"student-tests/unit/test_attachments_domain.py",
"student-tests/unit/test_authentication.py",
"student-tests/unit/test_cursor.py",
"student-tests/unit/test_domain.py",
"student-tests/unit/test_haproxy_errors.py",
"student-tests/unit/test_image_inspection.py",
"student-tests/unit/test_job_observer.py",
"student-tests/unit/test_nul_inputs.py",
"student-tests/unit/test_pipeline_domain.py",
"student-tests/unit/test_png_memory.py",
"student-tests/unit/test_png_transparency.py",
"student-tests/unit/test_png_upload.py",
"student-tests/unit/test_probe.py",
"student-tests/unit/test_reconcile_measurements.py",
"student-tests/unit/test_render_pool.py",
"student-tests/unit/test_renderer.py",
"student-tests/unit/test_scaling.py",
"student-tests/unit/test_session_cache.py",
"student-tests/unit/test_telemetry_platform.py",
"student-tests/unit/test_timeouts.py",
"student-tests/unit/test_worker_delivery.py",
"tools/plot_observability.py",
"tools/plot_track1.py",
"tools/summarize_reconciler.py",
"tools/summarize_upload_memory.py",
"uv.lock"
]
}
+26
View File
@@ -0,0 +1,26 @@
# Реальные проверки ЛР7
Проверки выполнены на локальном arm64 Docker-стенде13–14.11.2026. Не являются старыми нагрузочными измерениями ЛР6. Полные последовательности запуска и ограничения — [REPORT](../../REPORT.md).
| Файл | Что подтверждает |
| --- | --- |
| [clean-clone.txt](clean-clone.txt) | Чистый clone170a76a, новые CA/credentials/volumes, exit_code0 |
| [offline-final.txt](offline-final.txt) | make check validate lint; Ruff/format/ty |
| [student-tests-clean.txt](student-tests-clean.txt) | 997passed/2skipped/19deselected; чистое состояние fixtures |
| [public-final.txt](public-final.txt) | HTTPS public smoke, внешний TLS trust |
| [persistence.txt](persistence.txt) | restart с сохранением data/session |
| [failover-1.txt](failover-1.txt) | публичный resilience:129writes/0failed |
| [failover-each.json](failover-each.json) | отдельная остановка обеих реплик,60writes/0failed |
| [trust-audit-final.json](trust-audit-final.json) | 103positive/negativeTLS/ACL probes |
| [hardening.json](hardening.json) | реальный Docker inspect без env/secrets,23runtimecontainers |
| [kdf-budget.json](kdf-budget.json) | Argon2id100verify ×3concurrency |
| [lifecycle.json](lifecycle.json) | Bao sealed/restart/expiry/unseal, shared login limit |
| [broad-rights.json](broad-rights.json) | реальные broad operator permissions для сравнения |
| [worker-compromise.json](worker-compromise.json) | итоговые3повтора, revocation/rotation/recovery и counts публикаций |
| [worker-compromise-first-series.json](worker-compromise-first-series.json) | первая серия3повторов, сохранена отдельно |
| [telemetry-secrets.json](telemetry-secrets.json) | presence метрик/логов/traces и secret-value/canary scan |
| [image-scan.json](image-scan.json) | Trivy OS+Python итогового runtime image; [разбор](../../security/SCAN_TRIAGE.md) |
Промежуточные провалы сохранены: `student-tests.txt` (новая migration head и Rabbit fixture), `student-tests-final.txt` (оставшийся outbox между тестами), `student-tests-isolated.txt` (прерван после обнаружения отсутствующего импорта в fixture), `login-tests.txt` (reserved SQL identifier window), `trust-audit-initial.json` (S3 SigV4 probe без обязательного payload header), `telemetry-secrets-initial.json` (S3 identity ID ошибочно посчитан secret). Они не помечены как PASS. Соответствующие исправленные прогоны лежат рядом. `broad-rights.stderr` — первый probe попытался создать запрещённый тип transient queue; финальный тест использует изолированную exclusive queue.
Проверяем не только завершение скрипта: сравниваем код ответа/разрешённую операцию, SHA256, сохранённую сессию, конкретный остановленный instance, server-side ACL, новые соединения со старым cert/password. Подробные Docker logs с потенциальными идентификаторами не коммитятся; сохранили только агрегаты проверки и безопасный вывод.
+16
View File
@@ -0,0 +1,16 @@
Container mtusi2026lab7-api-1 Recreate
Container mtusi2026lab7-worker-1 Recreate
Container mtusi2026lab7-relay-1 Recreate
Container mtusi2026lab7-api-2 Recreate
Container mtusi2026lab7-relay-1 Recreated
Container mtusi2026lab7-worker-1 Recreated
Container mtusi2026lab7-api-1 Recreated
Container mtusi2026lab7-api-2 Recreated
Container mtusi2026lab7-worker-1 Starting
Container mtusi2026lab7-relay-1 Starting
Container mtusi2026lab7-api-1 Starting
Container mtusi2026lab7-relay-1 Started
Container mtusi2026lab7-worker-1 Started
Container mtusi2026lab7-api-1 Started
Container mtusi2026lab7-api-2 Starting
Container mtusi2026lab7-api-2 Started
+54
View File
@@ -0,0 +1,54 @@
Container mtusi2026lab7-prometheus-tls-1 Stopping
Container mtusi2026lab7-postgres-1 Stopping
Container mtusi2026lab7-broker-tls-1 Stopping
Container mtusi2026lab7-tempo-tls-1 Stopping
Container mtusi2026lab7-valkey-tls-1 Stopping
Container mtusi2026lab7-grafana-tls-1 Stopping
Container mtusi2026lab7-lb-1 Stopping
Container mtusi2026lab7-loki-tls-1 Stopping
Container mtusi2026lab7-relay-1 Stopping
Container mtusi2026lab7-api-1 Stopping
Container mtusi2026lab7-otel-collector-tls-1 Stopping
Container mtusi2026lab7-api-2 Stopping
Container mtusi2026lab7-worker-1 Stopping
Container mtusi2026lab7-postgres-1 Stopped
Container mtusi2026lab7-broker-tls-1 Stopped
Container mtusi2026lab7-broker-1 Stopping
Container mtusi2026lab7-lb-1 Stopped
Container mtusi2026lab7-worker-1 Stopped
Container mtusi2026lab7-relay-1 Stopped
Container mtusi2026lab7-broker-1 Stopped
Container mtusi2026lab7-valkey-tls-1 Stopped
Container mtusi2026lab7-valkey-1 Stopping
Container mtusi2026lab7-valkey-1 Stopped
Container mtusi2026lab7-api-1 Stopped
Container mtusi2026lab7-api-2 Stopped
Container mtusi2026lab7-migrate-1 Stopping
Container mtusi2026lab7-s3-init-1 Stopping
Container mtusi2026lab7-migrate-1 Stopped
Container mtusi2026lab7-openbao-1 Stopping
Container mtusi2026lab7-s3-init-1 Stopped
Container mtusi2026lab7-s3-tls-1 Stopping
Container mtusi2026lab7-s3-tls-1 Stopped
Container mtusi2026lab7-s3-1 Stopping
Container mtusi2026lab7-s3-1 Stopped
Container mtusi2026lab7-openbao-1 Stopped
Container mtusi2026lab7-bao-init-1 Stopping
Container mtusi2026lab7-bao-init-1 Stopped
Container mtusi2026lab7-grafana-tls-1 Stopped
Container mtusi2026lab7-grafana-1 Stopping
Container mtusi2026lab7-prometheus-tls-1 Stopped
Container mtusi2026lab7-loki-tls-1 Stopped
Container mtusi2026lab7-tempo-tls-1 Stopped
Container mtusi2026lab7-otel-collector-tls-1 Stopped
Container mtusi2026lab7-otel-collector-1 Stopping
Container mtusi2026lab7-grafana-1 Stopped
Container mtusi2026lab7-loki-1 Stopping
Container mtusi2026lab7-prometheus-1 Stopping
Container mtusi2026lab7-tempo-1 Stopping
Container mtusi2026lab7-otel-collector-1 Stopped
Container mtusi2026lab7-prometheus-1 Stopped
Container mtusi2026lab7-tempo-1 Stopped
Container mtusi2026lab7-loki-1 Stopped
Container mtusi2026lab7-healthtools-1 Stopping
Container mtusi2026lab7-healthtools-1 Stopped
+25
View File
@@ -0,0 +1,25 @@
{
"checks": [
{
"test": "broad-db:users-readable",
"passed": true
},
{
"test": "broad-db:ddl-allowed",
"passed": true
},
{
"test": "broad-s3:private-write",
"passed": true
},
{
"test": "broad-s3:private-delete",
"passed": true
},
{
"test": "broad-queue:configure-publish-consume-delete",
"passed": true
}
],
"passed": true
}
+87
View File
@@ -0,0 +1,87 @@
Unexpected connection close from remote "amqps://messenger-ops:******@broker:5671/messenger", Connection.Close(reply_code=541, reply_text='INTERNAL_ERROR - Feature `transient_nonexcl_queues` is deprecated.\nIts use is not permitted per the configuration (overriding the default, which is permitted):\n "deprecated_features.permit.transient_nonexcl_queues = false"\nIts use will not be permit...')
Cancelling cause reader exited abnormally
Traceback (most recent call last):
File "/opt/venv/lib/python3.14/site-packages/aiormq/connection.py", line 755, in __reader
await self.__handle_close(frame)
File "/opt/venv/lib/python3.14/site-packages/aiormq/connection.py", line 698, in __handle_close
raise exception
aiormq.exceptions.ConnectionInternalError: INTERNAL_ERROR - Feature `transient_nonexcl_queues` is deprecated.
Its use is not permitted per the configuration (overriding the default, which is permitted):
"deprecated_features.permit.transient_nonexcl_queues = false"
Its use will not be permit...
Traceback (most recent call last):
File "/opt/venv/lib/python3.14/site-packages/aiormq/abc.py", line 42, in __inner
return await self.task
^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aiormq/connection.py", line 808, in __heartbeat
await asyncio.sleep(heartbeat_timeout)
File "/usr/local/lib/python3.14/asyncio/tasks.py", line 705, in sleep
return await future
^^^^^^^^^^^^
asyncio.exceptions.CancelledError: INTERNAL_ERROR - Feature `transient_nonexcl_queues` is deprecated.
Its use is not permitted per the configuration (overriding the default, which is permitted):
"deprecated_features.permit.transient_nonexcl_queues = false"
Its use will not be permit...
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/audit/privilege_compare.py", line 50, in <module>
asyncio.run(compare(json.load(sys.stdin)))
~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.14/asyncio/runners.py", line 205, in run
return runner.run(main)
~~~~~~~~~~^^^^^^
File "/usr/local/lib/python3.14/asyncio/runners.py", line 128, in run
return self._loop.run_until_complete(task)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
File "/usr/local/lib/python3.14/asyncio/base_events.py", line 720, in run_until_complete
return future.result()
~~~~~~~~~~~~~^^
File "/audit/privilege_compare.py", line 41, in compare
queue = await channel.declare_queue("security-comparison", auto_delete=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aio_pika/channel.py", line 399, in declare_queue
await queue.declare(timeout=timeout)
File "/opt/venv/lib/python3.14/site-packages/aio_pika/queue.py", line 117, in declare
self.declaration_result = await channel.queue_declare(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<7 lines>...
)
^
File "/opt/venv/lib/python3.14/site-packages/aiormq/channel.py", line 1318, in queue_declare
return await self.rpc(
^^^^^^^^^^^^^^^
...<10 lines>...
)
^
File "/opt/venv/lib/python3.14/site-packages/aiormq/channel.py", line 64, in wrap
return await self.create_task(func(self, *args, **kwargs))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aiormq/abc.py", line 48, in __inner
raise self._exception
File "/usr/local/lib/python3.14/asyncio/tasks.py", line 724, in _wrap_awaitable
return await awaitable
^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aiormq/abc.py", line 48, in __inner
raise self._exception
File "/usr/local/lib/python3.14/asyncio/tasks.py", line 724, in _wrap_awaitable
return await awaitable
^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aiormq/abc.py", line 49, in __inner
raise self._exception from e
File "/usr/local/lib/python3.14/asyncio/tasks.py", line 724, in _wrap_awaitable
return await awaitable
^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aiormq/abc.py", line 42, in __inner
return await self.task
^^^^^^^^^^^^^^^
File "/opt/venv/lib/python3.14/site-packages/aiormq/connection.py", line 755, in __reader
await self.__handle_close(frame)
File "/opt/venv/lib/python3.14/site-packages/aiormq/connection.py", line 698, in __handle_close
raise exception
aiormq.exceptions.ConnectionInternalError: INTERNAL_ERROR - Feature `transient_nonexcl_queues` is deprecated.
Its use is not permitted per the configuration (overriding the default, which is permitted):
"deprecated_features.permit.transient_nonexcl_queues = false"
Its use will not be permit...
+364
View File
@@ -0,0 +1,364 @@
#1 [internal] load local bake definitions
#1 reading from stdin 432B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.95kB done
#2 DONE 0.0s
#3 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#3 DONE 0.0s
#4 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#4 DONE 3.0s
#5 [internal] load .dockerignore
#5 transferring context: 196B done
#5 DONE 0.0s
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
#6 DONE 0.0s
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
#7 CACHED
#8 [internal] load build context
#8 transferring context: 387.94kB 0.0s done
#8 DONE 0.0s
#9 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
#9 CACHED
#10 [builder 3/7] WORKDIR /build
#10 CACHED
#11 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
#11 DONE 0.0s
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
#12 0.231 Using CPython 3.14.7 interpreter at: /usr/local/bin/python3
#12 0.232 Creating virtual environment at: /opt/venv
#12 0.252 Resolved 100 packages in 9ms
#12 0.597 Downloading pillow (6.2MiB)
#12 0.597 Downloading aiohttp (1.8MiB)
#12 0.598 Downloading sqlalchemy (3.1MiB)
#12 0.622 Downloading pydantic-core (1.9MiB)
#12 0.622 Downloading uvloop (4.1MiB)
#12 0.622 Downloading asyncpg (3.6MiB)
#12 0.622 Downloading botocore (15.3MiB)
#12 4.418 Downloading aiohttp
#12 4.503 Downloading pydantic-core
#12 4.842 Downloading sqlalchemy
#12 4.894 Downloading asyncpg
#12 4.997 Downloading uvloop
#12 5.185 Downloading pillow
#12 5.735 Downloading botocore
#12 5.735 Prepared 79 packages in 5.47s
#12 5.824 Installed 79 packages in 89ms
#12 6.267 Bytecode compiled 2685 files in 442ms
#12 6.267 + aio-pika==10.2.1
#12 6.267 + aiobotocore==3.10.0
#12 6.267 + aiohappyeyeballs==2.7.1
#12 6.267 + aiohttp==3.14.5
#12 6.267 + aioitertools==0.13.0
#12 6.267 + aiormq==7.2.4
#12 6.267 + aiosignal==1.4.0
#12 6.267 + alembic==1.16.5
#12 6.267 + annotated-doc==0.0.5
#12 6.267 + annotated-types==0.8.0
#12 6.267 + anyio==4.15.1
#12 6.267 + argon2-cffi==25.1.0
#12 6.267 + argon2-cffi-bindings==26.1.0
#12 6.267 + asgiref==3.13.0
#12 6.267 + asyncpg==0.32.0
#12 6.267 + attrs==26.1.0
#12 6.267 + botocore==1.43.117
#12 6.267 + certifi==2026.10.23
#12 6.267 + cffi==2.1.1
#12 6.267 + charset-normalizer==3.5.2
#12 6.267 + click==8.5.0
#12 6.267 + dishka==1.10.1
#12 6.267 + dishka-faststream==0.7.0
#12 6.267 + fast-depends==3.0.10
#12 6.267 + fastapi==0.141.1
#12 6.267 + faststream==0.7.8
#12 6.267 + frozenlist==1.8.0
#12 6.268 + googleapis-common-protos==1.76.0
#12 6.268 + greenlet==3.5.6
#12 6.268 + gunicorn==26.2.0
#12 6.268 + h11==0.16.0
#12 6.268 + httptools==0.8.0
#12 6.268 + idna==3.20
#12 6.268 + jmespath==1.1.0
#12 6.268 + mako==1.4.3
#12 6.268 + markupsafe==3.0.4
#12 6.268 + multidict==7.2.1
#12 6.268 + opentelemetry-api==1.45.1
#12 6.268 + opentelemetry-exporter-http-transport==0.66b1
#12 6.268 + opentelemetry-exporter-otlp-common==0.66b1
#12 6.268 + opentelemetry-exporter-otlp-proto-common==1.45.1
#12 6.268 + opentelemetry-exporter-otlp-proto-http==1.45.1
#12 6.268 + opentelemetry-instrumentation==0.66b1
#12 6.268 + opentelemetry-instrumentation-asgi==0.66b1
#12 6.268 + opentelemetry-instrumentation-asyncpg==0.66b1
#12 6.268 + opentelemetry-instrumentation-fastapi==0.66b1
#12 6.268 + opentelemetry-proto==1.45.1
#12 6.268 + opentelemetry-sdk==1.45.1
#12 6.268 + opentelemetry-semantic-conventions==0.66b1
#12 6.268 + opentelemetry-util-http==0.66b1
#12 6.268 + orjson==3.12.0
#12 6.268 + packaging==26.3
#12 6.268 + pamqp==4.0.1
#12 6.268 + pillow==13.0.0
#12 6.268 + propcache==0.5.4
#12 6.268 + protobuf==7.36.2
#12 6.268 + pycparser==3.12
#12 6.268 + pydantic==2.13.5
#12 6.268 + pydantic-core==2.46.5
#12 6.268 + pydantic-settings==2.15.0
#12 6.268 + python-dateutil==2.9.0.post0
#12 6.268 + python-dotenv==1.2.3
#12 6.268 + pyyaml==6.0.3
#12 6.268 + redis==8.1.0
#12 6.268 + requests==2.34.2
#12 6.268 + six==1.17.0
#12 6.268 + sqlalchemy==2.0.45
#12 6.268 + starlette==1.7.0
#12 6.268 + structlog==26.1.0
#12 6.268 + typing-extensions==4.16.0
#12 6.268 + typing-inspection==0.4.4
#12 6.268 + urllib3==2.8.0
#12 6.268 + uvicorn==0.54.0
#12 6.268 + uvicorn-worker==0.4.0
#12 6.268 + uvloop==0.22.1
#12 6.268 + watchfiles==1.3.0
#12 6.268 + websockets==17.1
#12 6.268 + wrapt==2.5.1
#12 6.268 + yarl==1.25.1
#12 DONE 6.8s
#13 [runtime 2/9] RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends haproxy && rm -rf /var/lib/apt/lists/*
#13 0.314 Hit:1 http://deb.debian.org/debian trixie InRelease
#13 0.586 Get:2 http://deb.debian.org/debian trixie-updates InRelease [47.3 kB]
#13 0.747 Get:3 http://deb.debian.org/debian-security trixie-security InRelease [43.4 kB]
#13 ...
#14 [builder 6/7] COPY src ./src
#14 DONE 0.1s
#15 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
#15 0.147 Resolved 100 packages in 0.76ms
#15 0.149 Building mtusi-messenger @ file:///build
#15 2.056 Built mtusi-messenger @ file:///build
#15 2.060 Prepared 1 package in 1.91s
#15 2.065 Installed 1 package in 5ms
#15 2.149 Bytecode compiled 2807 files in 83ms
#15 2.149 + mtusi-messenger==1.7.0 (from file:///build)
#15 DONE 2.2s
#13 [runtime 2/9] RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends haproxy && rm -rf /var/lib/apt/lists/*
#13 7.943 Get:4 http://deb.debian.org/debian trixie/main arm64 Packages [9614 kB]
#13 8.084 Get:5 http://deb.debian.org/debian trixie-updates/main arm64 Packages [1108 B]
#13 8.310 Get:6 http://deb.debian.org/debian-security trixie-security/main arm64 Packages [303 kB]
#13 8.538 Fetched 10.0 MB in 8s (1189 kB/s)
#13 8.538 Reading package lists...
#13 8.852 Reading package lists...
#13 9.155 Building dependency tree...
#13 9.232 Reading state information...
#13 9.256 Calculating upgrade...
#13 9.384 The following packages will be upgraded:
#13 9.384 apt ca-certificates libapt-pkg7.0 liblzma5 libpcre2-8-0 libssl3t64 openssl
#13 9.384 openssl-provider-legacy tzdata zlib1g
#13 9.750 10 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
#13 9.750 Need to get 7977 kB of archives.
#13 9.750 After this operation, 0 B of additional disk space will be used.
#13 9.750 Get:1 http://deb.debian.org/debian-security trixie-security/main arm64 liblzma5 arm64 5.8.1-1+deb13u2 [303 kB]
#13 9.902 Get:2 http://deb.debian.org/debian-security trixie-security/main arm64 openssl-provider-legacy arm64 3.5.7-1~deb13u3 [314 kB]
#13 10.25 Get:3 http://deb.debian.org/debian-security trixie-security/main arm64 zlib1g arm64 1:1.3.dfsg+really1.3.1-1+deb13u1 [85.0 kB]
#13 10.93 Get:4 http://deb.debian.org/debian-security trixie-security/main arm64 libssl3t64 arm64 3.5.7-1~deb13u3 [2770 kB]
#13 11.27 Get:5 http://deb.debian.org/debian-security trixie-security/main arm64 libapt-pkg7.0 arm64 3.0.4 [978 kB]
#13 11.87 Get:6 http://deb.debian.org/debian-security trixie-security/main arm64 apt arm64 3.0.4 [1363 kB]
#13 12.06 Get:7 http://deb.debian.org/debian-security trixie-security/main arm64 libpcre2-8-0 arm64 10.46-1~deb13u3 [263 kB]
#13 12.26 Get:8 http://deb.debian.org/debian trixie-updates/main arm64 tzdata all 2026f-0+deb13u1 [268 kB]
#13 12.53 Get:9 http://deb.debian.org/debian-security trixie-security/main arm64 openssl arm64 3.5.7-1~deb13u3 [1473 kB]
#13 12.75 Get:10 http://deb.debian.org/debian-security trixie-security/main arm64 ca-certificates all 20250419+deb13u1 [161 kB]
#13 12.84 debconf: unable to initialize frontend: Dialog
#13 12.84 debconf: (TERM is not set, so the dialog frontend is not usable.)
#13 12.84 debconf: falling back to frontend: Readline
#13 12.84 debconf: unable to initialize frontend: Readline
#13 12.84 debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/aarch64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/aarch64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/aarch64-linux-gnu/perl-base /usr/lib/aarch64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8, <STDIN> line 10.)
#13 12.84 debconf: falling back to frontend: Teletype
#13 12.84 debconf: unable to initialize frontend: Teletype
#13 12.84 debconf: (This frontend requires a controlling tty.)
#13 12.84 debconf: falling back to frontend: Noninteractive
#13 13.10 Preconfiguring packages ...
#13 13.29 Fetched 7977 kB in 3s (2370 kB/s)
#13 13.31 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5657 files and directories currently installed.)
#13 13.32 Preparing to unpack .../liblzma5_5.8.1-1+deb13u2_arm64.deb ...
#13 13.32 Unpacking liblzma5:arm64 (5.8.1-1+deb13u2) over (5.8.1-1+deb13u1) ...
#13 13.34 Setting up liblzma5:arm64 (5.8.1-1+deb13u2) ...
#13 13.35 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5657 files and directories currently installed.)
#13 13.35 Preparing to unpack .../openssl-provider-legacy_3.5.7-1~deb13u3_arm64.deb ...
#13 13.36 Unpacking openssl-provider-legacy (3.5.7-1~deb13u3) over (3.5.7-1~deb13u2) ...
#13 13.37 Setting up openssl-provider-legacy (3.5.7-1~deb13u3) ...
#13 13.38 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5657 files and directories currently installed.)
#13 13.38 Preparing to unpack .../zlib1g_1%3a1.3.dfsg+really1.3.1-1+deb13u1_arm64.deb ...
#13 13.39 Unpacking zlib1g:arm64 (1:1.3.dfsg+really1.3.1-1+deb13u1) over (1:1.3.dfsg+really1.3.1-1+b1) ...
#13 13.40 Setting up zlib1g:arm64 (1:1.3.dfsg+really1.3.1-1+deb13u1) ...
#13 13.41 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5656 files and directories currently installed.)
#13 13.42 Preparing to unpack .../libssl3t64_3.5.7-1~deb13u3_arm64.deb ...
#13 13.42 Unpacking libssl3t64:arm64 (3.5.7-1~deb13u3) over (3.5.7-1~deb13u2) ...
#13 13.51 Setting up libssl3t64:arm64 (3.5.7-1~deb13u3) ...
#13 13.53 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5656 files and directories currently installed.)
#13 13.53 Preparing to unpack .../libapt-pkg7.0_3.0.4_arm64.deb ...
#13 13.53 Unpacking libapt-pkg7.0:arm64 (3.0.4) over (3.0.3) ...
#13 13.58 Setting up libapt-pkg7.0:arm64 (3.0.4) ...
#13 13.59 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5656 files and directories currently installed.)
#13 13.60 Preparing to unpack .../archives/apt_3.0.4_arm64.deb ...
#13 13.61 Unpacking apt (3.0.4) over (3.0.3) ...
#13 13.68 Setting up apt (3.0.4) ...
#13 13.78 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5656 files and directories currently installed.)
#13 13.78 Preparing to unpack .../libpcre2-8-0_10.46-1~deb13u3_arm64.deb ...
#13 13.79 Unpacking libpcre2-8-0:arm64 (10.46-1~deb13u3) over (10.46-1~deb13u2) ...
#13 13.81 Setting up libpcre2-8-0:arm64 (10.46-1~deb13u3) ...
#13 13.82 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5656 files and directories currently installed.)
#13 13.82 Preparing to unpack .../tzdata_2026f-0+deb13u1_all.deb ...
#13 13.83 Unpacking tzdata (2026f-0+deb13u1) over (2026c-0+deb13u1) ...
#13 14.13 Preparing to unpack .../openssl_3.5.7-1~deb13u3_arm64.deb ...
#13 14.13 Unpacking openssl (3.5.7-1~deb13u3) over (3.5.7-1~deb13u2) ...
#13 14.20 Preparing to unpack .../ca-certificates_20250419+deb13u1_all.deb ...
#13 14.20 Unpacking ca-certificates (20250419+deb13u1) over (20250419) ...
#13 14.33 Setting up tzdata (2026f-0+deb13u1) ...
#13 14.35 debconf: unable to initialize frontend: Dialog
#13 14.35 debconf: (TERM is not set, so the dialog frontend is not usable.)
#13 14.35 debconf: falling back to frontend: Readline
#13 14.35 debconf: unable to initialize frontend: Readline
#13 14.35 debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/aarch64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/aarch64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/aarch64-linux-gnu/perl-base /usr/lib/aarch64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8.)
#13 14.35 debconf: falling back to frontend: Teletype
#13 14.35 debconf: unable to initialize frontend: Teletype
#13 14.35 debconf: (This frontend requires a controlling tty.)
#13 14.35 debconf: falling back to frontend: Noninteractive
#13 14.37
#13 14.37 Current default time zone: 'Etc/UTC'
#13 14.37 Local time is now: Fri Nov 13 20:44:00 UTC 2026.
#13 14.37 Universal Time is now: Fri Nov 13 20:44:00 UTC 2026.
#13 14.37 Run 'dpkg-reconfigure tzdata' if you wish to change it.
#13 14.37
#13 14.38 Setting up openssl (3.5.7-1~deb13u3) ...
#13 14.38 Setting up ca-certificates (20250419+deb13u1) ...
#13 14.42 debconf: unable to initialize frontend: Dialog
#13 14.42 debconf: (TERM is not set, so the dialog frontend is not usable.)
#13 14.42 debconf: falling back to frontend: Readline
#13 14.42 debconf: unable to initialize frontend: Readline
#13 14.42 debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/aarch64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/aarch64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/aarch64-linux-gnu/perl-base /usr/lib/aarch64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8.)
#13 14.42 debconf: falling back to frontend: Teletype
#13 14.42 debconf: unable to initialize frontend: Teletype
#13 14.42 debconf: (This frontend requires a controlling tty.)
#13 14.42 debconf: falling back to frontend: Noninteractive
#13 14.86 Updating certificates in /etc/ssl/certs...
#13 15.11 rehash: warning: skipping ca-certificates.crt, it does not contain exactly one certificate or CRL
#13 15.11 0 added, 2 removed; done.
#13 15.12 Processing triggers for libc-bin (2.41-12+deb13u4) ...
#13 15.14 Processing triggers for ca-certificates (20250419+deb13u1) ...
#13 15.15 Updating certificates in /etc/ssl/certs...
#13 15.30 0 added, 0 removed; done.
#13 15.30 Running hooks in /etc/ca-certificates/update.d...
#13 15.30 done.
#13 15.31 Reading package lists...
#13 15.59 Building dependency tree...
#13 15.66 Reading state information...
#13 15.76 The following additional packages will be installed:
#13 15.76 libjemalloc2 liblua5.4-0 libopentracing-c-wrapper0t64 libopentracing1
#13 15.76 Suggested packages:
#13 15.76 vim-haproxy haproxy-doc
#13 15.78 The following NEW packages will be installed:
#13 15.78 haproxy libjemalloc2 liblua5.4-0 libopentracing-c-wrapper0t64
#13 15.78 libopentracing1
#13 16.28 0 upgraded, 5 newly installed, 0 to remove and 0 not upgraded.
#13 16.28 Need to get 2931 kB of archives.
#13 16.28 After this operation, 7673 kB of additional disk space will be used.
#13 16.28 Get:1 http://deb.debian.org/debian trixie/main arm64 libjemalloc2 arm64 5.3.0-3 [216 kB]
#13 16.51 Get:2 http://deb.debian.org/debian trixie/main arm64 liblua5.4-0 arm64 5.4.7-1+b2 [134 kB]
#13 16.76 Get:3 http://deb.debian.org/debian trixie/main arm64 libopentracing1 arm64 1.6.0-4+b2 [50.0 kB]
#13 17.02 Get:4 http://deb.debian.org/debian trixie/main arm64 libopentracing-c-wrapper0t64 arm64 1.1.3-3.1+b1 [28.3 kB]
#13 17.83 Get:5 http://deb.debian.org/debian-security trixie-security/main arm64 haproxy arm64 3.0.11-1+deb13u4 [2502 kB]
#13 17.89 debconf: unable to initialize frontend: Dialog
#13 17.89 debconf: (TERM is not set, so the dialog frontend is not usable.)
#13 17.89 debconf: falling back to frontend: Readline
#13 17.89 debconf: unable to initialize frontend: Readline
#13 17.89 debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC entries checked: /etc/perl /usr/local/lib/aarch64-linux-gnu/perl/5.40.1 /usr/local/share/perl/5.40.1 /usr/lib/aarch64-linux-gnu/perl5/5.40 /usr/share/perl5 /usr/lib/aarch64-linux-gnu/perl-base /usr/lib/aarch64-linux-gnu/perl/5.40 /usr/share/perl/5.40 /usr/local/lib/site_perl) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 8, <STDIN> line 5.)
#13 17.89 debconf: falling back to frontend: Teletype
#13 17.89 debconf: unable to initialize frontend: Teletype
#13 17.89 debconf: (This frontend requires a controlling tty.)
#13 17.89 debconf: falling back to frontend: Noninteractive
#13 18.20 Fetched 2931 kB in 2s (1430 kB/s)
#13 18.21 Selecting previously unselected package libjemalloc2:arm64.
#13 18.21 (Reading database ... ␍(Reading database ... 5%␍(Reading database ... 10%␍(Reading database ... 15%␍(Reading database ... 20%␍(Reading database ... 25%␍(Reading database ... 30%␍(Reading database ... 35%␍(Reading database ... 40%␍(Reading database ... 45%␍(Reading database ... 50%␍(Reading database ... 55%␍(Reading database ... 60%␍(Reading database ... 65%␍(Reading database ... 70%␍(Reading database ... 75%␍(Reading database ... 80%␍(Reading database ... 85%␍(Reading database ... 90%␍(Reading database ... 95%␍(Reading database ... 100%␍(Reading database ... 5654 files and directories currently installed.)
#13 18.21 Preparing to unpack .../libjemalloc2_5.3.0-3_arm64.deb ...
#13 18.22 Unpacking libjemalloc2:arm64 (5.3.0-3) ...
#13 18.24 Selecting previously unselected package liblua5.4-0:arm64.
#13 18.24 Preparing to unpack .../liblua5.4-0_5.4.7-1+b2_arm64.deb ...
#13 18.24 Unpacking liblua5.4-0:arm64 (5.4.7-1+b2) ...
#13 18.25 Selecting previously unselected package libopentracing1:arm64.
#13 18.25 Preparing to unpack .../libopentracing1_1.6.0-4+b2_arm64.deb ...
#13 18.26 Unpacking libopentracing1:arm64 (1.6.0-4+b2) ...
#13 18.27 Selecting previously unselected package libopentracing-c-wrapper0t64:arm64.
#13 18.27 Preparing to unpack .../libopentracing-c-wrapper0t64_1.1.3-3.1+b1_arm64.deb ...
#13 18.27 Unpacking libopentracing-c-wrapper0t64:arm64 (1.1.3-3.1+b1) ...
#13 18.28 Selecting previously unselected package haproxy.
#13 18.28 Preparing to unpack .../haproxy_3.0.11-1+deb13u4_arm64.deb ...
#13 18.29 Unpacking haproxy (3.0.11-1+deb13u4) ...
#13 18.37 Setting up libjemalloc2:arm64 (5.3.0-3) ...
#13 18.37 Setting up liblua5.4-0:arm64 (5.4.7-1+b2) ...
#13 18.38 Setting up libopentracing1:arm64 (1.6.0-4+b2) ...
#13 18.38 Setting up libopentracing-c-wrapper0t64:arm64 (1.1.3-3.1+b1) ...
#13 18.38 Setting up haproxy (3.0.11-1+deb13u4) ...
#13 18.45 invoke-rc.d: could not determine current runlevel
#13 18.45 invoke-rc.d: policy-rc.d denied execution of start.
#13 18.49 Processing triggers for libc-bin (2.41-12+deb13u4) ...
#13 DONE 18.5s
#16 [runtime 3/9] RUN python -m pip uninstall -y pip setuptools
#16 0.702 Found existing installation: pip 26.2.1
#16 0.718 Uninstalling pip-26.2.1:
#16 0.806 Successfully uninstalled pip-26.2.1
#16 0.811 WARNING: Skipping setuptools as it is not installed.
#16 0.811 WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.
#16 DONE 0.8s
#17 [runtime 4/9] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
#17 0.114 useradd warning: messenger's uid 10001 is greater than SYS_UID_MAX 999
#17 DONE 0.1s
#18 [runtime 5/9] COPY --from=builder --chown=root:root /opt/venv /opt/venv
#18 DONE 0.5s
#19 [runtime 6/9] WORKDIR /app
#19 DONE 0.1s
#20 [runtime 7/9] COPY --chown=root:root alembic.ini ./
#20 DONE 0.0s
#21 [runtime 8/9] COPY --chown=root:root alembic ./alembic
#21 DONE 0.0s
#22 [runtime 9/9] COPY --chown=root:root scripts/secure_runtime.py /app/secure_runtime.py
#22 DONE 0.0s
#23 exporting to image
#23 exporting layers
#23 exporting layers 3.5s done
#23 exporting manifest sha256:a0036abaf1dc26192b52ff2c091022ceab20cc9cea9cd610455ad8b805893ceb done
#23 exporting config sha256:803afd2b81699b01569b340023bdb8ce3b04bbb5516bf4932104131ec2f4b887 done
#23 exporting attestation manifest sha256:0943882cb81d076bc7efc4801fa7d7a6fd440c72672848ec0e88cae548a655d5 done
#23 exporting manifest list sha256:f9ed609edabbcad8fd8fad7c8bd936dac23004c7b4b0b0d795ddf0741ae65dd6 done
#23 naming to docker.io/library/mtusi-messenger:mtusi2026lab7 done
#23 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab7
#23 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab7 0.8s done
#23 DONE 4.3s
#24 resolving provenance for metadata file
#24 DONE 0.0s
api Built
+91
View File
@@ -0,0 +1,91 @@
#1 [internal] load local bake definitions
#1 reading from stdin 432B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.88kB done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#3 DONE 0.0s
#4 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#4 DONE 0.0s
#5 [internal] load .dockerignore
#5 transferring context: 196B done
#5 DONE 0.0s
#6 [internal] load build context
#6 transferring context: 73.97kB done
#6 DONE 0.0s
#7 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#7 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 0.0s done
#7 DONE 0.0s
#8 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#8 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d 0.0s done
#8 DONE 0.0s
#9 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
#9 CACHED
#10 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
#10 CACHED
#11 [builder 3/7] WORKDIR /build
#11 CACHED
#12 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
#12 CACHED
#13 [builder 6/7] COPY src ./src
#13 DONE 0.0s
#14 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
#14 0.199 Resolved 100 packages in 9ms
#14 0.215 Building mtusi-messenger @ file:///build
#14 2.103 Built mtusi-messenger @ file:///build
#14 2.107 Prepared 1 package in 1.89s
#14 2.111 Installed 1 package in 3ms
#14 2.236 Bytecode compiled 2807 files in 125ms
#14 2.236 + mtusi-messenger==1.7.0 (from file:///build)
#14 DONE 2.3s
#15 [runtime 2/8] RUN apt-get update && apt-get install -y --no-install-recommends haproxy && rm -rf /var/lib/apt/lists/*
#15 CACHED
#16 [runtime 3/8] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
#16 CACHED
#17 [runtime 4/8] COPY --from=builder --chown=root:root /opt/venv /opt/venv
#17 DONE 0.6s
#18 [runtime 5/8] WORKDIR /app
#18 DONE 0.0s
#19 [runtime 6/8] COPY --chown=root:root alembic.ini ./
#19 DONE 0.0s
#20 [runtime 7/8] COPY --chown=root:root alembic ./alembic
#20 DONE 0.0s
#21 [runtime 8/8] COPY --chown=root:root scripts/secure_runtime.py /app/secure_runtime.py
#21 DONE 0.0s
#22 exporting to image
#22 exporting layers
#22 exporting layers 3.0s done
#22 exporting manifest sha256:2dd9d90263b945b433decb3e4e88025e71db828435e834e192f7db244e7f9a03 done
#22 exporting config sha256:f45b3bc323b75ed7ed9e129a9984a941140c4e33470af9b28f8a0a93df7b2098 done
#22 exporting attestation manifest sha256:5d2feb213f2276f5fc3f53fd4c785818db775c52c6bbc5af3e89714cdb4d6239 done
#22 exporting manifest list sha256:65434998dd3e65d8f35ee6528ffccf302cbdefce28e3592295bbf535be95ea33 done
#22 naming to docker.io/library/mtusi-messenger:mtusi2026lab7 done
#22 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab7
#22 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab7 0.7s done
#22 DONE 3.8s
#23 resolving provenance for metadata file
#23 DONE 0.0s
api Built
File diff suppressed because it is too large Load Diff
+2
View File
@@ -0,0 +1,2 @@
Resolved 100 packages in 540ms
Updated certifi v2026.7.22 -> v2026.10.23
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 3.154}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+16
View File
@@ -0,0 +1,16 @@
[
{
"target": "d4eee6af7034",
"successful_writes": 30,
"failed_attempts": 0,
"first_recovery_seconds": 2.756,
"message_attachment_session_preserved": true
},
{
"target": "58f9d8b2ba51",
"successful_writes": 30,
"failed_attempts": 0,
"first_recovery_seconds": 2.734,
"message_attachment_session_preserved": true
}
]
+16
View File
@@ -0,0 +1,16 @@
[
{
"target": "d4eee6af7034",
"successful_writes": 30,
"failed_attempts": 0,
"first_recovery_seconds": 2.756,
"message_attachment_session_preserved": true
},
{
"target": "58f9d8b2ba51",
"successful_writes": 30,
"failed_attempts": 0,
"first_recovery_seconds": 2.734,
"message_attachment_session_preserved": true
}
]
+10
View File
@@ -0,0 +1,10 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
230 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
+7
View File
@@ -0,0 +1,7 @@
{
"tracked_files_checked": 593,
"known_private_values_checked": 48,
"matching_paths": [],
"private_paths_tracked": false,
"passed": true
}
+212
View File
@@ -0,0 +1,212 @@
{
"containers": [
{
"service": "api",
"user": "10001:10001",
"readonly": true,
"memory_bytes": 536870912,
"cpu_quota": 1.0,
"only_own_keys": true,
"passed": true
},
{
"service": "api",
"user": "10001:10001",
"readonly": true,
"memory_bytes": 536870912,
"cpu_quota": 1.0,
"only_own_keys": true,
"passed": true
},
{
"service": "broker",
"user": "100:101",
"readonly": true,
"memory_bytes": 536870912,
"cpu_quota": 1.0,
"only_own_keys": true,
"passed": true
},
{
"service": "broker-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "grafana",
"user": "472",
"readonly": false,
"memory_bytes": 402653184,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "grafana-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "lb",
"user": "haproxy",
"readonly": true,
"memory_bytes": 134217728,
"cpu_quota": 1.0,
"only_own_keys": true,
"passed": true
},
{
"service": "loki",
"user": "10001",
"readonly": false,
"memory_bytes": 402653184,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "loki-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "openbao",
"user": "100:1000",
"readonly": true,
"memory_bytes": 268435456,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "otel-collector",
"user": "10001:10001",
"readonly": false,
"memory_bytes": 402653184,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "otel-collector-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "postgres",
"user": "70:70",
"readonly": false,
"memory_bytes": 1073741824,
"cpu_quota": 2.0,
"only_own_keys": true,
"passed": true
},
{
"service": "prometheus",
"user": "nobody",
"readonly": false,
"memory_bytes": 536870912,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "prometheus-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "relay",
"user": "10001:10001",
"readonly": true,
"memory_bytes": 201326592,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "s3",
"user": "rustfs",
"readonly": true,
"memory_bytes": 536870912,
"cpu_quota": 1.0,
"only_own_keys": true,
"passed": true
},
{
"service": "s3-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "tempo",
"user": "10001:10001",
"readonly": false,
"memory_bytes": 402653184,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "tempo-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "valkey",
"user": "999:1000",
"readonly": false,
"memory_bytes": 134217728,
"cpu_quota": 0.5,
"only_own_keys": true,
"passed": true
},
{
"service": "valkey-tls",
"user": "haproxy",
"readonly": true,
"memory_bytes": 67108864,
"cpu_quota": 0.25,
"only_own_keys": true,
"passed": true
},
{
"service": "worker",
"user": "10001:10001",
"readonly": true,
"memory_bytes": 1073741824,
"cpu_quota": 2.0,
"only_own_keys": true,
"passed": true
}
],
"passed": true
}
+7
View File
@@ -0,0 +1,7 @@
2026-11-13T23:44:40+03:00 INFO [vuln] Vulnerability scanning is enabled
2026-11-13T23:44:41+03:00 INFO [python] Licenses acquired from one or more METADATA files may be subject to additional terms. Use `--debug` flag to see all affected packages.
2026-11-13T23:44:41+03:00 INFO Detected OS family="debian" version="13.7"
2026-11-13T23:44:41+03:00 INFO [debian] Detecting vulnerabilities... os_version="13" pkg_num=92
2026-11-13T23:44:41+03:00 INFO Number of language-specific files num=1
2026-11-13T23:44:41+03:00 INFO [python-pkg] Detecting vulnerabilities...
2026-11-13T23:44:41+03:00 WARN Using severities from other vendors for some vulnerabilities. Read https://trivy.dev/docs/v0.76/guide/scanner/vulnerability#severity-selection for details.
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+9
View File
@@ -0,0 +1,9 @@
2026-11-13T23:42:00+03:00 INFO [vuln] Vulnerability scanning is enabled
2026-11-13T23:42:00+03:00 WARN Third-party SBOM may lead to inaccurate vulnerability detection
2026-11-13T23:42:00+03:00 WARN Recommend using Trivy to generate SBOMs
2026-11-13T23:42:00+03:00 INFO [python] Licenses acquired from one or more METADATA files may be subject to additional terms. Use `--debug` flag to see all affected packages.
2026-11-13T23:42:01+03:00 INFO Detected OS family="debian" version="13.7"
2026-11-13T23:42:01+03:00 INFO [debian] Detecting vulnerabilities... os_version="13" pkg_num=92
2026-11-13T23:42:01+03:00 INFO Number of language-specific files num=1
2026-11-13T23:42:01+03:00 INFO [python-pkg] Detecting vulnerabilities...
2026-11-13T23:42:01+03:00 WARN Using severities from other vendors for some vulnerabilities. Read https://trivy.dev/docs/v0.76/guide/scanner/vulnerability#severity-selection for details.
+34
View File
@@ -0,0 +1,34 @@
{
"memory_kib": 19456,
"iterations": 2,
"parallelism": 1,
"results": [
{
"concurrency": 1,
"samples": 100,
"median_ms": 10.341,
"p95_ms": 11.651,
"cpu_ms_per_verify": 10.531,
"wall_seconds": 1.058,
"peak_rss_kib": 41632
},
{
"concurrency": 2,
"samples": 100,
"median_ms": 11.587,
"p95_ms": 65.264,
"cpu_ms_per_verify": 11.647,
"wall_seconds": 1.134,
"peak_rss_kib": 61088
},
{
"concurrency": 4,
"samples": 100,
"median_ms": 85.386,
"p95_ms": 94.708,
"cpu_ms_per_verify": 15.126,
"wall_seconds": 1.496,
"peak_rss_kib": 100000
}
]
}
View File
+17
View File
@@ -0,0 +1,17 @@
{
"lifecycle": {
"sealed_api_stop_seconds": 56.955,
"unseal_recovery_seconds": 12.595,
"bao_restart_stays_sealed": true,
"persistent_configuration": true,
"session_and_message_preserved": true
},
"login_limit": {
"attempts": 30,
"kdf_admitted": 20,
"rate_limited": 10,
"instances": 2,
"retry_after_present": true,
"elapsed_seconds": 0.32
}
}
+18
View File
@@ -0,0 +1,18 @@
OpenBao unsealed by operator; shares not displayed.
{
"lifecycle": {
"sealed_api_stop_seconds": 56.955,
"unseal_recovery_seconds": 12.595,
"bao_restart_stays_sealed": true,
"persistent_configuration": true,
"session_and_message_preserved": true
},
"login_limit": {
"attempts": 30,
"kdf_admitted": 20,
"rate_limited": 10,
"instances": 2,
"retry_after_present": true,
"elapsed_seconds": 0.32
}
}
+14
View File
@@ -0,0 +1,14 @@
.venv/bin/ruff check src student-tests scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
E501 Line too long (102 > 100)
--> scripts/security_operator.py:190:101
|
188 | for role in ["worker", "relay"]:
189 | sql(
190 | "GRANT SELECT, INSERT, UPDATE, DELETE ON attachments, attachment_variants, jobs, outbox, "
| ^^
191 | "job_events TO messenger_" + role + ";"
192 | )
|
Found 1 error.
make: *** [lint] Error 1
+2
View File
@@ -0,0 +1,2 @@
.. [100%]
2 passed in 1.80s
+135
View File
@@ -0,0 +1,135 @@
F. [100%]
=================================== FAILURES ===================================
______________ test_login_budget_shared_between_independent_apps _______________
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:526: in _prepare_and_execute
prepared_stmt, attributes = await adapt_connection._prepare(
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:773: in _prepare
prepared_stmt = await self._connection.prepare(
.venv/lib/python3.14/site-packages/asyncpg/connection.py:661: in prepare
return await self._prepare(
.venv/lib/python3.14/site-packages/asyncpg/connection.py:680: in _prepare
stmt = await self._get_statement(
.venv/lib/python3.14/site-packages/asyncpg/connection.py:466: in _get_statement
statement = await self._protocol.prepare(
asyncpg/protocol/protocol.pyx:161: in prepare
???
E asyncpg.exceptions.PostgresSyntaxError: syntax error at or near "window"
The above exception was the direct cause of the following exception:
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:1967: in _exec_single_context
self.dialect.do_execute(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/default.py:952: in do_execute
cursor.execute(statement, parameters)
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:585: in execute
self._adapt_connection.await_(
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:132: in await_only
return current.parent.switch(awaitable) # type: ignore[no-any-return,attr-defined] # noqa: E501
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:196: in greenlet_spawn
value = await result
^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:563: in _prepare_and_execute
self._handle_exception(error)
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:513: in _handle_exception
self._adapt_connection._handle_exception(error)
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:797: in _handle_exception
raise translated_error from error
E sqlalchemy.dialects.postgresql.asyncpg.AsyncAdapt_asyncpg_dbapi.ProgrammingError: <class 'asyncpg.exceptions.PostgresSyntaxError'>: syntax error at or near "window"
The above exception was the direct cause of the following exception:
student-tests/adapters/test_login_protection.py:20: in test_login_budget_shared_between_independent_apps
responses = await asyncio.gather(
.venv/lib/python3.14/site-packages/httpx/_client.py:1859: in post
return await self.request(
.venv/lib/python3.14/site-packages/httpx/_client.py:1540: in request
return await self.send(request, auth=auth, follow_redirects=follow_redirects)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/httpx/_client.py:1629: in send
response = await self._send_handling_auth(
.venv/lib/python3.14/site-packages/httpx/_client.py:1657: in _send_handling_auth
response = await self._send_handling_redirects(
.venv/lib/python3.14/site-packages/httpx/_client.py:1694: in _send_handling_redirects
response = await self._send_single_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/httpx/_client.py:1730: in _send_single_request
response = await transport.handle_async_request(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/httpx/_transports/asgi.py:170: in handle_async_request
await self.app(scope, receive, send)
.venv/lib/python3.14/site-packages/fastapi/applications.py:1163: in __call__
await super().__call__(scope, receive, send)
.venv/lib/python3.14/site-packages/starlette/applications.py:96: in __call__
await self.middleware_stack(scope, receive, send)
.venv/lib/python3.14/site-packages/starlette/middleware/errors.py:186: in __call__
raise exc
.venv/lib/python3.14/site-packages/starlette/middleware/errors.py:164: in __call__
await self.app(scope, receive, _send)
.venv/lib/python3.14/site-packages/dishka/integrations/starlette.py:68: in __call__
return await self.app(scope, receive, send)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
src/messenger/shared/entrypoints/pipeline.py:163: in __call__
rejection = await self._guard(scope) or self._oversized(scope, limit)
^^^^^^^^^^^^^^^^^^^^^^^^
src/messenger/identity/entrypoints/authentication.py:69: in guard
user = await verify(*credentials) if credentials else None
^^^^^^^^^^^^^^^^^^^^^^^^^^
src/messenger/identity/application/queries.py:41: in __call__
if self._maintenance and (retry := await self._maintenance.consume()):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
src/messenger/identity/infrastructure/login_protection.py:46: in consume
await session.execute(statement, {"window": self._window, "limit": self._limit})
.venv/lib/python3.14/site-packages/sqlalchemy/ext/asyncio/session.py:449: in execute
result = await greenlet_spawn(
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:201: in greenlet_spawn
result = context.throw(*sys.exc_info())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:2351: in execute
return self._execute_internal(
.venv/lib/python3.14/site-packages/sqlalchemy/orm/session.py:2258: in _execute_internal
result = conn.execute(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:1419: in execute
return meth(
.venv/lib/python3.14/site-packages/sqlalchemy/sql/elements.py:527: in _execute_on_connection
return connection._execute_clauseelement(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:1641: in _execute_clauseelement
ret = self._execute_context(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:1846: in _execute_context
return self._exec_single_context(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:1986: in _exec_single_context
self._handle_dbapi_exception(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:2363: in _handle_dbapi_exception
raise sqlalchemy_exception.with_traceback(exc_info[2]) from e
.venv/lib/python3.14/site-packages/sqlalchemy/engine/base.py:1967: in _exec_single_context
self.dialect.do_execute(
.venv/lib/python3.14/site-packages/sqlalchemy/engine/default.py:952: in do_execute
cursor.execute(statement, parameters)
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:585: in execute
self._adapt_connection.await_(
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:132: in await_only
return current.parent.switch(awaitable) # type: ignore[no-any-return,attr-defined] # noqa: E501
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:196: in greenlet_spawn
value = await result
^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:563: in _prepare_and_execute
self._handle_exception(error)
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:513: in _handle_exception
self._adapt_connection._handle_exception(error)
.venv/lib/python3.14/site-packages/sqlalchemy/dialects/postgresql/asyncpg.py:797: in _handle_exception
raise translated_error from error
E sqlalchemy.exc.ProgrammingError: (sqlalchemy.dialects.postgresql.asyncpg.ProgrammingError) <class 'asyncpg.exceptions.PostgresSyntaxError'>: syntax error at or near "window"
E [SQL:
E INSERT INTO login_budget (id, window, attempts)
E VALUES (1, floor(extract(epoch FROM statement_timestamp()) / $1), 1)
E ON CONFLICT (id) DO UPDATE SET
E window = EXCLUDED.window,
E attempts = CASE WHEN login_budget.window = EXCLUDED.window
E THEN login_budget.attempts + 1 ELSE 1 END
E WHERE login_budget.window <> EXCLUDED.window OR login_budget.attempts < $2
E RETURNING attempts
E ]
E [SQL parameters hidden due to hide_parameters=True]
E (Background on this error at: https://sqlalche.me/e/20/f405)
=========================== short test summary info ============================
FAILED student-tests/adapters/test_login_protection.py::test_login_budget_shared_between_independent_apps
1 failed, 1 passed in 1.95s
+10
View File
@@ -0,0 +1,10 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
230 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
+10
View File
@@ -0,0 +1,10 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
222 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
+32
View File
@@ -0,0 +1,32 @@
- Validating...
✓ spec/lab1-event-publication
✓ spec/lab1-messenger-chats
✓ spec/lab1-messenger-history
✓ spec/lab1-messenger-identity
✓ spec/lab1-service-operability
✓ spec/lab2-auth-sessions
✓ spec/lab2-dependency-failures
✓ spec/lab2-history-index
✓ spec/lab2-persistent-storage
✓ spec/lab2-session-cache
✓ spec/lab3-failover-operations
✓ spec/lab3-idempotent-messages
✓ spec/lab3-load-balancing
✓ spec/lab4-attachments
✓ spec/lab4-object-storage
✓ spec/lab4-storage-reconciler
✓ spec/lab4-upload-resilience
✓ spec/lab5-attachment-pipeline
✓ spec/lab5-image-variants
✓ spec/lab5-job-delivery
✓ spec/lab5-job-recovery
✓ spec/lab5-versioned-pipeline
✓ spec/lab6-observability-cost
✓ spec/lab6-observability-resilience
✓ spec/lab6-service-telemetry
✓ spec/lab6-telemetry-stack
✓ spec/lab7-login-defense
✓ spec/lab7-secret-lifecycle
✓ spec/lab7-transport-trust
✓ spec/lab7-worker-containment
Totals: 30 passed, 0 failed (30 items)
+32
View File
@@ -0,0 +1,32 @@
- Validating...
✓ spec/lab1-event-publication
✓ spec/lab1-messenger-chats
✓ spec/lab1-messenger-history
✓ spec/lab1-messenger-identity
✓ spec/lab1-service-operability
✓ spec/lab2-auth-sessions
✓ spec/lab2-dependency-failures
✓ spec/lab2-history-index
✓ spec/lab2-persistent-storage
✓ spec/lab2-session-cache
✓ spec/lab3-failover-operations
✓ spec/lab3-idempotent-messages
✓ spec/lab3-load-balancing
✓ spec/lab4-attachments
✓ spec/lab4-object-storage
✓ spec/lab4-storage-reconciler
✓ spec/lab4-upload-resilience
✓ spec/lab5-attachment-pipeline
✓ spec/lab5-image-variants
✓ spec/lab5-job-delivery
✓ spec/lab5-job-recovery
✓ spec/lab5-versioned-pipeline
✓ spec/lab6-observability-cost
✓ spec/lab6-observability-resilience
✓ spec/lab6-service-telemetry
✓ spec/lab6-telemetry-stack
✓ spec/lab7-login-defense
✓ spec/lab7-secret-lifecycle
✓ spec/lab7-transport-trust
✓ spec/lab7-worker-containment
Totals: 30 passed, 0 failed (30 items)
+1
View File
@@ -0,0 +1 @@
Resolved 100 packages in 931ms
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+6
View File
@@ -0,0 +1,6 @@
.venv/bin/ruff check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
230 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
+10
View File
@@ -0,0 +1,10 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
All checks passed!
.venv/bin/ruff format --check src student-tests scripts/security_lifecycle.py scripts/security_inventory.py scripts/security_evidence.py scripts/privilege_compare.py scripts/security_failover.py scripts/security_probe.py scripts/worker_compromise.py scripts/security_benchmark.py scripts/pki.py scripts/security_operator.py scripts/secure_runtime.py scripts/obs_cost_summary.py scripts/obs_cost.py scripts/obs_incidents.py scripts/obs_recovery.py scripts/capture_evidence.py alembic scripts/scaling.py tools scripts/probe.py scripts/limits.py scripts/lab4_scenarios.py scripts/lab4_seed.py scripts/upload_memory.py scripts/reconcile_check.py scripts/reconcile_track.py scripts/make_images.py scripts/pipeline_suite.py scripts/pipeline_evidence.py
230 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
+4
View File
@@ -0,0 +1,4 @@
python3 tests/smoke.py --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
python3 tests/security.py --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/smoke.py --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
python3 tests/security.py --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
+16
View File
@@ -0,0 +1,16 @@
Container mtusi2026lab7-worker-1 Recreate
Container mtusi2026lab7-relay-1 Recreate
Container mtusi2026lab7-api-2 Recreate
Container mtusi2026lab7-api-1 Recreate
Container mtusi2026lab7-worker-1 Recreated
Container mtusi2026lab7-relay-1 Recreated
Container mtusi2026lab7-api-2 Recreated
Container mtusi2026lab7-api-1 Recreated
Container mtusi2026lab7-relay-1 Starting
Container mtusi2026lab7-worker-1 Starting
Container mtusi2026lab7-api-2 Starting
Container mtusi2026lab7-api-2 Started
Container mtusi2026lab7-api-1 Starting
Container mtusi2026lab7-worker-1 Started
Container mtusi2026lab7-relay-1 Started
Container mtusi2026lab7-api-1 Started
+6
View File
@@ -0,0 +1,6 @@
Building mtusi-messenger @ file:///Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7
Built mtusi-messenger @ file:///Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7
Uninstalled 1 package in 1ms
Installed 1 package in 1ms
......................................... [100%]
41 passed in 66.61s (0:01:06)
+8
View File
@@ -0,0 +1,8 @@
scripts/restore.sh
Container mtusi2026lab7-api-1 Running
Container mtusi2026lab7-api-2 Starting
Container mtusi2026lab7-api-2 Started
Container mtusi2026lab7-api-1 Waiting
Container mtusi2026lab7-api-2 Waiting
Container mtusi2026lab7-api-1 Healthy
Container mtusi2026lab7-api-2 Healthy
+189
View File
@@ -0,0 +1,189 @@
Container mtusi2026lab7-bao-init-1 Starting
Container mtusi2026lab7-bao-init-1 Started
Container mtusi2026lab7-bao-init-1 Waiting
Container mtusi2026lab7-bao-init-1 Exited
Container mtusi2026lab7-openbao-1 Starting
Container mtusi2026lab7-openbao-1 Started
sh scripts/secure-up.sh
/Applications/Xcode.app/Contents/Developer/usr/bin/make secrets
sh scripts/secrets.sh
sh scripts/broker-secrets.sh
broker and S3 worker secrets ready in secrets/ (values not printed)
sh scripts/secrets-lab6.sh
secrets/grafana_admin_password is ready (value not printed)
python3 scripts/pki.py bootstrap
PKI operation completed; private material not displayed.
python3 scripts/security_operator.py passwords
#1 [internal] load local bake definitions
#1 reading from stdin 432B done
#1 DONE 0.0s
#2 [internal] load build definition from Dockerfile
#2 transferring dockerfile: 1.95kB done
#2 DONE 0.0s
#3 [internal] load metadata for docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#3 DONE 0.0s
#4 [internal] load metadata for ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#4 DONE 0.0s
#5 [internal] load .dockerignore
#5 transferring context: 196B done
#5 DONE 0.0s
#6 FROM ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142
#6 resolve ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 done
#6 DONE 0.0s
#7 [builder 1/7] FROM docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d
#7 resolve docker.io/library/python:3.14-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d done
#7 DONE 0.0s
#8 [internal] load build context
#8 transferring context: 22.34kB 0.0s done
#8 DONE 0.0s
#9 [runtime 7/9] COPY --chown=root:root alembic.ini ./
#9 CACHED
#10 [builder 2/7] COPY --from=ghcr.io/astral-sh/uv:0.9.7@sha256:ba4857bf2a068e9bc0e64eed8563b065908a4cd6bfb66b531a9c424c8e25e142 /uv /usr/local/bin/uv
#10 CACHED
#11 [builder 3/7] WORKDIR /build
#11 CACHED
#12 [builder 7/7] RUN uv sync --locked --no-dev --no-editable
#12 CACHED
#13 [builder 6/7] COPY src ./src
#13 CACHED
#14 [runtime 2/9] RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends haproxy && rm -rf /var/lib/apt/lists/*
#14 CACHED
#15 [runtime 4/9] RUN groupadd --system --gid 10001 messenger && useradd --system --uid 10001 --gid messenger --no-create-home --shell /usr/sbin/nologin messenger
#15 CACHED
#16 [runtime 6/9] WORKDIR /app
#16 CACHED
#17 [builder 5/7] RUN uv sync --locked --no-dev --no-install-project --no-editable
#17 CACHED
#18 [runtime 8/9] COPY --chown=root:root alembic ./alembic
#18 CACHED
#19 [runtime 3/9] RUN python -m pip uninstall -y pip setuptools
#19 CACHED
#20 [runtime 5/9] COPY --from=builder --chown=root:root /opt/venv /opt/venv
#20 CACHED
#21 [builder 4/7] COPY pyproject.toml uv.lock README.md ./
#21 CACHED
#22 [runtime 9/9] COPY --chown=root:root scripts/secure_runtime.py /app/secure_runtime.py
#22 CACHED
#23 exporting to image
#23 exporting layers done
#23 exporting manifest sha256:a0036abaf1dc26192b52ff2c091022ceab20cc9cea9cd610455ad8b805893ceb done
#23 exporting config sha256:803afd2b81699b01569b340023bdb8ce3b04bbb5516bf4932104131ec2f4b887 done
#23 exporting attestation manifest sha256:118fdb34ce34ee46b8ee68995a29e6a8f09343e1827baedfa738c093de2b8ccc done
#23 exporting manifest list sha256:ec5359bf4aaa4802f6a5d0bce04343e5778731cb1abadf1884bffcd34b59a726 done
#23 naming to docker.io/library/mtusi-messenger:mtusi2026lab7 done
#23 unpacking to docker.io/library/mtusi-messenger:mtusi2026lab7 done
#23 DONE 0.0s
#24 resolving provenance for metadata file
#24 DONE 0.0s
api Built
Container mtusi2026lab7-openbao-1 Running
Container mtusi2026lab7-prometheus-1 Starting
Container mtusi2026lab7-broker-1 Starting
Container mtusi2026lab7-valkey-1 Starting
Container mtusi2026lab7-s3-1 Starting
Container mtusi2026lab7-healthtools-1 Starting
Container mtusi2026lab7-bao-init-1 Starting
Container mtusi2026lab7-postgres-1 Starting
Container mtusi2026lab7-bao-init-1 Started
Container mtusi2026lab7-bao-init-1 Waiting
Container mtusi2026lab7-valkey-1 Started
Container mtusi2026lab7-s3-1 Started
Container mtusi2026lab7-healthtools-1 Started
Container mtusi2026lab7-healthtools-1 Waiting
Container mtusi2026lab7-healthtools-1 Waiting
Container mtusi2026lab7-healthtools-1 Waiting
Container mtusi2026lab7-prometheus-1 Started
Container mtusi2026lab7-postgres-1 Started
Container mtusi2026lab7-broker-1 Started
Container mtusi2026lab7-bao-init-1 Exited
Container mtusi2026lab7-healthtools-1 Exited
Container mtusi2026lab7-loki-1 Starting
Container mtusi2026lab7-healthtools-1 Exited
Container mtusi2026lab7-tempo-1 Starting
Container mtusi2026lab7-healthtools-1 Exited
Container mtusi2026lab7-otel-collector-1 Starting
Container mtusi2026lab7-tempo-1 Started
Container mtusi2026lab7-loki-1 Started
Container mtusi2026lab7-grafana-1 Starting
Container mtusi2026lab7-otel-collector-1 Started
Container mtusi2026lab7-grafana-1 Started
Container mtusi2026lab7-s3-1 Running
Container mtusi2026lab7-broker-1 Running
Container mtusi2026lab7-tempo-1 Running
Container mtusi2026lab7-loki-1 Running
Container mtusi2026lab7-otel-collector-1 Running
Container mtusi2026lab7-valkey-1 Running
Container mtusi2026lab7-prometheus-1 Running
Container mtusi2026lab7-grafana-1 Running
Container mtusi2026lab7-healthtools-1 Starting
Container mtusi2026lab7-prometheus-tls-1 Starting
Container mtusi2026lab7-s3-tls-1 Starting
Container mtusi2026lab7-broker-tls-1 Starting
Container mtusi2026lab7-valkey-tls-1 Starting
Container mtusi2026lab7-valkey-tls-1 Started
Container mtusi2026lab7-broker-tls-1 Started
Container mtusi2026lab7-s3-tls-1 Started
Container mtusi2026lab7-healthtools-1 Started
Container mtusi2026lab7-healthtools-1 Waiting
Container mtusi2026lab7-healthtools-1 Waiting
Container mtusi2026lab7-healthtools-1 Waiting
Container mtusi2026lab7-prometheus-tls-1 Started
Container mtusi2026lab7-healthtools-1 Exited
Container mtusi2026lab7-healthtools-1 Exited
Container mtusi2026lab7-healthtools-1 Exited
Container mtusi2026lab7-loki-tls-1 Starting
Container mtusi2026lab7-tempo-tls-1 Starting
Container mtusi2026lab7-otel-collector-tls-1 Starting
Container mtusi2026lab7-grafana-tls-1 Starting
Container mtusi2026lab7-grafana-tls-1 Started
Container mtusi2026lab7-tempo-tls-1 Started
Container mtusi2026lab7-otel-collector-tls-1 Started
Container mtusi2026lab7-loki-tls-1 Started
security.configuration.accepted
Scoped database roles applied.
s3-init: bucket=messenger-attachments anonymous=private policies=messenger-api,messenger-reconciler lifecycle=abort-incomplete-uploads
Container mtusi2026lab7-worker-1 Recreate
Container mtusi2026lab7-api-2 Recreate
Container mtusi2026lab7-relay-1 Recreate
Container mtusi2026lab7-api-1 Recreate
Container mtusi2026lab7-worker-1 Recreated
Container mtusi2026lab7-api-1 Recreated
Container mtusi2026lab7-relay-1 Recreated
Container mtusi2026lab7-api-2 Recreated
Container mtusi2026lab7-lb-1 Starting
Container mtusi2026lab7-worker-1 Starting
Container mtusi2026lab7-relay-1 Starting
Container mtusi2026lab7-api-1 Starting
Container mtusi2026lab7-relay-1 Started
Container mtusi2026lab7-worker-1 Started
Container mtusi2026lab7-api-1 Started
Container mtusi2026lab7-api-2 Starting
Container mtusi2026lab7-lb-1 Started
Container mtusi2026lab7-api-2 Started
python3 tests/smoke.py --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
PASS: lab7 public API scenario; infrastructure and grading evidence remain separate.
python3 tests/security.py --base-url "https://localhost:18443" --ca-file "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/secrets/pki/operator/ca.crt"
PASS: trusted HTTPS works; untrusted CA rejected. This does NOT verify internal mTLS, OpenBao or hardening.
+15
View File
@@ -0,0 +1,15 @@
........................................................................ [ 7%]
........................................................................ [ 14%]
........................................................................ [ 21%]
........................................................................ [ 28%]
........................................................................ [ 36%]
........................................................................ [ 43%]
........................................................................ [ 50%]
........................................................................ [ 57%]
........................................................................ [ 64%]
........................................................................ [ 72%]
................................................s....................... [ 79%]
........................................................................ [ 86%]
........................................................................ [ 93%]
......s........................................................ [100%]
997 passed, 2 skipped, 19 deselected in 173.99s (0:02:53)
+50
View File
@@ -0,0 +1,50 @@
........................................................................ [ 7%]
........................................................................ [ 14%]
........................................................................ [ 21%]
........................................................................ [ 28%]
............................................................F........... [ 36%]
........................................................................ [ 43%]
........................................................................ [ 50%]
........................................................................ [ 57%]
........................................................................ [ 64%]
........................................................................ [ 72%]
................................................s....................... [ 79%]
........................................................................ [ 86%]
........................................................................ [ 93%]
......s........................................................ [100%]
=================================== FAILURES ===================================
___________ test_context_survives_outbox_delivery_and_lease_recovery ___________
client = <httpx.AsyncClient object at 0x128a83590>
async def test_context_survives_outbox_delivery_and_lease_recovery(client: AsyncClient) -> None:
provider = TracerProvider(shutdown_on_exit=False)
tracer = provider.get_tracer("test")
with tracer.start_as_current_span("upload", kind=SpanKind.SERVER):
origin = capture_context()
_, _, attachment_id = await accepted(client)
job = await job_of(attachment_id)
async with sql_connection() as connection:
assert (
await connection.scalar(select(JobRow.trace_context).where(JobRow.job_id == job.id))
== origin
)
assert (
await connection.scalar(
select(OutboxRow.trace_context).where(OutboxRow.job_id == job.id)
)
== origin
)
pipeline = await pipeline_of(client)
await pipeline.publish()
> assert pipeline.publisher.contexts == [origin]
E AssertionError: assert [{}, {'tracep...0c95827a-03'}] == [{'traceparen...0c95827a-03'}]
E
E At index 0 diff: {} != {'traceparent': '00-c10c079f76aa2b71e389ac3cbb0e8676-3bb4c2360c95827a-03'}
E Left contains one more item: {'traceparent': '00-c10c079f76aa2b71e389ac3cbb0e8676-3bb4c2360c95827a-03'}
E Use -v to get more diff
student-tests/api/test_trace_context.py:37: AssertionError
=========================== short test summary info ============================
FAILED student-tests/api/test_trace_context.py::test_context_survives_outbox_delivery_and_lease_recovery
1 failed, 996 passed, 2 skipped, 19 deselected in 171.38s (0:02:51)
@@ -0,0 +1,87 @@
..!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! KeyboardInterrupt !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/Users/artemkostylev/.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/asyncio/runners.py:132: KeyboardInterrupt
(to show a full traceback on KeyboardInterrupt use --full-trace)
Traceback (most recent call last):
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/bin/pytest", line 10, in <module>
sys.exit(_console_main())
~~~~~~~~~~~~~^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/config/__init__.py", line 253, in _console_main
code = _main(prog=_get_prog_name(sys.argv))
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/config/__init__.py", line 229, in _main
ret: ExitCode | int = config.hook.pytest_cmdline_main(config=config)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
raise exception
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
res = hook_impl.function(*args)
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/main.py", line 377, in pytest_cmdline_main
return wrap_session(config, _main)
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/main.py", line 365, in wrap_session
config.hook.pytest_sessionfinish(
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
session=session, exitstatus=session.exitstatus
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
raise exception
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
teardown.throw(exception)
~~~~~~~~~~~~~~^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/logging.py", line 888, in pytest_sessionfinish
return (yield)
^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
teardown.throw(exception)
~~~~~~~~~~~~~~^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/terminal.py", line 961, in pytest_sessionfinish
result = yield
^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
teardown.throw(exception)
~~~~~~~~~~~~~~^^^^^^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/warnings.py", line 119, in pytest_sessionfinish
return (yield)
^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
res = hook_impl.function(*args)
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/runner.py", line 112, in pytest_sessionfinish
session._setupstate.teardown_exact(None)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/runner.py", line 568, in teardown_exact
fin()
~~~^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/fixtures.py", line 1166, in finish
raise exceptions[0]
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/fixtures.py", line 1156, in finish
fin()
~~~^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/.venv/lib/python3.14/site-packages/_pytest/fixtures.py", line 1014, in _teardown_yield_fixture
next(it)
~~~~^^^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/student-tests/adapters/test_rabbitmq.py", line 112, in rabbitmq
subprocess.run(["docker", "rm", "-f", name], capture_output=True, check=False)
~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/subprocess.py", line 557, in run
stdout, stderr = process.communicate(input, timeout=timeout)
~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/subprocess.py", line 1221, in communicate
stdout, stderr = self._communicate(input, endtime, timeout)
~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/artemkostylev/.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/subprocess.py", line 2153, in _communicate
ready = selector.select(timeout)
File "/Users/artemkostylev/.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/selectors.py", line 398, in select
fd_event_list = self._selector.poll(timeout)
KeyboardInterrupt
+233
View File
@@ -0,0 +1,233 @@
..EEEEE................................................................. [ 7%]
........................................................................ [ 14%]
....FFFFFFFFF..................FF..F.................................... [ 21%]
........................................................................ [ 28%]
........................................................................ [ 36%]
........................................................................ [ 43%]
........................................................................ [ 50%]
........................................................................ [ 57%]
........................................................................ [ 64%]
........................................................................ [ 72%]
................................................s....................... [ 79%]
........................................................................ [ 86%]
........................................................................ [ 93%]
......s........................................................ [100%]
==================================== ERRORS ====================================
__________ ERROR at setup of test_confirmed_job_is_consumed_and_acked __________
student-tests/adapters/test_rabbitmq.py:101: in rabbitmq
assert time.monotonic() < deadline, "RabbitMQ не стал готов"
E AssertionError: RabbitMQ не стал готов
E assert 1484532.602456583 < 1484531.669322458
E + where 1484532.602456583 = <built-in function monotonic>()
E + where <built-in function monotonic> = time.monotonic
_ ERROR at setup of test_message_off_the_schema_goes_to_the_dead_letter_queue __
student-tests/adapters/test_rabbitmq.py:101: in rabbitmq
assert time.monotonic() < deadline, "RabbitMQ не стал готов"
E AssertionError: RabbitMQ не стал готов
E assert 1484532.602456583 < 1484531.669322458
E + where 1484532.602456583 = <built-in function monotonic>()
E + where <built-in function monotonic> = time.monotonic
_____ ERROR at setup of test_failed_processing_is_requeued_and_retried[1] ______
student-tests/adapters/test_rabbitmq.py:101: in rabbitmq
assert time.monotonic() < deadline, "RabbitMQ не стал готов"
E AssertionError: RabbitMQ не стал готов
E assert 1484532.602456583 < 1484531.669322458
E + where 1484532.602456583 = <built-in function monotonic>()
E + where <built-in function monotonic> = time.monotonic
_____ ERROR at setup of test_failed_processing_is_requeued_and_retried[12] _____
student-tests/adapters/test_rabbitmq.py:101: in rabbitmq
assert time.monotonic() < deadline, "RabbitMQ не стал готов"
E AssertionError: RabbitMQ не стал готов
E assert 1484532.602456583 < 1484531.669322458
E + where 1484532.602456583 = <built-in function monotonic>()
E + where <built-in function monotonic> = time.monotonic
_____________ ERROR at setup of test_worker_account_cannot_publish _____________
student-tests/adapters/test_rabbitmq.py:101: in rabbitmq
assert time.monotonic() < deadline, "RabbitMQ не стал готов"
E AssertionError: RabbitMQ не стал готов
E assert 1484532.602456583 < 1484531.669322458
E + where 1484532.602456583 = <built-in function monotonic>()
E + where <built-in function monotonic> = time.monotonic
=================================== FAILURES ===================================
__________________ test_migrations_are_ordered_and_versioned ___________________
student-tests/api/test_migrations.py:283: in test_migrations_are_ordered_and_versioned
assert [script.revision for script in reversed(list(scripts.walk_revisions()))] == VERSIONS
E AssertionError: assert ['0001_init',...rantine', ...] == ['0001_init',...rantine', ...]
E
E Left contains one more item: '0008_login_budget'
E Use -v to get more diff
_______________________ test_fresh_schema_matches_models _______________________
student-tests/api/test_migrations.py:290: in test_fresh_schema_matches_models
assert (
E AssertionError: assert [[('modify_ty...GINT(), ...)]] == []
E
E Left contains one more item: [('modify_type', None, 'login_budget', 'window', {'existing_nullable': False, 'existing_server_default': False, 'existing_comment': None}, BIGINT(), ...)]
E Use -v to get more diff
__________________________ test_rerun_changes_nothing __________________________
student-tests/api/test_migrations.py:305: in test_rerun_changes_nothing
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
__________________ test_legacy_upgrade_preserves_every_row[1] __________________
student-tests/api/test_migrations.py:320: in test_legacy_upgrade_preserves_every_row
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
__________________ test_legacy_upgrade_preserves_every_row[2] __________________
student-tests/api/test_migrations.py:320: in test_legacy_upgrade_preserves_every_row
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
__________________ test_legacy_upgrade_preserves_every_row[3] __________________
student-tests/api/test_migrations.py:320: in test_legacy_upgrade_preserves_every_row
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
__________________ test_legacy_upgrade_preserves_every_row[4] __________________
student-tests/api/test_migrations.py:320: in test_legacy_upgrade_preserves_every_row
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
__________________ test_legacy_upgrade_preserves_every_row[5] __________________
student-tests/api/test_migrations.py:320: in test_legacy_upgrade_preserves_every_row
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
__________________ test_legacy_upgrade_preserves_every_row[6] __________________
student-tests/api/test_migrations.py:320: in test_legacy_upgrade_preserves_every_row
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
____________ test_simultaneous_migration_processes_serialize[False] ____________
student-tests/api/test_migrations.py:415: in test_simultaneous_migration_processes_serialize
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
____________ test_simultaneous_migration_processes_serialize[True] _____________
student-tests/api/test_migrations.py:415: in test_simultaneous_migration_processes_serialize
assert await revision(empty_database) == VERSIONS[-1]
E AssertionError: assert '0008_login_budget' == '0007_trace_context'
E
E - 0007_trace_context
E + 0008_login_budget
_______________ test_failed_revision_rolls_back_ddl_and_version ________________
.venv/lib/python3.14/site-packages/alembic/script/base.py:217: in _catch_revision_errors
yield
.venv/lib/python3.14/site-packages/alembic/script/base.py:425: in _upgrade_revs
for script in reversed(list(revs))
^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/script/revision.py:814: in iterate_revisions
revisions, heads = fn(
.venv/lib/python3.14/site-packages/alembic/script/revision.py:1447: in _collect_upgrade_revisions
for rev in self._parse_upgrade_target(
.venv/lib/python3.14/site-packages/alembic/script/revision.py:1234: in _parse_upgrade_target
return self.get_revisions(target)
^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/script/revision.py:544: in get_revisions
resolved_id, branch_label = self._resolve_revision_number(id_)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/script/revision.py:778: in _resolve_revision_number
current_head = self.get_current_head(branch_label)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/script/revision.py:508: in get_current_head
raise MultipleHeads(
E alembic.script.revision.MultipleHeads: Multiple heads are present for given argument 'head'; 0003_failure, 0008_login_budget
The above exception was the direct cause of the following exception:
student-tests/api/test_migrations.py:465: in test_failed_revision_rolls_back_ddl_and_version
await asyncio.to_thread(command.upgrade, config, "head")
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/asyncio/threads.py:25: in to_thread
return await loop.run_in_executor(None, func_call)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/concurrent/futures/thread.py:86: in run
result = ctx.run(self.task)
^^^^^^^^^^^^^^^^^^
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/concurrent/futures/thread.py:73: in run
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/command.py:483: in upgrade
script.run_env()
.venv/lib/python3.14/site-packages/alembic/script/base.py:549: in run_env
util.load_python_file(self.dir, "env.py")
.venv/lib/python3.14/site-packages/alembic/util/pyfiles.py:116: in load_python_file
module = load_module_py(module_id, path)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/util/pyfiles.py:136: in load_module_py
spec.loader.exec_module(module) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
<frozen importlib._bootstrap_external>:759: in exec_module
???
<frozen importlib._bootstrap>:491: in _call_with_frames_removed
???
/private/var/folders/mv/1d_0hqt94jj1lb13qxh20b8c0000gn/T/pytest-of-artemkostylev/pytest-317/test_failed_revision_rolls_bac0/alembic/env.py:111: in <module>
asyncio.run(run_async_migrations())
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/asyncio/runners.py:204: in run
return runner.run(main)
^^^^^^^^^^^^^^^^
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/asyncio/runners.py:127: in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/asyncio/base_events.py:719: in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
/private/var/folders/mv/1d_0hqt94jj1lb13qxh20b8c0000gn/T/pytest-of-artemkostylev/pytest-317/test_failed_revision_rolls_bac0/alembic/env.py:94: in run_async_migrations
await connection.run_sync(run_migrations)
.venv/lib/python3.14/site-packages/sqlalchemy/ext/asyncio/engine.py:888: in run_sync
return await greenlet_spawn(
.venv/lib/python3.14/site-packages/sqlalchemy/util/_concurrency_py3k.py:203: in greenlet_spawn
result = context.switch(value)
^^^^^^^^^^^^^^^^^^^^^
/private/var/folders/mv/1d_0hqt94jj1lb13qxh20b8c0000gn/T/pytest-of-artemkostylev/pytest-317/test_failed_revision_rolls_bac0/alembic/env.py:68: in run_migrations
context.run_migrations()
<string>:8: in run_migrations
???
.venv/lib/python3.14/site-packages/alembic/runtime/environment.py:946: in run_migrations
self.get_context().run_migrations(**kw)
.venv/lib/python3.14/site-packages/alembic/runtime/migration.py:615: in run_migrations
for step in self._migrations_fn(heads, self):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/command.py:472: in upgrade
return script._upgrade_revs(revision, rev)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.14/site-packages/alembic/script/base.py:413: in _upgrade_revs
with self._catch_revision_errors(
../../../../../.local/share/uv/python/cpython-3.14.5-macos-aarch64-none/lib/python3.14/contextlib.py:162: in __exit__
self.gen.throw(value)
.venv/lib/python3.14/site-packages/alembic/script/base.py:243: in _catch_revision_errors
raise util.CommandError(multiple_heads) from mh
E alembic.util.exc.CommandError: Multiple head revisions are present for given argument 'head'; please specify a specific target revision, '<branchname>@head' to narrow to a specific head, or 'heads' for all heads
=========================== short test summary info ============================
FAILED student-tests/api/test_migrations.py::test_migrations_are_ordered_and_versioned
FAILED student-tests/api/test_migrations.py::test_fresh_schema_matches_models
FAILED student-tests/api/test_migrations.py::test_rerun_changes_nothing - Ass...
FAILED student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[1]
FAILED student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[2]
FAILED student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[3]
FAILED student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[4]
FAILED student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[5]
FAILED student-tests/api/test_migrations.py::test_legacy_upgrade_preserves_every_row[6]
FAILED student-tests/api/test_migrations.py::test_simultaneous_migration_processes_serialize[False]
FAILED student-tests/api/test_migrations.py::test_simultaneous_migration_processes_serialize[True]
FAILED student-tests/api/test_migrations.py::test_failed_revision_rolls_back_ddl_and_version
ERROR student-tests/adapters/test_rabbitmq.py::test_confirmed_job_is_consumed_and_acked
ERROR student-tests/adapters/test_rabbitmq.py::test_message_off_the_schema_goes_to_the_dead_letter_queue
ERROR student-tests/adapters/test_rabbitmq.py::test_failed_processing_is_requeued_and_retried[1]
ERROR student-tests/adapters/test_rabbitmq.py::test_failed_processing_is_requeued_and_retried[12]
ERROR student-tests/adapters/test_rabbitmq.py::test_worker_account_cannot_publish
12 failed, 980 passed, 2 skipped, 19 deselected, 5 errors in 256.73s (0:04:16)
@@ -0,0 +1 @@
Broad permissions and telemetry leakage checks passed.
+7
View File
@@ -0,0 +1,7 @@
Traceback (most recent call last):
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/scripts/security_evidence.py", line 85, in <module>
leakage()
~~~~~~~^^
File "/Users/artemkostylev/projects/learning/HL/labs/mtusi2026lab7/scripts/security_evidence.py", line 80, in leakage
raise RuntimeError("Telemetry presence/leakage audit failed")
RuntimeError: Telemetry presence/leakage audit failed
@@ -0,0 +1,10 @@
{
"known_secret_values_checked": 31,
"compose_log_bytes": 421635,
"telemetry_bytes": 2152712,
"metric_series": 3062,
"log_streams": 1000,
"traces": 20,
"exact_secret_matches": 3,
"passed": false
}
+11
View File
@@ -0,0 +1,11 @@
{
"known_secret_values_checked": 26,
"nonsecret_s3_identity_matches": 3,
"compose_log_bytes": 473186,
"telemetry_bytes": 2166419,
"metric_series": 3084,
"log_streams": 1000,
"traces": 20,
"exact_secret_matches": 0,
"passed": true
}
+434
View File
@@ -0,0 +1,434 @@
{
"checks": [
{
"test": "mtls:api:8443:lb",
"passed": true
},
{
"test": "mtls:api:8443:None",
"passed": true
},
{
"test": "mtls:api:8443:outsider",
"passed": true
},
{
"test": "mtls:api:8443:foreign",
"passed": true
},
{
"test": "hostname:api:8443",
"passed": true
},
{
"test": "mtls:s3:9443:api",
"passed": true
},
{
"test": "mtls:s3:9443:None",
"passed": true
},
{
"test": "mtls:s3:9443:outsider",
"passed": true
},
{
"test": "mtls:s3:9443:foreign",
"passed": true
},
{
"test": "hostname:s3:9443",
"passed": true
},
{
"test": "mtls:broker:5671:worker",
"passed": true
},
{
"test": "mtls:broker:5671:None",
"passed": true
},
{
"test": "mtls:broker:5671:outsider",
"passed": true
},
{
"test": "mtls:broker:5671:foreign",
"passed": true
},
{
"test": "hostname:broker:5671",
"passed": true
},
{
"test": "mtls:otel-collector:44318:api",
"passed": true
},
{
"test": "mtls:otel-collector:44318:None",
"passed": true
},
{
"test": "mtls:otel-collector:44318:outsider",
"passed": true
},
{
"test": "mtls:otel-collector:44318:foreign",
"passed": true
},
{
"test": "hostname:otel-collector:44318",
"passed": true
},
{
"test": "mtls:prometheus:9443:grafana",
"passed": true
},
{
"test": "mtls:prometheus:9443:None",
"passed": true
},
{
"test": "mtls:prometheus:9443:outsider",
"passed": true
},
{
"test": "mtls:prometheus:9443:foreign",
"passed": true
},
{
"test": "hostname:prometheus:9443",
"passed": true
},
{
"test": "mtls:loki:9443:grafana",
"passed": true
},
{
"test": "mtls:loki:9443:None",
"passed": true
},
{
"test": "mtls:loki:9443:outsider",
"passed": true
},
{
"test": "mtls:loki:9443:foreign",
"passed": true
},
{
"test": "hostname:loki:9443",
"passed": true
},
{
"test": "mtls:tempo:9443:grafana",
"passed": true
},
{
"test": "mtls:tempo:9443:None",
"passed": true
},
{
"test": "mtls:tempo:9443:outsider",
"passed": true
},
{
"test": "mtls:tempo:9443:foreign",
"passed": true
},
{
"test": "hostname:tempo:9443",
"passed": true
},
{
"test": "mtls:tempo:44318:otel-collector",
"passed": true
},
{
"test": "mtls:tempo:44318:None",
"passed": true
},
{
"test": "mtls:tempo:44318:outsider",
"passed": true
},
{
"test": "mtls:tempo:44318:foreign",
"passed": true
},
{
"test": "hostname:tempo:44318",
"passed": true
},
{
"test": "mtls:grafana:9443:operator",
"passed": true
},
{
"test": "mtls:grafana:9443:None",
"passed": true
},
{
"test": "mtls:grafana:9443:outsider",
"passed": true
},
{
"test": "mtls:grafana:9443:foreign",
"passed": true
},
{
"test": "hostname:grafana:9443",
"passed": true
},
{
"test": "mtls:valkey:6380:api",
"passed": true
},
{
"test": "mtls:valkey:6380:None",
"passed": true
},
{
"test": "mtls:valkey:6380:outsider",
"passed": true
},
{
"test": "mtls:valkey:6380:foreign",
"passed": true
},
{
"test": "hostname:valkey:6380",
"passed": true
},
{
"test": "plaintext-closed:api:8080",
"passed": true
},
{
"test": "plaintext-closed:s3:9000",
"passed": true
},
{
"test": "plaintext-closed:broker:5672",
"passed": true
},
{
"test": "plaintext-closed:broker:15672",
"passed": true
},
{
"test": "plaintext-closed:broker:15692",
"passed": true
},
{
"test": "plaintext-closed:broker:25672",
"passed": true
},
{
"test": "plaintext-closed:broker:4369",
"passed": true
},
{
"test": "plaintext-closed:valkey:6379",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:4318",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:8888",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:13133",
"passed": true
},
{
"test": "plaintext-closed:prometheus:9090",
"passed": true
},
{
"test": "plaintext-closed:loki:3100",
"passed": true
},
{
"test": "plaintext-closed:loki:9096",
"passed": true
},
{
"test": "plaintext-closed:tempo:3200",
"passed": true
},
{
"test": "plaintext-closed:tempo:4318",
"passed": true
},
{
"test": "plaintext-closed:tempo:9095",
"passed": true
},
{
"test": "plaintext-closed:grafana:3000",
"passed": true
},
{
"test": "bao-scope:api",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "bao-scope:worker",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "bao-scope:relay",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "telemetry-query:prometheus",
"passed": true,
"bytes": 572
},
{
"test": "telemetry-query:loki",
"passed": true,
"bytes": 75
},
{
"test": "telemetry-query:tempo",
"passed": true,
"bytes": 1997
},
{
"test": "collector-read-denied:prometheus",
"passed": true
},
{
"test": "collector-read-denied:loki",
"passed": true
},
{
"test": "grafana-write-denied:prometheus",
"passed": true
},
{
"test": "grafana-write-denied:loki",
"passed": true
},
{
"test": "api-replica:172.25.0.13:lb",
"passed": true
},
{
"test": "api-replica:172.25.0.13:None",
"passed": true
},
{
"test": "api-replica:172.25.0.13:outsider",
"passed": true
},
{
"test": "api-replica:172.25.0.13:foreign",
"passed": true
},
{
"test": "api-replica:172.25.0.9:lb",
"passed": true
},
{
"test": "api-replica:172.25.0.9:None",
"passed": true
},
{
"test": "api-replica:172.25.0.9:outsider",
"passed": true
},
{
"test": "api-replica:172.25.0.9:foreign",
"passed": true
},
{
"test": "worker-db:jobs-readable",
"passed": true
},
{
"test": "worker-db:users-denied",
"passed": true
},
{
"test": "worker-db:sessions-denied",
"passed": true
},
{
"test": "worker-db:ddl-denied",
"passed": true
},
{
"test": "worker-s3:PUT:variants/security-probe",
"passed": true,
"status": 200
},
{
"test": "worker-s3:GET:variants/security-probe",
"passed": true,
"status": 200
},
{
"test": "worker-s3:PUT:staging/security-probe",
"passed": true,
"status": 403
},
{
"test": "worker-s3:GET:private/security-probe",
"passed": true,
"status": 403
},
{
"test": "worker-s3:DELETE:variants/security-probe",
"passed": true,
"status": 403
},
{
"test": "worker-queue:consume",
"passed": true
},
{
"test": "worker-queue:publish-denied",
"passed": true
},
{
"test": "worker-queue:configure-denied",
"passed": true
},
{
"test": "valkey:own-prefix",
"passed": true
},
{
"test": "valkey:foreign-prefix-denied",
"passed": true
},
{
"test": "valkey:admin-denied",
"passed": true
},
{
"test": "postgres:plaintext-denied",
"passed": true
},
{
"test": "postgres:untrusted-ca-denied",
"passed": true
}
],
"passed": true
}
+351
View File
@@ -0,0 +1,351 @@
{
"checks": [
{
"test": "mtls:api:8443:lb",
"passed": true
},
{
"test": "mtls:api:8443:None",
"passed": true
},
{
"test": "mtls:api:8443:outsider",
"passed": true
},
{
"test": "mtls:api:8443:foreign",
"passed": true
},
{
"test": "hostname:api:8443",
"passed": true
},
{
"test": "mtls:s3:9443:api",
"passed": true
},
{
"test": "mtls:s3:9443:None",
"passed": true
},
{
"test": "mtls:s3:9443:outsider",
"passed": true
},
{
"test": "mtls:s3:9443:foreign",
"passed": true
},
{
"test": "hostname:s3:9443",
"passed": true
},
{
"test": "mtls:broker:5671:worker",
"passed": true
},
{
"test": "mtls:broker:5671:None",
"passed": true
},
{
"test": "mtls:broker:5671:outsider",
"passed": true
},
{
"test": "mtls:broker:5671:foreign",
"passed": true
},
{
"test": "hostname:broker:5671",
"passed": true
},
{
"test": "mtls:otel-collector:44318:api",
"passed": true
},
{
"test": "mtls:otel-collector:44318:None",
"passed": true
},
{
"test": "mtls:otel-collector:44318:outsider",
"passed": true
},
{
"test": "mtls:otel-collector:44318:foreign",
"passed": true
},
{
"test": "hostname:otel-collector:44318",
"passed": true
},
{
"test": "mtls:prometheus:9443:grafana",
"passed": true
},
{
"test": "mtls:prometheus:9443:None",
"passed": true
},
{
"test": "mtls:prometheus:9443:outsider",
"passed": true
},
{
"test": "mtls:prometheus:9443:foreign",
"passed": true
},
{
"test": "hostname:prometheus:9443",
"passed": true
},
{
"test": "mtls:loki:9443:grafana",
"passed": true
},
{
"test": "mtls:loki:9443:None",
"passed": true
},
{
"test": "mtls:loki:9443:outsider",
"passed": true
},
{
"test": "mtls:loki:9443:foreign",
"passed": true
},
{
"test": "hostname:loki:9443",
"passed": true
},
{
"test": "mtls:tempo:9443:grafana",
"passed": true
},
{
"test": "mtls:tempo:9443:None",
"passed": true
},
{
"test": "mtls:tempo:9443:outsider",
"passed": true
},
{
"test": "mtls:tempo:9443:foreign",
"passed": true
},
{
"test": "hostname:tempo:9443",
"passed": true
},
{
"test": "mtls:tempo:44318:otel-collector",
"passed": true
},
{
"test": "mtls:tempo:44318:None",
"passed": true
},
{
"test": "mtls:tempo:44318:outsider",
"passed": true
},
{
"test": "mtls:tempo:44318:foreign",
"passed": true
},
{
"test": "hostname:tempo:44318",
"passed": true
},
{
"test": "mtls:grafana:9443:operator",
"passed": true
},
{
"test": "mtls:grafana:9443:None",
"passed": true
},
{
"test": "mtls:grafana:9443:outsider",
"passed": true
},
{
"test": "mtls:grafana:9443:foreign",
"passed": true
},
{
"test": "hostname:grafana:9443",
"passed": true
},
{
"test": "mtls:valkey:6380:api",
"passed": true
},
{
"test": "mtls:valkey:6380:None",
"passed": true
},
{
"test": "mtls:valkey:6380:outsider",
"passed": true
},
{
"test": "mtls:valkey:6380:foreign",
"passed": true
},
{
"test": "hostname:valkey:6380",
"passed": true
},
{
"test": "plaintext-closed:api:8080",
"passed": true
},
{
"test": "plaintext-closed:s3:9000",
"passed": true
},
{
"test": "plaintext-closed:broker:5672",
"passed": true
},
{
"test": "plaintext-closed:broker:15672",
"passed": true
},
{
"test": "plaintext-closed:broker:15692",
"passed": true
},
{
"test": "plaintext-closed:broker:25672",
"passed": true
},
{
"test": "plaintext-closed:broker:4369",
"passed": true
},
{
"test": "plaintext-closed:valkey:6379",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:4318",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:8888",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:13133",
"passed": true
},
{
"test": "plaintext-closed:prometheus:9090",
"passed": true
},
{
"test": "plaintext-closed:loki:3100",
"passed": true
},
{
"test": "plaintext-closed:loki:9096",
"passed": true
},
{
"test": "plaintext-closed:tempo:3200",
"passed": true
},
{
"test": "plaintext-closed:tempo:4318",
"passed": true
},
{
"test": "plaintext-closed:tempo:9095",
"passed": true
},
{
"test": "plaintext-closed:grafana:3000",
"passed": true
},
{
"test": "bao-scope:api",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "bao-scope:worker",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "bao-scope:relay",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "worker-db:jobs-readable",
"passed": true
},
{
"test": "worker-db:users-denied",
"passed": true
},
{
"test": "worker-db:sessions-denied",
"passed": true
},
{
"test": "worker-db:ddl-denied",
"passed": true
},
{
"test": "worker-s3:PUT:variants/security-probe",
"passed": false,
"status": 400
},
{
"test": "worker-s3:GET:variants/security-probe",
"passed": false,
"status": 400
},
{
"test": "worker-s3:PUT:staging/security-probe",
"passed": false,
"status": 400
},
{
"test": "worker-s3:GET:private/security-probe",
"passed": false,
"status": 400
},
{
"test": "worker-s3:DELETE:variants/security-probe",
"passed": false,
"status": 400
},
{
"test": "worker-queue:consume",
"passed": true
},
{
"test": "worker-queue:publish-denied",
"passed": true
},
{
"test": "worker-queue:configure-denied",
"passed": true
}
],
"passed": false
}
@@ -0,0 +1 @@
+351
View File
@@ -0,0 +1,351 @@
{
"checks": [
{
"test": "mtls:api:8443:lb",
"passed": true
},
{
"test": "mtls:api:8443:None",
"passed": true
},
{
"test": "mtls:api:8443:outsider",
"passed": true
},
{
"test": "mtls:api:8443:foreign",
"passed": true
},
{
"test": "hostname:api:8443",
"passed": true
},
{
"test": "mtls:s3:9443:api",
"passed": true
},
{
"test": "mtls:s3:9443:None",
"passed": true
},
{
"test": "mtls:s3:9443:outsider",
"passed": true
},
{
"test": "mtls:s3:9443:foreign",
"passed": true
},
{
"test": "hostname:s3:9443",
"passed": true
},
{
"test": "mtls:broker:5671:worker",
"passed": true
},
{
"test": "mtls:broker:5671:None",
"passed": true
},
{
"test": "mtls:broker:5671:outsider",
"passed": true
},
{
"test": "mtls:broker:5671:foreign",
"passed": true
},
{
"test": "hostname:broker:5671",
"passed": true
},
{
"test": "mtls:otel-collector:44318:api",
"passed": true
},
{
"test": "mtls:otel-collector:44318:None",
"passed": true
},
{
"test": "mtls:otel-collector:44318:outsider",
"passed": true
},
{
"test": "mtls:otel-collector:44318:foreign",
"passed": true
},
{
"test": "hostname:otel-collector:44318",
"passed": true
},
{
"test": "mtls:prometheus:9443:grafana",
"passed": true
},
{
"test": "mtls:prometheus:9443:None",
"passed": true
},
{
"test": "mtls:prometheus:9443:outsider",
"passed": true
},
{
"test": "mtls:prometheus:9443:foreign",
"passed": true
},
{
"test": "hostname:prometheus:9443",
"passed": true
},
{
"test": "mtls:loki:9443:grafana",
"passed": true
},
{
"test": "mtls:loki:9443:None",
"passed": true
},
{
"test": "mtls:loki:9443:outsider",
"passed": true
},
{
"test": "mtls:loki:9443:foreign",
"passed": true
},
{
"test": "hostname:loki:9443",
"passed": true
},
{
"test": "mtls:tempo:9443:grafana",
"passed": true
},
{
"test": "mtls:tempo:9443:None",
"passed": true
},
{
"test": "mtls:tempo:9443:outsider",
"passed": true
},
{
"test": "mtls:tempo:9443:foreign",
"passed": true
},
{
"test": "hostname:tempo:9443",
"passed": true
},
{
"test": "mtls:tempo:44318:otel-collector",
"passed": true
},
{
"test": "mtls:tempo:44318:None",
"passed": true
},
{
"test": "mtls:tempo:44318:outsider",
"passed": true
},
{
"test": "mtls:tempo:44318:foreign",
"passed": true
},
{
"test": "hostname:tempo:44318",
"passed": true
},
{
"test": "mtls:grafana:9443:operator",
"passed": true
},
{
"test": "mtls:grafana:9443:None",
"passed": true
},
{
"test": "mtls:grafana:9443:outsider",
"passed": true
},
{
"test": "mtls:grafana:9443:foreign",
"passed": true
},
{
"test": "hostname:grafana:9443",
"passed": true
},
{
"test": "mtls:valkey:6380:api",
"passed": true
},
{
"test": "mtls:valkey:6380:None",
"passed": true
},
{
"test": "mtls:valkey:6380:outsider",
"passed": true
},
{
"test": "mtls:valkey:6380:foreign",
"passed": true
},
{
"test": "hostname:valkey:6380",
"passed": true
},
{
"test": "plaintext-closed:api:8080",
"passed": true
},
{
"test": "plaintext-closed:s3:9000",
"passed": true
},
{
"test": "plaintext-closed:broker:5672",
"passed": true
},
{
"test": "plaintext-closed:broker:15672",
"passed": true
},
{
"test": "plaintext-closed:broker:15692",
"passed": true
},
{
"test": "plaintext-closed:broker:25672",
"passed": true
},
{
"test": "plaintext-closed:broker:4369",
"passed": true
},
{
"test": "plaintext-closed:valkey:6379",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:4318",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:8888",
"passed": true
},
{
"test": "plaintext-closed:otel-collector:13133",
"passed": true
},
{
"test": "plaintext-closed:prometheus:9090",
"passed": true
},
{
"test": "plaintext-closed:loki:3100",
"passed": true
},
{
"test": "plaintext-closed:loki:9096",
"passed": true
},
{
"test": "plaintext-closed:tempo:3200",
"passed": true
},
{
"test": "plaintext-closed:tempo:4318",
"passed": true
},
{
"test": "plaintext-closed:tempo:9095",
"passed": true
},
{
"test": "plaintext-closed:grafana:3000",
"passed": true
},
{
"test": "bao-scope:api",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "bao-scope:worker",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "bao-scope:relay",
"passed": true,
"own": 200,
"foreign": 403,
"admin": 403
},
{
"test": "worker-db:jobs-readable",
"passed": true
},
{
"test": "worker-db:users-denied",
"passed": true
},
{
"test": "worker-db:sessions-denied",
"passed": true
},
{
"test": "worker-db:ddl-denied",
"passed": true
},
{
"test": "worker-s3:PUT:variants/security-probe",
"passed": true,
"status": 200
},
{
"test": "worker-s3:GET:variants/security-probe",
"passed": true,
"status": 200
},
{
"test": "worker-s3:PUT:staging/security-probe",
"passed": true,
"status": 403
},
{
"test": "worker-s3:GET:private/security-probe",
"passed": true,
"status": 403
},
{
"test": "worker-s3:DELETE:variants/security-probe",
"passed": true,
"status": 403
},
{
"test": "worker-queue:consume",
"passed": true
},
{
"test": "worker-queue:publish-denied",
"passed": true
},
{
"test": "worker-queue:configure-denied",
"passed": true
}
],
"passed": true
}
+10
View File
@@ -0,0 +1,10 @@
........................................................................ [ 11%]
........................................................................ [ 22%]
........................................................................ [ 34%]
........................................................................ [ 45%]
........................................................................ [ 57%]
.....................................s.................................. [ 68%]
........................................................................ [ 80%]
...................................................................s.... [ 91%]
.................................................... [100%]
626 passed, 2 skipped in 10.89s
@@ -0,0 +1,104 @@
[
{
"run": 1,
"authorization_stop_seconds": 51.881,
"recovery_seconds": 12.897,
"queued": 3,
"ready": 3,
"checksums_preserved": true,
"revoked": {
"checks": [
{
"test": "revoked-cert:broker",
"passed": true
},
{
"test": "revoked-cert:s3",
"passed": true
},
{
"test": "revoked-cert:otel-collector",
"passed": true
},
{
"test": "revoked-bao-cert",
"passed": true,
"status": 400
},
{
"test": "rotated-db-password",
"passed": true
}
],
"passed": true
}
},
{
"run": 2,
"authorization_stop_seconds": 57.562,
"recovery_seconds": 13.133,
"queued": 3,
"ready": 3,
"checksums_preserved": true,
"revoked": {
"checks": [
{
"test": "revoked-cert:broker",
"passed": true
},
{
"test": "revoked-cert:s3",
"passed": true
},
{
"test": "revoked-cert:otel-collector",
"passed": true
},
{
"test": "revoked-bao-cert",
"passed": true,
"status": 400
},
{
"test": "rotated-db-password",
"passed": true
}
],
"passed": true
}
},
{
"run": 3,
"authorization_stop_seconds": 57.202,
"recovery_seconds": 13.181,
"queued": 3,
"ready": 3,
"checksums_preserved": true,
"revoked": {
"checks": [
{
"test": "revoked-cert:broker",
"passed": true
},
{
"test": "revoked-cert:s3",
"passed": true
},
{
"test": "revoked-cert:otel-collector",
"passed": true
},
{
"test": "revoked-bao-cert",
"passed": true,
"status": 400
},
{
"test": "rotated-db-password",
"passed": true
}
],
"passed": true
}
}
]
@@ -0,0 +1,3 @@
{"run": 1, "authorization_stop_seconds": 51.881, "recovery_seconds": 12.897, "queued": 3, "ready": 3, "checksums_preserved": true, "revoked": {"checks": [{"test": "revoked-cert:broker", "passed": true}, {"test": "revoked-cert:s3", "passed": true}, {"test": "revoked-cert:otel-collector", "passed": true}, {"test": "revoked-bao-cert", "passed": true, "status": 400}, {"test": "rotated-db-password", "passed": true}], "passed": true}}
{"run": 2, "authorization_stop_seconds": 57.562, "recovery_seconds": 13.133, "queued": 3, "ready": 3, "checksums_preserved": true, "revoked": {"checks": [{"test": "revoked-cert:broker", "passed": true}, {"test": "revoked-cert:s3", "passed": true}, {"test": "revoked-cert:otel-collector", "passed": true}, {"test": "revoked-bao-cert", "passed": true, "status": 400}, {"test": "rotated-db-password", "passed": true}], "passed": true}}
{"run": 3, "authorization_stop_seconds": 57.202, "recovery_seconds": 13.181, "queued": 3, "ready": 3, "checksums_preserved": true, "revoked": {"checks": [{"test": "revoked-cert:broker", "passed": true}, {"test": "revoked-cert:s3", "passed": true}, {"test": "revoked-cert:otel-collector", "passed": true}, {"test": "revoked-bao-cert", "passed": true, "status": 400}, {"test": "rotated-db-password", "passed": true}], "passed": true}}
+164
View File
@@ -0,0 +1,164 @@
[
{
"run": 1,
"authorization_stop_seconds": 60.285,
"recovery_seconds": 12.923,
"queued": 3,
"ready": 3,
"checksums_preserved": true,
"publications": [
{
"attachment_id": "01a1d6b1-0483-7309-b485-fba89094dd76",
"jobs": 1,
"variants": 3,
"published_version": 1
},
{
"attachment_id": "01a1d6b1-04a4-730a-9dec-a96456e3da54",
"jobs": 1,
"variants": 3,
"published_version": 1
},
{
"attachment_id": "01a1d6b1-04c8-728f-9a41-8e5e0740a8bc",
"jobs": 1,
"variants": 3,
"published_version": 1
}
],
"revoked": {
"checks": [
{
"test": "revoked-cert:broker",
"passed": true
},
{
"test": "revoked-cert:s3",
"passed": true
},
{
"test": "revoked-cert:otel-collector",
"passed": true
},
{
"test": "revoked-bao-cert",
"passed": true,
"status": 400
},
{
"test": "rotated-db-password",
"passed": true
}
],
"passed": true
}
},
{
"run": 2,
"authorization_stop_seconds": 56.717,
"recovery_seconds": 20.992,
"queued": 3,
"ready": 3,
"checksums_preserved": true,
"publications": [
{
"attachment_id": "01a1d6b2-1afc-7231-a4b6-d11ed9a73c78",
"jobs": 1,
"variants": 3,
"published_version": 1
},
{
"attachment_id": "01a1d6b2-1b1c-77c5-a6c0-022c0407f957",
"jobs": 1,
"variants": 3,
"published_version": 1
},
{
"attachment_id": "01a1d6b2-1b3a-711c-84c7-229afb945dbb",
"jobs": 1,
"variants": 3,
"published_version": 1
}
],
"revoked": {
"checks": [
{
"test": "revoked-cert:broker",
"passed": true
},
{
"test": "revoked-cert:s3",
"passed": true
},
{
"test": "revoked-cert:otel-collector",
"passed": true
},
{
"test": "revoked-bao-cert",
"passed": true,
"status": 400
},
{
"test": "rotated-db-password",
"passed": true
}
],
"passed": true
}
},
{
"run": 3,
"authorization_stop_seconds": 56.953,
"recovery_seconds": 12.906,
"queued": 3,
"ready": 3,
"checksums_preserved": true,
"publications": [
{
"attachment_id": "01a1d6b3-51bc-77de-8485-e6ef064ca92b",
"jobs": 1,
"variants": 3,
"published_version": 1
},
{
"attachment_id": "01a1d6b3-51d9-71c4-9945-08af74e833d9",
"jobs": 1,
"variants": 3,
"published_version": 1
},
{
"attachment_id": "01a1d6b3-51f1-766e-ac4c-c12e5b93bc7c",
"jobs": 1,
"variants": 3,
"published_version": 1
}
],
"revoked": {
"checks": [
{
"test": "revoked-cert:broker",
"passed": true
},
{
"test": "revoked-cert:s3",
"passed": true
},
{
"test": "revoked-cert:otel-collector",
"passed": true
},
{
"test": "revoked-bao-cert",
"passed": true,
"status": 400
},
{
"test": "rotated-db-password",
"passed": true
}
],
"passed": true
}
}
]
+3
View File
@@ -0,0 +1,3 @@
{"run": 1, "authorization_stop_seconds": 60.285, "recovery_seconds": 12.923, "queued": 3, "ready": 3, "checksums_preserved": true, "publications": [{"attachment_id": "01a1d6b1-0483-7309-b485-fba89094dd76", "jobs": 1, "variants": 3, "published_version": 1}, {"attachment_id": "01a1d6b1-04a4-730a-9dec-a96456e3da54", "jobs": 1, "variants": 3, "published_version": 1}, {"attachment_id": "01a1d6b1-04c8-728f-9a41-8e5e0740a8bc", "jobs": 1, "variants": 3, "published_version": 1}], "revoked": {"checks": [{"test": "revoked-cert:broker", "passed": true}, {"test": "revoked-cert:s3", "passed": true}, {"test": "revoked-cert:otel-collector", "passed": true}, {"test": "revoked-bao-cert", "passed": true, "status": 400}, {"test": "rotated-db-password", "passed": true}], "passed": true}}
{"run": 2, "authorization_stop_seconds": 56.717, "recovery_seconds": 20.992, "queued": 3, "ready": 3, "checksums_preserved": true, "publications": [{"attachment_id": "01a1d6b2-1afc-7231-a4b6-d11ed9a73c78", "jobs": 1, "variants": 3, "published_version": 1}, {"attachment_id": "01a1d6b2-1b1c-77c5-a6c0-022c0407f957", "jobs": 1, "variants": 3, "published_version": 1}, {"attachment_id": "01a1d6b2-1b3a-711c-84c7-229afb945dbb", "jobs": 1, "variants": 3, "published_version": 1}], "revoked": {"checks": [{"test": "revoked-cert:broker", "passed": true}, {"test": "revoked-cert:s3", "passed": true}, {"test": "revoked-cert:otel-collector", "passed": true}, {"test": "revoked-bao-cert", "passed": true, "status": 400}, {"test": "rotated-db-password", "passed": true}], "passed": true}}
{"run": 3, "authorization_stop_seconds": 56.953, "recovery_seconds": 12.906, "queued": 3, "ready": 3, "checksums_preserved": true, "publications": [{"attachment_id": "01a1d6b3-51bc-77de-8485-e6ef064ca92b", "jobs": 1, "variants": 3, "published_version": 1}, {"attachment_id": "01a1d6b3-51d9-71c4-9945-08af74e833d9", "jobs": 1, "variants": 3, "published_version": 1}, {"attachment_id": "01a1d6b3-51f1-766e-ac4c-c12e5b93bc7c", "jobs": 1, "variants": 3, "published_version": 1}], "revoked": {"checks": [{"test": "revoked-cert:broker", "passed": true}, {"test": "revoked-cert:s3", "passed": true}, {"test": "revoked-cert:otel-collector", "passed": true}, {"test": "revoked-bao-cert", "passed": true, "status": 400}, {"test": "rotated-db-password", "passed": true}], "passed": true}}
+18
View File
@@ -0,0 +1,18 @@
services:
relay:
restart: "no"
environment:
FAULT_POINT: ${RELAY_FAULT_POINT:-}
OUTBOX_POLL_SECONDS: "0.5"
SWEEP_INTERVAL_SECONDS: "1"
STAGING_RETENTION_SECONDS: ${FAULT_STAGING_RETENTION_SECONDS:-${STAGING_RETENTION_SECONDS:-86400}}
worker:
restart: "no"
environment:
FAULT_POINT: ${WORKER_FAULT_POINT:-}
FAULT_STALL_SECONDS: ${FAULT_STALL_SECONDS:-15}
JOB_TIMEOUT_SECONDS: ${FAULT_JOB_TIMEOUT_SECONDS:-5}
JOB_BACKOFF_BASE_SECONDS: ${FAULT_BACKOFF_BASE_SECONDS:-1}
RENDER_TIMEOUT_SECONDS: ${FAULT_RENDER_TIMEOUT_SECONDS:-${RENDER_TIMEOUT_SECONDS:-30}}
RENDER_MEMORY_MB: ${FAULT_RENDER_MEMORY_MB:-${RENDER_MEMORY_MB:-448}}
+863
View File
@@ -0,0 +1,863 @@
{
"uid": "messenger-api",
"title": "Messenger · API RED",
"description": "RED по endpoint, SLI/SLO, логи и трассы API.",
"tags": [
"messenger",
"lab6",
"red"
],
"timezone": "browser",
"editable": false,
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 41,
"version": 1,
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"fiscalYearStartMonth": 0,
"liveNow": false,
"links": [
{
"title": "Messenger",
"type": "dashboards",
"tags": [
"messenger"
],
"asDropdown": true,
"includeVars": false,
"keepTime": true
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"enable": true,
"iconColor": "red",
"name": "Firing alerts",
"expr": "ALERTS{alertstate=\"firing\"}",
"step": "15s",
"titleFormat": "{{alertname}}",
"textFormat": "{{alertname}} firing",
"useValueForTime": false
}
]
},
"templating": {
"list": [
{
"name": "route",
"label": "Маршрут",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"query": {
"query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, http_route)",
"refId": "PrometheusVariableQueryEditor-VariableQuery",
"qryType": 1
},
"definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, http_route)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"sort": 1
},
{
"name": "instance",
"label": "Реплика",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"query": {
"query": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery",
"qryType": 1
},
"definition": "label_values(http_server_request_duration_seconds_count{job=\"messenger-api\"}, instance)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"sort": 1
}
]
},
"panels": [
{
"type": "stat",
"title": "RPS бизнес-запросов",
"description": "Все маршруты, кроме /health/*; сумма по репликам.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\"}[$__rate_interval]))",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 1,
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "stat",
"title": "Доля 5xx (2m)",
"description": "SLI доступности: 5xx / все бизнес-запросы без /health/*. Неверный пароль (401) не считается отказом.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "messenger:http_errors:ratio_rate2m",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.01
},
{
"color": "red",
"value": 0.05
}
]
},
"decimals": 2
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 2,
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "stat",
"title": "p95 latency (5m)",
"description": "histogram_quantile по сумме buckets всех реплик, не среднее p95.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "messenger:http_request_duration_seconds:p95_5m",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "orange",
"value": 0.3
},
{
"color": "red",
"value": 0.5
}
]
}
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 3,
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "stat",
"title": "Доля не-5xx (окно 1h)",
"description": "Доля не-5xx бизнес-запросов за 1h; цель 99%.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "1 - messenger:http_errors:ratio_rate1h",
"legendFormat": "__auto",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"decimals": 3
},
"overrides": []
},
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"colorMode": "value",
"graphMode": "area",
"textMode": "auto"
},
"id": 4,
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
}
},
{
"type": "row",
"title": "Rate / Errors / Duration по endpoint",
"collapsed": false,
"panels": [],
"id": 5,
"gridPos": {
"x": 0,
"y": 4,
"w": 24,
"h": 1
}
},
{
"type": "timeseries",
"title": "RPS по маршруту и классу статуса",
"description": "Recording rule messenger:http_requests:rate2m; status class выводится из http.response.status_code.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (http_route, http_request_method, status_class) (messenger:http_requests:rate2m{http_route=~\"$route\"})",
"legendFormat": "{{http_request_method}} {{http_route}} {{status_class}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 6,
"gridPos": {
"x": 0,
"y": 5,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Доля ошибок",
"description": "Числитель SLI — только 5xx; 4xx показаны отдельно и в SLO не входят.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\", http_response_status_code=~\"5..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "5xx / все (SLI)",
"range": true,
"exemplar": false
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "B",
"expr": "(sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\", http_response_status_code=~\"4..\"}[$__rate_interval])) or vector(0)) / sum(rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "4xx / все (клиент)",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 7,
"gridPos": {
"x": 12,
"y": 5,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Latency p50 / p95 / p99",
"description": "Из histogram buckets (секунды) всех выбранных реплик; exemplars ведут в Tempo.",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "p50",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "p95",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "p99",
"range": true,
"exemplar": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 8,
"gridPos": {
"x": 0,
"y": 13,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "p95 по маршруту",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "{{http_route}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 9,
"gridPos": {
"x": 12,
"y": 13,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "RPS по реплике",
"description": "instance = service.instance.id (реплика + pid процесса gunicorn).",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (instance) (rate(http_server_request_duration_seconds_count{job=\"messenger-api\", http_route!~\"/health/.*\", http_route=~\"$route\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "{{instance}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 10,
"gridPos": {
"x": 0,
"y": 21,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Активные запросы",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (instance) (http_server_active_requests{job=\"messenger-api\"})",
"legendFormat": "{{instance}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "short",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 11,
"gridPos": {
"x": 12,
"y": 21,
"w": 12,
"h": 8
}
},
{
"type": "row",
"title": "Логи и трассы API",
"collapsed": false,
"panels": [],
"id": 12,
"gridPos": {
"x": 0,
"y": 29,
"w": 24,
"h": 1
}
},
{
"type": "logs",
"title": "Логи API (ошибки и предупреждения)",
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"refId": "A",
"expr": "{service_name=\"messenger-api\"} | severity_text=~\"(?i)error|warn.*\"",
"queryType": "range"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending",
"enableLogDetails": true,
"dedupStrategy": "none",
"prettifyLogMessage": false
},
"id": 13,
"gridPos": {
"x": 0,
"y": 30,
"w": 12,
"h": 10
}
},
{
"type": "table",
"title": "Трассы API с ошибкой",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"refId": "A",
"queryType": "traceql",
"query": "{ resource.service.name = \"messenger-api\" && status = error }",
"limit": 20,
"tableType": "traces"
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {},
"id": 14,
"gridPos": {
"x": 12,
"y": 30,
"w": 12,
"h": 10
}
}
]
}
@@ -0,0 +1,569 @@
{
"uid": "messenger-pipeline",
"title": "Messenger · Worker",
"description": "Задания worker: исходы, длительность обработки, логи и трассы.",
"tags": [
"messenger",
"lab6",
"worker"
],
"timezone": "browser",
"editable": false,
"graphTooltip": 1,
"refresh": "10s",
"schemaVersion": 41,
"version": 2,
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"fiscalYearStartMonth": 0,
"liveNow": false,
"links": [
{
"title": "Messenger",
"type": "dashboards",
"tags": [
"messenger"
],
"asDropdown": true,
"includeVars": false,
"keepTime": true
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"enable": true,
"iconColor": "red",
"name": "Firing alerts",
"expr": "ALERTS{alertstate=\"firing\"}",
"step": "15s",
"titleFormat": "{{alertname}}",
"textFormat": "{{alertname}} firing",
"useValueForTime": false
}
]
},
"templating": {
"list": []
},
"panels": [
{
"type": "row",
"title": "Worker",
"collapsed": false,
"panels": [],
"id": 8,
"gridPos": {
"x": 0,
"y": 0,
"w": 24,
"h": 1
}
},
{
"type": "timeseries",
"title": "Jobs по исходу",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "sum by (job_type, outcome) (rate(messenger_jobs_processed_total[$__rate_interval]))",
"legendFormat": "{{job_type}} {{outcome}}",
"range": true,
"exemplar": false
}
],
"fieldConfig": {
"defaults": {
"unit": "ops",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "normal",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 9,
"gridPos": {
"x": 0,
"y": 1,
"w": 12,
"h": 8
}
},
{
"type": "timeseries",
"title": "Длительность обработки job",
"description": "",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))",
"legendFormat": "p50 {{job_type}}",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "B",
"expr": "histogram_quantile(0.95, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))",
"legendFormat": "p95 {{job_type}}",
"range": true,
"exemplar": true
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "C",
"expr": "histogram_quantile(0.99, sum by (le, job_type) (rate(messenger_job_duration_seconds_bucket{}[$__rate_interval])))",
"legendFormat": "p99 {{job_type}}",
"range": true,
"exemplar": true
}
],
"fieldConfig": {
"defaults": {
"unit": "s",
"custom": {
"drawStyle": "line",
"lineWidth": 1,
"fillOpacity": 10,
"showPoints": "never",
"spanNulls": false,
"stacking": {
"mode": "none",
"group": "A"
}
},
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max"
]
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"id": 10,
"gridPos": {
"x": 12,
"y": 1,
"w": 12,
"h": 8
}
},
{
"type": "row",
"title": "Логи и трассы worker",
"collapsed": false,
"panels": [],
"id": 13,
"gridPos": {
"x": 0,
"y": 9,
"w": 24,
"h": 1
}
},
{
"type": "logs",
"title": "Логи worker",
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"refId": "A",
"expr": "{service_name=\"messenger-worker\"}",
"queryType": "range"
}
],
"options": {
"showTime": true,
"wrapLogMessage": true,
"sortOrder": "Descending",
"enableLogDetails": true,
"dedupStrategy": "none",
"prettifyLogMessage": false
},
"id": 14,
"gridPos": {
"x": 0,
"y": 10,
"w": 12,
"h": 10
}
},
{
"type": "table",
"title": "Трассы worker",
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"targets": [
{
"datasource": {
"type": "tempo",
"uid": "tempo"
},
"refId": "A",
"queryType": "traceql",
"query": "{ resource.service.name = \"messenger-worker\" }",
"limit": 20,
"tableType": "traces"
}
],
"fieldConfig": {
"defaults": {},
"overrides": []
},
"options": {},
"id": 15,
"gridPos": {
"x": 12,
"y": 10,
"w": 12,
"h": 10
}
},
{
"id": 16,
"type": "timeseries",
"title": "Durable queue depth (includes retry)",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 17,
"type": "timeseries",
"title": "Oldest durable job age",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 20,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_age_seconds)",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 18,
"type": "timeseries",
"title": "DB pool used / waiting / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 28,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (__name__,job) ({__name__=~\"messenger_db_pool_(used|waiting|capacity)\"})",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 19,
"type": "timeseries",
"title": "Worker busy / capacity",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 28,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (__name__) ({__name__=~\"messenger_worker_(busy|capacity)\"})",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 20,
"type": "timeseries",
"title": "Process CPU cores",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 36,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (job) (rate(process_cpu_time_seconds_total{job=~\"messenger-.*\"}[1m]))",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 21,
"type": "timeseries",
"title": "Process resident memory",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 36,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (job) (process_memory_usage_bytes{job=~\"messenger-.*\"})",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
},
{
"id": 22,
"type": "timeseries",
"title": "Time to ready p50 / p95 / p99",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 44,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "0.5"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "0.95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99,sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
"legendFormat": "0.99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 23,
"type": "timeseries",
"title": "Ready within 30s, SLO 95% / 1h",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 12,
"y": 44,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:ready_under_30s:ratio1h",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 24,
"type": "timeseries",
"title": "API availability, SLO 99% / 1h",
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"x": 0,
"y": 52,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "messenger:availability:ratio1h",
"legendFormat": "{{job}} {{__name__}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
}
]
}
@@ -0,0 +1 @@
apiVersion: 1
@@ -0,0 +1,14 @@
apiVersion: 1
providers:
- name: messenger
orgId: 1
folder: Messenger
folderUid: messenger
type: file
disableDeletion: true
allowUiUpdates: false
updateIntervalSeconds: 30
options:
path: /etc/grafana/dashboards
foldersFromFilesStructure: false
@@ -0,0 +1,79 @@
apiVersion: 1
prune: true
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://127.0.0.1:19090
isDefault: true
editable: false
jsonData:
httpMethod: POST
prometheusType: Prometheus
prometheusVersion: 3.15.1
timeInterval: 15s
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
urlDisplayLabel: Trace
- name: Loki
uid: loki
type: loki
access: proxy
url: http://127.0.0.1:13100
editable: false
jsonData:
maxLines: 1000
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: "$${__value.raw}"
urlDisplayLabel: Trace
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://127.0.0.1:13200
editable: false
jsonData:
nodeGraph:
enabled: true
search:
hide: false
traceQuery:
timeShiftEnabled: true
spanStartTimeShift: -30m
spanEndTimeShift: 30m
streamingEnabled:
search: true
metrics: false
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: true
filterBySpanID: false
tags:
- key: service.name
value: service_name
tracesToMetrics:
datasourceUid: prometheus
spanStartTimeShift: -5m
spanEndTimeShift: 5m
tags:
- key: service.name
value: job
queries:
- name: RPS маршрута
query: sum(rate(http_server_request_duration_seconds_count{$$__tags}[1m]))
- name: p95 маршрута
query: histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{$$__tags}[5m])))
lokiSearch:
datasourceUid: loki
@@ -0,0 +1,3 @@
apiVersion: 1
apps: []
+71
View File
@@ -0,0 +1,71 @@
global
log stdout format raw local0 info
maxconn 4096
hard-stop-after 5s
tune.bufsize 32768
tune.rcvbuf.client 65536
tune.rcvbuf.server 65536
tune.sndbuf.client 65536
tune.sndbuf.server 65536
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold nx 2s
hold other 2s
hold refused 2s
hold timeout 2s
defaults
mode http
log global
option dontlognull
option dontlog-normal
log-format "%ci [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] %HM %HPO"
option http-server-close
option redispatch 1
retries 2
retry-on conn-failure empty-response
timeout connect 500ms
timeout client 30s
timeout server 10s
timeout http-request 5s
timeout http-keep-alive 5s
timeout queue 3s
timeout check 1s
http-error status 502 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream replica failed","request_id":"%[var(txn.request_id),json(ascii)]"}}'
http-error status 503 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"No ready API replica","request_id":"%[var(txn.request_id),json(ascii)]"}}'
http-error status 504 content-type application/json hdr X-Request-Id "%[var(txn.request_id)]" lf-string '{"error":{"code":"unavailable","message":"Upstream timeout","request_id":"%[var(txn.request_id),json(ascii)]"}}'
frontend public
bind :8443 ssl crt /run/pki/tls.pem
option forwardfor
http-request set-var(txn.request_id) req.fhdr(X-Request-Id) if { req.fhdr(X-Request-Id) -m len 1:200 }
http-request set-var(txn.request_id) uuid(4) unless { var(txn.request_id) -m found }
http-request set-header X-Request-Id %[var(txn.request_id)]
default_backend api
backend api
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
http-check expect status 200
acl message_path path_reg ^/api/v1/chats/[^/]+/messages$
acl has_key req.hdr(Idempotency-Key) -m found
http-request set-var(txn.replay_safe) bool(1) if METH_GET
http-request set-var(txn.replay_safe) bool(1) if METH_POST message_path has_key
http-request disable-l7-retry unless { var(txn.replay_safe) -m bool }
default-server check init-state fully-down inter 1s fastinter 500ms downinter 2s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none maxconn "${API_SERVER_MAXCONN-64}"
server-template api 8 api:8443 ssl verify required ca-file /run/pki/ca.crt crt /run/pki/tls.pem verifyhost api sni str(api)
frontend stats
bind 127.0.0.1:8404
monitor-uri /healthz
monitor fail if { nbsrv(api) lt 1 }
http-request use-service prometheus-exporter if { path /metrics }
stats enable
stats uri /stats
stats refresh 5s
+106
View File
@@ -0,0 +1,106 @@
import http from 'k6/http';
import { check } from 'k6';
import encoding from 'k6/encoding';
import exec from 'k6/execution';
const BASE = __ENV.BASE_URL || 'http://lb:8080';
const RATE = Number(__ENV.RATE || 50);
const WARMUP = Number(__ENV.WARMUP || 10);
const DURATION = Number(__ENV.DURATION || 60);
const USERS = Math.max(2, Number(__ENV.USERS || 20));
const READ_RATIO = Number(__ENV.READ_RATIO || 0.2);
const VUS = Number(__ENV.VUS || Math.max(10, Math.ceil(RATE / 10)));
const MAX_VUS = Number(__ENV.MAX_VUS || Math.max(VUS, Math.min(VUS * 4, 1000)));
const RUN = __ENV.RUN_ID || Date.now().toString(36);
const SUMMARY = __ENV.SUMMARY || '/out/summary.json';
const JSON_HEADERS = { 'Content-Type': 'application/json' };
const phase = (startTime, duration) => ({
executor: 'constant-arrival-rate',
rate: RATE,
timeUnit: '1s',
startTime: `${startTime}s`,
duration: `${duration}s`,
preAllocatedVUs: VUS,
maxVUs: MAX_VUS,
gracefulStop: '5s',
});
const measured = ['http_reqs', 'http_req_failed', 'http_req_duration', 'dropped_iterations', 'iterations'];
export const options = {
setupTimeout: '180s',
summaryTrendStats: ['avg', 'min', 'med', 'p(90)', 'p(95)', 'p(99)', 'max'],
scenarios: Object.assign(WARMUP > 0 ? { warmup: phase(0, WARMUP) } : {}, { measure: phase(WARMUP, DURATION) }),
thresholds: Object.fromEntries([
...measured.map((metric) => [`${metric}{scenario:measure}`, []]),
['http_req_duration{scenario:measure,name:post_message}', []],
['http_req_duration{scenario:measure,name:get_history}', []],
]),
};
function created(response, what) {
if (response.status !== 201) {
throw new Error(`${what}: HTTP ${response.status}`);
}
return response.json();
}
const bearer = (token, extra) => Object.assign({ Authorization: `Bearer ${token}` }, extra || {});
export function setup() {
const actors = [];
for (let i = 0; i < USERS; i++) {
const username = `k6_${RUN}_${i}`.slice(0, 32);
const password = `K6-${RUN}-${i}-synthetic`;
const user = created(
http.post(`${BASE}/api/v1/users`, JSON.stringify({ username, display_name: `k6 ${i}`, password }), { headers: JSON_HEADERS }),
'register',
);
const basic = encoding.b64encode(`${username}:${password}`);
const session = created(
http.post(`${BASE}/api/v1/auth/sessions`, null, { headers: { Authorization: `Basic ${basic}` } }),
'login',
);
actors.push({ id: user.id, token: session.token });
}
const chats = [];
for (let i = 0; i < USERS; i += 2) {
const [a, b] = [actors[i], actors[(i + 1) % USERS]];
const body = JSON.stringify({ title: `k6 ${RUN} ${i}`, member_ids: [a.id, b.id] });
const chat = created(http.post(`${BASE}/api/v1/chats`, body, { headers: bearer(a.token, JSON_HEADERS) }), 'chat');
chats.push({ id: chat.id, tokens: [a.token, b.token] });
}
console.log('LOAD-START');
return { chats };
}
export default function (data) {
const n = exec.scenario.iterationInTest;
const chat = data.chats[n % data.chats.length];
const token = chat.tokens[Math.floor(n / data.chats.length) % 2];
if (Math.random() < READ_RATIO) {
const response = http.get(`${BASE}/api/v1/chats/${chat.id}/messages?limit=50`, {
headers: bearer(token),
tags: { name: 'get_history' },
timeout: '10s',
});
check(response, { 'history 200': (r) => r.status === 200 });
return;
}
const key = `${RUN}-${exec.scenario.name}-${n}`;
const response = http.post(`${BASE}/api/v1/chats/${chat.id}/messages`, JSON.stringify({ text: `k6 ${key}` }), {
headers: bearer(token, Object.assign({ 'Idempotency-Key': key }, JSON_HEADERS)),
tags: { name: 'post_message' },
timeout: '10s',
});
check(response, { 'message 201': (r) => r.status === 201 });
}
export function handleSummary(data) {
delete data.setup_data;
const reqs = data.metrics['http_reqs{scenario:measure}'];
const failed = data.metrics['http_req_failed{scenario:measure}'];
const line = `measure: ${reqs ? reqs.values.count : 0} requests, failed ratio ${failed ? failed.values.rate : 'n/a'}\n`;
return { [SUMMARY]: JSON.stringify(data, null, 1), stdout: line };
}
+78
View File
@@ -0,0 +1,78 @@
auth_enabled: false
server:
http_listen_address: 127.0.0.1
http_listen_port: 3100
grpc_listen_address: 127.0.0.1
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
replication_factor: 1
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: "2026-01-01"
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
ingester:
chunk_idle_period: 5m
max_chunk_age: 1h
limits_config:
allow_structured_metadata: true
volume_enabled: true
retention_period: 168h
reject_old_samples: true
reject_old_samples_max_age: 24h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
max_query_lookback: 168h
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes:
- service.name
- deployment.environment.name
- action: structured_metadata
attributes:
- service.instance.id
- service.version
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
compaction_interval: 10m
query_range:
results_cache:
cache:
embedded_cache:
enabled: true
max_size_mb: 64
pattern_ingester:
enabled: false
ruler:
enable_api: false
analytics:
reporting_enabled: false
+1
View File
@@ -0,0 +1 @@
path "secret/data/api" { capabilities = ["read"] }
+1
View File
@@ -0,0 +1 @@
path "secret/data/migrate" { capabilities = ["read"] }
+1
View File
@@ -0,0 +1 @@
path "secret/data/reconciler" { capabilities = ["read"] }
+1
View File
@@ -0,0 +1 @@
path "secret/data/relay" { capabilities = ["read"] }
+14
View File
@@ -0,0 +1,14 @@
ui = true
api_addr = "https://openbao:8200"
cluster_addr = "https://openbao:8201"
storage "raft" {
node_id = "lab7"
path = "/bao/data"
}
listener "tcp" {
address = "0.0.0.0:8200"
tls_cert_file = "/run/pki/tls.crt"
tls_key_file = "/run/pki/tls.key"
tls_client_ca_file = "/run/pki/ca.crt"
tls_min_version = "tls12"
}
+1
View File
@@ -0,0 +1 @@
path "secret/data/worker" { capabilities = ["read"] }
+122
View File
@@ -0,0 +1,122 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 8388608
processors:
tail_sampling:
decision_wait: 10s
num_traces: 2000
expected_new_traces_per_sec: 30
decision_cache:
sampled_cache_size: 10000
non_sampled_cache_size: 10000
policies:
- name: errors
type: status_code
status_code: {status_codes: [ERROR]}
- name: slow
type: latency
latency: {threshold_ms: 300}
- name: baseline
type: probabilistic
probabilistic: {sampling_percentage: 5}
memory_limiter:
check_interval: 1s
limit_percentage: 75
spike_limit_percentage: 20
redaction:
allow_all_keys: true
summary: silent
blocked_key_patterns:
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
- "^http\\.(request|response)\\.header\\..+"
- "^url\\.(query|full)$"
- "^http\\.(url|target)$"
- "^(messenger\\.)?(message\\.)?(text|body)$"
- "^(file\\.)?filename$"
blocked_values:
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
otlp_http/tempo:
traces_endpoint: http://tempo:4318/v1/traces
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/prometheus:
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 50000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/loki:
logs_endpoint: http://loki:3100/otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
without_type_suffix: false
without_units: false
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, redaction, tail_sampling, batch]
exporters: [otlp_http/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/loki]
+122
View File
@@ -0,0 +1,122 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 8388608
processors:
tail_sampling:
decision_wait: 10s
num_traces: 2000
expected_new_traces_per_sec: 30
decision_cache:
sampled_cache_size: 10000
non_sampled_cache_size: 10000
policies:
- name: errors
type: status_code
status_code: {status_codes: [ERROR]}
- name: slow
type: latency
latency: {threshold_ms: 300}
- name: baseline
type: probabilistic
probabilistic: {sampling_percentage: 10}
memory_limiter:
check_interval: 1s
limit_percentage: 75
spike_limit_percentage: 20
redaction:
allow_all_keys: true
summary: silent
blocked_key_patterns:
- "(?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$"
- "^http\\.(request|response)\\.header\\..+"
- "^url\\.(query|full)$"
- "^http\\.(url|target)$"
- "^(messenger\\.)?(message\\.)?(text|body)$"
- "^(file\\.)?filename$"
blocked_values:
- "(?i)bearer\\s+[A-Za-z0-9._~+/=-]{8,}"
- "(?i)basic\\s+[A-Za-z0-9+/=]{8,}"
- "(?i)x-amz-(signature|credential)=[^&\\s]+"
- "-----BEGIN [A-Z ]*PRIVATE KEY-----"
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
otlp_http/tempo:
traces_endpoint: http://tempo:4318/v1/traces
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/prometheus:
metrics_endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 50000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlp_http/loki:
logs_endpoint: http://loki:3100/otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: info
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
without_type_suffix: false
without_units: false
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, redaction, tail_sampling, batch]
exporters: [otlp_http/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, redaction, batch]
exporters: [otlp_http/loki]
+129
View File
@@ -0,0 +1,129 @@
extensions:
health_check:
endpoint: 127.0.0.1:13133
receivers:
otlp:
protocols:
http:
endpoint: 127.0.0.1:4318
max_request_body_size: 8388608
processors:
memory_limiter:
check_interval: 1s
limit_percentage: 75
spike_limit_percentage: 20
redaction:
allow_all_keys: true
summary: silent
blocked_key_patterns:
- (?i)^(.*[._-])?(password|passwd|secret|token|authorization|cookie|api[._-]?key|private[._-]?key|credentials?)$
- ^http\.(request|response)\.header\..+
- ^url\.(query|full)$
- ^http\.(url|target)$
- ^(messenger\.)?(message\.)?(text|body)$
- ^(file\.)?filename$
blocked_values:
- (?i)bearer\s+[A-Za-z0-9._~+/=-]{8,}
- (?i)basic\s+[A-Za-z0-9+/=]{8,}
- (?i)x-amz-(signature|credential)=[^&\s]+
- '-----BEGIN [A-Z ]*PRIVATE KEY-----'
batch:
send_batch_size: 1024
send_batch_max_size: 2048
timeout: 2s
exporters:
otlp_http/tempo:
traces_endpoint: https://tempo:44318/v1/traces
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
tls:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
otlp_http/prometheus:
metrics_endpoint: https://prometheus:9443/api/v1/otlp/v1/metrics
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 50000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
tls:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
otlp_http/loki:
logs_endpoint: https://loki:9443/otlp/v1/logs
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
sizer: items
queue_size: 20000
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
tls:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
service:
extensions:
- health_check
telemetry:
logs:
level: info
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 127.0.0.1
port: 8888
without_type_suffix: false
without_units: false
pipelines:
traces:
receivers:
- otlp
processors:
- memory_limiter
- redaction
- batch
exporters:
- otlp_http/tempo
metrics:
receivers:
- otlp
processors:
- memory_limiter
- redaction
- batch
exporters:
- otlp_http/prometheus
logs:
receivers:
- otlp
processors:
- memory_limiter
- redaction
- batch
exporters:
- otlp_http/loki
+62
View File
@@ -0,0 +1,62 @@
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 15s
rule_files:
- /etc/prometheus/rules/*.yml
otlp:
translation_strategy: UnderscoreEscapingWithSuffixes
promote_resource_attributes:
- service.name
- service.version
- deployment.environment.name
- container.name
- compose.project
- compose.service
keep_identifying_resource_attributes: true
storage:
tsdb:
out_of_order_time_window: 30m
exemplars:
max_exemplars: 100000
scrape_configs:
- job_name: prometheus
static_configs:
- targets:
- localhost:9090
- job_name: otel-collector
static_configs:
- targets:
- otel-collector:48888
scheme: https
tls_config:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
- job_name: loki
static_configs:
- targets:
- loki:9443
scheme: https
tls_config:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
- job_name: tempo
static_configs:
- targets:
- tempo:9443
scheme: https
tls_config:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
- job_name: grafana
static_configs:
- targets:
- grafana:9443
scheme: https
tls_config:
ca_file: /run/pki/ca.crt
cert_file: /run/pki/tls.crt
key_file: /run/pki/tls.key
@@ -0,0 +1,30 @@
groups:
- name: messenger-slo
interval: 5s
rules:
- record: messenger:http_errors:ratio_rate1m
expr: |
(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*",http_response_status_code=~"5.."}[1m])) or vector(0))
/ clamp_min(sum(rate(http_server_request_duration_seconds_count{job="messenger-api",http_route!~"/health/.*"}[1m])), 0.001)
- record: messenger:availability:ratio1h
expr: 1 - messenger:http_errors:ratio_rate1h
- record: messenger:ready_under_30s:ratio1h
expr: sum(increase(messenger_job_time_to_ready_seconds_bucket{le="30"}[1h])) / clamp_min(sum(increase(messenger_job_time_to_ready_seconds_count[1h])), 1)
- alert: MessengerHighErrorRate
expr: messenger:http_errors:ratio_rate1m > 0.05
for: 15s
labels:
severity: warning
annotations:
summary: More than 5% business HTTP 5xx over 1 minute
description: Availability target is 99% over 1 hour; health and client 4xx are excluded from failures.
runbook: infra/prometheus/runbooks.md#messengerhigherrorrate
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_age_seconds{queue_name="attachments.process"}) > 15 and max(messenger_queue_depth{queue_name="attachments.process"}) > 0
for: 15s
labels:
severity: warning
annotations:
summary: Durable attachment backlog has waited more than 15 seconds
description: Target is 95% ready within 30 seconds over 1 hour; depth includes retry and leased work.
runbook: infra/prometheus/runbooks.md#messengerqueuelag

Some files were not shown because too many files have changed in this diff Show More