Author SHA1 Message Date
AndrewNikkk ccb463aea1 lab6 2026-11-16 18:47:37 +03:00
109 changed files with 64105 additions and 32 deletions
+2
View File
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
build
+31 -9
View File
@@ -11,20 +11,42 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
# Same Valkey instance serves sessions and Streams.
BROKER_URL_FILE=/run/secrets/cache_url
WORKER_CONCURRENCY=1
# One initial attempt plus five retries.
JOB_MAX_ATTEMPTS=6
JOB_RECLAIM_IDLE_MS=5000
JOB_RETRY_BASE_MS=1000
JOB_RETRY_MAX_MS=60000
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
OTEL_SERVICE_NAME=messenger-api
# Worker must set a distinct service.name.
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
STAGING_CLEANUP_INTERVAL_SECONDS=60
# Local Compose bootstrap; choose your own values and matching secret files.
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change_me_locally
POSTGRES_DB=messenger
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=change_me_minio_root_locally
MINIO_USER_ACCESS_KEY=messenger-s3
MINIO_USER_SECRET_KEY=change_me_s3_user_locally
MAX_CONCURRENT_UPLOADS=3
DIRECT_UPLOAD_TTL_SECONDS=300
# Observability: distinct service.name is set by each binary.
OTEL_SDK_DISABLED=false
GRAFANA_ADMIN_USER=admin
API_PORT=8080
GRAFANA_PORT=3000
PROMETHEUS_PORT=9090
# Optional Collector profile (Track 1 overrides this per trial).
OTEL_COLLECTOR_CONFIG=./infra/otel/collector.yaml
+146
View File
@@ -0,0 +1,146 @@
cmake_minimum_required(VERSION 3.16)
project(mtusi_messenger CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# Official MinIO C++ SDK (S3-compatible). Pinned tag; not a hand-rolled SigV4.
set(MTUSI_MINIO_CPP_VERSION "v0.3.0" CACHE STRING "minio-cpp git tag")
find_package(Threads REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PQXX REQUIRED IMPORTED_TARGET libpqxx>=7.8)
pkg_check_modules(HIREDIS REQUIRED IMPORTED_TARGET hiredis)
pkg_check_modules(SODIUM REQUIRED IMPORTED_TARGET libsodium)
pkg_check_modules(EXIF REQUIRED IMPORTED_TARGET libexif)
pkg_check_modules(WEBP REQUIRED IMPORTED_TARGET libwebp)
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ImportMinioDeps.cmake")
include(FetchContent)
set(MINIO_CPP_TEST OFF CACHE BOOL "" FORCE)
set(MINIO_CPP_MAKE_DOC OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
miniocpp
GIT_REPOSITORY https://github.com/minio/minio-cpp.git
GIT_TAG ${MTUSI_MINIO_CPP_VERSION}
GIT_SHALLOW TRUE
)
# Inject find_package stubs before miniocpp configures.
function(_mtusi_provide_minio_find_packages)
# miniocpp calls find_package(unofficial-curlpp CONFIG) etc.
# We already created ALIAS targets; create tiny config files in a prefix.
set(_cfg_dir "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps")
file(MAKE_DIRECTORY "${_cfg_dir}")
file(WRITE "${_cfg_dir}/unofficial-curlpp-config.cmake" "")
file(WRITE "${_cfg_dir}/unofficial-inih-config.cmake" "")
list(PREPEND CMAKE_PREFIX_PATH "${_cfg_dir}")
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
endfunction()
_mtusi_provide_minio_find_packages()
FetchContent_GetProperties(miniocpp)
if(NOT miniocpp_POPULATED)
FetchContent_Populate(miniocpp)
# Neutralize CONFIG find_package calls by providing empty configs + targets.
set(CMAKE_FIND_PACKAGE_REDIRECTS_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-find-redirects")
file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
# CMake 3.24+ redirects; for older, patch via policy: set variables that
# make find_package(CONFIG) succeed when targets already exist.
set(unofficial-curlpp_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
set(unofficial-inih_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
# Provide package config bodies that are no-ops (targets already defined).
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-curlpp-config.cmake"
"# provided by ImportMinioDeps.cmake\n")
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-inih-config.cmake"
"# provided by ImportMinioDeps.cmake\n")
# Skip miniocpp install(EXPORT) — our INTERFACE system deps are not exportable.
set(CMAKE_SKIP_INSTALL_RULES ON)
add_subdirectory(${miniocpp_SOURCE_DIR} ${miniocpp_BINARY_DIR})
set(CMAKE_SKIP_INSTALL_RULES OFF)
endif()
# OpenTelemetry C++ v1.21.0, OTLP HTTP/protobuf only (no gRPC).
set(WITH_OTLP_HTTP ON CACHE BOOL "" FORCE)
set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE)
set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE)
set(WITH_FUNC_TESTS OFF CACHE BOOL "" FORCE)
FetchContent_Declare(opentelemetry_cpp
GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git
GIT_TAG v1.21.0 # b9cf499ff5715433848b316059714b5c59af1f2c
GIT_SHALLOW TRUE
GIT_SUBMODULES "")
FetchContent_MakeAvailable(opentelemetry_cpp)
add_library(messenger_core STATIC
src/telemetry.cpp
src/observability_monitor.cpp
src/db.cpp
src/auth.cpp
src/user_repo.cpp
src/chat_repo.cpp
src/message_repo.cpp
src/redis_pool.cpp
src/session_repo.cpp
src/pg_session_repo.cpp
src/s3_client.cpp
src/image_util.cpp
src/attachment_repo.cpp
src/attachment_queue.cpp
src/outbox.cpp
src/staging_cleaner.cpp
src/job_repo.cpp
src/image_pipeline.cpp
src/direct_upload_repo.cpp
)
target_include_directories(messenger_core PUBLIC
src
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
)
target_compile_definitions(messenger_core PUBLIC
MTUSI_MINIO_CPP_VERSION="${MTUSI_MINIO_CPP_VERSION}"
)
target_link_libraries(messenger_core PUBLIC
Threads::Threads
PkgConfig::PQXX
PkgConfig::HIREDIS
PkgConfig::SODIUM
PkgConfig::EXIF
PkgConfig::WEBP
opentelemetry_exporter_otlp_http
opentelemetry_exporter_otlp_http_metric
opentelemetry_exporter_otlp_http_log
miniocpp::miniocpp
)
add_executable(messenger src/main.cpp)
target_link_libraries(messenger PRIVATE messenger_core)
add_executable(worker src/worker_main.cpp)
target_link_libraries(worker PRIVATE messenger_core)
add_executable(s3_smoke tools/s3_smoke.cpp)
target_link_libraries(s3_smoke PRIVATE messenger_core)
add_executable(image_smoke tools/image_smoke.cpp)
target_link_libraries(image_smoke PRIVATE messenger_core)
target_compile_definitions(image_smoke PRIVATE
IMAGE_SMOKE_FIXTURES="${CMAKE_SOURCE_DIR}/tools/fixtures"
)
add_executable(thumbnail_smoke tools/thumbnail_smoke.cpp)
target_link_libraries(thumbnail_smoke PRIVATE messenger_core)
add_executable(variants_smoke tools/variants_smoke.cpp)
target_link_libraries(variants_smoke PRIVATE messenger_core)
add_executable(job_admin src/job_admin_main.cpp)
target_link_libraries(job_admin PRIVATE messenger_core)
add_executable(telemetry_test student-tests/telemetry_test.cpp)
target_link_libraries(telemetry_test PRIVATE messenger_core)
target_compile_options(telemetry_test PRIVATE -UNDEBUG)
+26
View File
@@ -0,0 +1,26 @@
{
"version": 6,
"cmakeMinimumRequired": {
"major": 3,
"minor": 16,
"patch": 0
},
"configurePresets": [
{
"name": "release",
"displayName": "Release with minio-cpp",
"generator": "Unix Makefiles",
"binaryDir": "${sourceDir}/build",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"MTUSI_MINIO_CPP_VERSION": "v0.3.0"
}
}
],
"buildPresets": [
{
"name": "release",
"configurePreset": "release"
}
]
}
+123
View File
@@ -0,0 +1,123 @@
# Clean-clone build: Ubuntu 24.04 + libpqxx from a pinned git tag.
# S3 client: official minio-cpp v0.3.0 via FetchContent (library SigV4).
# No dependency on mtusi2026lab3-api or other prior-lab images.
ARG PQXX_VERSION=7.10.0
ARG MTUSI_MINIO_CPP_VERSION=v0.3.0
# ---------------------------------------------------------------------------
# Stage: build libpqxx into /opt/libpqxx
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS pqxx
ARG PQXX_VERSION
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake git ca-certificates pkg-config \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
RUN git clone --depth 1 --branch "${PQXX_VERSION}" \
https://github.com/jtv/libpqxx.git libpqxx-src \
&& cmake -S libpqxx-src -B libpqxx-build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/opt/libpqxx \
-DBUILD_SHARED_LIBS=ON \
-DSKIP_BUILD_TEST=ON \
&& cmake --build libpqxx-build -j"$(nproc)" \
&& cmake --install libpqxx-build \
&& rm -rf /tmp/libpqxx-src /tmp/libpqxx-build
# ---------------------------------------------------------------------------
# Stage: build messenger + worker + smokes
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS builder
ARG MTUSI_MINIO_CPP_VERSION
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake pkg-config git ca-certificates \
libpq-dev \
libhiredis-dev \
libsodium-dev libexif-dev libwebp-dev \
libprotobuf-dev protobuf-compiler \
nlohmann-json3-dev \
libcurl4-openssl-dev libssl-dev zlib1g-dev \
libpugixml-dev \
libcurlpp-dev \
libinih-dev \
&& rm -rf /var/lib/apt/lists/*
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
ENV PKG_CONFIG_PATH=/opt/libpqxx/lib/pkgconfig:/opt/libpqxx/lib/x86_64-linux-gnu/pkgconfig
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
ENV CMAKE_PREFIX_PATH=/opt/libpqxx
WORKDIR /src
COPY CMakeLists.txt CMakePresets.json ./
COPY cmake ./cmake
COPY src ./src
COPY tools ./tools
COPY student-tests/telemetry_test.cpp ./student-tests/telemetry_test.cpp
COPY migrations ./migrations
COPY third_party ./third_party
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \
-DMTUSI_MINIO_CPP_VERSION=${MTUSI_MINIO_CPP_VERSION} \
&& cmake --build build --target messenger worker job_admin s3_smoke image_smoke thumbnail_smoke variants_smoke telemetry_test -j2
# ---------------------------------------------------------------------------
# Runtime
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS runtime
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
libprotobuf32t64 \
libpq5 \
libhiredis1.1.0 \
libsodium23 libexif12 libwebp7 \
libssl3t64 \
libcurl4t64 \
zlib1g \
libpugixml1v5 \
libcurlpp0t64 \
libinih1 \
libinireader0 \
util-linux \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --gid 10001 app \
&& useradd --uid 10001 --gid app --shell /usr/sbin/nologin --no-create-home app
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
WORKDIR /app
COPY --from=builder /src/build/_deps/opentelemetry_cpp-build/libopentelemetry_proto.so /usr/local/lib/libopentelemetry_proto.so
COPY --from=builder /src/build/messenger /app/messenger
COPY --from=builder /src/build/worker /app/worker
COPY --from=builder /src/build/telemetry_test /app/telemetry_test
COPY --from=builder /src/build/job_admin /app/job_admin
COPY --from=builder /src/build/s3_smoke /app/s3_smoke
COPY --from=builder /src/build/image_smoke /app/image_smoke
COPY --from=builder /src/build/thumbnail_smoke /app/thumbnail_smoke
COPY --from=builder /src/build/variants_smoke /app/variants_smoke
COPY --from=builder /src/migrations /app/migrations
COPY tools/fixtures /app/fixtures
COPY docker-entrypoint.sh /docker-entrypoint.sh
RUN chmod 0755 /docker-entrypoint.sh \
&& chown -R app:app /app \
&& ldconfig
# Entrypoint starts as root to stage S3 secret files for uid 10001, then drops.
USER root
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=3 \
CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["/app/messenger"]
+13
View File
@@ -28,3 +28,16 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: test-observability test-incidents
COMPOSE ?= docker compose
test-observability:
$(COMPOSE) exec -T api /app/telemetry_test
$(PYTHON) student-tests/observability.py --base-url "$(BASE_URL)"
test-incidents:
COMPOSE="$(COMPOSE)" $(PYTHON) student-tests/incidents.py --base-url "$(BASE_URL)"
.PHONY: benchmark-telemetry
benchmark-telemetry:
COMPOSE="$(COMPOSE)" $(PYTHON) scripts/benchmark_telemetry.py
+24 -1
View File
@@ -2,7 +2,30 @@
Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку.
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
В репозитории реализован C++ сервер на основе lab5 и стек наблюдаемости уровня 4:
OpenTelemetry C++ → Collector → Prometheus / Loki / Tempo → Grafana.
Бизнес-API сохранён. Архитектура, метрики, ограничения и результаты проверок — в [REPORT.md](REPORT.md).
```sh
python3 scripts/bootstrap-local.py
docker compose up -d --build
# После готовности http://localhost:8080/health/ready:
make test
make test-observability
```
Grafana: [localhost:3000](http://localhost:3000/d/messenger-observability), login `admin`,
пароль — локальный `secrets/grafana_admin_password`. Bootstrap не перезаписывает существующие секреты.
Контролируемые отказы с автоматическим восстановлением: `make test-incidents`.
Track 1 (9 прогонов по 1000 uploads, затем автоматическое восстановление telemetry):
```sh
python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3
```
Результаты: `evidence/track1-benchmark.md` и подробные обезличенные samples в `evidence/track1-benchmark.json`.
Ниже сохранены исходные требования и критерии лабораторной.
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
+541 -14
View File
@@ -1,37 +1,564 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Никитин Андрей Александрович, БВТ2301
- Целевая оценка; выбранный трек 5, если нужен: 5; Track 1 — цена наблюдаемости.
- Commit/PR: реализация находится в рабочем дереве; отдельный commit/PR ещё не опубликован. Baseline шаблона: `470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1`.
- Использованные источники, библиотеки и помощь ИИ: README и контракты курса; первичная документация OpenTelemetry, W3C Trace Context и Grafana; opentelemetry-cpp, minio-cpp, libpqxx, hiredis, libsodium, nlohmann/json, libexif, libwebp и stb. Помощь ИИ: Codex — реализация инструментации, конфигурации, скрипты проверок/бенчмарка и оформление отчёта. Версии и ссылки приведены ниже.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Linux amd64, Docker Engine с Compose v2+, Python 3.9+ и доступ к
Ubuntu/GitHub/registry при первой сборке. Приложения собираются как C++17;
компилятор, CMake и native-зависимости устанавливаются внутри Dockerfile.
Прежние готовые API-образы lab5 не требуются.
| Компонент | Закреплённая версия |
| --- | --- |
| Build/runtime OS | Ubuntu 24.04 |
| OpenTelemetry C++ / MinIO C++ / libpqxx | v1.21.0 / v0.3.0 / 7.10.0 |
| PostgreSQL / Valkey / Nginx | 16.15-alpine / 8.1.10-alpine / 1.28.0-alpine |
| MinIO / mc | RELEASE.2025-04-22T22-12-26Z / RELEASE.2025-04-16T18-13-26Z |
| Collector Contrib / Prometheus | 0.119.0 / 3.2.1 |
| Loki / Tempo / Grafana | 3.4.2 / 2.7.2 / 11.6.0 |
```sh
python3 scripts/bootstrap-local.py
docker compose config --quiet
docker compose up -d --build
# Дождаться HTTP 200:
curl -fsS http://localhost:8080/health/ready
make check
make test
make test-observability
make test-incidents
```
Grafana: http://localhost:3000/d/messenger-observability, пользователь `admin`,
пароль в локальном `secrets/grafana_admin_password`. Dashboard: папка Messenger.
Prometheus: http://localhost:9090, alerts http://localhost:9090/alerts.
В Grafana Explore выбрать Tempo, найти trace, перейти к Loki logs; либо выбрать
Loki `{service_name="messenger-worker"}` и фильтровать по `trace_id`.
API → HTTP → worker → S3/DB проверяется `student-tests/observability.py`.
`API_PORT`, `GRAFANA_PORT`, `PROMETHEUS_PORT` настраиваются в `.env`; при их
изменении URL тестов задаются параметрами соответствующих Python scripts.
`OTEL_SDK_DISABLED=true docker compose up -d --force-recreate api worker` отключает
SDK для сравнения; бизнес-процессы продолжают работать.
Если в окружении Docker Snap команда `docker compose` не найдена, используйте
установленный CLI plugin напрямую: `make test-observability COMPOSE=/path/to/docker-compose`.
Первый запуск требует действий оператора: запустить bootstrap, выполнить build/up,
дождаться HTTP 200 readiness и затем запустить проверки. Bootstrap создаёт `.env`
и `secrets/` со случайными учебными credentials без вывода значений. Если они уже
существуют, скрипт сохраняет их и завершает работу: для повторного запуска достаточно
`docker compose up -d`. Секреты в отчёт и evidence не включаются.
API и Worker применяют SQL-миграции автоматически под PostgreSQL advisory lock.
Сервисы `minio-init` и `minio-user-init` идемпотентно создают приватный bucket,
policy и ограниченного S3 пользователя; Worker создаёт Redis consumer group.
Grafana автоматически загружает datasources/dashboard. Ручное выполнение SQL
и ручной импорт dashboard не требуются.
Безопасная остановка с сохранением named volumes:
```sh
docker compose down
```
Намеренное удаление **всех учебных данных этого Compose-проекта**: БД, Redis,
MinIO, Prometheus, Loki, Tempo и состояния Grafana. Ниже приведена только инструкция;
в рамках работы эта команда не выполнялась:
```sh
# Только для осознанного сброса собственного учебного стенда:
docker compose down --volumes
```
Локальные `.env` и `secrets/` эта команда не удаляет.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
### Основа и границы изменений
В исходном lab6 был skeleton без C++ приложения, CMakeLists.txt и рабочего compose.yaml.
В этот репозиторий перенесены исходники lab5 из соседнего checkout
(commit `580b33efe867d348285785bcccd63681ec7ae962`), миграции, MinIO bootstrap,
Nginx, инструменты проверки изображений и vendored stb. Соседний lab5 не изменён.
Сохранены API, авторизация/сессии, схемы БД, Redis Streams, transactional outbox,
retry/backoff, дедупликация, ACK после durable результата, проверка изображений,
MinIO original/variants и direct-upload расширение lab5. Изменения в job payload:
только предусмотренные контрактом optional поля `traceparent`/`tracestate`.
Когда SDK отключён, отсутствующий traceparent не записывается как пустая строка.
### Процессы, хранилища и версии SDK
```mermaid
flowchart LR
API[API x2 / opentelemetry-cpp] -->|OTLP HTTP protobuf :4318| C[OTel Collector]
W[Worker / opentelemetry-cpp] -->|OTLP HTTP protobuf :4318| C
API --> O[PostgreSQL outbox + W3C context]
O --> P[Publisher producer span]
P --> R[Redis Stream / JSON + traceparent]
R --> W
C -->|Prometheus exporter :9464| M[Prometheus / 7 days]
C -->|OTLP HTTP /otlp/v1/logs| L[Loki / 7 days]
C -->|OTLP HTTP /v1/traces| T[Tempo / 7 days]
G[Grafana] --> M
G --> L
G --> T
```
API и worker инициализируют SDK в своих `main()`. В каждом процессе отдельные
TracerProvider, MeterProvider, LoggerProvider, resource attributes:
`service.name=messenger-api|messenger-worker`, `service.version=lab6-1.0`,
`service.instance.id=<hostname контейнера>`, `deployment.environment.name=lab`.
Instrumentation scope `messenger`, version `1.0`. Используются имена OTel HTTP/messaging
и локальные документированные метрики worker/queue/pool; это не заявление о полном
соответствии всем экспериментальным semantic conventions конкретного schema URL.
SDK закреплён на `opentelemetry-cpp v1.21.0` (commit
`b9cf499ff5715433848b316059714b5c59af1f2c`), ABI 1. Экспортер только OTLP
HTTP/protobuf: `WITH_OTLP_HTTP=ON`, `WITH_OTLP_GRPC=OFF`.
Collector Contrib 0.119.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.2,
Grafana 11.6.0. PostgreSQL 16.15, Valkey 8.1.10, Nginx 1.28.0;
MinIO RELEASE.2025-04-22T22-12-26Z, mc RELEASE.2025-04-16T18-13-26Z.
Теги и конфиги закреплены в Compose. Loki использует native OTLP endpoint,
а не удалённый Loki exporter. Collector не является долговременным хранилищем;
Grafana строит запросы и визуализирует данные хранилищ. Все четыре хранилища
состояния telemetry имеют отдельные named volumes.
### Источник истины и транзакционные границы
PostgreSQL — источник истины для пользователей, чатов, сообщений, вложений,
состояний jobs и durable outbox; MinIO хранит бинарные оригиналы/варианты.
Valkey используется для Redis Streams и, при `SESSION_STORE=redis`, сессий;
в проверенной `.env` выбран `SESSION_STORE=postgres`. Брокер доставляет задания,
а не заменяет durable состояние job в PostgreSQL.
Создание job и outbox записи выполняется в одной PostgreSQL-транзакции.
Publisher отмечает outbox published после XADD; повторная доставка допускается.
Worker сохраняет durable результат/следующую попытку до ACK. Публикация ссылки на
готовые S3-объекты и статуса ready происходит транзакционно в БД. Эти правила lab5
сохранены; telemetry не участвует в принятии бизнес-решений.
### Соответствие `.env.example` конфигурации
| Переменные | Реальное применение |
| --- | --- |
| `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` | Bootstrap PostgreSQL; `secrets/database_url` должен содержать согласованный connection string. Генератор создаёт согласованные значения. |
| `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE`, S3 `*_FILE` | Compose монтирует file secrets в `/run/secrets`; entrypoint переносит их в `/tmp/app-secrets` для uid 10001. Worker и API используют один Redis/Valkey endpoint. |
| `SESSION_STORE`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` | Выбор хранилища/TTL сессий и идемпотентности API. |
| `API_REPLICAS`, `API_PORT` | Число API-реплик и внешний порт lb; `APP_HOST`/`APP_PORT` внутри Compose фиксированы как `0.0.0.0:8080`. |
| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`, MinIO credentials | Endpoint/bucket приложения и идемпотентный S3 bootstrap; приложения используют отдельного ограниченного пользователя. |
| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS`, `MAX_CONCURRENT_UPLOADS`, `DIRECT_UPLOAD_TTL_SECONDS` | Ограничения upload/image/direct-upload из lab5. |
| `JOB_MAX_ATTEMPTS`, `JOB_RECLAIM_IDLE_MS`, `JOB_RETRY_BASE_MS`, `JOB_RETRY_MAX_MS`, `JOB_TIMEOUT_SECONDS` | Retry/reclaim/backoff и watchdog Worker. |
| `STAGING_RETENTION_SECONDS`, `STAGING_CLEANUP_INTERVAL_SECONDS` | Очистка staging. |
| `WORKER_CONCURRENCY=1` | Описывает реализованный последовательный Worker; изменение этой переменной не включает дополнительные потоки. Метрика capacity равна 1. |
| `OTEL_SDK_DISABLED` | Runtime on/off SDK в обоих приложениях. |
| `OTEL_COLLECTOR_CONFIG` | Путь к Collector profile: по умолчанию 100%; для Track 1 используется отдельный 10% traces profile. |
| `GRAFANA_ADMIN_USER`, `GRAFANA_PORT`, `PROMETHEUS_PORT` | Учётная запись Grafana и loopback-порты UI. Пароль Grafana читается из отдельного file secret. |
| `APP_ENV` | В этом стенде окружение фиксировано как `deployment.environment.name=lab`; переменная не переключает ресурсные атрибуты. |
OTLP endpoint задаётся Compose как `http://otel-collector:4318`; service.name и
service.version — кодом инициализации SDK. Настройки HTTP/DB pool и CPU/RAM имеют
дополнительные defaults в Compose; проверенные лимиты приведены в экспериментах.
### Сквозная передача контекста
1. После разбора заголовков API извлекает только W3C `traceparent`/`tracestate`
через `HttpTraceContext::Extract`, создаёт SERVER span, активирует Scope.
Невалидный traceparent не становится родителем. Baggage не переносится.
2. `enqueue_attachment_job()` создаёт PRODUCER span `attachment.enqueue` и
в той же транзакции, что и job, записывает текущие W3C поля в JSON outbox.
Это существенно: позднее publisher работает в другом потоке и не имеет HTTP Scope.
3. Publisher извлекает durable контекст из JSON, создаёт PRODUCER span
`jobs:attachments publish`, подставляет **его** traceparent в публикуемый Redis JSON.
Исходный durable payload остаётся неизменным, повторная публикация получает
новый producer span с прежним причинным родителем.
4. Worker извлекает контекст из полученного JSON и создаёт CONSUMER span
`attachment.process` с родителем Redis producer. S3 операции и DB публикация
ready — дочерние CLIENT spans. В спаны не записываются object keys и SQL-параметры.
5. Если JobRepo планирует retry, `enqueue_attachment_job()` вызывается под текущим
consumer Scope. Следующая попытка продолжает тот же trace через новый outbox
producer; исходные правила retry/ACK не меняются.
Root sampling — always on, дочерние spans используют ParentBasedSampler и
уважают sampled flag родителя. API и worker ведут реальные spans операций,
а не фиксированные демонстрационные traces. Логи завершения HTTP и job получают
тот же trace_id/span_id; HTTP `request_id` сохраняется отдельно и соответствует
`X-Request-Id`, уникальность между репликами задаёт `service.instance.id`.
### Сигналы и имена в Prometheus
| SDK instrument | Prometheus | Единицы / смысл |
|---|---|---|
| http.server.requests | http_server_requests_total | бизнес-запросы; method, route template, status class |
| http.server.request.duration | http_server_request_duration_seconds_bucket/count/sum | секунды, включая чтение запроса |
| worker.jobs | worker_jobs_total | доставки; job.type, outcome |
| worker.job.duration | worker_job_duration_seconds_bucket/count/sum | секунды от consume до завершения обработки/ACK |
| attachment.ready.duration | attachment_ready_duration_seconds_bucket/count/sum | durable job creation → транзакция публикации ready |
| queue.depth | queue_depth | незавершённые durable jobs |
| queue.oldest.age | queue_oldest_age | возраст старейшей незавершённой job, секунды |
| queue.observation.timestamp | queue_observation_timestamp | Unix seconds последнего успешного наблюдения |
| db.pool.used/waiting/capacity | db_pool_used/waiting/capacity | занятые, ожидающие и все соединения PgPool процесса |
| worker.busy/capacity | worker_busy/capacity | активные задачи / 1 последовательный worker |
| process.cpu.time | process_cpu_time | user+system CPU seconds; rate = использованные ядра |
| process.memory.usage | process_memory_usage | текущий RSS bytes из /proc/self/statm |
| telemetry.sdk.errors | telemetry_sdk_errors | накопленные внутренние SDK warning/error diagnostics |
| telemetry.sdk.dropped.spans | telemetry_sdk_dropped_spans | накопленные предупреждения переполнения span batch queue |
Health endpoints исключены из RPS/errors/duration и логов завершения запроса.
Duration записывается и на ранних HTTP-ошибках; неподдержанные/неизвестные маршруты
получают `unmatched`, неизвестные методы `_OTHER`. Идентификаторы и query strings
не попадают в metric labels. Границы histogram в секундах:
`0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120`.
Worker outcomes: `success`, `failed`, `retry`, `busy`, `superseded`.
Это счётчик **попыток доставки**, а не unique jobs: повторное чтение ready job
учитывается как ещё одна успешная доставка. `retry` означает неудачную попытку
с durable перепланированием, `failed` — terminal failure/невалидная ссылка либо
исключение, оставившее доставку pending. Не смешивайте эти категории с числом
уникально опубликованных attachments.
Queue observer раз в 5 секунд читает `jobs WHERE status IN ('queued','processing')`
по отдельному соединению PostgreSQL, только в API. `max` объединяет одинаковое
наблюдение от двух реплик. Метрика включает отложенные retry и непубликованный
outbox, не включает terminal jobs и историю ACKed stream entries. Это **логический
backlog**, а не XLEN и не время обработки. При недоступной БД последнее значение
может устареть — для этого есть timestamp, панель возраста наблюдения и alert.
Соединение observer не входит в PgPool; фоновые outbox/cleaner PgPool входят.
Dashboard `Messenger — Observability`, UID `messenger-observability`, импортируется
автоматически из `infra/grafana/dashboards/messenger.json`. Есть RPS, p50/p95/p99
HTTP/worker/time-to-ready, error ratio, queue depth/age, worker saturation,
DB pools, CPU/RSS, диагностика Collector и связанные логи.
Пример percentile всех реплик:
```promql
histogram_quantile(0.95,
sum by(le)(rate(http_server_request_duration_seconds_bucket[5m])))
```
Error ratio: `5xx / все бизнес-запросы`. Ответы 4xx в numerator не входят.
При отсутствии серии 5xx numerator равен нулю (`or vector(0)`).
Worker saturation: `sum(worker_busy) / clamp_min(sum(worker_capacity), 1)`.
Histogram quantiles при отсутствии наблюдений дают NaN; dashboard не подменяет
его средним или выдуманным percentile. OTLP Prometheus exporter может ещё хранить
последние серии остановленного процесса до истечения настроенного expiration interval
20s (затем требуется ещё один scrape Prometheus).
### SLI/SLO и alerts
Учебный availability SLI = `1 - 5xx / все бизнес-запросы`, health исключён.
Цель: ≥99% за скользящие 30 минут при ненулевом трафике. Практический сигнал
быстрой диагностики: error ratio >5% за 1 минуту, `for: 30s` (`ApiHighErrorRate`).
Это локальный диагностический alert, а не burn-rate модель production SLO.
`QueueLagGrowing`: oldest age >30s, depth >0, delta(age[1m]) >0, `for: 30s`.
`QueueObservationStale`: наблюдение старше 30s или отсутствует, `for: 30s`.
Все правила, пороги, окна и пояснения сохранены в `infra/prometheus/alerts.yaml`.
Локальное состояние доступно в Prometheus `/alerts`; внешних уведомлений нет.
### Отказоустойчивость и безопасность
SDK batch queues: максимум 1024 span records и 1024 log records на процесс,
пачка 128, интервал 500ms, timeout HTTP 500ms, одна попытка экспорта.
Экспорт выполняется в фоновых потоках SDK; периодический reader метрик — 5s,
export timeout 1s. При недоступном Collector приложения продолжают работу,
возможна потеря telemetry. Graceful shutdown ограничивает ожидание каждой
из трёх providers двумя секундами.
Collector: memory_limiter первым процессором (192 MiB, spike 32 MiB, 1s),
batch 256, max 512, timeout 1s. Очередь каждого backend exporter ограничена
256 batches, 2 consumers, HTTP timeout 2s, retry с backoff 1–5s, общий лимит
15s. Docker memory limit 256 MiB и GOMEMLIMIT 200MiB ограничивают процесс.
API/worker не зависят через depends_on от Collector/Loki/Tempo/Grafana.
При отказе backend сохраняются работа API/worker и экспорт метрик в Prometheus;
Collector регистрирует ошибки/потери и после восстановления возобновляет экспорт.
Диагностика потерь: `otelcol_exporter_send_failed_spans`,
`otelcol_exporter_send_failed_log_records`, `otelcol_exporter_enqueue_failed_spans`,
`otelcol_processor_refused_spans`, `telemetry_sdk_errors`,
`telemetry_sdk_dropped_spans`. Не складывайте эти значения как точное количество
уникально потерянных spans: неуспешный экспорт может повторяться при retry.
SDK 1.21 не сообщает отдельный точный счётчик переполнения log batch queue;
он может терять логи молча. Точный аудит всех потерь потребовал бы отдельного
processor/exporter wrapper. Падение/kill процесса до flush также теряет буферы.
Телеметрия строится по allowlist: статические имена событий/операций, route
шаблоны, метод, status class/code, job.type/outcome и resource attributes.
Не экспортируются Authorization/Basic/Bearer, пароли, session tokens, тела
сообщений/запросов, filename, binary image, SQL-параметры, object keys,
signed URL и тексты исключений. SDK diagnostics считаются без вывода текста.
Отдельный privacy test ищет canary сообщения, пароль, token и base64 картинки
в свежих логах и upload trace; это проверка выбранного сценария, не доказательство
отсутствия утечек для всех возможных входов.
Пароли генерирует bootstrap, `.env` и `secrets/` исключены из Git и Docker context.
Bootstrap сохраняет существующие credentials, а не перезаписывает их.
Secret directory 0700, app secrets 0600; entrypoint копирует их в 0400 файлы
для uid 10001 и сбрасывает root. Grafana secret 0444 внутри защищённого 0700
каталога нужен для bind mount в Grafana uid 472. Telemetry backends и OTLP
доступны только в Compose network; Grafana/Prometheus/API опубликованы на loopback.
### Ключевые файлы
- `src/telemetry.hpp`, `src/telemetry.cpp`: SDK lifecycle, RAII spans, propagator,
HTTP/jobs metrics и структурные логи.
- `src/main.cpp`, `src/worker_main.cpp`: инициализация SDK и границы операций.
- `src/outbox.cpp`: durable W3C context, producer publication и retry propagation.
- `src/db.cpp`, `src/s3_client.cpp`, `src/job_repo.cpp`: pool и dependency spans,
публикация ready и end-to-end histogram.
- `src/observability_monitor.*`: отдельное наблюдение очереди, process CPU/RSS.
- `CMakeLists.txt`, `Dockerfile`, `docker-entrypoint.sh`: SDK, зависимости,
сборка и безопасное чтение Compose secrets непривилегированными процессами.
- `compose.yaml`, `infra/{otel,prometheus,loki,tempo,grafana}`: весь telemetry stack,
persistent volumes, provisioning, dashboard и alert rules.
- `scripts/bootstrap-local.py`, `student-tests/*`, `Makefile`: bootstrap и проверки.
Первичные справочные материалы:
[OTel C++ exporters](https://opentelemetry.io/docs/languages/cpp/exporters/),
[Loki native OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
[W3C Trace Context](https://www.w3.org/TR/trace-context/).
## Доказательства критериев
Проверки реализации выполнены 16.11.2026. Таблица связывает отдельные заявленные
критерии с воспроизводимыми командами, наблюдениями и сохранёнными файлами.
Офлайн `make check` проверяет структуру, но не доказывает работу приложения.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke; сохранение бизнес-API lab5 | `make test` | PASS: пользователи/сессии, чат, сообщения, идемпотентность, upload → ready, original/thumbnail и доступы | [evidence/verification.md](evidence/verification.md), публичный `tests/smoke.py` |
| Уровень 3: API-реплики, БД, S3, broker и Worker | `docker compose up -d --build`; `docker compose ps` | Две API-реплики, Worker и бизнес-хранилища запущены; API readiness healthy; upload обрабатывается | `compose.yaml`, [evidence/verification.md](evidence/verification.md) |
| Уровень 3: Collector и хранилища трёх сигналов в общем стенде | `docker compose config --quiet`; Collector `validate`; запуск Compose | OTLP HTTP receiver и три pipelines; Prometheus/Loki/Tempo/Grafana работают, конфиги и named volumes заданы | `compose.yaml`, `infra/otel/collector.yaml`, `infra/{prometheus,loki,tempo,grafana}`, [evidence/verification.md](evidence/verification.md) |
| Уровень 3: SDK API/Worker, HTTP/job counters и histogram | `make test-observability` | Реальные метрики обоих приложений; проверены имена и запросы dashboard | [evidence/observability.json](evidence/observability.json), `src/telemetry.cpp` |
| Уровень 3: реальные traces и структурные логи с корреляцией | `make test-observability` | Upload trace содержит API/Worker и S3/DB spans; Loki events обоих service.name найдены по trace ID | [evidence/observability.json](evidence/observability.json) |
| Уровень 3: низкая кардинальность и отсутствие приватных payloads в проверенном сценарии | Privacy checks внутри `make test-observability` | Нет raw chat ID/запрещённых labels, canary сообщения, пароля, token и base64 изображения в проверенных logs/trace | [evidence/observability.json](evidence/observability.json), allowlist в `src/telemetry.cpp` |
| Уровень 3: импортируемые dashboard/datasources | `make test-observability` | Provisioned dashboard найден; 25 PromQL queries проверены | `infra/grafana/dashboards/messenger.json`, `infra/grafana/provisioning`, [evidence/observability.json](evidence/observability.json) |
| Уровень 4: W3C API → outbox → Redis → Worker | `make test-observability`; `/app/telemetry_test` | Consumer parent — producer, producer продолжает durable outbox context; invalid context и route privacy self-test PASS | [evidence/observability.json](evidence/observability.json), `student-tests/telemetry_test.cpp` |
| Уровень 4: цепочка до S3/DB, включая retry | `make test-incidents`, ограниченный отказ S3 | Retry сохранил trace ID, следующая попытка дошла до ready; parent-child цепочка проверена | [evidence/incidents.json](evidence/incidents.json), событие `retry_same_trace_ready_verified` |
| Уровень 4: p50/p95/p99 из histogram всех реплик | `make test-observability` | Запросы с `histogram_quantile` и `sum by(le)` выполняются; единицы seconds документированы | `infra/grafana/dashboards/messenger.json`, [evidence/observability.json](evidence/observability.json) |
| Уровень 4: queue depth/age, DB pool, Worker saturation, process CPU/RAM и time-to-ready | `make test-observability`; остановка Worker | Метрики присутствуют; при остановке Worker depth=1, oldest age=61.10s, после запуска backlog=0 | [evidence/observability.json](evidence/observability.json), [evidence/incidents.json](evidence/incidents.json) |
| Уровень 4: проверяемый SLI/SLO и alert rules | `promtool check config /etc/prometheus/prometheus.yaml`; `promtool check rules /etc/prometheus/alerts.yaml` в контейнере Prometheus | Конфиг и 3 правила валидны; задан availability ≥99% / 30min; достижение SLO за полное окно отдельно не доказывалось | `infra/prometheus/alerts.yaml`, [evidence/verification.md](evidence/verification.md), подраздел SLI/SLO |
| Уровень 4: alerts при росте queue lag и DB outage | `make test-incidents` | `QueueLagGrowing` и `ApiHighErrorRate` прошли firing → resolved | [evidence/incidents.json](evidence/incidents.json), [evidence/verification.md](evidence/verification.md) |
| Уровень 4: ограниченные очереди и отказ telemetry backend | `make test-incidents`, остановка Tempo/Loki | Upload дошёл до ready, сообщения записывались; +46 failed span export attempts, RSS Collector 179634176 bytes; сервисы восстановлены | [evidence/incidents.json](evidence/incidents.json), `infra/otel/collector.yaml`, `src/telemetry.cpp` |
| Уровень 4: локальные runbooks двух инцидентов | Диагностика Worker stopped / DB unavailable по подразделу ниже | Описаны сигналы, различение причин, команды восстановления и проверка resolved | Подраздел «Диагностика контролируемых инцидентов», [evidence/verification.md](evidence/verification.md) |
| Уровень 5, Track 1: одинаковая нагрузка Off / On 100% / Collector 10% | `python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3` | 9 прогонов ×1000 uploads, concurrency 4; все 9000 HTTP 202/jobs ready, docker stats и ingest counters сохранены | [evidence/track1-benchmark.md](evidence/track1-benchmark.md), [evidence/track1-benchmark.json](evidence/track1-benchmark.json) |
| Уровень 5, Track 1: цена сигналов, потери редких ошибок, выбор конфигурации | Тот же benchmark и отдельный ERROR probe | API CPU +22.5% по медианам средних; C сохранил 10.92% spans; в Tempo найдены 9 из 100 synthetic ERROR traces; компромиссы объяснены | Подраздел «Track 1: Цена наблюдаемости», [evidence/track1-benchmark.md](evidence/track1-benchmark.md) |
| Восстановление после эксперимента и проверки инструмента | `python3 scripts/benchmark_telemetry.py --self-test`; затем `make test`, `make test-observability` | Парсеры/trace-only config self-test PASS; SDK on и Collector 100% восстановлены; повторные smoke PASS | [evidence/track1-benchmark.json](evidence/track1-benchmark.json), скрипт benchmark, [evidence/observability.json](evidence/observability.json) |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Сохранённые результаты проверок
Track 1: методика и итоговая таблица — [evidence/track1-benchmark.md](evidence/track1-benchmark.md);
исходные latency/Docker stats/counters — [evidence/track1-benchmark.json](evidence/track1-benchmark.json).
Проверено 16.11.2026 на финальном образе: Docker build, Compose config,
Collector validate, promtool (конфиг и 3 правила), C++ self-test, публичный
`make test` с telemetry on/off — PASS. Сквозная проверка реального upload,
связанных логов, privacy canaries и 25 запросов provisioned dashboard — PASS.
Полный поставляемый сценарий инцидентов на финальном образе — PASS: оба alerts
firing/resolved, retry в том же trace до ready, работа API/worker при остановленных
Loki/Tempo и последующее восстановление сервисов.
Нагрузка: 10338 успешных POST за 30.026s, 0 ошибок,
344.305 successful RPS, клиентские p50/p95/p99 =
5.666/52.505/61.702 ms.
Это closed-loop тест, concurrency 4, percentile только успешных запросов;
он не доказывает предельную производительность или соблюдение 30-минутного SLO.
Даты/времена инцидентов и итоговые проверки: `evidence/verification.md`.
Длительный memory soak и исчерпывающий privacy-аудит не проводились.
Автоматические evidence файлы не содержат токены, пароли или тела сообщений:
`evidence/observability.json` — реальный trace ID, имена spans, проверенные метрики,
связь producer/consumer и query checks; `evidence/incidents.json` — времена
firing/resolved, агрегированные response codes и RSS Collector.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Нагрузочный smoke
Хост: Intel Core i5-12450H, 12 logical CPU, 14.85 GiB RAM. Лимиты API —
2 ×0.5 CPU/256 MiB, Worker — 0.5 CPU/512 MiB; образы и SDK закреплены выше.
Сценарий `python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json`:
closed-loop POST сообщений без автоматических retry. Получено 10338 успешных
запросов за 30.026s, ошибок 0, 344.305 RPS, p50/p95/p99 успешных запросов
5.666/52.505/61.702ms. Это один smoke-прогон, не измерение предельной мощности;
CPU/RAM этим генератором не снимались. Полный сравнительный эксперимент вложений,
включая CPU/RAM и прогрев/повторы, приведён в Track 1.
### Контролируемые отказы
Команда `make test-incidents` останавливает/восстанавливает реальные Compose-сервисы
`worker`, `db`, `minio`, `tempo`, `loki`; volumes не удаляет. Для S3 retry применяется
pause/unpause уже запущенного Worker. Это прямые действия скрипта, а не
проверка restart/failover hook; отдельные hooks в этой работе не заявляются.
Времена ниже рассчитаны по `evidence/incidents.json`, дата — 16.11.2026.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановка `worker`, затем upload | HTTP 202/queued, рост backlog/age, alert; после запуска job ready | `QueueLagGrowing` firing через 74.1s после принятия upload; depth=1, oldest age=61.10s; затем backlog=0 и resolved | От firing до drained/resolved 20.6s; та же attachment дошла до ready |
| Остановка `db`, валидные reads | 5xx, ошибки DB dependency, `ApiHighErrorRate` firing | Зафиксированы 1×503 и 87×502 на внешнем API; alert firing через 44.0s, после запуска БД чтение восстановлено | Firing → resolved за 60.1s, включая вымывание ошибок из минутного окна; отдельный аудит всех COMMIT не проводился |
| Остановка `minio` при существующем Worker | Неудачная попытка → durable retry → ready в прежнем trace | `s3_failure_scheduled_retry` → `retry_same_trace_ready_verified`, trace `997e005d6cc34eaa869fc3ced3e479ee` | Между этими событиями 1.23s; job дошёл до ready без смены trace |
| Остановка `tempo` и `loki` | API/Worker работают, экспорт ошибается, память/очереди ограничены | Во время outage upload ready, сообщения сохранялись; +46 failed span export attempts, RSS Collector 171.31 MiB при Docker limit 256 MiB | Работа приложения подтверждена через 21.3s намеренного outage; backends запущены, последующие smoke PASS. Потеря части telemetry допустима; failed attempts не равны уникальным потерям spans |
Точное время команд restart/start не измерялось отдельно: указанные интервалы
связаны с сохранёнными событиями наблюдения, а не обещанием recovery SLA.
### Диагностика контролируемых инцидентов
`make test-incidents` создаёт только синтетические данные. Скрипт временно
останавливает worker, БД и затем Loki/Tempo по очереди; каждый остановленный
сервис запускается обратно в `finally`. Volumes не удаляются.
Не запускайте сценарий на общем/боевом стенде с реальными пользователями.
1. **Worker stopped.** Upload остаётся 202/queued, depth и oldest age растут,
API продолжает отвечать, нет свежих consumer spans после producer.
Проверить QueueLagGrowing firing в Prometheus, worker container state и logs.
`docker compose start worker`; та же attachment становится ready, backlog
уменьшается до 0, alert resolved. Низкий busy сам по себе не доказывает
отсутствие очереди: проверять также age и свежесть наблюдения.
2. **DB unavailable.** Валидные аутентифицированные reads возвращают 5xx,
ApiHighErrorRate firing. В Tempo SERVER span error, задержка/ошибка
`postgresql.pool.acquire`; `db_pool_waiting` позволяет отличить ожидание
соединения от медленного S3. Queue observation становится stale; это не
означает нулевой backlog. После `docker compose start db` pool переподключается,
запросы снова успешны, alert resolved после вымывания ошибок из минутного окна.
3. **Tempo/Loki unavailable.** Upload проходит до ready, сообщения сохраняются.
Collector exporter failures растут; очередь ограничена, RSS доступен в
`otelcol_process_memory_rss`. После возврата backend exporter восстанавливается;
пакеты, истёкшие по retry/queue limit, не восстанавливаются из application data.
### Track 1: Цена наблюдаемости
**Гипотеза.** Telemetry 100% увеличивает CPU/RAM API и Worker. Sampling 10% в
Collector уменьшает экспорт в Tempo примерно на 90%, но не стоимость SDK и не
входящий OTLP ingest. Ожидание «CPU +20–40%» проверяется, а не подставляется в результаты.
**Методика.** 16.11.2026, Intel Core i5-12450H, 12 logical CPU, 14.85 GiB RAM.
Две API-реплики по 0.5 CPU/256 MiB, Worker 0.5 CPU/512 MiB; одинаковый binary
и бизнес-зависимости. По три прогона каждого варианта, порядок ABC / BCA / CAB.
Перед каждым — 20 warm-up uploads, затем **1000 POST вложений, concurrency 4**:
PNG 320×200, 111817 bytes. Все 9000 измеренных uploads вернули 202 и дошли до ready;
очередь между прогонами полностью обработана, volumes не удаляются.
A: SDK disabled; B: SDK ParentBased(AlwaysOn), Collector 100%; C: SDK остаётся
100%, только traces pipeline получает `probabilistic_sampler/track1`
(`sampling_percentage: 10`, `hash_seed: 22`) после memory_limiter и перед batch.
Внешний traceparent не задаётся. Логи и метрики в C сохраняются на 100%.
Off оставляет no-op обёртки и background observer: это runtime SDK on/off,
а не сравнение со сборкой без инструментации.
Latency/mean/RPS измерены клиентом до HTTP 202, не до ready; p50/p99 — nearest rank.
CPU/RAM — средние `docker stats` за POST + полный drain Worker; API = сумма двух
реплик, 100% CPU = одно logical core, RAM = Docker working set в MiB, не peak RSS.
Spans/s = delta `otelcol_receiver_accepted_spans` / окно POST + drain + финальный flush.
Collector пересоздаётся перед прогоном; delta берётся внутри одного его запуска.
Ниже медианы трёх прогонов, включая медиану индивидуальных p50/p99, не pooled percentile.
| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |
|---|---:|---:|---:|---:|---:|---:|---:|---:|
| A: Off | 84.37 | 27.30 | 202.70 | 10.32% | 14.22 | 34.79% | 6.88 | 0.00 |
| B: On, SDK 100% / Collector 100% | 75.44 | 30.46 | 288.30 | 12.64% | 17.05 | 38.48% | 7.81 | 391.01 |
| C: On, SDK 100% / Collector 10% | 75.28 | 30.48 | 204.05 | 13.31% | 16.58 | 39.12% | 7.88 | 412.19 |
Медианное **среднее HTTP-время** A/B/C: **46.89 / 52.56 / 52.82 ms**.
Отдельные прогоны, дополнительные counters и длительности —
[evidence/track1-benchmark.md](evidence/track1-benchmark.md);
исходные latency samples, Docker stats, SHA256 конфигов и hardware/limits —
[evidence/track1-benchmark.json](evidence/track1-benchmark.json).
**Цена CPU/RAM.** B против A: средний CPU API **+22.5%**, Worker **+10.6%**;
RAM API **+19.9%** (+2.83 MiB на две реплики), Worker **+13.5%** (+0.93 MiB).
Но CPU% усредняется за окна разной длины. Для одинакового объёма работы оценка
CPU·мс/job = (CPU% / 100) × pipeline_seconds × 1000 / jobs_count; медиана её
парного прироста по повторам — **+9.9% API, +2.2% Worker**.
Диапазон +20–40% наблюдался у среднего CPU API, но не является универсальной
надбавкой SDK и не подтвердился для нормированного CPU Worker.
**Ingest и экспорт.** В B и C принято примерно 28.2 тыс. spans на 1000 uploads.
Медианные accepted/sent: B **28231 / 28230**, C **28218 / 3083**.
C сохраняет медианно **10.92%** spans, отбрасывая около **89.08%** перед Tempo.
Это вероятностный отбор по trace ID, а не точная квота. Исходящий поток в Tempo
снизился с **391.00 до 44.78 spans/s**. Входящий ingest не уменьшился: SDK всё
создаёт и отправляет; разница его rate связана с длительностью окна.
Это количества spans, **не bytes**. В каждом on-прогоне принято 3000 log records:
экономия относится к traces, а не ко всем трём сигналам. Разгрузка CPU/RAM
приложений вариантом C не доказана.
**Какие редкие ошибки теряются.** Отдельно от замеров API отправлены 100
синтетических ERROR-traces со status 503: в Tempo найдены **9**, отсутствуют **91**.
Это проверка sampler, не 100 реальных отказов API. Чистый probabilistic sampler
не учитывает тяжесть ошибки: единичный 5xx, неудачный Worker/S3 retry или редкий
медленный запрос теряет подробный trace примерно с вероятностью 90%.
Вероятность потерять все пять независимых редких error traces — 0.9⁵ ≈ 59%.
Ошибки в метриках Prometheus и ERROR logs Loki остаются; ссылка из сохранённого
лога может не найти trace в Tempo. Поведение соответствует
[probabilistic sampler 0.119.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.119.0/processor/probabilisticsamplerprocessor/README.md).
**Выбор для production.** Из A/B/C — **C при ограниченном бюджете хранения**:
полные метрики/логи и около десятой части traces. A лишает систему наблюдаемости;
B полезен при малом трафике и диагностике, но сохраняет все traces. Для критичных
потоков чистого C недостаточно: предпочтительнее error/slow-aware tail sampling,
сохраняющий ERROR и медленные traces плюс 10% обычных. Это рекомендация, не
дополнительная реализованная политика. Для async Worker decision window должен
учитывать очередь/retry и иметь ограниченную память. Для снижения именно CPU SDK
или OTLP ingress sampling потребуется в SDK, с риском потерять ошибки ещё до Collector.
**Качество и ограничения.** Во всех прогонах HTTP errors, refused spans и failed
span exports = 0; внутри on-окон delta SDK diagnostics = 0, максимум счётчика
переполнения span batch queue = 0. До начала некоторых окон SDK diagnostics уже
равнялся 4: startup/reconfiguration исключены из steady-state замеров. Разница
accepted/sent в B до нескольких spans объяснима асинхронными границами и фоновыми
health/outbox spans; counters не являются строгим аудитом всех возможных потерь SDK.
Closed-loop, один хост, три повтора, без CPU pinning и изоляции фоновой нагрузки.
БД/S3 растут; порядок вариантов ротируется. В третьем повторе pipeline A/B занял
140.02/137.77s вместо примерно 64–80s в других прогонах; причина отдельно не
профилировалась, результаты не удалялись. p99 отдельных прогонов:
A 201.98–413.17ms, B 200.24–437.94ms, C 202.22–212.64ms. По медианному p99 нельзя
доказывать, что C быстрее B; весь рост CPU% нельзя приписать SDK. Startup cost,
точные OTLP bytes и backend CPU/RAM не измерялись. После завершения восстановлены
SDK on и Collector 100%; API/observability smoke повторно прошли.
Воспроизведение:
```sh
python3 scripts/benchmark_telemetry.py --self-test
python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3
```
Для Docker Snap при необходимости передать `--compose /path/to/docker-compose`
и `--docker /path/to/docker`. Скрипт восстанавливает baseline в `finally`.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- API/Worker выдержали ограниченные остановки Worker, БД, S3 и telemetry backends
с описанным восстановлением; это не HA-гарантия для БД, lb, MinIO или всего хоста.
Persistent volumes заданы, но отдельный полный persistence/failover hook в lab6
не проверялся. Достижение availability SLO ≥99% за 30 минут отдельно не измерялось.
- Временные ошибки экспорта, переполнение bounded buffers и аварийный kill могут
терять telemetry. SDK 1.21 не даёт точного счётчика потерь log batch queue;
export failures/SDK diagnostics не являются строгим аудитом уникальных потерь.
- Queue depth — логические незавершённые jobs из PostgreSQL, не Redis XLEN.
При отказе БД значения устаревают, что отмечается отдельным alert. PgPool metrics
не включают dedicated connection observer. Worker последовательный, capacity=1.
- Low-cardinality allowlist и privacy canaries проверены для синтетического сценария;
исчерпывающий privacy/security audit и длительный memory soak не проводились.
Наблюдение RSS за 21.3s outage не заменяет длительный тест ограниченности памяти.
- Track 1 — closed-loop, один хост, 3 повтора без CPU pinning/изоляции фоновой нагрузки,
с растущей БД/S3. Все результаты, включая медленные третьи прогоны, сохранены.
Выводы относятся к этим лимитам/PNG/нагрузке, не к произвольному production.
- Подтверждено: Collector sampling уменьшает число отправленных в Tempo spans
примерно на 89%, сохраняя metrics/logs; не подтверждено снижение CPU SDK,
OTLP bytes или backend CPU/стоимости хранения в деньгах. Сохранение только 9 из
100 synthetic ERROR traces показывает отсутствие error-aware гарантии.
- Следующие шаги: отдельный хост/CPU pinning и более длинные повторы для устойчивой
оценки overhead; измерение OTLP bytes и ingest/storage CPU; эксперимент с SDK
sampling, если нужна экономия приложения. Error/slow-aware tail sampling с bounded
memory и окном, учитывающим async queue/retry, остаётся рекомендацией и требует
отдельной проверки. Также нужны длительный memory soak и полный аудит потерь.
+55
View File
@@ -0,0 +1,55 @@
# Provide the imported target names expected by minio-cpp's CMakeLists
# (unofficial-curlpp / unofficial-inih) using system packages from apt.
find_package(OpenSSL REQUIRED)
find_package(ZLIB REQUIRED)
find_package(CURL REQUIRED)
# nlohmann_json (header-only; Ubuntu ships /usr/include/nlohmann/json.hpp)
if(NOT TARGET nlohmann_json::nlohmann_json)
find_package(nlohmann_json QUIET CONFIG)
endif()
if(NOT TARGET nlohmann_json::nlohmann_json)
add_library(nlohmann_json INTERFACE)
add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json)
find_path(NLOHMANN_JSON_INCLUDE_DIR nlohmann/json.hpp)
if(NOT NLOHMANN_JSON_INCLUDE_DIR)
message(FATAL_ERROR "nlohmann/json.hpp not found (install nlohmann-json3-dev)")
endif()
target_include_directories(nlohmann_json INTERFACE "${NLOHMANN_JSON_INCLUDE_DIR}")
endif()
# pugixml
if(NOT TARGET pugixml)
find_package(pugixml QUIET CONFIG)
endif()
if(NOT TARGET pugixml AND NOT TARGET pugixml::pugixml)
pkg_check_modules(PUGIXML REQUIRED IMPORTED_TARGET pugixml)
add_library(pugixml INTERFACE)
target_link_libraries(pugixml INTERFACE PkgConfig::PUGIXML)
endif()
if(TARGET pugixml::pugixml AND NOT TARGET pugixml)
add_library(pugixml ALIAS pugixml::pugixml)
endif()
# curlpp → unofficial::curlpp::curlpp
if(NOT TARGET unofficial::curlpp::curlpp)
find_library(CURLPP_LIBRARY NAMES curlpp REQUIRED)
find_path(CURLPP_INCLUDE_DIR curlpp/cURLpp.hpp REQUIRED)
add_library(unofficial_curlpp INTERFACE)
target_include_directories(unofficial_curlpp INTERFACE "${CURLPP_INCLUDE_DIR}")
target_link_libraries(unofficial_curlpp INTERFACE "${CURLPP_LIBRARY}" CURL::libcurl)
add_library(unofficial::curlpp::curlpp ALIAS unofficial_curlpp)
endif()
# inih INIReader → unofficial::inih::inireader
if(NOT TARGET unofficial::inih::inireader)
find_library(INIREADER_LIBRARY NAMES INIReader REQUIRED)
find_library(INIH_LIBRARY NAMES inih REQUIRED)
find_path(INIREADER_INCLUDE_DIR INIReader.h REQUIRED)
add_library(unofficial_inih_inireader INTERFACE)
target_include_directories(unofficial_inih_inireader INTERFACE "${INIREADER_INCLUDE_DIR}")
target_link_libraries(unofficial_inih_inireader INTERFACE
"${INIREADER_LIBRARY}" "${INIH_LIBRARY}")
add_library(unofficial::inih::inireader ALIAS unofficial_inih_inireader)
endif()
+340
View File
@@ -0,0 +1,340 @@
services:
db:
image: postgres:16.15-alpine
command: postgres -c max_connections=200 -c shared_buffers=256MB
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
cpus: ${DB_CPU_LIMIT:-1.0}
mem_limit: 512m
stop_grace_period: 20s
cache:
image: valkey/valkey:8.1.10-alpine
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
start_period: 5s
cpus: "0.50"
mem_limit: 256m
stop_grace_period: 15s
api: &api
build: .
image: mtusi2026lab6-api:local
depends_on:
db:
condition: service_healthy
cache:
condition: service_healthy
minio:
condition: service_healthy
minio-user-init:
condition: service_completed_successfully
environment:
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
APP_HOST: 0.0.0.0
APP_PORT: "8080"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-redis}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
BROKER_URL_FILE: /run/secrets/cache_url
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
HTTP_WORKERS: ${HTTP_WORKERS:-8}
DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
DIRECT_UPLOAD_TTL_SECONDS: ${DIRECT_UPLOAD_TTL_SECONDS:-300}
# Optional: sign URLs for a host-reachable MinIO (compose.presign.yaml).
S3_PRESIGN_ENDPOINT: ${S3_PRESIGN_ENDPOINT:-}
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: ${API_CPU_LIMIT:-0.50}
memory: ${API_MEM_LIMIT:-256m}
cpus: ${API_CPU_LIMIT:-0.50} # для совместимости с docker compose up
mem_limit: ${API_MEM_LIMIT:-256m}
stop_grace_period: 15s
worker:
<<: *api
# Keep the shared entrypoint: it stages secrets and drops root privileges.
entrypoint: ["/docker-entrypoint.sh", "/app/worker"]
command: []
restart: unless-stopped
environment:
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-6}
JOB_RECLAIM_IDLE_MS: ${JOB_RECLAIM_IDLE_MS:-5000}
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000}
JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-60000}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60}
healthcheck:
disable: true
deploy:
replicas: 1
resources:
limits:
cpus: "0.50"
memory: 512m
cpus: "0.50"
mem_limit: 512m
lb:
image: nginx:1.28.0-alpine
depends_on:
api:
condition: service_healthy
ports:
- "127.0.0.1:${API_PORT:-8080}:8080"
environment:
NGINX_WORKERS: ${NGINX_WORKERS:-1}
NGINX_ENVSUBST_TEMPLATE_DIR: /etc/nginx/templates
NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx
NGINX_ENVSUBST_FILTER: "^NGINX_"
volumes:
- ./lb/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
cpus: ${LB_CPU_LIMIT:-0.25}
mem_limit: 128m
stop_grace_period: 10s
# Private S3-compatible store. Ports 9000/9001 are NOT published to the host.
# Image: pinned MinIO RELEASE.2025-04-22T22-12-26Z via local Dockerfile
# (official binary from GitHub; docker.io hub may be unreachable).
minio:
build:
context: ./infra/minio
args:
MINIO_VERSION: RELEASE.2025-04-22T22-12-26Z
image: mtusi2026lab5-minio:RELEASE.2025-04-22T22-12-26Z
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
volumes:
- miniodata:/data
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
cpus: ${MINIO_CPU_LIMIT:-0.50}
mem_limit: ${MINIO_MEM_LIMIT:-512m}
stop_grace_period: 15s
# Idempotent bucket bootstrap: create if missing, force private ACL.
minio-init:
build:
context: ./infra/mc
args:
MC_VERSION: RELEASE.2025-04-16T18-13-26Z
image: mtusi2026lab5-mc:RELEASE.2025-04-16T18-13-26Z
depends_on:
minio:
condition: service_healthy
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
echo "minio-init: waiting for endpoint $$S3_ENDPOINT"
i=0
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
i=$$((i + 1))
if [ "$$i" -ge 30 ]; then
echo "minio-init: cannot reach MinIO" >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing "local/$$S3_BUCKET"
mc anonymous set none "local/$$S3_BUCKET"
echo "minio-init: bucket listing:"
mc ls "local/$$S3_BUCKET" || true
echo "minio-init: bucket $$S3_BUCKET ready (anonymous=none)"
restart: "no"
# Idempotent limited S3 user for the API (not MinIO root).
minio-user-init:
image: mtusi2026lab5-mc:RELEASE.2025-04-16T18-13-26Z
depends_on:
minio:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
MINIO_USER_ACCESS_KEY: ${MINIO_USER_ACCESS_KEY}
MINIO_USER_SECRET_KEY: ${MINIO_USER_SECRET_KEY}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
POLICY_NAME: messenger-attachments-rw
volumes:
- ./infra/minio/policy-attachments.json:/policy.template.json:ro
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
echo "minio-user-init: alias $$S3_ENDPOINT"
i=0
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
i=$$((i + 1))
if [ "$$i" -ge 30 ]; then
echo "minio-user-init: cannot reach MinIO" >&2
exit 1
fi
sleep 1
done
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy.template.json > /tmp/policy.json
if mc admin policy info local "$$POLICY_NAME" >/dev/null 2>&1; then
mc admin policy remove local "$$POLICY_NAME" >/dev/null 2>&1 || true
fi
mc admin policy create local "$$POLICY_NAME" /tmp/policy.json
if mc admin user info local "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1; then
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY already exists"
else
mc admin user add local "$$MINIO_USER_ACCESS_KEY" "$$MINIO_USER_SECRET_KEY"
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY created"
fi
mc admin policy attach local "$$POLICY_NAME" --user "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1 || true
echo "minio-user-init: policy $$POLICY_NAME attached to $$MINIO_USER_ACCESS_KEY"
restart: "no"
otel-collector:
image: otel/opentelemetry-collector-contrib:0.119.0
command: [--config=/etc/otelcol/config.yaml]
restart: unless-stopped
mem_limit: 256m
environment:
GOMEMLIMIT: 200MiB
volumes:
- ${OTEL_COLLECTOR_CONFIG:-./infra/otel/collector.yaml}:/etc/otelcol/config.yaml:ro
prometheus:
image: prom/prometheus:v3.2.1
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d]
restart: unless-stopped
mem_limit: 512m
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"]
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheusdata:/prometheus
loki:
image: grafana/loki:3.4.2
command: [-config.file=/etc/loki/config.yaml]
restart: unless-stopped
mem_limit: 512m
volumes:
- ./infra/loki/loki.yaml:/etc/loki/config.yaml:ro
- lokidata:/loki
tempo:
image: grafana/tempo:2.7.2
user: "0:0"
command: [-config.file=/etc/tempo/config.yaml]
restart: unless-stopped
mem_limit: 512m
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/config.yaml:ro
- tempodata:/var/tempo
grafana:
image: grafana/grafana:11.6.0
restart: unless-stopped
mem_limit: 256m
ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"]
environment:
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
GF_USERS_ALLOW_SIGN_UP: "false"
GF_PLUGINS_PREINSTALL_DISABLED: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
secrets: [grafana_admin_password]
volumes:
- grafanadata:/var/lib/grafana
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
secrets:
grafana_admin_password:
file: ./secrets/grafana_admin_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata: {}
valkeydata: {}
miniodata: {}
prometheusdata: {}
lokidata: {}
tempodata: {}
grafanadata: {}
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
set -eu
# Compose bind-backed secrets retain host permissions. Stage for application uid.
if [ "$(id -u)" = "0" ]; then
mkdir -p /tmp/app-secrets
chmod 0700 /tmp/app-secrets
for name in database_url cache_url s3_access_key_id s3_secret_access_key; do
if [ -r "/run/secrets/$name" ]; then
cp "/run/secrets/$name" "/tmp/app-secrets/$name"
chown 10001:10001 "/tmp/app-secrets/$name"
chmod 0400 "/tmp/app-secrets/$name"
fi
done
chown 10001:10001 /tmp/app-secrets
export DATABASE_URL_FILE=/tmp/app-secrets/database_url
export CACHE_URL_FILE=/tmp/app-secrets/cache_url
export BROKER_URL_FILE=/tmp/app-secrets/cache_url
export S3_ACCESS_KEY_ID_FILE=/tmp/app-secrets/s3_access_key_id
export S3_SECRET_ACCESS_KEY_FILE=/tmp/app-secrets/s3_secret_access_key
exec setpriv --reuid=10001 --regid=10001 --clear-groups -- "$@"
fi
exec "$@"
+13
View File
@@ -3,3 +3,16 @@
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
Результаты реализации lab6:
- `verification.md` — итоговые проверки и время инцидентов (Москва).
- `observability.json` — реальный trace, parent-child связь, проверенные метрики/PromQL.
- `incidents.json` — firing/resolved, retry trace и ограниченный отказ telemetry backend.
- `load.json` — 30-секундный closed-loop прогон синтетических POST сообщений.
Запросы Grafana находятся в `infra/grafana/dashboards/messenger.json`, методика — в `REPORT.md`.
Track 1: `track1-benchmark.md` содержит сравнение A/B/C, `track1-benchmark.json` —
все отдельные прогоны, latency samples, Docker stats, counters Collector и параметры хоста.
Методика: `python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3`.
+56
View File
@@ -0,0 +1,56 @@
[
{
"event": "worker_stopped_upload_accepted",
"timestamp": "2026-11-16T07:30:35.969738+00:00"
},
{
"event": "QueueLagGrowing_firing",
"timestamp": "2026-11-16T07:31:50.091878+00:00",
"depth": 1.0,
"oldest_seconds": 61.100729
},
{
"event": "QueueLagGrowing_resolved_backlog_drained",
"timestamp": "2026-11-16T07:32:10.680368+00:00"
},
{
"event": "database_stopped",
"timestamp": "2026-11-16T07:32:10.928124+00:00"
},
{
"event": "ApiHighErrorRate_firing",
"timestamp": "2026-11-16T07:32:54.959429+00:00",
"response_counts": {
"503": 1,
"502": 87
}
},
{
"event": "ApiHighErrorRate_resolved_database_recovered",
"timestamp": "2026-11-16T07:33:55.054932+00:00"
},
{
"event": "s3_failure_scheduled_retry",
"timestamp": "2026-11-16T07:33:57.449574+00:00",
"trace_id": "997e005d6cc34eaa869fc3ced3e479ee"
},
{
"event": "retry_same_trace_ready_verified",
"timestamp": "2026-11-16T07:33:58.678216+00:00",
"trace_id": "997e005d6cc34eaa869fc3ced3e479ee"
},
{
"event": "tempo_loki_stopped",
"timestamp": "2026-11-16T07:34:08.938180+00:00"
},
{
"event": "telemetry_outage_api_worker_healthy",
"timestamp": "2026-11-16T07:34:30.248711+00:00",
"failed_span_exports": 46.0,
"collector_rss_bytes": 179634176.0
},
{
"event": "tempo_loki_restarted",
"timestamp": "2026-11-16T07:34:30.765794+00:00"
}
]
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 6,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.026,
"concurrency": 4,
"successful_requests": 10338,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 344.305,
"successful_latency_ms": {
"p50": 5.666,
"p95": 52.505,
"p99": 61.702
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+39
View File
@@ -0,0 +1,39 @@
{
"timestamp": "2026-11-16T15:21:12.712973+00:00",
"result": "PASS",
"trace_id": "c3280a5a6f0d494f8f62834603b982a0",
"span_names": [
"POST /api/v1/chats/{chat_id}/attachments",
"attachment.enqueue",
"attachment.process",
"jobs:attachments publish",
"postgresql.pool.acquire",
"postgresql.publish_ready",
"s3.delete_object",
"s3.get_object",
"s3.put_object",
"s3.stat_object"
],
"consumer_parent_is_producer": true,
"log_services": [
"messenger-api",
"messenger-worker"
],
"metric_names_checked": [
"http_server_requests_total",
"http_server_request_duration_seconds_bucket",
"worker_jobs_total",
"worker_job_duration_seconds_bucket",
"attachment_ready_duration_seconds_bucket",
"queue_depth",
"queue_oldest_age",
"db_pool_used",
"db_pool_waiting",
"worker_busy",
"worker_capacity",
"process_memory_usage",
"process_cpu_time"
],
"dashboard_queries_checked": 25,
"privacy_canaries": "absent in recent logs and upload trace"
}
File diff suppressed because it is too large Load Diff
+59
View File
@@ -0,0 +1,59 @@
# Track 1: Цена наблюдаемости
Результат: **PASS**. Начало UTC: 2026-11-16T14:52:30.645168+00:00.
По 3 прогона каждого варианта, 1000 POST uploads, concurrency 4, warm-up 20 uploads.
Порядок ABC / BCA / CAB. PNG: 111817 bytes. Каждый job дошёл до ready; очередь между прогонами пуста.
Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).
RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.
CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.
| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |
|---|---:|---:|---:|---:|---:|---:|---:|---:|
| A: Off | 84.37 | 27.30 | 202.70 | 10.32% | 14.22 | 34.79% | 6.88 | 0.00 |
| B: On, SDK 100% / Collector 100% | 75.44 | 30.46 | 288.30 | 12.64% | 17.05 | 38.48% | 7.81 | 391.01 |
| C: On, SDK 100% / Collector 10% | 75.28 | 30.48 | 204.05 | 13.31% | 16.58 | 39.12% | 7.88 | 412.19 |
| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |
|---|---:|---:|---:|---:|---:|---:|---:|
| A | 46.89 | 79.95 | 0 | 0 | 0.00 | 7.80 | 27.81 |
| B | 52.56 | 69.12 | 28231 | 28230 | 391.00 | 8.57 | 26.60 |
| C | 52.82 | 65.37 | 28218 | 3083 | 44.78 | 8.75 | 25.82 |
*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.
Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.
В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.
| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |
|---|---:|---:|---:|---:|---:|---:|
| A/1 | 79.03 | 50.13 | 201.98 | 79.95 | 320 / 160 | 0 / 0 |
| B/1 | 78.94 | 50.20 | 200.24 | 69.12 | 276 / 138 | 0 / 0 |
| C/1 | 75.28 | 52.82 | 204.05 | 65.37 | 260 / 130 | 0 / 0 |
| B/2 | 75.44 | 52.56 | 288.30 | 65.36 | 260 / 130 | 0 / 0 |
| C/2 | 77.95 | 50.86 | 202.22 | 64.91 | 258 / 129 | 0 / 0 |
| A/2 | 85.62 | 46.41 | 202.70 | 63.87 | 254 / 127 | 0 / 0 |
| C/3 | 68.84 | 57.60 | 212.64 | 73.28 | 292 / 146 | 0 / 0 |
| A/3 | 84.37 | 46.89 | 413.17 | 140.02 | 560 / 280 | 0 / 0 |
| B/3 | 69.37 | 57.14 | 437.94 | 137.77 | 550 / 275 | 0 / 0 |
SDK audit: delta warning/error diagnostics в on-окнах: [0.0]; значения перед началом окон: [0.0, 4.0]. Startup/reconfiguration не входит в steady-state замер.
Максимум telemetry_sdk_dropped_spans в интервале эксперимента: 0.0. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.
Отдельный synthetic ERROR probe (вне замеров API): {"type": "synthetic ERROR spans, separate from upload benchmark", "sent": 100, "found_in_tempo": 9, "missing_in_tempo": 91}.
## Воспроизведение
```sh
python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3
```
Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.
Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.
В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.
## Ограничения
Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.
Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.
Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.
Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.
+34
View File
@@ -0,0 +1,34 @@
# Фактическая проверка — 16.11.2026
- Clean Dockerfile build: PASS, runtime loader dependency included.
- Compose config and Collector validate: PASS.
- Prometheus config + 3 alert rules (promtool): PASS.
- C++ W3C context/invalid-context/route-privacy self-test: PASS.
- Public lab6 API smoke with telemetry on and off: PASS.
- Disabled SDK durable payload: traceparent absent, not an empty invalid field.
- Real upload trace parent-child chain API/outbox/Redis/worker/S3/DB: PASS.
- Loki events from both services correlated by trace ID: PASS.
- Provisioned dashboard: 25 PromQL queries checked.
- Privacy canaries absent in recent Loki events and selected upload trace.
- Whole delivered incidents.py run against final image: PASS.
- All affected services restored; SDK enabled.
Load: 10338 successes / 0 errors, 344.305 RPS; client p50/p95/p99 5.666/52.505/61.702 ms.
Closed-loop synthetic message load, concurrency 4, 30 seconds; not a capacity or 30-minute SLO proof.
| Событие | Москва, UTC+3 |
|---|---|
| worker_stopped_upload_accepted | 10:30:35 |
| QueueLagGrowing_firing | 10:31:50 |
| QueueLagGrowing_resolved_backlog_drained | 10:32:10 |
| database_stopped | 10:32:10 |
| ApiHighErrorRate_firing | 10:32:54 |
| ApiHighErrorRate_resolved_database_recovered | 10:33:55 |
| s3_failure_scheduled_retry | 10:33:57 |
| retry_same_trace_ready_verified | 10:33:58 |
| tempo_loki_stopped | 10:34:08 |
| telemetry_outage_api_worker_healthy | 10:34:30 |
| tempo_loki_restarted | 10:34:30 |
Evidence contains only synthetic technical IDs, timestamps and aggregate metrics. Full backend payloads, credentials and message bodies were not saved.
Limits: no prolonged memory soak or exhaustive privacy audit; SDK log-queue drops are not counted exactly (see REPORT.md).
+30 -4
View File
@@ -1,7 +1,33 @@
# Инфраструктура студента
# Инфраструктура lab6
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Рабочая топология — в корневом `compose.yaml`. `compose.example.yaml` остаётся
учебным worksheet, его запускать не нужно. Bootstrap: `python3 scripts/bootstrap-local.py`,
затем `docker compose up -d --build`. Повторный запуск сохраняет volumes и данные.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
| Роль | Compose service | Persistent volume | Host port |
|---|---|---|---|
| C++ API (2 реплики) | api | — | через lb |
| Reverse proxy | lb | — | 127.0.0.1:8080 |
| PostgreSQL 16.15 | db | pgdata | нет |
| Valkey 8.1.10 / Redis Streams | cache | valkeydata | нет |
| MinIO | minio | miniodata | нет |
| Bucket / least-privilege user bootstrap | minio-init, minio-user-init | — | нет |
| C++ последовательный worker | worker | — | нет |
| Collector 0.119.0 | otel-collector | — | нет |
| Prometheus 3.2.1 | prometheus | prometheusdata | 127.0.0.1:9090 |
| Loki 3.4.2 | loki | lokidata | нет |
| Tempo 2.7.2 | tempo | tempodata | нет |
| Grafana 11.6.0 | grafana | grafanadata | 127.0.0.1:3000 |
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
API/worker строятся из исходников этого репозитория; MinIO/mc — из закреплённых
официальных release binaries с проверкой SHA256. PostgreSQL/Valkey/MinIO имеют
healthchecks; API ждёт готовности бизнес-зависимостей и bootstrap S3, lb ждёт API.
Telemetry backends не блокируют запуск или работу бизнес-процессов.
Credentials хранятся в локальных `.env` и `secrets/`, исключённых из Git/context.
API/worker читают Docker file secrets после staging для uid 10001; Grafana читает
пароль из отдельного file secret. Корневые MinIO credentials нужны только store/bootstrap;
приложения используют ограниченного S3 пользователя. Все сервисы находятся в default
Compose network. Метрики, логи, трассы и состояние Grafana переживают restart.
Provisioning, версии, SLO, runbooks и реальные проверки: [REPORT.md](../REPORT.md).
+485
View File
@@ -0,0 +1,485 @@
{
"uid": "messenger-observability",
"title": "Messenger — Observability",
"schemaVersion": 40,
"version": 1,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "API RPS (business requests)",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by(http_route)(rate(http_server_requests_total[$__rate_interval]))",
"legendFormat": "{{http_route}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"options": {}
},
{
"id": 2,
"title": "API p50 / p95 / p99 (all replicas)",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by(le)(rate(http_server_request_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95, sum by(le)(rate(http_server_request_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99, sum by(le)(rate(http_server_request_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 3,
"title": "API error rate (5xx / all business requests)",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(http_server_requests_total{http_response_status_class=\"5xx\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[$__rate_interval])),0.001)",
"legendFormat": "5xx ratio"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {}
},
{
"id": 4,
"title": "Queue depth (durable outstanding jobs)",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "max(queue_depth)",
"legendFormat": "jobs"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 5,
"title": "Oldest outstanding job / observation age",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "max(queue_oldest_age)",
"legendFormat": "oldest"
},
{
"refId": "B",
"expr": "time()-max(queue_observation_timestamp)",
"legendFormat": "observation age"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 6,
"title": "Worker saturation (busy / capacity)",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(worker_busy) / clamp_min(sum(worker_capacity),1)",
"legendFormat": "saturation"
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
},
"options": {}
},
{
"id": 7,
"title": "Worker outcomes per second",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum by(outcome)(rate(worker_jobs_total[$__rate_interval]))",
"legendFormat": "{{outcome}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
},
"options": {}
},
{
"id": 8,
"title": "Worker p50 / p95 / p99",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5,sum by(le)(rate(worker_job_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by(le)(rate(worker_job_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99,sum by(le)(rate(worker_job_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 9,
"title": "DB pools (application + outbox)",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(db_pool_used)",
"legendFormat": "used"
},
{
"refId": "B",
"expr": "sum(db_pool_waiting)",
"legendFormat": "waiting"
},
{
"refId": "C",
"expr": "sum(db_pool_capacity)",
"legendFormat": "capacity"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 10,
"title": "Acceptance to ready p50 / p95 / p99",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5,sum by(le)(rate(attachment_ready_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p50"
},
{
"refId": "B",
"expr": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p95"
},
{
"refId": "C",
"expr": "histogram_quantile(0.99,sum by(le)(rate(attachment_ready_duration_seconds_bucket[$__rate_interval])))",
"legendFormat": "p99"
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"options": {}
},
{
"id": 11,
"title": "Process resident memory",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "process_memory_usage",
"legendFormat": "{{service_name}} / {{service_instance_id}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"options": {}
},
{
"id": 12,
"title": "Process CPU (cores)",
"type": "timeseries",
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "rate(process_cpu_time[$__rate_interval])",
"legendFormat": "{{service_name}} / {{service_instance_id}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 13,
"title": "Collector rejected/failed signal export",
"type": "timeseries",
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"targets": [
{
"refId": "A",
"expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval]))",
"legendFormat": "exporter_send_failed_spans"
},
{
"refId": "B",
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))",
"legendFormat": "exporter_send_failed_log_records"
},
{
"refId": "C",
"expr": "sum(rate(otelcol_processor_refused_spans[$__rate_interval]))",
"legendFormat": "processor_refused_spans"
},
{
"refId": "D",
"expr": "sum(rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))",
"legendFormat": "exporter_enqueue_failed_spans"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
},
{
"id": 14,
"title": "Correlated application events",
"type": "logs",
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"datasource": {
"type": "loki",
"uid": "loki"
},
"targets": [
{
"refId": "A",
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"legendFormat": "logs"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"options": {}
}
],
"tags": [
"lab6"
],
"annotations": {
"list": []
}
}
@@ -0,0 +1,7 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,33 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: TraceID
matcherType: label
matcherRegex: trace_id
datasourceUid: tempo
url: '$${__value.raw}'
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: '-1m'
spanEndTimeShift: '1m'
tags: [{key: service.name, value: service_name}]
filterByTraceID: true
filterBySpanID: true
+31
View File
@@ -0,0 +1,31 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+19
View File
@@ -0,0 +1,19 @@
# Local MinIO Client (mc) from the official GitHub release binary (pinned).
# Fallback when docker.io/minio/mc cannot be pulled.
FROM ubuntu:24.04
ARG MC_VERSION=RELEASE.2025-04-16T18-13-26Z
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
&& curl -fsSL -o /tmp/mc.bin \
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}" \
&& curl -fsSL -o /tmp/mc.sha256sum \
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}.sha256sum" \
&& expected="$(awk '{print $1}' /tmp/mc.sha256sum)" \
&& echo "${expected} /tmp/mc.bin" | sha256sum -c - \
&& install -m 0755 /tmp/mc.bin /usr/local/bin/mc \
&& rm -f /tmp/mc.bin /tmp/mc.sha256sum \
&& rm -rf /var/lib/apt/lists/*
ENTRYPOINT ["mc"]
+22
View File
@@ -0,0 +1,22 @@
# Local MinIO image from the official GitHub release binary (pinned).
# Fallback when docker.io/minio/minio cannot be pulled.
FROM ubuntu:24.04
ARG MINIO_VERSION=RELEASE.2025-04-22T22-12-26Z
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
&& curl -fsSL -o /tmp/minio.bin \
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}" \
&& curl -fsSL -o /tmp/minio.sha256sum \
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}.sha256sum" \
&& expected="$(awk '{print $1}' /tmp/minio.sha256sum)" \
&& echo "${expected} /tmp/minio.bin" | sha256sum -c - \
&& install -m 0755 /tmp/minio.bin /usr/local/bin/minio \
&& rm -f /tmp/minio.bin /tmp/minio.sha256sum \
&& rm -rf /var/lib/apt/lists/*
VOLUME ["/data"]
EXPOSE 9000 9001
ENTRYPOINT ["minio"]
CMD ["server", "/data", "--console-address", ":9001"]
+29
View File
@@ -0,0 +1,29 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucketMultipartUploads",
"s3:GetBucketLocation",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__/*"
]
}
]
}
+36 -4
View File
@@ -1,7 +1,39 @@
# Конфигурация наблюдаемости — реализовать
# OpenTelemetry Collector
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
`collector.yaml` рассчитан на Collector Contrib **0.119.0**.
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
- Receiver: OTLP HTTP/protobuf, `0.0.0.0:4318`, только Compose network.
- Все pipelines: `memory_limiter` → `batch`.
- Metrics → Prometheus exporter `:9464`; process diagnostics `:8888`.
- Traces → Tempo `http://tempo:4318/v1/traces`.
- Logs → Loki native OTLP `http://loki:3100/otlp/v1/logs`.
- Ограниченные очереди 256 batches, retry не более 15s; Docker limit 256 MiB.
- Метрики остановленных SDK instances истекают через 20s, чтобы pool/capacity
не считались повторно после смены контейнеров.
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
API/worker экспортируют в Collector асинхронно и не зависят от его readiness.
Payloads не печатаются debug exporter-ом. Подробности и ограничения потерь —
в [REPORT.md](../../REPORT.md).
Проверка:
```sh
docker compose exec -T otel-collector /otelcol-contrib validate --config=/etc/otelcol/config.yaml
```
## Track 1
`collector-sampling10.yaml` отличается только traces processor:
`memory_limiter → probabilistic_sampler/track1 (10%, hash_seed=22) → batch`.
Логи и метрики сохраняются без sampling. Переключение:
```sh
OTEL_COLLECTOR_CONFIG=./infra/otel/collector-sampling10.yaml docker compose up -d otel-collector
# Вернуть исходную конфигурацию:
OTEL_COLLECTOR_CONFIG=./infra/otel/collector.yaml docker compose up -d otel-collector
```
Пересоздание Collector сбрасывает его собственные counters: измерять delta только
внутри одного прогона. `otelcol_receiver_accepted_spans` отражает вход до sampling;
эффект 10% проверяется также через `otelcol_exporter_sent_spans`.
Полный эксперимент и автоматическое восстановление: `scripts/benchmark_telemetry.py`.
+67
View File
@@ -0,0 +1,67 @@
# Track 1: sample traces only; metrics and logs remain at 100%.
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
probabilistic_sampler/track1:
hash_seed: 22
sampling_percentage: 10
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:9464
metric_expiration: 20s
resource_to_telemetry_conversion:
enabled: true
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 15s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 15s
service:
telemetry:
logs:
level: warn
metrics:
address: 0.0.0.0:8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, probabilistic_sampler/track1, batch]
exporters: [otlphttp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
+63
View File
@@ -0,0 +1,63 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 192
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:9464
metric_expiration: 20s
resource_to_telemetry_conversion:
enabled: true
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 15s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 2s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 256
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 5s
max_elapsed_time: 15s
service:
telemetry:
logs:
level: warn
metrics:
address: 0.0.0.0:8888
pipelines:
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
+27
View File
@@ -0,0 +1,27 @@
groups:
- name: messenger-slo
rules:
- alert: ApiHighErrorRate
expr: (sum(rate(http_server_requests_total{http_response_status_class="5xx"}[1m])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.001) > 0.05
for: 30s
labels:
severity: warning
annotations:
summary: API 5xx ratio above 5% for 30 seconds (1 minute window)
description: Check server spans, postgresql.pool.acquire and DB pool metrics. Runbook in REPORT.md.
- alert: QueueLagGrowing
expr: max(queue_oldest_age) > 30 and max(queue_depth) > 0 and max(delta(queue_oldest_age[1m])) > 0
for: 30s
labels:
severity: warning
annotations:
summary: Durable backlog is ageing beyond 30 seconds
description: Check worker capacity/busy, worker logs and Redis/DB/S3 spans. Runbook in REPORT.md.
- alert: QueueObservationStale
expr: time() - max(queue_observation_timestamp) > 30 or absent(queue_observation_timestamp)
for: 30s
labels:
severity: warning
annotations:
summary: Queue observer unavailable
description: Queue values may be stale; check PostgreSQL and API observer logs.
+11
View File
@@ -0,0 +1,11 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
rule_files: [/etc/prometheus/alerts.yaml]
scrape_configs:
- job_name: messenger
static_configs:
- targets: [otel-collector:9464]
- job_name: collector
static_configs:
- targets: [otel-collector:8888]
+22
View File
@@ -0,0 +1,22 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false
+44
View File
@@ -0,0 +1,44 @@
worker_processes ${NGINX_WORKERS};
error_log /dev/stderr warn;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 2s;
upstream api_upstream {
zone api_upstream 64k;
server api:8080 resolve max_fails=1 fail_timeout=5s;
keepalive 32;
}
server {
listen 8080;
# Lab4 attachments ≤10 MiB; allow a little headroom for multipart/headers.
client_max_body_size 12m;
client_body_timeout 60s;
location / {
proxy_pass http://api_upstream;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 10s;
proxy_connect_timeout 2s;
# Uploads/downloads of ≤10 MiB need longer than JSON chat traffic.
proxy_read_timeout 120s;
proxy_send_timeout 120s;
}
}
}
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX messages_chat_keyset_idx
ON messages (chat_id, created_at, id);
+6
View File
@@ -0,0 +1,6 @@
CREATE TABLE sessions (
token text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
+22
View File
@@ -0,0 +1,22 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename text NOT NULL
CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL
CHECK (size_bytes >= 1 AND size_bytes <= 10485760),
sha256 text NOT NULL
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
status text NOT NULL
CHECK (status IN ('ready')),
s3_key text NOT NULL UNIQUE,
error_code text NULL,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
CREATE INDEX attachments_created_at_idx ON attachments (created_at);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id),
position integer NOT NULL CHECK (position >= 0 AND position < 10),
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX message_attachments_attachment_idx
ON message_attachments (attachment_id);
CREATE UNIQUE INDEX message_attachments_order_idx
ON message_attachments (message_id, position);
+26
View File
@@ -0,0 +1,26 @@
-- Track 1 (grade 5): pending direct/presigned upload sessions.
-- Attachment rows stay status=ready only; incomplete uploads live here.
CREATE TABLE direct_uploads (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename text NOT NULL
CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CHECK (content_type IN ('image/png', 'image/jpeg')),
expected_size integer NOT NULL
CHECK (expected_size >= 1 AND expected_size <= 10485760),
expected_sha256 text NOT NULL
CHECK (expected_sha256 ~ '^[a-f0-9]{64}$'),
s3_key text NOT NULL UNIQUE,
status text NOT NULL
CHECK (status IN ('pending', 'completed', 'aborted', 'expired')),
attachment_id uuid NULL REFERENCES attachments(id),
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX direct_uploads_status_expires_idx
ON direct_uploads (status, expires_at);
CREATE INDEX direct_uploads_owner_idx ON direct_uploads (owner_id);
+12
View File
@@ -0,0 +1,12 @@
-- Track 1: presigned PUT lands in staging/; complete copies to attachments/.
-- Existing rows keep the old key as staging so in-flight sessions still resolve.
ALTER TABLE direct_uploads ADD COLUMN staging_key text;
UPDATE direct_uploads
SET staging_key = s3_key
WHERE staging_key IS NULL;
ALTER TABLE direct_uploads ALTER COLUMN staging_key SET NOT NULL;
ALTER TABLE direct_uploads
ADD CONSTRAINT direct_uploads_staging_key_key UNIQUE (staging_key);
+21
View File
@@ -0,0 +1,21 @@
-- Keep existing lab4 originals ready; new jobs set queued explicitly.
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status_check
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'::jsonb
CHECK (jsonb_typeof(variants) = 'array');
CREATE TABLE jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE
REFERENCES attachments(id) ON DELETE CASCADE,
status text NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
retries integer NOT NULL DEFAULT 0 CHECK (retries >= 0),
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX jobs_status_created_at_idx ON jobs (status, created_at);
+20
View File
@@ -0,0 +1,20 @@
ALTER TABLE jobs ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now();
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
payload jsonb NOT NULL CHECK (jsonb_typeof(payload) = 'object'),
published boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
available_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE NOT published;
-- Upgrade jobs accepted before outbox existed; duplicates are safe.
INSERT INTO outbox (payload, available_at)
SELECT jsonb_build_object(
'schema_version', 1, 'type', 'attachment.process.v1',
'job_id', id::text, 'attachment_id', attachment_id::text,
'pipeline_version', 1, 'attempt', retries + 1,
'created_at', to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')
), next_attempt_at FROM jobs WHERE status IN ('queued', 'processing');
+22
View File
@@ -0,0 +1,22 @@
ALTER TABLE attachments
ADD COLUMN staging_key text,
ADD COLUMN staging_cleaned_at timestamptz;
UPDATE attachments a SET staging_key='staging/' || a.chat_id::text || '/' || a.id::text
WHERE EXISTS (SELECT 1 FROM jobs j WHERE j.attachment_id=a.id);
CREATE INDEX attachments_staging_cleanup_idx ON attachments(status, updated_at)
WHERE staging_key IS NOT NULL AND staging_cleaned_at IS NULL;
ALTER TABLE jobs ADD COLUMN retry_base integer NOT NULL DEFAULT 0
CHECK (retry_base >= 0 AND retry_base <= retries);
CREATE TABLE job_replays (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs(id),
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100),
reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500),
previous_error_code text,
retries_before integer NOT NULL CHECK (retries_before >= 0),
created_at timestamptz NOT NULL DEFAULT now()
);
+41
View File
@@ -0,0 +1,41 @@
ALTER TABLE attachments
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
ADD COLUMN published_version integer CHECK (published_version > 0),
ADD CONSTRAINT attachments_version_order CHECK (published_version <= pipeline_version);
UPDATE attachments SET published_version=1 WHERE status='ready';
ALTER TABLE jobs
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
ADD COLUMN source_key text;
UPDATE jobs j SET source_key=a.s3_key FROM attachments a WHERE a.id=j.attachment_id;
ALTER TABLE jobs ALTER COLUMN source_key SET NOT NULL;
ALTER TABLE jobs DROP CONSTRAINT jobs_attachment_id_key;
ALTER TABLE jobs ADD CONSTRAINT jobs_attachment_version_key UNIQUE (attachment_id, pipeline_version);
ALTER TABLE jobs DROP CONSTRAINT jobs_status_check;
ALTER TABLE jobs ADD CONSTRAINT jobs_status_check
CHECK (status IN ('queued','processing','ready','failed','superseded'));
-- One immutable logical result per (attachment, pipeline version).
CREATE TABLE attachment_results (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
job_id uuid UNIQUE REFERENCES jobs(id),
original_key text NOT NULL,
variants jsonb NOT NULL CHECK (jsonb_typeof(variants)='array'),
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version)
);
INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants)
SELECT a.id,1,j.id,a.s3_key,a.variants FROM attachments a
LEFT JOIN jobs j ON j.attachment_id=a.id AND j.pipeline_version=1 WHERE a.status='ready';
CREATE TABLE pipeline_requests (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
job_id uuid NOT NULL UNIQUE REFERENCES jobs(id),
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100),
reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500),
created_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(attachment_id,pipeline_version)
);
+461
View File
@@ -0,0 +1,461 @@
#!/usr/bin/env python3
"""Track 1: fixed-count attachment benchmark; stdlib only, no credentials in evidence.
Creates synthetic durable data. Never deletes volumes or changes business code.
Always restores SDK on / Collector 100% when the experiment finishes or fails.
"""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import re
import shlex
import signal
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
VARIANTS = {'A': ('Off', True, 'collector.yaml'),
'B': ('On, SDK 100% / Collector 100%', False, 'collector.yaml'),
'C': ('On, SDK 100% / Collector 10%', False, 'collector-sampling10.yaml')}
def percentile(values, quantile):
return sorted(values)[max(0, math.ceil(len(values) * quantile) - 1)]
def memory_mib(text):
number, unit = re.fullmatch(r'([\d.]+)\s*([kKMGT]?i?B)', text.strip()).groups()
factors = {'B': 1, 'kB': 1000, 'MB': 1000**2, 'GB': 1000**3, 'TB': 1000**4,
'KiB': 1024, 'MiB': 1024**2, 'GiB': 1024**3, 'TiB': 1024**4}
return float(number) * factors[unit] / 1024**2
def parse_metrics(text):
result = {}
for line in text.splitlines():
if not line or line.startswith('#'):
continue
name = line.split('{', 1)[0].split(' ', 1)[0]
result[name] = result.get(name, 0) + float(line.rsplit(' ', 1)[1])
return result
class Bench:
def __init__(self, args):
self.args = args
self.compose = shlex.split(args.compose)
self.docker = shlex.split(args.docker)
self.output = ROOT / args.output
self.output.mkdir(parents=True, exist_ok=True)
self.content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
self.report = {'started_utc': datetime.now(timezone.utc).isoformat(),
'method': {'requests': args.requests, 'concurrency': args.concurrency,
'repetitions': args.repetitions, 'warmup': args.warmup,
'fixture_bytes': len(self.content),
'fixture_sha256': hashlib.sha256(self.content).hexdigest(),
'stats_source': 'docker stats stream; 100% CPU = one logical core',
'cpu_ram_window': 'uploads plus complete worker drain, excluding final export wait',
'percentile': 'nearest rank; HTTP upload acceptance, client wall clock',
'config_sha256': {name: hashlib.sha256((ROOT/'infra/otel'/name).read_bytes()).hexdigest() for name in ('collector.yaml', 'collector-sampling10.yaml')},
'order': 'ABC / BCA / CAB (rotated each repetition)'},
'host': {'machine': platform.machine(), 'logical_cpus': os.cpu_count(),
'cpu_model': next((line.split(':',1)[1].strip() for line in Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'unknown'),
'memory_kib': int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')))},
'trials': []}
def run(self, cmd, *, env=None, data=None):
completed = subprocess.run(cmd, cwd=ROOT, env=env, input=data, text=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if completed.returncode:
# Subprocess output can contain connection credentials; do not forward it.
raise RuntimeError('Command failed: ' + Path(cmd[0]).name + ' ' + cmd[1] +
' (exit ' + str(completed.returncode) + ')')
return completed.stdout
def compose_run(self, *args, **kwargs):
return self.run(self.compose + list(args), **kwargs)
def sql(self, sql):
return self.compose_run('exec', '-T', 'db', 'sh', '-c',
'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tA', data=sql+'\n').strip()
def wait_jobs(self, chat=None, expected=None):
deadline = time.monotonic() + self.args.drain_timeout
while time.monotonic() < deadline:
if chat:
uuid.UUID(chat) # Only synthetic UUIDs may enter the query.
row = self.sql("SELECT count(*),count(*) FILTER (WHERE j.status='ready'),"
"count(*) FILTER (WHERE j.status='failed') FROM jobs j JOIN attachments a "
"ON a.id=j.attachment_id WHERE a.chat_id='" + chat + "'::uuid;")
total, ready, failed = map(int, row.split('|'))
if failed:
raise RuntimeError('Worker produced failed benchmark jobs')
if total == expected and ready == expected:
return
elif int(self.sql("SELECT count(*) FROM jobs WHERE status IN ('queued','processing');")) == 0:
return
time.sleep(1)
raise RuntimeError('Worker backlog did not drain; evidence is incomplete')
def configure(self, variant):
_, disabled, filename = VARIANTS[variant]
env = dict(os.environ, OTEL_SDK_DISABLED=str(disabled).lower(),
OTEL_COLLECTOR_CONFIG=str(ROOT / 'infra/otel' / filename))
self.compose_run('config', '--quiet', env=env)
self.compose_run('up', '-d', '--no-deps', '--force-recreate', '--wait',
'--wait-timeout', '120', 'otel-collector', 'api', 'worker', env=env)
self.compose_run('exec', '-T', 'otel-collector', '/otelcol-contrib', 'validate',
'--config=/etc/otelcol/config.yaml', env=env)
deadline = time.monotonic() + 60
while True:
try:
Client(self.args.base_url).call('GET', '/health/ready', schema='Health')
self.collector()
return
except (Failure, RuntimeError):
if time.monotonic() >= deadline:
raise RuntimeError('API/Collector did not become ready') from None
time.sleep(1)
def collector(self):
return parse_metrics(self.compose_run('exec', '-T', '--index', '1', 'api',
'curl', '-fsS', '--max-time', '5', 'http://otel-collector:8888/metrics'))
def upload(self, count, actor, chat):
barrier = threading.Barrier(self.args.concurrency)
def work(slot):
client = Client(self.args.base_url, timeout=60)
results = []
barrier.wait()
for index in range(slot, count, self.args.concurrency):
started = time.monotonic()
try:
# No supplied traceparent: the SDK's AlwaysOn root policy is exercised.
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
expected=202, actor=actor, raw=self.content,
headers={'Content-Type': 'image/png'}, schema='Attachment')
results.append((time.monotonic()-started, True))
except Failure:
results.append((time.monotonic()-started, False))
return results
start = time.monotonic()
with ThreadPoolExecutor(max_workers=self.args.concurrency) as pool:
rows = [r for batch in pool.map(work, range(self.args.concurrency)) for r in batch]
return rows, time.monotonic()-start
def sample_stats(self, ids, stop, rows, errors):
process = subprocess.Popen(self.docker + ['stats', '--format', '{{json .}}'] + list(ids),
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
def terminate():
stop.wait()
process.terminate()
closer = threading.Thread(target=terminate, daemon=True)
closer.start()
try:
for line in process.stdout:
clean = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', line).strip()
if not clean:
continue
if stop.is_set() and not clean.endswith('}'):
continue
obj = json.loads(clean)
ident = obj['ID']
if ident not in ids:
ident = next((key for key in ids if key.startswith(ident)), '')
if ident:
rows.append({'t': time.monotonic(), 'container': ident, 'service': ids[ident],
'cpu_percent': float(obj['CPUPerc'].rstrip('%')),
'ram_mib': memory_mib(obj['MemUsage'].split('/')[0])})
if not stop.is_set():
errors.append('docker stats stopped unexpectedly')
except Exception as error:
errors.append(type(error).__name__ + ': ' + str(error))
finally:
stop.set()
process.terminate()
process.wait(timeout=10)
closer.join(timeout=10)
def trial(self, variant, repetition):
print(f'Preparing {variant}, repetition {repetition}', flush=True)
self.wait_jobs()
self.configure(variant)
client = Client(self.args.base_url)
alice, _, _, chat = client.bootstrap()
warm, _ = self.upload(self.args.warmup, alice, chat)
if not all(ok for _, ok in warm):
raise RuntimeError('Warm-up HTTP errors')
self.wait_jobs(chat['id'], self.args.warmup)
time.sleep(8) # SDK reader/processor + Collector batch settle before baseline.
ids = {}
for service in ('api', 'worker'):
for ident in self.compose_run('ps', '-q', service).split():
ids[ident] = service
self.report['container_limits'] = {service: self.run(self.docker + ['inspect', '--format', '{{.Config.Image}}|{{.Image}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}', ident]).strip() for ident, service in ids.items()}
if sum(s == 'api' for s in ids.values()) != 2 or sum(s == 'worker' for s in ids.values()) != 1:
raise RuntimeError('Benchmark requires two API replicas and one worker')
rows, errors, stop = [], [], threading.Event()
sampler = threading.Thread(target=self.sample_stats, args=(ids, stop, rows, errors))
sampler.start()
try:
time.sleep(3)
before = self.collector()
start = time.monotonic()
measurements, http_seconds = self.upload(self.args.requests, alice, chat)
if any(not ok for _, ok in measurements):
raise RuntimeError('HTTP upload errors invalidate the comparison')
self.wait_jobs(chat['id'], self.args.warmup+self.args.requests)
end = time.monotonic()
finally:
stop.set()
sampler.join(timeout=15)
if sampler.is_alive() or errors:
raise RuntimeError('docker stats sampling failed: ' + ', '.join(errors))
# Export is asynchronous. Wait for SDK batches and for backend queues to empty.
time.sleep(3)
after = self.collector()
for _ in range(10):
if after.get('otelcol_exporter_queue_size', 0) == 0:
break
time.sleep(1)
after = self.collector()
metric_seconds = time.monotonic()-start
def delta(name):
return after.get(name, 0)-before.get(name, 0)
samples = [r for r in rows if start <= r['t'] <= end]
resource = {}
for service in ('api', 'worker'):
per_container = []
for ident, kind in ids.items():
if kind != service:
continue
series = [r for r in samples if r['container'] == ident]
if len(series) < 3:
raise RuntimeError('Too few docker stats samples; use more requests')
per_container.append({'cpu_percent_mean': statistics.mean(r['cpu_percent'] for r in series),
'ram_mib_mean': statistics.mean(r['ram_mib'] for r in series),
'ram_mib_max': max(r['ram_mib'] for r in series), 'samples': len(series)})
resource[service] = {key: sum(r[key] for r in per_container)
for key in ('cpu_percent_mean', 'ram_mib_mean', 'ram_mib_max', 'samples')}
resource[service]['cpu_seconds_per_job_estimate'] = resource[service]['cpu_percent_mean']/100*(end-start)/self.args.requests
good = [seconds*1000 for seconds, ok in measurements if ok]
failures = sum(not ok for _, ok in measurements)
counters = {name: delta(name) for name in (
'otelcol_receiver_accepted_spans', 'otelcol_exporter_sent_spans',
'otelcol_receiver_refused_spans', 'otelcol_exporter_send_failed_spans',
'otelcol_receiver_accepted_log_records', 'otelcol_receiver_accepted_metric_points')}
if failures or not good or counters['otelcol_receiver_refused_spans'] or counters['otelcol_exporter_send_failed_spans']:
raise RuntimeError('HTTP or Collector failures invalidate the comparison')
if variant == 'A' and counters['otelcol_receiver_accepted_spans'] != 0:
raise RuntimeError('Telemetry Off still exported spans')
retention = counters['otelcol_exporter_sent_spans']/max(counters['otelcol_receiver_accepted_spans'], 1)
if variant == 'B' and not 0.95 <= retention <= 1.05:
raise RuntimeError('100% trace export did not match receiver ingest')
if variant == 'C' and not 0.04 <= retention <= 0.18:
raise RuntimeError('Collector sampling did not reduce export to approximately 10%')
result = {'variant': variant, 'repetition': repetition, 'completed_utc': datetime.now(timezone.utc).isoformat(),
'successful_uploads': len(good), 'http_errors': failures, 'http_seconds': http_seconds,
'pipeline_seconds': end-start, 'collector_window_seconds': metric_seconds,
'rps': len(good)/http_seconds, 'latency_mean_ms': statistics.mean(good),
'latency_p50_ms': percentile(good, .5), 'latency_p99_ms': percentile(good, .99),
'latencies_ms': good,
'resources': resource, 'collector_deltas': counters,
'ingest_spans_per_second': counters['otelcol_receiver_accepted_spans']/metric_seconds,
'tempo_spans_per_second': counters['otelcol_exporter_sent_spans']/metric_seconds,
'trace_retention': retention, 'stats_samples': [{**r, 't': r['t']-start} for r in samples]}
self.report['trials'].append(result)
self.save()
print(f'{variant}/{repetition}: {result["rps"]:.2f} RPS, p99 {result["latency_p99_ms"]:.2f} ms, '
f'accepted {counters["otelcol_receiver_accepted_spans"]:.0f}, exported {counters["otelcol_exporter_sent_spans"]:.0f}', flush=True)
def save(self):
(self.output / 'track1-benchmark.json').write_text(json.dumps(self.report, indent=2)+'\n')
def write_summary(self):
def med(variant, getter):
return statistics.median(getter(t) for t in self.report['trials'] if t['variant'] == variant)
lines = ['# Track 1: Цена наблюдаемости', '',
'Результат: **' + self.report.get('result', 'INCOMPLETE') + '**. Начало UTC: ' + self.report['started_utc'] + '.', '',
f"По {self.args.repetitions} прогона каждого варианта, {self.args.requests} POST uploads, concurrency {self.args.concurrency}, warm-up {self.args.warmup} uploads.",
'Порядок ABC / BCA / CAB. PNG: ' + str(len(self.content)) + ' bytes. Каждый job дошёл до ready; очередь между прогонами пуста.',
'Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).',
'RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.',
'CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.', '',
'| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |',
'|---|---:|---:|---:|---:|---:|---:|---:|---:|']
for v in 'ABC':
values = [med(v, lambda t: t[k]) for k in ('rps', 'latency_p50_ms', 'latency_p99_ms')]
api = [med(v, lambda t: t['resources']['api'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
worker = [med(v, lambda t: t['resources']['worker'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
lines.append(f'| {v}: {VARIANTS[v][0]} | {values[0]:.2f} | {values[1]:.2f} | {values[2]:.2f} | {api[0]:.2f}% | {api[1]:.2f} | {worker[0]:.2f}% | {worker[1]:.2f} | {med(v, lambda t:t["ingest_spans_per_second"]):.2f} |')
lines += ['', '| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |',
'|---|---:|---:|---:|---:|---:|---:|---:|']
for v in 'ABC':
lines.append(f'| {v} | {med(v, lambda t:t["latency_mean_ms"]):.2f} | {med(v, lambda t:t["pipeline_seconds"]):.2f} | {med(v, lambda t:t["collector_deltas"]["otelcol_receiver_accepted_spans"]):.0f} | {med(v, lambda t:t["collector_deltas"]["otelcol_exporter_sent_spans"]):.0f} | {med(v, lambda t:t["tempo_spans_per_second"]):.2f} | {1000*med(v, lambda t:t["resources"]["api"]["cpu_seconds_per_job_estimate"]):.2f} | {1000*med(v, lambda t:t["resources"]["worker"]["cpu_seconds_per_job_estimate"]):.2f} |')
lines += ['', '*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.',
'Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.',
'В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.', '',
'| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |',
'|---|---:|---:|---:|---:|---:|---:|']
for t in self.report['trials']:
lines.append(f'| {t["variant"]}/{t["repetition"]} | {t["rps"]:.2f} | {t["latency_mean_ms"]:.2f} | {t["latency_p99_ms"]:.2f} | {t["pipeline_seconds"]:.2f} | {t["resources"]["api"]["samples"]} / {t["resources"]["worker"]["samples"]} | {t["collector_deltas"]["otelcol_receiver_refused_spans"]:.0f} / {t["collector_deltas"]["otelcol_exporter_send_failed_spans"]:.0f} |')
if all('sdk_diagnostic_audit' in t for t in self.report['trials']):
starts = sorted({t['sdk_diagnostic_audit']['warning_error_start'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_start'] is not None})
deltas = sorted({t['sdk_diagnostic_audit']['warning_error_delta'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_delta'] is not None})
drops = self.report.get('sdk_diagnostic_audit', {}).get('span_queue_drop_max_during_experiment')
lines += ['', f'SDK audit: delta warning/error diagnostics в on-окнах: {deltas}; значения перед началом окон: {starts}. Startup/reconfiguration не входит в steady-state замер.',
f'Максимум telemetry_sdk_dropped_spans в интервале эксперимента: {drops}. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.']
probe = self.report.get('error_probe', {})
lines += ['', 'Отдельный synthetic ERROR probe (вне замеров API): ' + json.dumps(probe, ensure_ascii=False) + '.', '',
'## Воспроизведение', '', '```sh',
'python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3', '```', '',
'Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.',
'Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.',
'В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.', '',
'## Ограничения', '',
'Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.',
'Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.',
'Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.',
'Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.']
(self.output/'track1-benchmark.md').write_text('\n'.join(lines)+'\n')
def audit_sdk(self):
# Read the same externally scraped diagnostics for each trial's own SDK instances.
def query(expr, timestamp):
url = self.args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr, 'time': timestamp})
with urllib.request.urlopen(url, timeout=15) as response:
rows = json.load(response)['data']['result']
return float(rows[0]['value'][1]) if rows else None
end = 0
for trial in self.report['trials']:
ids = '|'.join(sorted({s['container'][:12] for s in trial['stats_samples']}))
end = datetime.fromisoformat(trial['completed_utc']).timestamp()
start = end-trial['collector_window_seconds']
expr = 'sum(telemetry_sdk_errors{service_instance_id=~"' + ids + '"})'
a, b = query(expr, start), query(expr, end)
trial['sdk_diagnostic_audit'] = {'source': 'Prometheus instant queries at trial start/end, restricted to trial container hostnames',
'warning_error_start': a, 'warning_error_end': b,
'warning_error_delta': b-a if a is not None and b is not None else None}
seconds = math.ceil(end-datetime.fromisoformat(self.report['started_utc']).timestamp())
self.report['sdk_diagnostic_audit'] = {
'span_queue_drop_max_during_experiment': query(f'max(max_over_time(telemetry_sdk_dropped_spans[{seconds}s]))', end),
'note': 'Warning/error diagnostics are not counts of lost spans. Compare per-trial deltas; sampling discard is sent/accepted.'}
self.save()
if any(t['sdk_diagnostic_audit']['warning_error_delta'] not in (None, 0) for t in self.report['trials']):
raise RuntimeError('SDK diagnostics increased during measured windows')
def probe_errors(self):
# Separate from measured trials: synthetic ERROR spans verify that probability sampling is not error-aware.
self.configure('C')
traces = [uuid.uuid4().hex for _ in range(100)]
now = time.time_ns()
spans = [{'traceId': ident, 'spanId': '0123456789abcdef', 'name': 'track1.synthetic-error-probe',
'kind': 2, 'startTimeUnixNano': str(now), 'endTimeUnixNano': str(now+1000000),
'status': {'code': 2}, 'attributes': [{'key': 'http.response.status_code', 'value': {'intValue': '503'}}]}
for ident in traces]
data = json.dumps({'resourceSpans': [{'resource': {'attributes': [{'key': 'service.name',
'value': {'stringValue': 'track1-sampling-probe'}}]}, 'scopeSpans': [{'spans': spans}]}]})
self.compose_run('exec', '-T', '--index', '1', 'api', 'curl', '-fsS', '--max-time', '10',
'-H', 'Content-Type: application/json', '--data-binary', '@-',
'http://otel-collector:4318/v1/traces', data=data)
time.sleep(5)
auth = base64.b64encode((os.environ.get('GRAFANA_ADMIN_USER', 'admin')+':'+(ROOT/'secrets/grafana_admin_password').read_text().strip()).encode()).decode()
def retained(ident):
request = urllib.request.Request(self.args.grafana_url+'/api/datasources/proxy/uid/tempo/api/traces/'+ident,
headers={'Authorization': 'Basic '+auth})
try:
with urllib.request.urlopen(request, timeout=15) as response:
return response.status == 200
except urllib.error.HTTPError as error:
if error.code == 404:
return False
raise RuntimeError('Tempo probe query failed') from None
with ThreadPoolExecutor(max_workers=4) as pool:
found = sum(pool.map(retained, traces))
if not 0 < found < len(traces):
raise RuntimeError('Error sampling probe failed')
self.report['error_probe'] = {'type': 'synthetic ERROR spans, separate from upload benchmark',
'sent': len(traces), 'found_in_tempo': found, 'missing_in_tempo': len(traces)-found}
self.save()
print(f'Error probe: {found}/100 ERROR traces retained', flush=True)
def self_test():
assert re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', '\x1b[?25l\x1b[H{}\x1b[?25h') == '{}'
assert percentile([1, 2, 3, 4], .5) == 2
assert percentile(list(range(1, 101)), .99) == 99
assert memory_mib('1024KiB') == 1
assert memory_mib('1GiB') == 1024
assert abs(memory_mib('1048576B')-1) < 1e-9
assert parse_metrics('# comment\nx{a="b"} 2\nx{a="c"} 3\ny 4\n') == {'x': 5, 'y': 4}
base = (ROOT/'infra/otel/collector.yaml').read_text()
sampled = (ROOT/'infra/otel/collector-sampling10.yaml').read_text()
expected = base.replace(' batch:\n', ' probabilistic_sampler/track1:\n hash_seed: 22\n sampling_percentage: 10\n batch:\n', 1)
expected = expected.replace('processors: [memory_limiter, batch]\n exporters: [otlphttp/tempo]',
'processors: [memory_limiter, probabilistic_sampler/track1, batch]\n exporters: [otlphttp/tempo]')
assert sampled.split('\n', 1)[1] == expected, 'Sampling must be the only Collector configuration difference'
assert 'sampling_percentage: 10' in sampled
assert sampled.count('processors: [memory_limiter, probabilistic_sampler/track1, batch]') == 1
assert sampled.count('processors: [memory_limiter, batch]') == base.count('processors: [memory_limiter, batch]')-1
print('PASS: percentile, docker memory units, metric sums, trace-only sampler configuration')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--requests', type=int, default=1000)
parser.add_argument('--concurrency', type=int, default=4)
parser.add_argument('--repetitions', type=int, default=3)
parser.add_argument('--warmup', type=int, default=20)
parser.add_argument('--drain-timeout', type=int, default=600)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--compose', default=os.environ.get('COMPOSE', 'docker compose'))
parser.add_argument('--docker', default=os.environ.get('DOCKER', 'docker'))
parser.add_argument('--output', default='evidence')
parser.add_argument('--self-test', action='store_true')
args = parser.parse_args()
if args.self_test:
return self_test()
if not (args.requests >= 100 and 1 <= args.concurrency <= 16 and 1 <= args.repetitions <= 10 and args.warmup >= args.concurrency):
parser.error('requests >=100, concurrency 1..16, repetitions 1..10, warmup >= concurrency required')
def interrupted(signum, frame):
raise KeyboardInterrupt('Benchmark interrupted; restoring baseline')
signal.signal(signal.SIGTERM, interrupted)
bench = Bench(args)
try:
for repetition in range(args.repetitions):
order = 'ABC'[repetition % 3:] + 'ABC'[:repetition % 3]
for variant in order:
bench.trial(variant, repetition+1)
bench.audit_sdk()
bench.probe_errors()
bench.report['result'] = 'PASS'
finally:
print('Restoring SDK on / Collector 100%', flush=True)
bench.configure('B')
bench.report['restored'] = 'SDK enabled, default Collector configuration'
bench.save()
bench.write_summary()
print('PASS: all trials complete; raw sanitized evidence saved', flush=True)
if __name__ == '__main__':
main()
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
"""Generate isolated local lab credentials without printing secret values."""
import os
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
os.umask(0o077)
if (root / '.env').exists() or (root / 'secrets').exists():
raise SystemExit('Existing .env/secrets preserved; use the existing credentials.')
password, minio_root, s3, grafana = [secrets.token_hex(24) for _ in range(4)]
text = (root / '.env.example').read_text()
for key, value in {'POSTGRES_PASSWORD': password, 'MINIO_ROOT_PASSWORD': minio_root,
'MINIO_USER_SECRET_KEY': s3}.items():
import re
text = re.sub(r'^' + key + r'=.*$', key + '=' + value, text, flags=re.M)
(root / '.env').write_text(text)
(root / 'secrets').mkdir(mode=0o700)
for name, value in {'database_url': f'postgresql://messenger:{password}@db:5432/messenger',
'cache_url': 'redis://cache:6379', 's3_access_key_id': 'messenger-s3',
's3_secret_access_key': s3, 'grafana_admin_password': grafana}.items():
(root / 'secrets' / name).write_text(value + '\n')
# Parent directory stays 0700; bind-mounted file must be readable by Grafana uid 472.
(root / 'secrets' / 'grafana_admin_password').chmod(0o444)
print('Created .env and secrets/. Grafana user: admin; password: secrets/grafana_admin_password')
+94
View File
@@ -0,0 +1,94 @@
#include "telemetry.hpp"
#include "attachment_queue.hpp"
#include "errors.hpp"
#include <memory>
void AttachmentQueue::publish(const std::string& payload) {
auto lease = pool_.acquire();
std::unique_ptr<redisReply, decltype(&freeReplyObject)> reply(
static_cast<redisReply*>(redisCommand(
lease.ctx(), "XADD jobs:attachments * payload %b",
payload.data(), payload.size())), &freeReplyObject);
if (!reply || reply->type != REDIS_REPLY_STRING || reply->len == 0) {
throw DependencyUnavailable("attachment queue publish failed");
}
}
namespace {
using Reply = std::unique_ptr<redisReply, decltype(&freeReplyObject)>;
Reply checked(redisReply* reply) {
Reply owned(reply, &freeReplyObject);
if (!owned || owned->type == REDIS_REPLY_ERROR) {
throw DependencyUnavailable("attachment stream command failed");
}
return owned;
}
JobDelivery delivery(const redisReply* entry) {
if (entry->type != REDIS_REPLY_ARRAY || entry->elements != 2 ||
entry->element[0]->type != REDIS_REPLY_STRING ||
entry->element[1]->type != REDIS_REPLY_ARRAY) {
throw DependencyUnavailable("invalid stream response");
}
JobDelivery result{std::string(entry->element[0]->str, entry->element[0]->len), {}};
const auto* fields = entry->element[1];
for (std::size_t i = 0; i + 1 < fields->elements; i += 2) {
if (fields->element[i]->type == REDIS_REPLY_STRING &&
std::string(fields->element[i]->str, fields->element[i]->len) == "payload" &&
fields->element[i + 1]->type == REDIS_REPLY_STRING && fields->element[i + 1]->len <= 4096) {
result.payload.assign(fields->element[i + 1]->str, fields->element[i + 1]->len);
}
}
return result;
}
} // namespace
void AttachmentQueue::ensure_group() {
auto lease = pool_.acquire();
Reply reply(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XGROUP CREATE jobs:attachments attachment-workers 0 MKSTREAM")), &freeReplyObject);
if (!reply || (reply->type == REDIS_REPLY_ERROR &&
std::string(reply->str, reply->len).rfind("BUSYGROUP", 0) != 0)) {
throw DependencyUnavailable("attachment consumer group unavailable");
}
}
std::optional<JobDelivery> AttachmentQueue::read(const std::string& consumer, int reclaim_idle_ms) {
auto lease = pool_.acquire();
auto pending = checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XAUTOCLAIM jobs:attachments attachment-workers %s %d %s COUNT 1",
consumer.c_str(), reclaim_idle_ms, claim_cursor_.c_str())));
if (pending->type != REDIS_REPLY_ARRAY || pending->elements < 2 ||
pending->element[0]->type != REDIS_REPLY_STRING ||
pending->element[1]->type != REDIS_REPLY_ARRAY) {
throw DependencyUnavailable("invalid pending response");
}
claim_cursor_.assign(pending->element[0]->str, pending->element[0]->len);
if (pending->element[1]->elements) return delivery(pending->element[1]->element[0]);
auto fresh = checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XREADGROUP GROUP attachment-workers %s COUNT 1 BLOCK 1000 STREAMS jobs:attachments >",
consumer.c_str())));
if (fresh->type == REDIS_REPLY_NIL) return std::nullopt;
if (fresh->type != REDIS_REPLY_ARRAY || fresh->elements != 1 ||
fresh->element[0]->type != REDIS_REPLY_ARRAY || fresh->element[0]->elements != 2 ||
fresh->element[0]->element[1]->type != REDIS_REPLY_ARRAY ||
fresh->element[0]->element[1]->elements != 1) {
throw DependencyUnavailable("invalid stream read response");
}
return delivery(fresh->element[0]->element[1]->element[0]);
}
void AttachmentQueue::ack(const std::string& stream_id) {
auto lease = pool_.acquire();
checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XACK jobs:attachments attachment-workers %s", stream_id.c_str())));
}
void AttachmentQueue::reject(const std::string& stream_id) {
{
auto lease = pool_.acquire();
checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XADD jobs:attachments:invalid * stream_id %s error_code invalid_job", stream_id.c_str())));
}
ack(stream_id);
}
+27
View File
@@ -0,0 +1,27 @@
#pragma once
#include "redis_pool.hpp"
#include <string>
#include <optional>
struct JobDelivery {
std::string stream_id;
std::string payload;
};
class AttachmentQueue {
public:
explicit AttachmentQueue(RedisPool& pool) : pool_(pool) {}
void publish(const std::string& payload);
void ensure_group();
std::optional<JobDelivery> read(const std::string& consumer, int reclaim_idle_ms);
void ack(const std::string& stream_id);
void reject(const std::string& stream_id);
bool ping() { return pool_.ping(); }
private:
RedisPool& pool_;
std::string claim_cursor_ = "0-0";
};
+156
View File
@@ -0,0 +1,156 @@
#include "attachment_repo.hpp"
#include "outbox.hpp"
#include <pqxx/pqxx>
#include <nlohmann/json.hpp>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <sstream>
namespace {
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch())
.count();
return std::to_string(secs);
}
Attachment row_to_attachment(const pqxx::row& r) {
Attachment a;
a.id = r["id"].as<std::string>();
a.chat_id = r["chat_id"].as<std::string>();
a.owner_id = r["owner_id"].as<std::string>();
a.filename = r["filename"].as<std::string>();
a.content_type = r["content_type"].as<std::string>();
a.size_bytes = r["size_bytes"].as<int>();
a.sha256 = r["sha256"].as<std::string>();
a.status = r["status"].as<std::string>();
a.s3_key = r["s3_key"].as<std::string>();
if (r["error_code"].is_null()) {
a.error_code = std::nullopt;
} else {
a.error_code = r["error_code"].as<std::string>();
}
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
for (const auto& v : nlohmann::json::parse(r["variants"].as<std::string>())) {
a.variants.push_back(Variant{v.at("name").get<std::string>(),
v.at("content_type").get<std::string>(), v.at("size_bytes").get<int>(),
v.at("sha256").get<std::string>(), v.at("width").get<int>(),
v.at("height").get<int>(), v.value("storage_key", "")});
}
return a;
}
constexpr const char* kAttachmentColumns =
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, error_code, variants, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
} // namespace
Attachment AttachmentRepo::insert(const Attachment& in) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
pqxx::result r;
if (in.error_code) {
r = tx.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, "
" to_timestamp($11::bigint), to_timestamp($12::bigint)"
") RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, *in.error_code, ts, ts});
} else {
r = tx.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
") RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, ts, ts});
}
tx.commit();
return row_to_attachment(r[0]);
}
std::optional<Attachment> AttachmentRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kAttachmentColumns) +
" FROM attachments WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to_attachment(r[0]);
}
Attachment AttachmentRepo::insert_queued(const Attachment& in,
const std::string& job_id,
const std::function<void()>& save_original) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
tx.exec("SELECT pg_advisory_xact_lock(hashtextextended('staging:' || $1::uuid::text, 0))",
pqxx::params{in.id});
save_original();
auto r = tx.exec(
"INSERT INTO attachments ("
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, staging_key, created_at, updated_at) VALUES ("
"$1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, 'queued', $8, $8, now(), now()) "
"RETURNING " + std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.s3_key});
tx.exec("INSERT INTO jobs (id, attachment_id, source_key) VALUES ($1::uuid, $2::uuid, $3)",
pqxx::params{job_id, in.id, in.s3_key});
enqueue_attachment_job(tx, job_id);
Attachment saved = row_to_attachment(r[0]);
tx.commit();
return saved;
}
bool AttachmentRepo::all_belong_to_chat(const std::string& chat_id,
const std::vector<std::string>& ids) {
if (ids.empty()) {
return true;
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
for (const auto& id : ids) {
auto r = tx.exec(
"SELECT 1 FROM attachments "
"WHERE id = $1::uuid AND chat_id = $2::uuid AND status <> 'failed'",
pqxx::params{id, chat_id});
if (r.empty()) {
return false;
}
}
return true;
}
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <functional>
#include <string>
#include <vector>
class AttachmentRepo {
public:
explicit AttachmentRepo(PgPool& pool) : pool_(pool) {}
Attachment insert(const Attachment& a);
Attachment insert_queued(const Attachment& a, const std::string& job_id,
const std::function<void()>& save_original);
std::optional<Attachment> find_by_id(const std::string& id);
// All IDs must exist, belong to chat_id, and not have status=failed.
// Returns false if any id is missing/foreign/failed.
bool all_belong_to_chat(const std::string& chat_id,
const std::vector<std::string>& ids);
private:
PgPool& pool_;
};
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <stdexcept>
#include <string>
inline std::string attachment_variant_key(const std::string& chat_id,
const std::string& id,
const std::string& variant) {
const char* suffix = variant == "thumbnail" ? "_256.jpg" :
variant == "crop" ? "_128.jpg" :
variant == "optimized" ? "_optimized.webp" : nullptr;
if (!suffix) throw std::invalid_argument("unknown variant");
return "attachments/" + chat_id + "/" + id + suffix;
}
inline std::string pipeline_prefix(const std::string& chat_id, const std::string& id,
int version) {
if (version < 1) throw std::invalid_argument("invalid pipeline version");
return "attachments/" + chat_id + "/" + id + "/v" + std::to_string(version) + "/";
}
+39
View File
@@ -0,0 +1,39 @@
#include "auth.hpp"
#include <sodium.h>
#include <stdexcept>
namespace {
struct SodiumInit {
SodiumInit() {
if (sodium_init() < 0) {
throw std::runtime_error("sodium_init failed");
}
}
};
const SodiumInit g_sodium_init;
}
std::string hash_password(const std::string& plain) {
char hashed[crypto_pwhash_STRBYTES];
if (crypto_pwhash_str(
hashed,
plain.data(), plain.size(),
crypto_pwhash_OPSLIMIT_INTERACTIVE,
crypto_pwhash_MEMLIMIT_INTERACTIVE) != 0) {
throw std::runtime_error("crypto_pwhash_str failed (out of memory?)");
}
return std::string(hashed);
}
bool verify_password(const std::string& plain, const std::string& hash) {
if (hash.empty() || hash.size() >= crypto_pwhash_STRBYTES) {
return false;
}
return crypto_pwhash_str_verify(
hash.c_str(),
plain.data(), plain.size()) == 0;
}
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#include <string>
std::string hash_password(const std::string& plain);
bool verify_password(const std::string& plain, const std::string& hash);
+186
View File
@@ -0,0 +1,186 @@
#include "chat_repo.hpp"
#include "cursor.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <random>
#include <sstream>
#include <string>
namespace {
std::string make_uuid() {
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
Chat row_to_chat(const pqxx::row& r) {
Chat c;
c.id = r["id"].as<std::string>();
c.title = r["title"].as<std::string>();
c.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return c;
}
std::vector<std::string> load_member_ids(pqxx::work& tx,
const std::string& chat_id) {
std::vector<std::string> ids;
auto r = tx.exec(
"SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id",
pqxx::params{chat_id});
ids.reserve(r.size());
for (const auto& row : r) {
ids.push_back(row["user_id"].as<std::string>());
}
return ids;
}
constexpr const char* kChatSelect =
"id, title, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
}
std::optional<Chat> ChatRepo::create(const std::string& title,
const std::vector<std::string>& member_ids) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string chat_id = make_uuid();
const std::string created = now_epoch_str();
try {
tx.exec(
"INSERT INTO chats (id, title, created_at) "
"VALUES ($1, $2, to_timestamp($3::bigint))",
pqxx::params{chat_id, title, created});
for (const auto& uid : member_ids) {
tx.exec(
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
pqxx::params{chat_id, uid});
}
} catch (const pqxx::foreign_key_violation&) {
tx.abort();
return std::nullopt;
} catch (const pqxx::unique_violation&) {
tx.abort();
return std::nullopt;
}
auto r = tx.exec(
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
pqxx::params{chat_id});
Chat c = row_to_chat(r[0]);
c.member_ids = load_member_ids(tx, chat_id);
tx.commit();
return c;
}
std::optional<Chat> ChatRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
pqxx::params{id});
if (r.empty()) {
tx.commit();
return std::nullopt;
}
Chat c = row_to_chat(r[0]);
c.member_ids = load_member_ids(tx, id);
tx.commit();
return c;
}
std::optional<ChatPage> ChatRepo::list_for_user(
const std::string& user_id,
int limit,
const std::optional<std::string>& cursor) {
std::optional<Cursor> cur;
if (cursor) {
cur = parse_cursor(*cursor);
if (!cur) {
return std::nullopt;
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
if (cur) {
auto owner_check = tx.exec(
"SELECT 1 FROM chats WHERE id = $1 AND id IN "
"(SELECT chat_id FROM chat_members WHERE user_id = $2)",
pqxx::params{cur->id, user_id});
if (owner_check.empty()) {
tx.commit();
return std::nullopt;
}
}
std::string sql =
"SELECT " + std::string(kChatSelect) + ", " + kCreatedUsSql + " "
"FROM chats "
"WHERE id IN (SELECT chat_id FROM chat_members WHERE user_id = $1) ";
if (cur) {
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
}
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
pqxx::result r;
if (cur) {
r = tx.exec(sql, pqxx::params{user_id, limit + 1, cur->created_us, cur->id});
} else {
r = tx.exec(sql, pqxx::params{user_id, limit + 1});
}
ChatPage page;
const bool has_more = (static_cast<int>(r.size()) > limit);
const int take = has_more ? limit : static_cast<int>(r.size());
page.items.reserve(take);
for (int i = 0; i < take; ++i) {
Chat c = row_to_chat(r[static_cast<std::size_t>(i)]);
c.member_ids = load_member_ids(tx, c.id);
page.items.push_back(std::move(c));
}
if (has_more && take > 0) {
const auto& last = r[static_cast<std::size_t>(take - 1)];
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
last["id"].as<std::string>());
}
tx.commit();
return page;
}
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
class ChatRepo {
public:
explicit ChatRepo(PgPool& pool) : pool_(pool) {}
std::optional<Chat> create(const std::string& title,
const std::vector<std::string>& member_ids);
std::optional<Chat> find_by_id(const std::string& id);
std::optional<ChatPage> list_for_user(const std::string& user_id, int limit, const std::optional<std::string>& cursor);
private:
PgPool& pool_;
};
+47
View File
@@ -0,0 +1,47 @@
#pragma once
#include <algorithm>
#include <cctype>
#include <optional>
#include <regex>
#include <string>
// Непрозрачный для клиента курсор keyset-пагинации: "<created_at в мкс от epoch>|<uuid>".
// Микросекунды сохраняют точный created_at строки, даже если в JSON время округлено до секунд.
struct Cursor {
std::string created_us;
std::string id;
};
// Выражение SQL, восстанавливающее timestamptz из created_us без потери точности.
constexpr const char* kCursorTimestampSql =
"(timestamptz 'epoch' + $3::bigint * interval '1 microsecond')";
constexpr const char* kCreatedUsSql =
"(EXTRACT(EPOCH FROM created_at) * 1000000)::bigint AS created_us";
inline bool is_uuid(const std::string& s) {
static const std::regex re(
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
return std::regex_match(s, re);
}
inline std::optional<Cursor> parse_cursor(const std::string& raw) {
const auto bar = raw.find('|');
if (bar == std::string::npos) {
return std::nullopt;
}
Cursor c{raw.substr(0, bar), raw.substr(bar + 1)};
const bool digits = !c.created_us.empty() && c.created_us.size() <= 18 &&
std::all_of(c.created_us.begin(), c.created_us.end(),
[](unsigned char ch) { return std::isdigit(ch); });
if (!digits || !is_uuid(c.id)) {
return std::nullopt;
}
return c;
}
inline std::string make_cursor(const std::string& created_us, const std::string& id) {
return created_us + "|" + id;
}
+370
View File
@@ -0,0 +1,370 @@
#include "telemetry.hpp"
#include "db.hpp"
#include "errors.hpp"
#include <chrono>
#include <fstream>
#include <future>
#include <thread>
#include <sys/socket.h>
#include <sys/time.h>
#include <stdexcept>
#include <algorithm>
#include <filesystem>
#include <vector>
#include <set>
#include <iostream>
#include <nlohmann/json.hpp>
namespace fs = std::filesystem;
namespace {
constexpr int kConnectTimeoutSeconds = 2;
constexpr int kTcpUserTimeoutMs = 5000;
constexpr int kStatementTimeoutMs = 5000;
constexpr int kClientIoTimeoutSeconds = 5;
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
std::string add_conn_param(std::string conninfo, const std::string& key,
const std::string& value) {
if (conninfo.find(key + "=") != std::string::npos) {
return conninfo;
}
const bool is_url = conninfo.rfind("postgres://", 0) == 0 ||
conninfo.rfind("postgresql://", 0) == 0;
if (is_url) {
conninfo += (conninfo.find('?') == std::string::npos ? "?" : "&");
} else {
conninfo += " ";
}
return conninfo + key + "=" + value;
}
bool is_alive(pqxx::connection* conn) {
std::promise<void> done;
auto fut = done.get_future();
std::thread th([conn, &done]() {
try {
pqxx::nontransaction tx(*conn);
tx.exec("SELECT 1");
done.set_value();
} catch (...) {
done.set_exception(std::current_exception());
}
});
if (fut.wait_for(std::chrono::seconds(kClientIoTimeoutSeconds)) !=
std::future_status::ready) {
const int fd = conn->sock();
if (fd >= 0) {
::shutdown(fd, SHUT_RDWR);
}
th.join();
return false;
}
th.join();
try {
fut.get();
return true;
} catch (...) {
return false;
}
}
struct Migration {
int version;
std::string path;
};
constexpr int kMigrationsAdvisoryLockKey = 0x6d747573;
Migration parse_migration_name(const fs::path& p) {
const std::string name = p.filename().string();
const auto dot = name.find('.');
if (dot == std::string::npos || name.substr(dot) != ".sql") {
throw std::runtime_error("migration file must end with .sql: " + name);
}
const auto us = name.find('_');
if (us == std::string::npos || us == 0) {
throw std::runtime_error(
"migration file must be named NNN_name.sql: " + name);
}
const std::string num = name.substr(0, us);
if (!std::all_of(num.begin(), num.end(),
[](unsigned char c){ return std::isdigit(c); })) {
throw std::runtime_error(
"migration file prefix must be digits: " + name);
}
Migration m;
m.version = std::stoi(num);
m.path = p.string();
return m;
}}
void apply_migrations(pqxx::connection& conn, const std::string& dir) {
{
pqxx::work tx(conn);
tx.exec("SELECT pg_advisory_lock($1)",
pqxx::params{kMigrationsAdvisoryLockKey});
tx.commit();
}
std::clog << "migration advisory lock acquired\n";
struct Unlock {
pqxx::connection& c;
~Unlock() {
try {
pqxx::work tx(c);
tx.exec("SELECT pg_advisory_unlock($1)",
pqxx::params{kMigrationsAdvisoryLockKey});
tx.commit();
} catch (...) {}
}
} unlock_guard{conn};
{
pqxx::work tx(conn);
tx.exec(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version int PRIMARY KEY,"
" applied_at timestamptz NOT NULL DEFAULT now()"
")");
tx.commit();
}
if (!fs::exists(dir) || !fs::is_directory(dir)) {
throw std::runtime_error("migrations dir not found: " + dir);
}
std::vector<Migration> files;
for (const auto& entry : fs::directory_iterator(dir)) {
if (entry.is_regular_file()) {
files.push_back(parse_migration_name(entry.path()));
}
}
std::sort(files.begin(), files.end(),
[](const Migration& a, const Migration& b){
return a.version < b.version;
});
for (std::size_t i = 1; i < files.size(); ++i) {
if (files[i].version == files[i-1].version) {
throw std::runtime_error(
"duplicate migration version: " + std::to_string(files[i].version));
}
}
std::set<int> applied;
{
pqxx::work tx(conn);
for (const auto& row : tx.exec("SELECT version FROM schema_migrations")) {
applied.insert(row[0].as<int>());
}
tx.commit();
}
for (const auto& m : files) {
if (applied.count(m.version)) {
continue;
}
std::ifstream f(m.path, std::ios::binary);
if (!f) {
throw std::runtime_error("cannot open migration: " + m.path);
}
std::stringstream buf;
buf << f.rdbuf();
const std::string sql = buf.str();
pqxx::work tx(conn);
tx.exec(sql);
tx.exec("INSERT INTO schema_migrations(version) VALUES ($1)",
m.version);
tx.commit();
std::clog << "applied migration " << m.version
<< " (" << m.path << ")\n";
}
}
std::string read_secret_file(const char* env_name) {
const char* path = std::getenv(env_name);
if (path == nullptr || *path == '\0') {
throw std::runtime_error(
std::string("environment variable not set: ") + env_name);
}
std::ifstream f(path, std::ios::binary);
if (!f) {
throw std::runtime_error(
std::string("cannot open secret file: ") + path);
}
std::string s;
std::getline(f, s);
if (!s.empty() && s.back() == '\r') {
s.pop_back();
}
if (s.empty()) {
throw std::runtime_error(
std::string("secret file is empty: ") + path);
}
return s;
}
bool is_dependency_failure(const std::exception& e) {
if (dynamic_cast<const DependencyUnavailable*>(&e) != nullptr ||
dynamic_cast<const pqxx::broken_connection*>(&e) != nullptr ||
dynamic_cast<const pqxx::in_doubt_error*>(&e) != nullptr) {
return true;
}
if (const auto* sql = dynamic_cast<const pqxx::sql_error*>(&e)) {
// 08xxx — ошибки соединения, 57xxx — отмена (statement_timeout) и остановка сервера.
const std::string& state = sql->sqlstate();
return state.rfind("08", 0) == 0 || state.rfind("57", 0) == 0;
}
return false;
}
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
bool statement_timeout) {
std::string full = add_conn_param(conninfo, "connect_timeout",
std::to_string(kConnectTimeoutSeconds));
full = add_conn_param(full, "tcp_user_timeout", std::to_string(kTcpUserTimeoutMs));
auto conn = std::make_unique<pqxx::connection>(full);
const int fd = conn->sock();
if (fd >= 0) {
timeval tv{};
tv.tv_sec = kClientIoTimeoutSeconds;
tv.tv_usec = 0;
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
}
if (statement_timeout) {
pqxx::nontransaction tx(*conn);
tx.exec("SET statement_timeout = " + std::to_string(kStatementTimeoutMs));
}
return conn;
}
PgPool::PgPool(std::string conninfo, std::size_t n)
: conninfo_(std::move(conninfo)) {
if (n == 0) {
throw std::runtime_error("PgPool: n must be > 0");
}
all_.reserve(n);
idle_.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
auto conn = open_pg_connection(conninfo_, true);
idle_.push_back(conn.get());
all_.push_back(std::move(conn));
}
telemetry::delta("db.pool.capacity", n);
}
PgPool::~PgPool() {
telemetry::delta("db.pool.capacity", -static_cast<long>(all_.size()));
{
std::lock_guard<std::mutex> lk(mu_);
shutting_down_ = true;
}
cv_.notify_all();
}
pqxx::connection* PgPool::take() {
telemetry::Span acquire_span("postgresql.pool.acquire", opentelemetry::trace::SpanKind::kClient);
struct Waiting {
Waiting() { telemetry::delta("db.pool.waiting", 1); }
~Waiting() { telemetry::delta("db.pool.waiting", -1); }
} waiting;
std::unique_lock<std::mutex> lk(mu_);
if (!cv_.wait_for(lk, kAcquireTimeout,
[&]() { return shutting_down_ || !idle_.empty(); })) {
throw DependencyUnavailable("postgres pool: acquire timeout");
}
if (shutting_down_) {
throw DependencyUnavailable("postgres pool is shutting down");
}
pqxx::connection* conn = idle_.back();
idle_.pop_back();
telemetry::delta("db.pool.used", 1);
lk.unlock();
if (is_alive(conn)) {
return conn;
}
std::unique_ptr<pqxx::connection> fresh;
try {
fresh = open_pg_connection(conninfo_, true);
} catch (const std::exception& e) {
// Мёртвое соединение возвращается в пул: следующий take() снова
// попробует переподключиться, и пул не теряет слот.
give_back(conn);
throw DependencyUnavailable(std::string("postgres reconnect failed: ") + e.what());
}
std::lock_guard<std::mutex> lk2(mu_);
for (auto& up : all_) {
if (up.get() == conn) {
up = std::move(fresh);
return up.get();
}
}
throw std::logic_error("PgPool: connection not owned by pool");
}
void PgPool::give_back(pqxx::connection* conn) {
{
std::lock_guard<std::mutex> lk(mu_);
idle_.push_back(conn);
telemetry::delta("db.pool.used", -1);
}
cv_.notify_one();
}
PgPool::Lease::Lease(PgPool& pool) : pool_(&pool), conn_(pool.take()) {}
PgPool::Lease::~Lease() {
if (pool_ != nullptr && conn_ != nullptr) {
pool_->give_back(conn_);
}
}
PgPool::Lease::Lease(Lease&& other) noexcept
: pool_(other.pool_), conn_(other.conn_) {
other.pool_ = nullptr;
other.conn_ = nullptr;
}
PgPool::Lease PgPool::acquire() {
return Lease(*this);
}
bool PgPool::ping() {
try {
auto lease = acquire();
pqxx::work tx(*lease);
tx.exec("SELECT 1");
tx.commit();
return true;
} catch (...) {
return false;
}
}
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include <condition_variable>
#include <pqxx/pqxx>
#include <cstddef>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
std::string read_secret_file(const char* env_name);
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
bool statement_timeout);
void apply_migrations(pqxx::connection& conn, const std::string& dir);
class PgPool {
public:
bool ping();
PgPool(std::string conninfo, std::size_t n);
~PgPool();
PgPool(const PgPool&) = delete;
PgPool& operator=(const PgPool&) = delete;
class Lease {
public:
explicit Lease(PgPool& pool);
~Lease();
Lease(Lease&& other) noexcept;
Lease& operator=(Lease&&) = delete;
Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
pqxx::connection& operator*() const { return *conn_; }
pqxx::connection* operator->() const { return conn_; }
private:
PgPool* pool_;
pqxx::connection* conn_;
};
Lease acquire();
private:
friend class Lease;
pqxx::connection* take();
void give_back(pqxx::connection* conn);
std::string conninfo_;
std::vector<std::unique_ptr<pqxx::connection>> all_;
std::vector<pqxx::connection*> idle_;
std::mutex mu_;
std::condition_variable cv_;
bool shutting_down_ = false;
};
+292
View File
@@ -0,0 +1,292 @@
#include "direct_upload_repo.hpp"
#include "outbox.hpp"
#include <pqxx/pqxx>
#include <nlohmann/json.hpp>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <sstream>
namespace {
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch())
.count();
return std::to_string(secs);
}
DirectUpload row_to(const pqxx::row& r) {
DirectUpload d;
d.id = r["id"].as<std::string>();
d.chat_id = r["chat_id"].as<std::string>();
d.owner_id = r["owner_id"].as<std::string>();
d.filename = r["filename"].as<std::string>();
d.content_type = r["content_type"].as<std::string>();
d.expected_size = r["expected_size"].as<int>();
d.expected_sha256 = r["expected_sha256"].as<std::string>();
d.s3_key = r["s3_key"].as<std::string>();
d.staging_key = r["staging_key"].as<std::string>();
d.status = r["status"].as<std::string>();
d.overdue = r["overdue"].as<bool>();
if (r["attachment_id"].is_null()) {
d.attachment_id = std::nullopt;
} else {
d.attachment_id = r["attachment_id"].as<std::string>();
}
d.expires_at = epoch_to_rfc3339(r["expires_at_epoch"].as<long long>());
d.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
d.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
return d;
}
constexpr const char* kCols =
"id, chat_id, owner_id, filename, content_type, expected_size, "
"expected_sha256, s3_key, staging_key, status, attachment_id, "
"(status = 'pending' AND expires_at < now()) AS overdue, "
"EXTRACT(EPOCH FROM expires_at)::bigint AS expires_at_epoch, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
constexpr const char* kAttachmentColumns =
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, error_code, variants, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
Attachment attachment_from_row(const pqxx::row& r) {
Attachment a;
a.id = r["id"].as<std::string>();
a.chat_id = r["chat_id"].as<std::string>();
a.owner_id = r["owner_id"].as<std::string>();
a.filename = r["filename"].as<std::string>();
a.content_type = r["content_type"].as<std::string>();
a.size_bytes = r["size_bytes"].as<int>();
a.sha256 = r["sha256"].as<std::string>();
a.status = r["status"].as<std::string>();
a.s3_key = r["s3_key"].as<std::string>();
if (r["error_code"].is_null()) {
a.error_code = std::nullopt;
} else {
a.error_code = r["error_code"].as<std::string>();
}
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
for (const auto& v : nlohmann::json::parse(r["variants"].as<std::string>())) {
a.variants.push_back(Variant{v.at("name").get<std::string>(),
v.at("content_type").get<std::string>(), v.at("size_bytes").get<int>(),
v.at("sha256").get<std::string>(), v.at("width").get<int>(),
v.at("height").get<int>(), v.value("storage_key", "")});
}
return a;
}
} // namespace
DirectUpload DirectUploadRepo::insert(const DirectUpload& in) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
// expires_at is already an absolute epoch string from the caller.
auto r = tx.exec(
"INSERT INTO direct_uploads ("
" id, chat_id, owner_id, filename, content_type, expected_size, "
" expected_sha256, s3_key, staging_key, status, attachment_id, "
" expires_at, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, NULL, "
" to_timestamp($11::bigint), to_timestamp($12::bigint), "
" to_timestamp($13::bigint)"
") RETURNING " +
std::string(kCols),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.expected_size, in.expected_sha256,
in.s3_key, in.staging_key, in.status, in.expires_at_epoch,
ts, ts});
tx.commit();
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::find_by_id(
const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kCols) +
" FROM direct_uploads WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::mark_completed(
const std::string& id, const std::string& attachment_id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'completed', "
" attachment_id = $2::uuid, "
" updated_at = to_timestamp($3::bigint) "
"WHERE id = $1::uuid AND status = 'pending' "
"RETURNING " +
std::string(kCols),
pqxx::params{id, attachment_id, ts});
if (r.empty()) {
return std::nullopt;
}
tx.commit();
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::mark_aborted(
const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'aborted', "
" updated_at = to_timestamp($2::bigint) "
"WHERE id = $1::uuid AND status = 'pending' "
"RETURNING " +
std::string(kCols),
pqxx::params{id, ts});
if (r.empty()) {
return std::nullopt;
}
tx.commit();
return row_to(r[0]);
}
std::vector<DirectUpload> DirectUploadRepo::expire_stale() {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'expired', "
" updated_at = to_timestamp($1::bigint) "
"WHERE status = 'pending' AND expires_at < now() "
"RETURNING " +
std::string(kCols),
pqxx::params{ts});
tx.commit();
std::vector<DirectUpload> out;
out.reserve(r.size());
for (const auto& row : r) {
out.push_back(row_to(row));
}
return out;
}
std::vector<std::string> DirectUploadRepo::pending_s3_keys() {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT staging_key FROM direct_uploads WHERE status = 'pending'");
std::vector<std::string> out;
out.reserve(r.size());
for (const auto& row : r) {
out.push_back(row["staging_key"].as<std::string>());
}
return out;
}
void DirectUploadRepo::with_lock(const std::function<void(UploadLock&)>& fn) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
UploadLock lock(tx);
fn(lock);
}
std::optional<DirectUpload> UploadLock::lock(const std::string& id) {
auto r = tx_.exec(
"SELECT " + std::string(kCols) +
" FROM direct_uploads WHERE id = $1::uuid FOR UPDATE",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to(r[0]);
}
std::optional<Attachment> UploadLock::find_attachment(const std::string& id) {
auto r = tx_.exec(
"SELECT " + std::string(kAttachmentColumns) +
" FROM attachments WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return attachment_from_row(r[0]);
}
std::optional<Attachment> UploadLock::insert_queued(const Attachment& in,
bool& inserted) {
inserted = false;
const std::string ts = now_epoch_str();
auto r = tx_.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, staging_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $9, NULL, "
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
") ON CONFLICT (id) DO NOTHING RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, ts, ts});
if (r.empty()) {
return std::nullopt;
}
tx_.exec("INSERT INTO jobs(id, attachment_id, source_key) VALUES($1::uuid, $1::uuid, $2)",
pqxx::params{in.id, in.s3_key});
enqueue_attachment_job(tx_, in.id);
inserted = true;
return attachment_from_row(r[0]);
}
void UploadLock::mark_completed(const std::string& id,
const std::string& attachment_id) {
const std::string ts = now_epoch_str();
auto r = tx_.exec(
"UPDATE direct_uploads SET status = 'completed', "
" attachment_id = $2::uuid, "
" updated_at = to_timestamp($3::bigint) "
"WHERE id = $1::uuid AND status = 'pending'",
pqxx::params{id, attachment_id, ts});
if (r.affected_rows() != 1) {
throw std::runtime_error("direct upload complete lost the pending row");
}
}
void UploadLock::mark_expired(const std::string& id) {
const std::string ts = now_epoch_str();
tx_.exec(
"UPDATE direct_uploads SET status = 'expired', "
" updated_at = to_timestamp($2::bigint) "
"WHERE id = $1::uuid AND status = 'pending'",
pqxx::params{id, ts});
}
void UploadLock::commit() {
if (committed_) {
return;
}
tx_.commit();
committed_ = true;
}
+63
View File
@@ -0,0 +1,63 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <functional>
#include <optional>
#include <string>
#include <vector>
class UploadLock;
class DirectUploadRepo {
public:
explicit DirectUploadRepo(PgPool& pool) : pool_(pool) {}
DirectUpload insert(const DirectUpload& in);
std::optional<DirectUpload> find_by_id(const std::string& id);
// Mark completed and link attachment_id. Returns nullopt if not pending
// (or already finished / wrong owner handled by caller).
std::optional<DirectUpload> mark_completed(const std::string& id,
const std::string& attachment_id);
std::optional<DirectUpload> mark_aborted(const std::string& id);
// pending → expired where expires_at < now; returns expired rows.
std::vector<DirectUpload> expire_stale();
// Staging keys of still-pending sessions (orphan GC must keep these).
std::vector<std::string> pending_s3_keys();
// One PG transaction. Callback may call S3 while the row lock is held.
// Commit explicitly via UploadLock::commit(); otherwise the work rolls back.
void with_lock(const std::function<void(UploadLock&)>& fn);
private:
PgPool& pool_;
};
// Row lock for a single complete(). Not safe to use after with_lock returns.
class UploadLock {
public:
std::optional<DirectUpload> lock(const std::string& id);
std::optional<Attachment> find_attachment(const std::string& id);
// Inserts attachment and its job in this transaction; job_id equals attachment id.
// ON CONFLICT (id) DO NOTHING. inserted=false and nullopt → row already exists.
std::optional<Attachment> insert_queued(const Attachment& in, bool& inserted);
void mark_completed(const std::string& id, const std::string& attachment_id);
void mark_expired(const std::string& id);
void commit();
private:
friend class DirectUploadRepo;
explicit UploadLock(pqxx::work& tx) : tx_(tx) {}
pqxx::work& tx_;
bool committed_ = false;
};
+15
View File
@@ -0,0 +1,15 @@
#pragma once
#include <exception>
#include <stdexcept>
// PostgreSQL или Valkey недоступны либо не ответили вовремя: клиент получает 503.
struct DependencyUnavailable : std::runtime_error {
using std::runtime_error::runtime_error;
};
struct AttachmentFailed : std::runtime_error {
AttachmentFailed() : std::runtime_error("attachment processing failed") {}
};
bool is_dependency_failure(const std::exception& e);
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include <cstdlib>
#include <cstring>
// Explicit local-test opt-in; no HTTP parameter can enable fault injection.
inline void lab_crash_at(const char* point) {
const char* enabled = std::getenv("LAB_ENABLE_FAULT_INJECTION");
const char* selected = std::getenv("LAB_FAULT_POINT");
if (enabled && std::strcmp(enabled, "1") == 0 && selected &&
std::strcmp(selected, point) == 0) std::_Exit(86);
}
#include <chrono>
#include <filesystem>
#include <fstream>
#include <string>
#include <thread>
// One injection per container/job/point; marker survives a process restart.
inline void lab_worker_fault(const char* point, const std::string& job_id, int version) {
const char* enabled = std::getenv("LAB_ENABLE_FAULT_INJECTION");
const char* selected = std::getenv("LAB_FAULT_POINT");
if (!enabled || std::strcmp(enabled, "1") || !selected || std::strcmp(selected, point)) return;
const char* job = std::getenv("LAB_FAULT_JOB_ID");
const char* target_version = std::getenv("LAB_FAULT_PIPELINE_VERSION");
if ((job && job_id != job) || (target_version && std::to_string(version) != target_version)) return;
const std::string base = "/tmp/lab-fault/" + std::string(point) + "." + job_id;
std::filesystem::create_directories("/tmp/lab-fault");
if (std::filesystem::exists(base + ".reached")) return;
std::ofstream marker(base + ".reached");
marker << version << '\n';
marker.close();
const char* action = std::getenv("LAB_FAULT_ACTION");
if (action && std::strcmp(action, "pause") == 0) {
while (!std::filesystem::exists(base + ".release")) {
std::this_thread::sleep_for(std::chrono::milliseconds(25));
}
} else {
std::_Exit(86);
}
}
+151
View File
@@ -0,0 +1,151 @@
#include "image_pipeline.hpp"
#include "image_util.hpp"
#include <stb_image.h>
#define STB_IMAGE_RESIZE_IMPLEMENTATION
#include <stb_image_resize.h>
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include <stb_image_write.h>
#include <libexif/exif-data.h>
#include <webp/encode.h>
#include <algorithm>
#include <cstring>
#include <memory>
namespace {
int orientation(const std::vector<std::uint8_t>& bytes) {
const unsigned char* data = bytes.data();
std::size_t size = bytes.size();
std::vector<unsigned char> exif;
// PNG eXIf stores a TIFF header without the JPEG Exif prefix.
if (size >= 8 && std::memcmp(data, "\x89PNG\r\n\x1a\n", 8) == 0) {
for (std::size_t pos = 8; pos + 12 <= size;) {
const std::uint32_t len = (std::uint32_t(data[pos]) << 24) |
(std::uint32_t(data[pos + 1]) << 16) |
(std::uint32_t(data[pos + 2]) << 8) | data[pos + 3];
if (len > size - pos - 12) break;
if (std::memcmp(data + pos + 4, "eXIf", 4) == 0) {
exif.assign({'E', 'x', 'i', 'f', 0, 0});
exif.insert(exif.end(), data + pos + 8, data + pos + 8 + len);
data = exif.data();
size = exif.size();
break;
}
pos += std::size_t(len) + 12;
}
}
std::unique_ptr<ExifData, decltype(&exif_data_unref)> metadata(
exif_data_new_from_data(data, static_cast<unsigned int>(size)), exif_data_unref);
if (!metadata) return 1;
const auto* entry = exif_content_get_entry(metadata->ifd[EXIF_IFD_0], EXIF_TAG_ORIENTATION);
if (!entry || entry->format != EXIF_FORMAT_SHORT || entry->size < 2) return 1;
const int result = exif_get_short(entry->data, exif_data_get_byte_order(metadata.get()));
return result >= 1 && result <= 8 ? result : 1;
}
struct Raster {
std::vector<unsigned char> pixels;
int width;
int height;
};
Raster decode(const std::vector<std::uint8_t>& bytes) {
const ImageLimits limits;
if (bytes.empty() || bytes.size() > static_cast<std::size_t>(limits.max_bytes) ||
!sniff_content_type(bytes.data(), bytes.size())) {
throw InvalidImage("invalid_image");
}
int w = 0, h = 0, channels = 0;
if (!stbi_info_from_memory(bytes.data(), static_cast<int>(bytes.size()), &w, &h, &channels) ||
w <= 0 || h <= 0) throw InvalidImage("invalid_image");
if (std::int64_t(w) * h > limits.max_pixels) throw InvalidImage("image_too_large");
std::unique_ptr<stbi_uc, decltype(&stbi_image_free)> rgba(
stbi_load_from_memory(bytes.data(), static_cast<int>(bytes.size()), &w, &h, &channels, 4),
stbi_image_free);
if (!rgba) throw InvalidImage("invalid_image");
const int o = orientation(bytes);
const int width = o >= 5 ? h : w;
const int height = o >= 5 ? w : h;
std::vector<unsigned char> rgb(std::size_t(width) * height * 3);
for (int y = 0; y < h; ++y) {
for (int x = 0; x < w; ++x) {
int dx = x, dy = y;
switch (o) {
case 2: dx = w - 1 - x; break;
case 3: dx = w - 1 - x; dy = h - 1 - y; break;
case 4: dy = h - 1 - y; break;
case 5: dx = y; dy = x; break;
case 6: dx = h - 1 - y; dy = x; break;
case 7: dx = h - 1 - y; dy = w - 1 - x; break;
case 8: dx = y; dy = w - 1 - x; break;
}
const auto* pixel = rgba.get() + (std::size_t(y) * w + x) * 4;
auto* target = rgb.data() + (std::size_t(dy) * width + dx) * 3;
for (int c = 0; c < 3; ++c) {
target[c] = static_cast<unsigned char>(
(pixel[c] * pixel[3] + 255 * (255 - pixel[3]) + 127) / 255);
}
}
}
rgba.reset();
return Raster{std::move(rgb), width, height};
}
Raster resize(const unsigned char* pixels, int width, int height, int stride,
int out_w, int out_h) {
Raster result{std::vector<unsigned char>(std::size_t(out_w) * out_h * 3), out_w, out_h};
if (!stbir_resize_uint8(pixels, width, height, stride,
result.pixels.data(), out_w, out_h, 0, 3)) {
throw std::runtime_error("resize failed");
}
return result;
}
Raster fit(const Raster& input, int limit) {
const int extent = std::max(input.width, input.height);
const int w = extent > limit ? std::max<std::int64_t>(1, std::int64_t(input.width) * limit / extent) : input.width;
const int h = extent > limit ? std::max<std::int64_t>(1, std::int64_t(input.height) * limit / extent) : input.height;
return resize(input.pixels.data(), input.width, input.height, 0, w, h);
}
EncodedImage jpeg(const Raster& raster, int quality = 85) {
EncodedImage image{{}, raster.width, raster.height};
const auto write = [](void* context, void* data, int size) {
static_cast<std::string*>(context)->append(static_cast<const char*>(data), size);
};
if (!stbi_write_jpg_to_func(write, &image.bytes, raster.width, raster.height,
3, raster.pixels.data(), quality)) {
throw std::runtime_error("JPEG encoding failed");
}
return image;
}
EncodedImage webp(const Raster& raster, float quality) {
unsigned char* output = nullptr;
const auto size = WebPEncodeRGB(raster.pixels.data(), raster.width, raster.height,
raster.width * 3, quality, &output);
std::unique_ptr<unsigned char, decltype(&WebPFree)> owned(output, WebPFree);
if (!size) throw std::runtime_error("WebP encoding failed");
return EncodedImage{std::string(reinterpret_cast<const char*>(output), size),
raster.width, raster.height};
}
} // namespace
EncodedImage make_thumbnail(const std::vector<std::uint8_t>& bytes) {
return jpeg(fit(decode(bytes), 256));
}
std::vector<EncodedVariant> make_variants(const std::vector<std::uint8_t>& bytes, int version) {
if (version != 1 && version != 2) throw std::invalid_argument("unsupported pipeline version");
const int quality = version == 1 ? 85 : 90;
const auto raster = decode(bytes);
std::vector<EncodedVariant> result;
result.push_back({"thumbnail", "image/jpeg", jpeg(fit(raster, 256), quality)});
const int side = std::min(raster.width, raster.height);
const int left = (raster.width - side) / 2;
const int top = (raster.height - side) / 2;
const auto* center = raster.pixels.data() + (std::size_t(top) * raster.width + left) * 3;
result.push_back({"crop", "image/jpeg", jpeg(resize(center, side, side, raster.width * 3, 128, 128), quality)});
result.push_back({"optimized", "image/webp", webp(fit(raster, 1280), version == 1 ? 80.0f : 90.0f)});
return result;
}
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <cstdint>
#include <stdexcept>
#include <string>
#include <vector>
struct InvalidImage : std::runtime_error {
explicit InvalidImage(const char* code) : std::runtime_error(code) {}
};
struct EncodedImage {
std::string bytes;
int width;
int height;
};
struct EncodedVariant {
std::string name;
std::string content_type;
EncodedImage image;
};
std::vector<EncodedVariant> make_variants(const std::vector<std::uint8_t>& original, int version = 1);
EncodedImage make_thumbnail(const std::vector<std::uint8_t>& original);
+90
View File
@@ -0,0 +1,90 @@
#include "image_util.hpp"
#define STB_IMAGE_IMPLEMENTATION
#define STBI_NO_STDIO
#define STBI_ONLY_JPEG
#define STBI_ONLY_PNG
#include "stb_image.h"
#include <cstring>
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
std::size_t len) {
if (!data || len < 3) {
return std::nullopt;
}
// JPEG: FF D8 FF
if (data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) {
return std::string("image/jpeg");
}
// PNG: 89 50 4E 47
if (len >= 4 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E &&
data[3] == 0x47) {
return std::string("image/png");
}
return std::nullopt;
}
std::string_view image_reject_code(ImageReject r) {
switch (r) {
case ImageReject::unsupported_media_type:
return "unsupported_media_type";
case ImageReject::payload_too_large:
return "payload_too_large";
case ImageReject::invalid_image:
return "invalid_image";
}
return "invalid_image";
}
std::optional<ImageReject> validate_image(const std::uint8_t* data,
std::size_t len,
const ImageLimits& limits,
ImageInfo& out) {
if (!data || len == 0) {
return ImageReject::unsupported_media_type;
}
if (static_cast<std::int64_t>(len) > limits.max_bytes) {
return ImageReject::payload_too_large;
}
auto sniffed = sniff_content_type(data, len);
if (!sniffed) {
return ImageReject::unsupported_media_type;
}
int w = 0;
int h = 0;
int comp = 0;
if (!stbi_info_from_memory(reinterpret_cast<const stbi_uc*>(data),
static_cast<int>(len), &w, &h, &comp)) {
return ImageReject::invalid_image;
}
if (w <= 0 || h <= 0) {
return ImageReject::invalid_image;
}
const std::int64_t pixels =
static_cast<std::int64_t>(w) * static_cast<std::int64_t>(h);
if (pixels > limits.max_pixels) {
return ImageReject::payload_too_large;
}
int load_w = 0;
int load_h = 0;
int load_comp = 0;
stbi_uc* decoded = stbi_load_from_memory(
reinterpret_cast<const stbi_uc*>(data), static_cast<int>(len), &load_w,
&load_h, &load_comp, 0);
if (!decoded) {
return ImageReject::invalid_image;
}
// Integrity only — do not retain decoded raster.
stbi_image_free(decoded);
out.content_type = *sniffed;
out.width = w;
out.height = h;
out.pixel_count = pixels;
return std::nullopt;
}
+41
View File
@@ -0,0 +1,41 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
struct ImageLimits {
std::int64_t max_bytes = 10'485'760; // 10 MiB
std::int64_t max_pixels = 40'000'000; // decoded width*height
};
enum class ImageReject {
unsupported_media_type, // → 415
payload_too_large, // → 413 (bytes or pixels)
invalid_image, // → 415 (corrupt / undecodable)
};
struct ImageInfo {
std::string content_type; // image/png | image/jpeg
int width = 0;
int height = 0;
std::int64_t pixel_count = 0;
};
// Magic sniff only (does not decode). Returns nullopt if neither PNG nor JPEG.
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
std::size_t len);
// Full validation pipeline from contracts:
// sniff → stbi_info (dims) → pixel limit → stbi_load (integrity) → free.
// On success fills `out`. On failure returns ImageReject; decoded pixels are
// never retained after return.
std::optional<ImageReject> validate_image(const std::uint8_t* data,
std::size_t len,
const ImageLimits& limits,
ImageInfo& out);
// Human-readable error_code for Attachment failed path / API body.
std::string_view image_reject_code(ImageReject r);
+110
View File
@@ -0,0 +1,110 @@
#include "db.hpp"
#include "outbox.hpp"
#include "s3_client.hpp"
#include <sodium.h>
#include <iostream>
#include <regex>
#include <nlohmann/json.hpp>
namespace {
void verify_source(S3Client& s3, const std::string& key, std::size_t expected,
const std::string& checksum) {
crypto_hash_sha256_state state;
crypto_hash_sha256_init(&state);
std::size_t received = 0;
s3.get_object(key, [&](const char* data, std::size_t n) {
if (received > expected || n > expected - received) throw std::runtime_error("source mismatch");
crypto_hash_sha256_update(&state, reinterpret_cast<const unsigned char*>(data), n);
received += n;
});
unsigned char digest[crypto_hash_sha256_BYTES];
crypto_hash_sha256_final(&state, digest);
char hex[crypto_hash_sha256_BYTES * 2 + 1];
sodium_bin2hex(hex, sizeof(hex), digest, sizeof(digest));
if (received != expected || checksum != hex) throw std::runtime_error("source checksum mismatch");
}
}
int main(int argc, char** argv) {
const bool replay = argc == 7 && std::string(argv[1]) == "replay" &&
std::string(argv[3]) == "--actor" && std::string(argv[5]) == "--reason";
const bool upgrade = argc == 9 && std::string(argv[1]) == "upgrade" &&
std::string(argv[3]) == "--version" && std::string(argv[5]) == "--actor" &&
std::string(argv[7]) == "--reason";
if (!replay && !upgrade) {
std::cerr << "usage: job_admin replay JOB_UUID --actor NAME --reason REASON\n"
" job_admin upgrade ATTACHMENT_UUID --version 2 --actor NAME --reason REASON\n";
return 2;
}
const std::string id = argv[2], actor = argv[replay ? 4 : 6], reason = argv[replay ? 6 : 8];
const std::regex uuid("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
if (!std::regex_match(id, uuid) || actor.empty() || actor.size() > 100 || reason.empty() ||
reason.size() > 500 || actor.find_first_not_of(" \t\r\n") == std::string::npos ||
reason.find_first_not_of(" \t\r\n") == std::string::npos) return 2;
if (upgrade && std::string(argv[4]) != "1" && std::string(argv[4]) != "2") return 2;
try {
if (sodium_init() < 0) return 1;
auto conn = open_pg_connection(read_secret_file("DATABASE_URL_FILE"), true);
S3Client s3(S3Client::from_env());
pqxx::work tx(*conn);
if (upgrade) {
const int version = std::stoi(argv[4]);
tx.exec("SELECT pg_advisory_xact_lock(hashtextextended('pipeline:' || $1::uuid::text,0))", pqxx::params{id});
auto rows = tx.exec("SELECT pipeline_version,s3_key,size_bytes,sha256 FROM attachments "
"WHERE id=$1::uuid FOR UPDATE", pqxx::params{id});
if (rows.empty() || version < rows[0]["pipeline_version"].as<int>()) return 1;
if (version == rows[0]["pipeline_version"].as<int>()) {
const auto jobs = tx.exec("SELECT id FROM jobs WHERE attachment_id=$1::uuid AND pipeline_version=$2",
pqxx::params{id, version});
if (jobs.empty()) return 1;
std::cout << nlohmann::json{{"job_id", jobs[0][0].as<std::string>()},
{"pipeline_version", version}, {"created", false}}.dump() << '\n';
return 0;
}
const auto& row = rows[0];
verify_source(s3, row["s3_key"].as<std::string>(), row["size_bytes"].as<std::size_t>(), row["sha256"].as<std::string>());
const auto job = tx.exec("INSERT INTO jobs(id,attachment_id,pipeline_version,source_key) "
"VALUES(gen_random_uuid(),$1::uuid,$2,$3) RETURNING id",
pqxx::params{id, version, row["s3_key"].as<std::string>()})[0][0].as<std::string>();
tx.exec("UPDATE attachments SET pipeline_version=$2, "
"status=CASE WHEN published_version IS NULL THEN 'queued' ELSE status END, "
"error_code=CASE WHEN published_version IS NULL THEN NULL ELSE error_code END, "
"updated_at=CASE WHEN published_version IS NULL THEN now() ELSE updated_at END WHERE id=$1::uuid",
pqxx::params{id, version});
tx.exec("INSERT INTO pipeline_requests(attachment_id,pipeline_version,job_id,actor,reason) "
"VALUES($1::uuid,$2,$3::uuid,$4,$5)", pqxx::params{id, version, job, actor, reason});
enqueue_attachment_job(tx, job);
tx.commit();
std::cout << nlohmann::json{{"job_id", job}, {"pipeline_version", version}, {"created", true}}.dump() << '\n';
return 0;
}
if (!tx.exec("SELECT pg_try_advisory_xact_lock(hashtextextended($1::uuid::text,0))",
pqxx::params{id})[0][0].as<bool>()) return 1;
const auto rows = tx.exec(
"SELECT j.attachment_id,j.status,j.retries,j.source_key,j.pipeline_version, "
"a.pipeline_version AS desired_version,a.staging_cleaned_at,a.size_bytes,a.sha256 "
"FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{id});
if (rows.empty() || rows[0]["status"].as<std::string>() != "failed" ||
rows[0]["pipeline_version"].as<int>() != rows[0]["desired_version"].as<int>()) return 1;
const auto& row = rows[0];
const std::string source = row["source_key"].as<std::string>();
if (source.rfind("staging/",0) == 0 && !row["staging_cleaned_at"].is_null()) return 1;
verify_source(s3, source, row["size_bytes"].as<std::size_t>(), row["sha256"].as<std::string>());
tx.exec("INSERT INTO job_replays(job_id,actor,reason,previous_error_code,retries_before) "
"SELECT id,$2,$3,error_code,retries FROM jobs WHERE id=$1::uuid", pqxx::params{id, actor, reason});
tx.exec("UPDATE jobs SET status='queued',retry_base=retries,error_code=NULL, "
"next_attempt_at=now(),updated_at=now() WHERE id=$1::uuid", pqxx::params{id});
tx.exec("UPDATE attachments SET status='queued',s3_key=$2,error_code=NULL,variants='[]'::jsonb,updated_at=now() "
"WHERE id=$1::uuid AND published_version IS NULL",
pqxx::params{row["attachment_id"].as<std::string>(), source});
enqueue_attachment_job(tx, id);
tx.commit();
std::cout << "replayed job=" << id << '\n';
return 0;
} catch (const std::exception&) {
std::cerr << "operator action failed; transaction was not confirmed\n";
return 1;
}
}
+139
View File
@@ -0,0 +1,139 @@
#include "telemetry.hpp"
#include "job_repo.hpp"
#include "image_pipeline.hpp"
#include "outbox.hpp"
#include "fault_injection.hpp"
#include <iostream>
#include <optional>
#include <algorithm>
#include <random>
JobRepo::Result JobRepo::run(const std::string& job_id, const std::string& attachment_id,
int version, int attempt, const std::function<JobResult(const JobInput&)>& process) {
auto lease = pool_.acquire();
{
pqxx::nontransaction tx(*lease);
if (!tx.exec("SELECT pg_try_advisory_lock(hashtextextended($1::uuid::text, 0))",
pqxx::params{job_id})[0][0].as<bool>()) return Result::Busy;
}
struct Unlock {
pqxx::connection& conn;
const std::string& id;
~Unlock() {
try {
pqxx::nontransaction tx(conn);
tx.exec("SELECT pg_advisory_unlock(hashtextextended($1::uuid::text, 0))", pqxx::params{id});
} catch (...) {}
}
} unlock{*lease, job_id};
JobInput input;
{
pqxx::work tx(*lease);
auto rows = tx.exec(
"SELECT j.status, j.retries, j.retry_base, j.source_key, j.pipeline_version AS job_version, "
"a.pipeline_version AS desired_version, j.next_attempt_at<=now() AS due, "
"a.status AS attachment_status FROM jobs j "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid AND j.attachment_id=$2::uuid FOR UPDATE OF j, a",
pqxx::params{job_id, attachment_id});
if (rows.empty()) return Result::Missing;
const auto& row = rows[0];
if (row["job_version"].as<int>() != version) return Result::Missing;
const std::string status = row["status"].as<std::string>();
if (version < row["desired_version"].as<int>() || status == "superseded") {
tx.exec("UPDATE jobs SET status='superseded', updated_at=now() "
"WHERE id=$1::uuid AND status IN ('queued','processing')", pqxx::params{job_id});
tx.commit();
return Result::Superseded;
}
if (version > row["desired_version"].as<int>()) return Result::Missing;
if (status == "ready" || status == "failed") return Result::Done;
input = JobInput{version, row["source_key"].as<std::string>()};
const int retries = row["retries"].as<int>();
// A previous processing state with no advisory lock means its worker died.
if (status == "processing") {
return fail(tx, job_id, attachment_id, version, retries + 1, row["retry_base"].as<int>(), "processing_failed", false);
}
// A newer attempt is already durable in outbox/the stream; discard an old delivery.
if (attempt < retries + 1) return Result::Scheduled;
if (attempt > retries + 1) return Result::Missing;
if (!row["due"].as<bool>()) return Result::Busy;
tx.exec("UPDATE jobs SET status='processing', error_code=NULL, updated_at=now() "
"WHERE id=$1::uuid", pqxx::params{job_id});
tx.exec("UPDATE attachments SET status='processing', error_code=NULL, updated_at=now() "
"WHERE id=$1::uuid AND pipeline_version=$2 AND published_version IS NULL",
pqxx::params{attachment_id, version});
tx.commit();
}
try {
lab_worker_fault("worker_before_effect", job_id, version);
const JobResult result = process(input);
telemetry::Span commit_span("postgresql.publish_ready", opentelemetry::trace::SpanKind::kClient);
pqxx::work tx(*lease);
const auto current = tx.exec(
"SELECT a.pipeline_version, extract(epoch FROM now()-j.created_at) AS ready_seconds FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{job_id});
tx.exec("INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants) "
"VALUES($1::uuid,$2,$3::uuid,$4,$5::jsonb) ON CONFLICT DO NOTHING",
pqxx::params{attachment_id, version, job_id, result.original_key, result.variants_json});
const bool publish = current[0][0].as<int>() == version;
if (publish) {
// The pointer and complete result become visible in one transaction.
tx.exec("UPDATE attachments a SET status='ready', published_version=$2, "
"s3_key=r.original_key, variants=r.variants, error_code=NULL, updated_at=now() "
"FROM attachment_results r WHERE a.id=$1::uuid AND a.pipeline_version=$2 "
"AND r.attachment_id=a.id AND r.pipeline_version=$2",
pqxx::params{attachment_id, version});
}
tx.exec("UPDATE jobs SET status=$2, error_code=NULL, updated_at=now() WHERE id=$1::uuid",
pqxx::params{job_id, publish ? "ready" : "superseded"});
tx.commit();
if (publish) telemetry::duration("attachment.ready.duration", current[0]["ready_seconds"].as<double>());
lab_worker_fault("worker_after_commit", job_id, version);
return publish ? Result::Done : Result::Superseded;
} catch (const std::exception& e) {
const auto* invalid = dynamic_cast<const InvalidImage*>(&e);
const std::string code = invalid ? invalid->what() : "processing_failed";
pqxx::work tx(*lease);
auto rows = tx.exec("SELECT j.status,j.retries,j.retry_base,a.pipeline_version FROM jobs j "
"JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1::uuid FOR UPDATE OF j,a",
pqxx::params{job_id});
if (rows.empty()) return Result::Missing;
if (rows[0]["pipeline_version"].as<int>() != version) {
tx.exec("UPDATE jobs SET status='superseded',updated_at=now() WHERE id=$1::uuid "
"AND status IN ('queued','processing')", pqxx::params{job_id});
tx.commit();
return Result::Superseded;
}
const std::string status = rows[0]["status"].as<std::string>();
// A lost COMMIT reply must not roll back an already published ready result.
if (status == "ready" || status == "failed") return Result::Done;
return fail(tx, job_id, attachment_id, version, rows[0]["retries"].as<int>() + 1,
rows[0]["retry_base"].as<int>(), code, invalid != nullptr);
}
}
JobRepo::Result JobRepo::fail(pqxx::work& tx, const std::string& job_id,
const std::string& attachment_id, int version, int retries, int retry_base,
const std::string& code, bool permanent) {
const bool failed = permanent || retries - retry_base >= policy_.max_attempts;
long long ceiling = policy_.base_delay_ms;
for (int i = 1; i < retries - retry_base; ++i) ceiling = std::min<long long>(policy_.max_delay_ms, ceiling * 2);
ceiling = std::min<long long>(policy_.max_delay_ms, ceiling);
thread_local std::mt19937 rng(std::random_device{}());
const int delay = failed ? 0 : std::uniform_int_distribution<int>(
static_cast<int>((ceiling + 1) / 2), static_cast<int>(ceiling))(rng);
tx.exec("UPDATE jobs SET status=$2, retries=$3, error_code=$4, "
"next_attempt_at=now()+($5::bigint * interval '1 millisecond'), updated_at=now() "
"WHERE id=$1::uuid",
pqxx::params{job_id, failed ? "failed" : "queued", retries, code, delay});
tx.exec("UPDATE attachments SET status=$2, error_code=$3, updated_at=now() "
"WHERE id=$1::uuid AND status='processing' AND pipeline_version=$4 AND published_version IS NULL",
pqxx::params{attachment_id, failed ? "failed" : "queued",
failed ? std::optional<std::string>(code) : std::nullopt, version});
if (!failed) enqueue_attachment_job(tx, job_id);
tx.commit();
telemetry::event(failed ? "worker.attempt.failed" : "worker.attempt.retry", true);
return failed ? Result::Done : Result::Scheduled;
}
+35
View File
@@ -0,0 +1,35 @@
#pragma once
#include "db.hpp"
#include <functional>
#include <string>
struct JobInput {
int pipeline_version;
std::string source_key;
};
struct JobResult {
std::string original_key;
std::string variants_json;
};
struct RetryPolicy {
int max_attempts = 6;
int base_delay_ms = 1000;
int max_delay_ms = 60000;
};
class JobRepo {
public:
enum class Result { Done, Scheduled, Busy, Missing, Superseded };
JobRepo(PgPool& pool, RetryPolicy policy) : pool_(pool), policy_(policy) {}
Result run(const std::string& job_id, const std::string& attachment_id,
int version, int attempt, const std::function<JobResult(const JobInput&)>& process);
private:
Result fail(pqxx::work& tx, const std::string& job_id,
const std::string& attachment_id, int version, int retries, int retry_base,
const std::string& code, bool permanent);
PgPool& pool_;
RetryPolicy policy_;
};
+2361
View File
File diff suppressed because it is too large Load Diff
+377
View File
@@ -0,0 +1,377 @@
#include "message_repo.hpp"
#include "cursor.hpp"
#include "errors.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <cstdlib>
#include <ctime>
#include <iomanip>
#include <mutex>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <sodium.h>
#include <vector>
namespace {
std::string make_uuid() {
static std::mutex rng_mu;
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::lock_guard<std::mutex> lock(rng_mu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string sha256_hex(const std::string& s) {
unsigned char out[crypto_hash_sha256_BYTES];
crypto_hash_sha256(out, reinterpret_cast<const unsigned char*>(s.data()), s.size());
static const char hex[] = "0123456789abcdef";
std::string r;
r.resize(sizeof(out) * 2);
for (std::size_t i = 0; i < sizeof(out); ++i) {
r[i * 2] = hex[out[i] >> 4];
r[i * 2 + 1] = hex[out[i] & 0x0f];
}
return r;
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
Message row_to_message(const pqxx::row& r) {
Message m;
m.id = r["id"].as<std::string>();
m.chat_id = r["chat_id"].as<std::string>();
m.sender_id = r["sender_id"].as<std::string>();
m.text = r["text"].as<std::string>();
m.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return m;
}
constexpr const char* kMessageSelect =
"id, chat_id, sender_id, text, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
std::string canonical_message_body(const std::string& text,
const std::vector<std::string>& attachment_ids) {
std::ostringstream out;
out << "text=" << text << "\nattachment_ids=";
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
if (i > 0) {
out << ',';
}
out << attachment_ids[i];
}
return out.str();
}
int idempotency_ttl_seconds() {
const char* env = std::getenv("IDEMPOTENCY_TTL_SECONDS");
if (env == nullptr || *env == '\0') {
return 86400;
}
try {
const int v = std::stoi(env);
if (v <= 0) {
return 86400;
}
return v;
} catch (...) {
return 86400;
}
}
bool attachments_belong_to_chat(pqxx::work& tx,
const std::string& chat_id,
const std::vector<std::string>& attachment_ids) {
bool failed = false;
for (const auto& id : attachment_ids) {
auto r = tx.exec(
"SELECT status FROM attachments "
"WHERE id = $1 AND chat_id = $2 FOR SHARE",
pqxx::params{id, chat_id});
if (r.empty()) {
return false;
}
failed = failed || r[0]["status"].as<std::string>() == "failed";
}
if (failed) {
throw AttachmentFailed();
}
return true;
}
void link_attachments(pqxx::work& tx,
const std::string& message_id,
const std::vector<std::string>& attachment_ids) {
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
tx.exec(
"INSERT INTO message_attachments (message_id, attachment_id, position) "
"VALUES ($1, $2, $3)",
pqxx::params{message_id, attachment_ids[i], static_cast<int>(i)});
}
}
std::vector<std::string> load_attachment_ids(pqxx::work& tx,
const std::string& message_id) {
auto r = tx.exec(
"SELECT attachment_id FROM message_attachments "
"WHERE message_id = $1 ORDER BY position ASC",
pqxx::params{message_id});
std::vector<std::string> ids;
ids.reserve(r.size());
for (const auto& row : r) {
ids.push_back(row["attachment_id"].as<std::string>());
}
return ids;
}
void fill_attachment_ids(pqxx::work& tx, Message& m) {
m.attachment_ids = load_attachment_ids(tx, m.id);
}
void fill_attachment_ids(pqxx::work& tx, std::vector<Message>& items) {
for (auto& m : items) {
fill_attachment_ids(tx, m);
}
}
Message insert_message(pqxx::work& tx,
const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids) {
const std::string id = make_uuid();
const std::string created = now_epoch_str();
auto r = tx.exec(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
"RETURNING " + std::string(kMessageSelect),
pqxx::params{id, chat_id, sender_id, text, created});
Message m = row_to_message(r[0]);
link_attachments(tx, m.id, attachment_ids);
m.attachment_ids = attachment_ids;
return m;
}
bool insert_idempotency_key(pqxx::work& tx,
const std::string& user_id,
const std::string& chat_id,
const std::string& idem_key,
const std::string& request_hash,
const std::string& message_id) {
auto inserted = tx.exec(
"INSERT INTO idempotency_keys "
"(user_id, chat_id, key, request_hash, message_id) "
"VALUES ($1, $2, $3, $4, $5) "
"ON CONFLICT (user_id, chat_id, key) DO NOTHING "
"RETURNING message_id",
pqxx::params{user_id, chat_id, idem_key, request_hash, message_id});
return !inserted.empty();
}
Message load_message(pqxx::work& tx, const std::string& message_id) {
auto r = tx.exec(
"SELECT " + std::string(kMessageSelect) + " FROM messages WHERE id = $1",
pqxx::params{message_id});
if (r.empty()) {
throw std::runtime_error("idempotent message row is missing");
}
Message m = row_to_message(r[0]);
fill_attachment_ids(tx, m);
return m;
}
} // namespace
std::optional<Message> MessageRepo::create(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto member_check = tx.exec(
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
pqxx::params{chat_id, sender_id});
if (member_check.empty()) {
tx.commit();
return std::nullopt;
}
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return std::nullopt;
}
Message created = insert_message(tx, chat_id, sender_id, text, attachment_ids);
tx.commit();
return created;
}
std::optional<MessagePage> MessageRepo::list_page(
const std::string& chat_id,
int limit,
const std::optional<std::string>& cursor) {
std::optional<Cursor> cur;
if (cursor) {
cur = parse_cursor(*cursor);
if (!cur) {
return std::nullopt;
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
if (cur) {
auto owner_check = tx.exec(
"SELECT 1 FROM messages WHERE id = $1 AND chat_id = $2",
pqxx::params{cur->id, chat_id});
if (owner_check.empty()) {
tx.commit();
return std::nullopt;
}
}
std::string sql =
"SELECT " + std::string(kMessageSelect) + ", " + kCreatedUsSql + " "
"FROM messages "
"WHERE chat_id = $1 ";
if (cur) {
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
}
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
pqxx::result r;
if (cur) {
r = tx.exec(sql, pqxx::params{chat_id, limit + 1, cur->created_us, cur->id});
} else {
r = tx.exec(sql, pqxx::params{chat_id, limit + 1});
}
MessagePage page;
const bool has_more = (static_cast<int>(r.size()) > limit);
const int take = has_more ? limit : static_cast<int>(r.size());
page.items.reserve(take);
for (int i = 0; i < take; ++i) {
page.items.push_back(row_to_message(r[static_cast<std::size_t>(i)]));
}
fill_attachment_ids(tx, page.items);
if (has_more && take > 0) {
const auto& last = r[static_cast<std::size_t>(take - 1)];
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
last["id"].as<std::string>());
}
tx.commit();
return page;
}
CreateResult MessageRepo::create_idempotent(
const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids,
const std::string& idem_key) {
const int ttl = idempotency_ttl_seconds();
const std::string hash = sha256_hex(canonical_message_body(text, attachment_ids));
for (int attempt = 0; attempt < 3; ++attempt) {
{
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto member_check = tx.exec(
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
pqxx::params{chat_id, sender_id});
if (member_check.empty()) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
const Message created =
insert_message(tx, chat_id, sender_id, text, attachment_ids);
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
created.id)) {
tx.commit();
return CreateResult{CreateResult::Status::Created, created};
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto existing = tx.exec(
"SELECT request_hash, message_id, "
"EXTRACT(EPOCH FROM (now() - created_at))::bigint AS age_seconds "
"FROM idempotency_keys "
"WHERE user_id = $1 AND chat_id = $2 AND key = $3 "
"FOR UPDATE",
pqxx::params{sender_id, chat_id, idem_key});
if (existing.empty()) {
tx.commit();
continue;
}
const std::string existing_hash = existing[0]["request_hash"].as<std::string>();
const long long age = existing[0]["age_seconds"].as<long long>();
const std::string message_id = existing[0]["message_id"].as<std::string>();
if (age < ttl) {
if (existing_hash != hash) {
tx.commit();
return CreateResult{CreateResult::Status::KeyConflict, std::nullopt};
}
const Message replayed = load_message(tx, message_id);
tx.commit();
return CreateResult{CreateResult::Status::Replayed, replayed};
}
tx.exec(
"DELETE FROM idempotency_keys "
"WHERE user_id = $1 AND chat_id = $2 AND key = $3",
pqxx::params{sender_id, chat_id, idem_key});
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
const Message created =
insert_message(tx, chat_id, sender_id, text, attachment_ids);
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
created.id)) {
tx.commit();
return CreateResult{CreateResult::Status::Created, created};
}
}
throw std::runtime_error("idempotency retry exhausted");
}
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
struct CreateResult {
enum class Status { Created, Replayed, NotFound, KeyConflict };
Status status;
std::optional<Message> message;
};
class MessageRepo {
public:
explicit MessageRepo(PgPool& pool) : pool_(pool) {}
std::optional<Message> create(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids);
std::optional<MessagePage> list_page(
const std::string& chat_id,
int limit,
const std::optional<std::string>& cursor);
CreateResult create_idempotent(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids,
const std::string& idem_key);
private:
PgPool& pool_;
};
+86
View File
@@ -0,0 +1,86 @@
#pragma once
#include <optional>
#include <string>
#include <vector>
struct User {
std::string id;
std::string username;
std::string display_name;
std::string password_hash;
std::string created_at;
};
struct Chat {
std::string id;
std::string title;
std::vector<std::string> member_ids;
std::string created_at;
};
struct Message {
std::string id;
std::string chat_id;
std::string sender_id;
std::string text;
std::string created_at;
std::vector<std::string> attachment_ids;
};
struct MessagePage {
std::vector<Message> items;
std::optional<std::string> next_cursor;
};
struct ChatPage {
std::vector<Chat> items;
std::optional<std::string> next_cursor;
};
// Public metadata; storage keys are derived internally.
struct Variant {
std::string name;
std::string content_type;
int size_bytes = 0;
std::string sha256;
int width = 0;
int height = 0;
std::string storage_key; // internal; filtered from API JSON
};
struct Attachment {
std::string id;
std::string chat_id;
std::string owner_id;
std::string filename;
std::string content_type;
int size_bytes = 0;
std::string sha256;
std::string status; // queued|processing|ready|failed
std::vector<Variant> variants;
std::optional<std::string> error_code;
std::string s3_key; // internal; never exposed in API JSON
std::string created_at;
std::string updated_at;
};
// Track 1: pending presigned PUT session (not a public attachment until complete).
struct DirectUpload {
std::string id;
std::string chat_id;
std::string owner_id;
std::string filename;
std::string content_type;
int expected_size = 0;
std::string expected_sha256;
std::string s3_key; // final key: attachments/{chat}/{id}
std::string staging_key; // presigned PUT target: staging/{chat}/{id}
std::string status; // pending|completed|aborted|expired
bool overdue = false; // pending && expires_at < now(); not a column
std::optional<std::string> attachment_id;
std::string expires_at; // RFC3339 for JSON
long long expires_at_epoch = 0; // for INSERT
std::string created_at;
std::string updated_at;
};
+37
View File
@@ -0,0 +1,37 @@
#include "observability_monitor.hpp"
#include "db.hpp"
#include "telemetry.hpp"
#include <fstream>
#include <sys/resource.h>
#include <unistd.h>
ObservabilityMonitor::ObservabilityMonitor(std::string db_url, bool queue)
: thread_([this, db_url, queue] {
std::unique_ptr<pqxx::connection> conn;
while (!stopping_) {
try {
rusage usage{};
if (getrusage(RUSAGE_SELF, &usage) == 0)
telemetry::gauge("process.cpu.time", usage.ru_utime.tv_sec + usage.ru_stime.tv_sec +
(usage.ru_utime.tv_usec + usage.ru_stime.tv_usec)/1e6);
long total, resident;
std::ifstream statm("/proc/self/statm");
if (statm >> total >> resident)
telemetry::gauge("process.memory.usage", resident * sysconf(_SC_PAGESIZE));
if (queue) {
if (!conn) conn = open_pg_connection(db_url, true);
pqxx::nontransaction tx(*conn);
// Logical durable backlog includes outbox, Redis pending and delayed retries,
// without counting duplicate stream entries or ACKed historical entries.
const auto rows = tx.exec("SELECT count(*), coalesce(extract(epoch FROM now()-min(created_at)),0) "
"FROM jobs WHERE status IN ('queued','processing')");
telemetry::gauge("queue.depth", rows[0][0].as<double>());
telemetry::gauge("queue.oldest.age", rows[0][1].as<double>());
telemetry::gauge("queue.observation.timestamp", std::chrono::duration<double>(
std::chrono::system_clock::now().time_since_epoch()).count());
}
} catch (...) { conn.reset(); telemetry::event("telemetry.queue.observation.failed", true); }
for (int i=0; i<50 && !stopping_; ++i) std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}) {}
ObservabilityMonitor::~ObservabilityMonitor() { stopping_ = true; thread_.join(); }
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include <atomic>
#include <string>
#include <thread>
// Dedicated connection: telemetry queries never consume the application's pool.
class ObservabilityMonitor {
public:
ObservabilityMonitor(std::string db_url, bool queue);
~ObservabilityMonitor();
private:
std::atomic<bool> stopping_{false};
std::thread thread_;
};
+80
View File
@@ -0,0 +1,80 @@
#include "telemetry.hpp"
#include "outbox.hpp"
#include "attachment_queue.hpp"
#include "fault_injection.hpp"
#include <chrono>
#include <nlohmann/json.hpp>
#include <iostream>
#include <utility>
void enqueue_attachment_job(pqxx::work& tx, const std::string& job_id) {
telemetry::Span producer("attachment.enqueue", opentelemetry::trace::SpanKind::kProducer);
producer.span->SetAttribute("messaging.system", "redis");
producer.span->SetAttribute("messaging.destination.name", "jobs:attachments");
const auto rows = tx.exec(
"INSERT INTO outbox(payload, available_at) "
"SELECT jsonb_strip_nulls(jsonb_build_object('traceparent', nullif($2::text,''), 'tracestate', $3::text, 'schema_version', 1, 'type', 'attachment.process.v1', "
"'job_id', id::text, 'attachment_id', attachment_id::text, 'pipeline_version', pipeline_version, "
"'attempt', retries + 1, 'created_at', "
"to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"'))), "
"next_attempt_at FROM jobs WHERE id=$1::uuid RETURNING id", pqxx::params{job_id, telemetry::inject(), telemetry::inject("tracestate")});
if (rows.size() != 1) throw std::runtime_error("outbox job missing");
}
OutboxPublisher::OutboxPublisher(std::string db_url, std::string broker_url)
: thread_([this, db_url = std::move(db_url), broker_url = std::move(broker_url)] {
run(db_url, broker_url);
}) {}
OutboxPublisher::~OutboxPublisher() {
stopping_.store(true);
cv_.notify_all();
thread_.join();
}
bool OutboxPublisher::wait_interval() {
std::unique_lock<std::mutex> lock(mutex_);
return cv_.wait_for(lock, std::chrono::seconds(1), [this] { return stopping_.load(); });
}
void OutboxPublisher::run(const std::string& db_url, const std::string& broker_url) {
while (!stopping_.load()) {
try {
// Dedicated pools keep broker failure/reconnect off the HTTP path.
PgPool pg(db_url, 1);
RedisPool redis(broker_url, 1);
AttachmentQueue queue(redis);
while (!stopping_.load()) {
{
auto lease = pg.acquire();
pqxx::work tx(*lease);
const auto rows = tx.exec(
"SELECT id, payload FROM outbox WHERE NOT published AND available_at<=now() "
"ORDER BY available_at, id LIMIT 32 FOR UPDATE SKIP LOCKED");
for (const auto& row : rows) {
auto payload = nlohmann::json::parse(row["payload"].as<std::string>());
telemetry::Span publish("jobs:attachments publish", opentelemetry::trace::SpanKind::kProducer,
payload.value("traceparent", ""), payload.value("tracestate", ""));
publish.span->SetAttribute("messaging.system", "redis");
publish.span->SetAttribute("messaging.destination.name", "jobs:attachments");
const auto parent = telemetry::inject();
if (!parent.empty()) payload["traceparent"] = parent;
else payload.erase("traceparent");
payload["tracestate"] = telemetry::inject("tracestate");
queue.publish(payload.dump());
telemetry::event("queue.published");
lab_crash_at("outbox_after_publish");
tx.exec("UPDATE outbox SET published=true WHERE id=$1",
pqxx::params{row["id"].as<long long>()});
}
tx.commit();
}
if (wait_interval()) break;
}
} catch (const std::exception&) {
telemetry::event("queue.publish.unavailable", true);
if (wait_interval()) break;
}
}
}
+26
View File
@@ -0,0 +1,26 @@
#pragma once
#include "db.hpp"
#include <atomic>
#include <condition_variable>
#include <mutex>
#include <string>
#include <thread>
// Must share the transaction that inserts/updates the job.
void enqueue_attachment_job(pqxx::work& tx, const std::string& job_id);
class OutboxPublisher {
public:
OutboxPublisher(std::string db_url, std::string broker_url);
~OutboxPublisher();
OutboxPublisher(const OutboxPublisher&) = delete;
OutboxPublisher& operator=(const OutboxPublisher&) = delete;
private:
void run(const std::string& db_url, const std::string& broker_url);
bool wait_interval();
std::atomic<bool> stopping_{false};
std::mutex mutex_;
std::condition_variable cv_;
std::thread thread_;
};
+66
View File
@@ -0,0 +1,66 @@
#include "pg_session_repo.hpp"
#include <pqxx/pqxx>
#include <sodium.h>
#include <ctime>
#include <stdexcept>
#include <string>
namespace {
std::string random_token_hex() {
unsigned char buf[32];
randombytes_buf(buf, sizeof(buf));
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(sizeof(buf) * 2);
for (std::size_t i = 0; i < sizeof(buf); ++i) {
out[i * 2] = hex[buf[i] >> 4];
out[i * 2 + 1] = hex[buf[i] & 0x0f];
}
return out;
}
std::int64_t now_unix() {
return static_cast<std::int64_t>(std::time(nullptr));
}
}
ISessionRepo::Created PgSessionRepo::create(const std::string& user_id) {
const std::string token = random_token_hex();
const std::int64_t exp = now_unix() + ttl_;
auto lease = pool_.acquire();
pqxx::work tx(*lease);
tx.exec(
"INSERT INTO sessions (token, user_id, expires_at) "
"VALUES ($1, $2, to_timestamp($3::bigint))",
pqxx::params{token, user_id, exp});
tx.commit();
return Created{token, exp};
}
std::optional<std::string> PgSessionRepo::lookup(const std::string& token) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT user_id FROM sessions "
"WHERE token = $1 AND expires_at > now()",
pqxx::params{token});
tx.commit();
if (r.empty()) return std::nullopt;
return r[0]["user_id"].as<std::string>();
}
bool PgSessionRepo::revoke(const std::string& token) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"DELETE FROM sessions WHERE token = $1",
pqxx::params{token});
tx.commit();
return r.affected_rows() > 0;
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "db.hpp"
#include "session_repo_base.hpp"
class PgSessionRepo : public ISessionRepo {
public:
PgSessionRepo(PgPool& pool, int ttl_seconds)
: pool_(pool), ttl_(ttl_seconds) {}
Created create(const std::string& user_id) override;
std::optional<std::string> lookup(const std::string& token) override;
bool revoke(const std::string& token) override;
private:
PgPool& pool_;
int ttl_;
};
+216
View File
@@ -0,0 +1,216 @@
#include "redis_pool.hpp"
#include "errors.hpp"
#include <chrono>
#include <cstdlib>
#include <stdexcept>
#include <string>
namespace {
constexpr timeval kConnectTimeout{2, 0};
constexpr timeval kCommandTimeout{2, 0};
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
struct RedisUrl {
std::string host = "127.0.0.1";
int port = 6379;
std::string password;
int db = 0;
};
RedisUrl parse_redis_url(const std::string& url) {
RedisUrl out;
const std::string prefix = "redis://";
if (url.compare(0, prefix.size(), prefix) != 0) {
throw std::runtime_error("redis url must start with redis://");
}
std::string rest = url.substr(prefix.size());
const auto slash = rest.find('/');
if (slash != std::string::npos) {
const std::string db_str = rest.substr(slash + 1);
rest.resize(slash);
if (!db_str.empty()) {
out.db = std::atoi(db_str.c_str());
}
}
const auto at = rest.find('@');
std::string hostport = rest;
if (at != std::string::npos) {
const std::string creds = rest.substr(0, at);
hostport = rest.substr(at + 1);
const auto colon = creds.find(':');
if (colon != std::string::npos) {
out.password = creds.substr(colon + 1);
} else {
out.password = creds;
}
}
const auto colon = hostport.find(':');
if (colon == std::string::npos) {
out.host = hostport;
} else {
out.host = hostport.substr(0, colon);
out.port = std::atoi(hostport.substr(colon + 1).c_str());
}
return out;
}
redisContext* make_connection(const RedisUrl& u) {
redisContext* c = redisConnectWithTimeout(u.host.c_str(), u.port, kConnectTimeout);
if (c == nullptr) {
throw std::runtime_error("redisConnect: allocation failed");
}
if (c->err) {
const std::string err = c->errstr;
redisFree(c);
throw DependencyUnavailable("redisConnect failed: " + err);
}
if (redisSetTimeout(c, kCommandTimeout) != REDIS_OK) {
redisFree(c);
throw DependencyUnavailable("redisSetTimeout failed");
}
if (!u.password.empty()) {
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "AUTH %s", u.password.c_str()));
if (r == nullptr) {
redisFree(c);
throw std::runtime_error("redis AUTH: no reply");
}
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
redisFree(c);
throw std::runtime_error("redis AUTH failed");
}
}
if (u.db != 0) {
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "SELECT %d", u.db));
if (r == nullptr) {
redisFree(c);
throw std::runtime_error("redis SELECT: no reply");
}
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
redisFree(c);
throw std::runtime_error("redis SELECT failed");
}
}
return c;
}
}
RedisPool::RedisPool(std::string url, std::size_t n)
: url_(std::move(url)) {
if (n == 0) {
throw std::runtime_error("RedisPool: n must be > 0");
}
const RedisUrl parsed = parse_redis_url(url_);
all_.reserve(n);
idle_.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
ConnPtr c(make_connection(parsed));
idle_.push_back(c.get());
all_.push_back(std::move(c));
}
}
RedisPool::~RedisPool() {
{
std::lock_guard<std::mutex> lk(mu_);
shutting_down_ = true;
}
cv_.notify_all();
}
RedisPool::Lease::Lease(RedisPool& pool)
: pool_(&pool), ctx_(pool.take()) {}
RedisPool::Lease::~Lease() {
if (pool_ != nullptr && ctx_ != nullptr) {
pool_->give_back(ctx_);
}
}
RedisPool::Lease::Lease(Lease&& other) noexcept
: pool_(other.pool_), ctx_(other.ctx_) {
other.pool_ = nullptr;
other.ctx_ = nullptr;
}
RedisPool::Lease RedisPool::acquire() {
return Lease(*this);
}
redisContext* RedisPool::take() {
std::unique_lock<std::mutex> lk(mu_);
if (!cv_.wait_for(lk, kAcquireTimeout,
[&]() { return shutting_down_ || !idle_.empty(); })) {
throw DependencyUnavailable("redis pool: acquire timeout");
}
if (shutting_down_) {
throw DependencyUnavailable("redis pool is shutting down");
}
redisContext* c = idle_.back();
idle_.pop_back();
lk.unlock();
bool alive = false;
if (!c->err) {
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
if (r != nullptr && r->type == REDIS_REPLY_STATUS) {
alive = true;
}
if (r != nullptr) freeReplyObject(r);
}
if (alive) {
return c;
}
ConnPtr fresh;
try {
fresh.reset(make_connection(parse_redis_url(url_)));
} catch (const std::exception& e) {
// Слот возвращается в пул, иначе после нескольких неудач пул опустеет навсегда.
give_back(c);
throw DependencyUnavailable(std::string("redis reconnect failed: ") + e.what());
}
std::lock_guard<std::mutex> lk2(mu_);
for (auto& up : all_) {
if (up.get() == c) {
up = std::move(fresh);
return up.get();
}
}
throw std::logic_error("RedisPool: connection not owned by pool");
}
void RedisPool::give_back(redisContext* c) {
{
std::lock_guard<std::mutex> lk(mu_);
idle_.push_back(c);
}
cv_.notify_one();
}
bool RedisPool::ping() {
try {
auto lease = acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
if (r == nullptr) return false;
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
return ok;
} catch (...) {
return false;
}
}
+59
View File
@@ -0,0 +1,59 @@
#pragma once
#include <hiredis/hiredis.h>
#include <condition_variable>
#include <cstddef>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
class RedisPool {
public:
RedisPool(std::string url, std::size_t n);
~RedisPool();
RedisPool(const RedisPool&) = delete;
RedisPool& operator=(const RedisPool&) = delete;
class Lease {
public:
explicit Lease(RedisPool& pool);
~Lease();
Lease(Lease&& other) noexcept;
Lease& operator=(Lease&&) = delete;
Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
redisContext* ctx() const { return ctx_; }
private:
RedisPool* pool_;
redisContext* ctx_;
};
Lease acquire();
bool ping();
private:
friend class Lease;
struct RedisDeleter {
void operator()(redisContext* c) const noexcept {
if (c != nullptr) ::redisFree(c);
}
};
using ConnPtr = std::unique_ptr<redisContext, RedisDeleter>;
redisContext* take();
void give_back(redisContext* c);
std::string url_;
std::vector<ConnPtr> all_;
std::vector<redisContext*> idle_;
std::mutex mu_;
std::condition_variable cv_;
bool shutting_down_ = false;
};
+290
View File
@@ -0,0 +1,290 @@
#include "telemetry.hpp"
#include "s3_client.hpp"
#include <miniocpp/args.h>
#include <miniocpp/client.h>
#include <miniocpp/providers.h>
#include <miniocpp/request.h>
#include <miniocpp/response.h>
#include <cstdlib>
#include <fstream>
#include <stdexcept>
namespace {
std::string read_file_trim(const std::string& path) {
std::ifstream in(path);
if (!in) {
throw std::runtime_error("cannot open secret/file: " + path);
}
std::string s((std::istreambuf_iterator<char>(in)),
std::istreambuf_iterator<char>());
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) {
s.pop_back();
}
return s;
}
std::string env_or(const char* key, const char* fallback = "") {
const char* v = std::getenv(key);
return (v && *v) ? std::string(v) : std::string(fallback);
}
void parse_endpoint(const std::string& endpoint, std::string& host, unsigned& port,
bool& https) {
https = true;
host = endpoint;
port = 0;
if (host.rfind("http://", 0) == 0) {
https = false;
host = host.substr(7);
} else if (host.rfind("https://", 0) == 0) {
https = true;
host = host.substr(8);
}
// strip path
const auto slash = host.find('/');
if (slash != std::string::npos) {
host = host.substr(0, slash);
}
const auto colon = host.rfind(':');
if (colon != std::string::npos) {
port = static_cast<unsigned>(std::stoul(host.substr(colon + 1)));
host = host.substr(0, colon);
}
if (port == 0) {
port = https ? 443u : 80u;
}
}
} // namespace
struct S3Client::Impl {
minio::s3::BaseUrl base_url;
minio::s3::BaseUrl presign_url;
std::unique_ptr<minio::creds::StaticProvider> provider;
std::unique_ptr<minio::s3::Client> client;
std::unique_ptr<minio::s3::Client> presign_client; // null → use client
};
S3Client::Config S3Client::from_env() {
Config c;
// Ops endpoint (Stat/Get/Put/Delete). Presign may use S3_PRESIGN_ENDPOINT
// (read in ctor) so signed Host matches the client-reachable address.
c.endpoint = env_or("S3_ENDPOINT", "http://minio:9000");
c.region = env_or("S3_REGION", "us-east-1");
c.bucket = env_or("S3_BUCKET");
if (c.bucket.empty()) {
throw std::runtime_error("S3_BUCKET is required");
}
const std::string key_file = env_or("S3_ACCESS_KEY_ID_FILE");
const std::string secret_file = env_or("S3_SECRET_ACCESS_KEY_FILE");
if (key_file.empty() || secret_file.empty()) {
throw std::runtime_error(
"S3_ACCESS_KEY_ID_FILE and S3_SECRET_ACCESS_KEY_FILE are required");
}
c.access_key = read_file_trim(key_file);
c.secret_key = read_file_trim(secret_file);
if (c.access_key.empty() || c.secret_key.empty()) {
throw std::runtime_error("S3 credentials files are empty");
}
return c;
}
S3Client::S3Client(Config cfg) : cfg_(std::move(cfg)), impl_(std::make_unique<Impl>()) {
if (cfg_.endpoint.empty() || cfg_.bucket.empty() ||
cfg_.access_key.empty() || cfg_.secret_key.empty()) {
throw std::runtime_error("S3Client: incomplete config");
}
std::string host;
unsigned port = 0;
bool https = false;
parse_endpoint(cfg_.endpoint, host, port, https);
impl_->base_url = minio::s3::BaseUrl(host, https, cfg_.region);
impl_->base_url.port = port;
// MinIO requires path-style addressing.
impl_->base_url.virtual_style = false;
impl_->provider = std::make_unique<minio::creds::StaticProvider>(
cfg_.access_key, cfg_.secret_key);
impl_->client =
std::make_unique<minio::s3::Client>(impl_->base_url, impl_->provider.get());
// Separate BaseUrl for presigning when S3_PRESIGN_ENDPOINT differs.
const std::string presign_ep = env_or("S3_PRESIGN_ENDPOINT");
if (!presign_ep.empty() && presign_ep != cfg_.endpoint) {
std::string ph;
unsigned pp = 0;
bool phttps = false;
parse_endpoint(presign_ep, ph, pp, phttps);
impl_->presign_url = minio::s3::BaseUrl(ph, phttps, cfg_.region);
impl_->presign_url.port = pp;
impl_->presign_url.virtual_style = false;
impl_->presign_client = std::make_unique<minio::s3::Client>(
impl_->presign_url, impl_->provider.get());
}
}
S3Client::~S3Client() = default;
void S3Client::put_object(const std::string& key,
std::istream& body,
std::int64_t size_bytes,
const std::string& content_type) {
telemetry::Span operation("s3.put_object", opentelemetry::trace::SpanKind::kClient);
if (size_bytes < 0) {
throw std::invalid_argument("put_object: negative size");
}
// part_size=0 → library chooses multipart sizing; streams from `body`.
minio::s3::PutObjectArgs args(body, static_cast<long>(size_bytes), 0);
args.bucket = cfg_.bucket;
args.object = key;
if (!content_type.empty()) {
args.content_type = content_type;
}
minio::s3::PutObjectResponse resp = impl_->client->PutObject(args);
if (!resp) {
throw std::runtime_error("S3 PutObject failed: " + resp.Error().String());
}
}
void S3Client::get_object(const std::string& key, std::ostream& out) {
telemetry::Span operation("s3.get_object", opentelemetry::trace::SpanKind::kClient);
get_object(key, [&](const char* data, std::size_t n) {
out.write(data, static_cast<std::streamsize>(n));
if (!out) {
throw std::runtime_error("get_object: write failed");
}
});
}
std::int64_t S3Client::get_object(
const std::string& key,
const std::function<void(const char*, std::size_t)>& on_chunk) {
telemetry::Span operation("s3.get_object", opentelemetry::trace::SpanKind::kClient);
std::int64_t total = 0;
minio::s3::GetObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
args.datafunc = [&](minio::http::DataFunctionArgs da) -> bool {
on_chunk(da.datachunk.data(), da.datachunk.size());
total += static_cast<std::int64_t>(da.datachunk.size());
return true;
};
minio::s3::GetObjectResponse resp = impl_->client->GetObject(args);
if (!resp) {
throw std::runtime_error("S3 GetObject failed: " + resp.Error().String());
}
return total;
}
void S3Client::copy_object(const std::string& src_key,
const std::string& dst_key) {
telemetry::Span operation("s3.copy_object", opentelemetry::trace::SpanKind::kClient);
if (src_key.empty() || dst_key.empty() || src_key == dst_key) {
throw std::invalid_argument("copy_object: bad keys");
}
minio::s3::CopyObjectArgs args;
args.bucket = cfg_.bucket;
args.object = dst_key;
args.source.bucket = cfg_.bucket;
args.source.object = src_key;
minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args);
if (!resp) {
throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String());
}
}
void S3Client::delete_object(const std::string& key) {
telemetry::Span operation("s3.delete_object", opentelemetry::trace::SpanKind::kClient);
minio::s3::RemoveObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
minio::s3::RemoveObjectResponse resp = impl_->client->RemoveObject(args);
if (!resp) {
throw std::runtime_error("S3 DeleteObject failed: " + resp.Error().String());
}
}
bool S3Client::stat_object(const std::string& key, std::int64_t& size_bytes,
std::string& etag) {
telemetry::Span operation("s3.stat_object", opentelemetry::trace::SpanKind::kClient);
minio::s3::StatObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
minio::s3::StatObjectResponse resp = impl_->client->StatObject(args);
if (!resp) {
// NoSuchKey / Not Found → absent
const std::string err = resp.Error().String();
if (resp.code == "NoSuchKey" || resp.status_code == 404) {
return false;
}
throw std::runtime_error("S3 StatObject failed: " + err);
}
size_bytes = static_cast<std::int64_t>(resp.size);
etag = resp.etag;
return true;
}
std::string S3Client::presign_put(const std::string& key,
unsigned expiry_seconds,
const std::string& content_type) {
telemetry::Span operation("s3.presign_put", opentelemetry::trace::SpanKind::kClient);
if (expiry_seconds < 1 || expiry_seconds > 7 * 24 * 3600u) {
throw std::invalid_argument("presign_put: bad expiry");
}
minio::s3::Client* c =
impl_->presign_client ? impl_->presign_client.get() : impl_->client.get();
// Note: minio-cpp query-presign signs method/host/path/query only — it does
// not bind Content-Type into the signature. Clients should still send
// Content-Type so the object metadata is correct; complete() re-validates
// magic/sha256 against the session.
(void)content_type;
minio::s3::GetPresignedObjectUrlArgs args;
args.bucket = cfg_.bucket;
args.object = key;
args.method = minio::http::Method::kPut;
args.expiry_seconds = expiry_seconds;
minio::s3::GetPresignedObjectUrlResponse resp = c->GetPresignedObjectUrl(args);
if (!resp) {
throw std::runtime_error("S3 PresignPut failed: " + resp.Error().String());
}
return resp.url;
}
std::vector<std::string> S3Client::old_staging(const std::string& cursor,
int retention_seconds,
std::string& next_cursor) {
telemetry::Span operation("s3.old_staging", opentelemetry::trace::SpanKind::kClient);
minio::s3::ListObjectsArgs args;
args.bucket = cfg_.bucket;
args.prefix = "staging/";
args.recursive = true;
args.max_keys = 100;
args.start_after = cursor;
auto cutoff = minio::utils::UtcTime::Now();
cutoff.Add(-retention_seconds);
auto result = impl_->client->ListObjects(args);
std::vector<std::string> keys;
next_cursor.clear();
for (int count = 0; result && count < 100; ++count) {
const auto& item = *result;
if (!item) throw std::runtime_error("S3 staging listing failed");
if (!item.is_prefix && item.last_modified <= cutoff) keys.push_back(item.name);
if (count == 99) {
next_cursor = item.name;
break;
}
++result;
}
return keys;
}
+71
View File
@@ -0,0 +1,71 @@
#pragma once
#include <cstdint>
#include <functional>
#include <istream>
#include <memory>
#include <ostream>
#include <string>
#include <vector>
// Thin wrapper over AWS SDK for C++ (S3 only). Credentials and SigV4 come from
// the SDK — no hand-rolled signing.
class S3Client {
public:
struct Config {
std::string endpoint; // e.g. http://minio:9000
std::string region; // e.g. us-east-1
std::string bucket;
std::string access_key;
std::string secret_key;
};
explicit S3Client(Config cfg);
~S3Client();
S3Client(const S3Client&) = delete;
S3Client& operator=(const S3Client&) = delete;
static Config from_env();
const std::string& bucket() const { return cfg_.bucket; }
// Stream upload. `body` must yield exactly `size_bytes` bytes.
void put_object(const std::string& key,
std::istream& body,
std::int64_t size_bytes,
const std::string& content_type);
// Stream download into `out` (chunked reads from SDK body stream).
void get_object(const std::string& key, std::ostream& out);
// Stream download with a consumer callback (chunk, size). Returns total bytes.
std::int64_t get_object(const std::string& key,
const std::function<void(const char*, std::size_t)>& on_chunk);
void delete_object(const std::string& key);
// At most 100 entries examined per call; cursor wraps to empty at end.
std::vector<std::string> old_staging(const std::string& cursor,
int retention_seconds, std::string& next_cursor);
// Server-side copy inside the bucket (complete: staging → final).
void copy_object(const std::string& src_key, const std::string& dst_key);
// HeadObject-style metadata. Returns false if the object does not exist.
bool stat_object(const std::string& key, std::int64_t& size_bytes,
std::string& etag);
// Presigned PUT URL via minio-cpp (SigV4). Optional required Content-Type
// is bound into the signature when non-empty.
// expiry_seconds: typically 60–600 for Track 1.
std::string presign_put(const std::string& key,
unsigned expiry_seconds,
const std::string& content_type);
private:
struct Impl;
Config cfg_;
std::unique_ptr<Impl> impl_;
};
+87
View File
@@ -0,0 +1,87 @@
#include "session_repo.hpp"
#include "errors.hpp"
#include <sodium.h>
#include <ctime>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
std::string random_token_hex() {
unsigned char buf[32];
randombytes_buf(buf, sizeof(buf));
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(sizeof(buf) * 2);
for (std::size_t i = 0; i < sizeof(buf); ++i) {
out[i * 2] = hex[buf[i] >> 4];
out[i * 2 + 1] = hex[buf[i] & 0x0f];
}
return out;
}
std::int64_t now_unix() {
return static_cast<std::int64_t>(std::time(nullptr));
}
redisReply* check_reply(redisContext* c, redisReply* r) {
if (r == nullptr) {
const std::string err = c->err ? c->errstr : "no reply";
throw DependencyUnavailable("redis command failed: " + err);
}
return r;
}
}
ISessionRepo::Created SessionRepo::create(const std::string& user_id) {
const std::string token = random_token_hex();
const std::int64_t exp = now_unix() + ttl_;
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "SET session:%s %s EX %d",
token.c_str(), user_id.c_str(), ttl_));
check_reply(c, r);
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
throw std::runtime_error("redis SET returned non-status");
}
return Created{token, exp};
}
std::optional<std::string> SessionRepo::lookup(const std::string& token) {
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "GET session:%s", token.c_str()));
check_reply(c, r);
std::optional<std::string> out;
if (r->type == REDIS_REPLY_STRING) {
out = std::string(r->str, r->len);
}
freeReplyObject(r);
return out;
}
bool SessionRepo::revoke(const std::string& token) {
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "DEL session:%s", token.c_str()));
check_reply(c, r);
const bool existed = (r->type == REDIS_REPLY_INTEGER && r->integer == 1);
freeReplyObject(r);
return existed;
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "redis_pool.hpp"
#include "session_repo_base.hpp"
class SessionRepo : public ISessionRepo {
public:
SessionRepo(RedisPool& pool, int ttl_seconds)
: pool_(pool), ttl_(ttl_seconds) {}
Created create(const std::string& user_id) override;
std::optional<std::string> lookup(const std::string& token) override;
bool revoke(const std::string& token) override;
private:
RedisPool& pool_;
int ttl_;
};
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string>
class ISessionRepo {
public:
virtual ~ISessionRepo() = default;
struct Created {
std::string token;
std::int64_t expires_at; // unix-время в секундах UTC
};
virtual Created create(const std::string& user_id) = 0;
virtual std::optional<std::string> lookup(const std::string& token) = 0;
virtual bool revoke(const std::string& token) = 0;
};
+77
View File
@@ -0,0 +1,77 @@
#include "staging_cleaner.hpp"
#include "attachment_storage.hpp"
#include <nlohmann/json.hpp>
#include <regex>
void StagingCleaner::clean(const std::string& key) {
static const std::regex pattern(
"^staging/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/"
"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$");
std::smatch match;
if (!std::regex_match(key, match, pattern)) return;
const std::string id = match[1];
auto lease = pg_.acquire();
pqxx::work tx(*lease);
if (!tx.exec("SELECT pg_try_advisory_xact_lock(hashtextextended('staging:' || $1::uuid::text, 0))",
pqxx::params{id})[0][0].as<bool>()) return;
auto rows = tx.exec(
"SELECT a.chat_id, a.staging_key, a.s3_key, a.size_bytes, a.variants, a.status, j.id AS job_id, "
"a.updated_at < now()-($2::bigint * interval '1 second') AS expired "
"FROM attachments a LEFT JOIN jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version "
"WHERE a.id=$1::uuid FOR UPDATE OF a SKIP LOCKED", pqxx::params{id, retention_seconds_});
if (rows.empty()) {
// An existing locked row is not an orphan.
if (!tx.exec("SELECT 1 FROM attachments WHERE id=$1::uuid", pqxx::params{id}).empty()) return;
} else {
const auto& row = rows[0];
if (row["staging_key"].is_null() || row["staging_key"].as<std::string>() != key) return;
const std::string status = row["status"].as<std::string>();
if (status != "ready" && !(status == "failed" && row["expired"].as<bool>())) return;
if (!row["job_id"].is_null() && !tx.exec(
"SELECT pg_try_advisory_xact_lock(hashtextextended($1::uuid::text, 0))",
pqxx::params{row["job_id"].as<std::string>()})[0][0].as<bool>()) return;
if (!tx.exec("SELECT 1 FROM jobs WHERE attachment_id=$1::uuid AND source_key=$2 "
"AND status IN ('queued','processing')", pqxx::params{id,key}).empty()) return;
if (status == "ready") {
std::int64_t size = 0;
std::string etag;
if (!s3_.stat_object(row["s3_key"].as<std::string>(), size, etag) ||
size != row["size_bytes"].as<int>()) return;
for (const auto& variant : nlohmann::json::parse(row["variants"].as<std::string>())) {
const std::string object_key = variant.value("storage_key",
attachment_variant_key(row["chat_id"].as<std::string>(), id, variant.at("name").get<std::string>()));
if (!s3_.stat_object(object_key, size, etag) ||
size != variant.at("size_bytes").get<std::int64_t>()) return;
}
}
}
// A still-valid presigned URL can recreate the key: wait for its expiry.
if (!tx.exec("SELECT 1 FROM direct_uploads WHERE staging_key=$1 AND expires_at>now()",
pqxx::params{key}).empty()) return;
s3_.delete_object(key);
if (!rows.empty()) {
// Internal cleanup does not change public metadata/updated_at.
tx.exec("UPDATE attachments SET staging_cleaned_at=now() WHERE id=$1::uuid", pqxx::params{id});
}
tx.commit();
}
void StagingCleaner::sweep() {
std::vector<std::string> known;
{
auto lease = pg_.acquire();
pqxx::work tx(*lease);
for (const auto& row : tx.exec(
"SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL "
"AND staging_cleaned_at IS NULL AND (status='ready' OR "
"(status='failed' AND updated_at<now()-($1::bigint * interval '1 second'))) "
"ORDER BY updated_at, id LIMIT 32", pqxx::params{retention_seconds_})) {
known.push_back(row[0].as<std::string>());
}
}
for (const auto& key : known) clean(key);
std::string next;
for (const auto& key : s3_.old_staging(cursor_, retention_seconds_, next)) clean(key);
cursor_ = std::move(next);
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "db.hpp"
#include "s3_client.hpp"
#include <string>
class StagingCleaner {
public:
StagingCleaner(PgPool& pg, S3Client& s3, int retention_seconds)
: pg_(pg), s3_(s3), retention_seconds_(retention_seconds) {}
void clean(const std::string& key);
void sweep();
private:
PgPool& pg_;
S3Client& s3_;
int retention_seconds_;
std::string cursor_;
};
+270
View File
@@ -0,0 +1,270 @@
#include "telemetry.hpp"
#include <opentelemetry/context/propagation/text_map_propagator.h>
#include <opentelemetry/trace/propagation/http_trace_context.h>
#include <opentelemetry/exporters/otlp/otlp_http_exporter_factory.h>
#include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_factory.h>
#include <opentelemetry/exporters/otlp/otlp_http_log_record_exporter_factory.h>
#include <opentelemetry/sdk/trace/batch_span_processor_factory.h>
#include <opentelemetry/sdk/trace/tracer_provider.h>
#include <opentelemetry/sdk/trace/samplers/parent.h>
#include <opentelemetry/sdk/metrics/meter_provider.h>
#include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_factory.h>
#include <opentelemetry/sdk/metrics/aggregation/aggregation_config.h>
#include <opentelemetry/sdk/logs/batch_log_record_processor_factory.h>
#include <opentelemetry/sdk/logs/logger_provider.h>
#include <opentelemetry/logs/log_record.h>
#include <opentelemetry/metrics/observer_result.h>
#include <opentelemetry/sdk/common/global_log_handler.h>
#include <array>
#include <atomic>
#include <cstdlib>
#include <exception>
#include <map>
#include <mutex>
#include <regex>
#include <unistd.h>
namespace telemetry {
namespace ot = opentelemetry;
namespace ts = ot::sdk::trace;
namespace ms = ot::sdk::metrics;
namespace ls = ot::sdk::logs;
namespace ex = ot::exporter::otlp;
namespace {
std::shared_ptr<ts::TracerProvider> traces;
std::shared_ptr<ms::MeterProvider> metrics;
std::shared_ptr<ls::LoggerProvider> logs;
ot::nostd::shared_ptr<ot::logs::Logger> logger;
ot::nostd::unique_ptr<ot::metrics::Counter<uint64_t>> requests, jobs;
ot::nostd::unique_ptr<ot::metrics::Histogram<double>> http_duration, job_duration, ready_duration;
struct Gauge {
std::atomic<double> value{0};
ot::nostd::shared_ptr<ot::metrics::ObservableInstrument> instrument;
};
std::map<std::string, Gauge> gauges;
thread_local HttpRequest* current_http = nullptr;
struct Carrier : ot::context::propagation::TextMapCarrier {
std::map<std::string, std::string> fields;
ot::nostd::string_view Get(ot::nostd::string_view key) const noexcept override {
auto i = fields.find(std::string(key));
return i == fields.end() ? ot::nostd::string_view{} : ot::nostd::string_view(i->second);
}
void Set(ot::nostd::string_view key, ot::nostd::string_view value) noexcept override {
fields[std::string(key)] = std::string(value);
}
};
ot::trace::propagation::HttpTraceContext propagation;
class Diagnostics : public ot::sdk::common::internal_log::LogHandler {
public:
void Handle(ot::sdk::common::internal_log::LogLevel level, const char*, int, const char* message,
const ot::sdk::common::AttributeMap&) noexcept override {
// Count diagnostics without copying backend errors, URLs or payloads into logs.
delta("telemetry.sdk.errors", 1);
if (message && std::string_view(message).find("BatchSpanProcessor queue is full") != std::string_view::npos)
delta("telemetry.sdk.dropped.spans", 1);
}
};
void observe(ot::metrics::ObserverResult result, void* state) {
ot::nostd::get<ot::nostd::shared_ptr<ot::metrics::ObserverResultT<double>>>(result)
->Observe(static_cast<Gauge*>(state)->value.load());
}
}
Runtime::Runtime(const char* service) noexcept {
try {
if (const auto* disabled = std::getenv("OTEL_SDK_DISABLED"); disabled && std::string(disabled) == "true") return;
// Do not dump exporter payloads or third party error strings to stderr.
ot::sdk::common::internal_log::GlobalLogHandler::SetLogLevel(
ot::sdk::common::internal_log::LogLevel::None);
char host[256]{};
gethostname(host, sizeof(host)-1);
auto resource = ot::sdk::resource::Resource::Create({
{"service.name", service}, {"service.version", "lab6-1.0"},
{"service.instance.id", host}, {"deployment.environment.name", "lab"}});
ex::OtlpHttpExporterOptions to;
to.timeout = std::chrono::milliseconds(500);
to.console_debug = false;
to.retry_policy_max_attempts = 1;
to.content_type = ex::HttpRequestContentType::kBinary;
ts::BatchSpanProcessorOptions batch;
batch.max_queue_size = 1024;
batch.max_export_batch_size = 128;
batch.schedule_delay_millis = std::chrono::milliseconds(500);
traces = std::make_shared<ts::TracerProvider>(
ts::BatchSpanProcessorFactory::Create(ex::OtlpHttpExporterFactory::Create(to), batch), resource,
std::make_unique<ts::ParentBasedSampler>(std::make_shared<ts::AlwaysOnSampler>()));
std::shared_ptr<ot::trace::TracerProvider> api_traces = traces;
ot::trace::Provider::SetTracerProvider(api_traces);
ex::OtlpHttpMetricExporterOptions mo;
mo.timeout = std::chrono::milliseconds(500);
mo.console_debug = false;
mo.retry_policy_max_attempts = 1;
mo.content_type = ex::HttpRequestContentType::kBinary;
metrics = std::make_shared<ms::MeterProvider>(std::make_unique<ms::ViewRegistry>(), resource);
for (auto name : {"http.server.request.duration", "worker.job.duration", "attachment.ready.duration"}) {
auto config = std::make_shared<ms::HistogramAggregationConfig>();
config->boundaries_ = {0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60,120};
metrics->AddView(std::make_unique<ms::InstrumentSelector>(ms::InstrumentType::kHistogram, name, "s"),
std::make_unique<ms::MeterSelector>("messenger", "1.0", ""),
std::make_unique<ms::View>(name, "", "s", ms::AggregationType::kHistogram, config));
}
ms::PeriodicExportingMetricReaderOptions ro;
ro.export_interval_millis = std::chrono::milliseconds(5000);
ro.export_timeout_millis = std::chrono::milliseconds(1000);
metrics->AddMetricReader(ms::PeriodicExportingMetricReaderFactory::Create(
ex::OtlpHttpMetricExporterFactory::Create(mo), ro));
auto meter = metrics->GetMeter("messenger", "1.0");
requests = meter->CreateUInt64Counter("http.server.requests", "Business HTTP requests", "{request}");
jobs = meter->CreateUInt64Counter("worker.jobs", "Delivery outcomes", "{job}");
http_duration = meter->CreateDoubleHistogram("http.server.request.duration", "Request latency", "s");
job_duration = meter->CreateDoubleHistogram("worker.job.duration", "Delivery processing latency", "s");
ready_duration = meter->CreateDoubleHistogram("attachment.ready.duration", "Durable acceptance to ready", "s");
for (auto name : {"queue.depth", "queue.oldest.age", "queue.observation.timestamp",
"db.pool.used", "db.pool.waiting", "db.pool.capacity", "worker.busy", "worker.capacity",
"process.cpu.time", "process.memory.usage", "telemetry.sdk.errors", "telemetry.sdk.dropped.spans"}) {
auto& g = gauges[name];
g.instrument = meter->CreateDoubleObservableGauge(name);
g.instrument->AddCallback(observe, &g);
}
ex::OtlpHttpLogRecordExporterOptions lo;
lo.timeout = std::chrono::milliseconds(500);
lo.console_debug = false;
lo.retry_policy_max_attempts = 1;
lo.content_type = ex::HttpRequestContentType::kBinary;
ls::BatchLogRecordProcessorOptions lb;
lb.max_queue_size = 1024;
lb.max_export_batch_size = 128;
lb.schedule_delay_millis = std::chrono::milliseconds(500);
logs = std::make_shared<ls::LoggerProvider>(ls::BatchLogRecordProcessorFactory::Create(
ex::OtlpHttpLogRecordExporterFactory::Create(lo), lb), resource);
logger = logs->GetLogger("messenger", "messenger", "1.0", "", ot::common::NoopKeyValueIterable());
ot::sdk::common::internal_log::GlobalLogHandler::SetLogHandler(
ot::nostd::shared_ptr<ot::sdk::common::internal_log::LogHandler>(new Diagnostics));
ot::sdk::common::internal_log::GlobalLogHandler::SetLogLevel(
ot::sdk::common::internal_log::LogLevel::Warning);
event("service.started");
} catch (...) { /* Telemetry setup must not prevent application startup. */ }
}
Runtime::~Runtime() {
if (metrics) metrics->Shutdown(std::chrono::seconds(2));
if (logs) logs->Shutdown(std::chrono::seconds(2));
if (traces) traces->Shutdown(std::chrono::seconds(2));
}
Span::Span(const std::string& name, ot::trace::SpanKind kind,
const std::string& parent, const std::string& state) : exceptions_(std::uncaught_exceptions()) {
ot::trace::StartSpanOptions options;
options.kind = kind;
if (!parent.empty()) {
Carrier carrier;
if (parent.size() <= 256) carrier.fields["traceparent"] = parent;
if (state.size() <= 512) carrier.fields["tracestate"] = state;
auto root = ot::context::Context{};
options.parent = propagation.Extract(carrier, root);
}
span = ot::trace::Provider::GetTracerProvider()->GetTracer("messenger", "1.0")->StartSpan(name, options);
scope_ = std::make_unique<ot::trace::Scope>(span);
}
Span::~Span() {
if (std::uncaught_exceptions() > exceptions_) error();
span->End();
}
void Span::error() noexcept { span->SetStatus(ot::trace::StatusCode::kError); }
std::string inject(const char* field) {
Carrier carrier;
auto context = ot::context::RuntimeContext::GetCurrent();
propagation.Inject(carrier, context);
return carrier.fields[field];
}
void event(const char* name, bool failed, const std::string& request_id) noexcept {
if (!logger) return;
auto record = logger->CreateLogRecord();
record->SetTimestamp(std::chrono::system_clock::now());
record->SetSeverity(failed ? ot::logs::Severity::kError : ot::logs::Severity::kInfo);
record->SetBody(name);
record->SetAttribute("event", name);
if (!request_id.empty()) record->SetAttribute("request_id", request_id);
const auto context = ot::trace::GetSpan(ot::context::RuntimeContext::GetCurrent())->GetContext();
if (context.IsValid()) {
record->SetTraceId(context.trace_id());
record->SetSpanId(context.span_id());
record->SetTraceFlags(context.trace_flags());
}
logger->EmitLogRecord(std::move(record));
}
void gauge(const char* name, double value) noexcept {
auto i = gauges.find(name);
if (i != gauges.end()) i->second.value.store(value);
}
void delta(const char* name, long value) noexcept {
auto i = gauges.find(name);
if (i != gauges.end()) {
auto old = i->second.value.load();
while (!i->second.value.compare_exchange_weak(old, old + value)) {}
}
}
void duration(const char*, double seconds) noexcept {
if (ready_duration) ready_duration->Record(seconds, ot::context::Context{});
}
std::string route_template(const std::string& path) {
// Whitelist full templates; unknown/hostile paths never enter telemetry.
static const std::pair<std::regex, const char*> routes[] = {
{std::regex("^/api/v1/chats/[^/]+/messages$"), "/api/v1/chats/{chat_id}/messages"},
{std::regex("^/api/v1/chats/[^/]+/attachments$"), "/api/v1/chats/{chat_id}/attachments"},
{std::regex("^/api/v1/chats/[^/]+/direct-uploads$"), "/api/v1/chats/{chat_id}/direct-uploads"},
{std::regex("^/api/v1/chats/[^/]+$"), "/api/v1/chats/{chat_id}"},
{std::regex("^/api/v1/attachments/[^/]+/content$"), "/api/v1/attachments/{attachment_id}/content"},
{std::regex("^/api/v1/attachments/[^/]+$"), "/api/v1/attachments/{attachment_id}"},
{std::regex("^/api/v1/direct-uploads/[^/]+/complete$"), "/api/v1/direct-uploads/{upload_id}/complete"},
{std::regex("^/api/v1/direct-uploads/[^/]+$"), "/api/v1/direct-uploads/{upload_id}"}};
for (auto exact : {"/api/v1/users", "/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/health/live", "/health/ready"})
if (path == exact) return exact;
for (const auto& route : routes) if (std::regex_match(path, route.first)) return route.second;
return "unmatched";
}
HttpRequest::HttpRequest() : start_(std::chrono::steady_clock::now()), previous_(current_http) { current_http = this; }
void HttpRequest::configure(const std::string& method, const std::string& path,
const std::string& parent, const std::string& state, const std::string& id) {
for (auto known : {"GET","POST","PUT","DELETE","PATCH","HEAD","OPTIONS"}) if (method == known) method_ = known;
status = 500;
route_ = route_template(path);
id_ = id;
span_ = std::make_unique<Span>(method_ + " " + route_, ot::trace::SpanKind::kServer, parent, state);
span_->span->SetAttribute("http.request.method", method_);
span_->span->SetAttribute("http.route", route_);
}
HttpRequest::~HttpRequest() {
current_http = previous_;
if (status == 0 || route_.rfind("/health/", 0) == 0) return;
const std::map<std::string, std::string> attrs = {{"http.request.method", method_},
{"http.route", route_}, {"http.response.status_class", std::to_string(status/100)+"xx"}};
if (requests) requests->Add(1, attrs);
if (http_duration) http_duration->Record(std::chrono::duration<double>(std::chrono::steady_clock::now()-start_).count(), attrs, ot::context::Context{});
if (span_) {
span_->span->SetAttribute("http.response.status_code", status);
if (status >= 500) span_->error();
}
event("http.request.completed", status >= 500, id_);
}
void response(const std::string& bytes) noexcept {
if (current_http && bytes.size() >= 12 && bytes.compare(0,9,"HTTP/1.1 ") == 0 &&
bytes[9] >= '1' && bytes[9] <= '5' && bytes[10] >= '0' && bytes[10] <= '9' && bytes[11] >= '0' && bytes[11] <= '9')
current_http->status = (bytes[9]-'0')*100+(bytes[10]-'0')*10+bytes[11]-'0';
}
Job::Job(const std::string& parent, const std::string& state)
: span_("attachment.process", ot::trace::SpanKind::kConsumer, parent, state), start_(std::chrono::steady_clock::now()) {
span_.span->SetAttribute("messaging.system", "redis");
span_.span->SetAttribute("messaging.destination.name", "jobs:attachments");
delta("worker.busy", 1);
}
void Job::finish(const char* outcome) { outcome_ = outcome; }
Job::~Job() {
if (std::uncaught_exceptions() > exceptions_) outcome_ = "failed";
delta("worker.busy", -1);
const std::map<std::string,std::string> attrs{{"job.type","attachment.process.v1"},{"outcome",outcome_}};
if (jobs) jobs->Add(1, attrs);
if (job_duration) job_duration->Record(std::chrono::duration<double>(std::chrono::steady_clock::now()-start_).count(), attrs, ot::context::Context{});
bool failed = std::string(outcome_) == "failed" || std::string(outcome_) == "retry";
span_.span->SetAttribute("job.outcome", outcome_);
if (failed) span_.error();
event("worker.job.completed", failed);
}
}
+60
View File
@@ -0,0 +1,60 @@
#pragma once
#include <opentelemetry/trace/provider.h>
#include <opentelemetry/trace/scope.h>
#include <opentelemetry/context/context.h>
#include <chrono>
#include <exception>
#include <memory>
#include <string>
namespace telemetry {
class Runtime {
public:
explicit Runtime(const char* service) noexcept;
~Runtime();
};
// Only trusted constants and bounded route templates may be passed as names/attributes.
class Span {
public:
explicit Span(const std::string& name,
opentelemetry::trace::SpanKind kind = opentelemetry::trace::SpanKind::kInternal,
const std::string& traceparent = "", const std::string& tracestate = "");
~Span();
void error() noexcept;
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span> span;
private:
int exceptions_;
std::unique_ptr<opentelemetry::trace::Scope> scope_;
};
std::string inject(const char* field = "traceparent");
std::string route_template(const std::string& path);
void event(const char* name, bool error = false, const std::string& request_id = "") noexcept;
void gauge(const char* name, double value) noexcept;
void delta(const char* name, long value) noexcept;
void duration(const char* name, double seconds) noexcept;
class HttpRequest {
public:
HttpRequest();
~HttpRequest();
void configure(const std::string& method, const std::string& path,
const std::string& parent, const std::string& state, const std::string& id);
int status = 0;
private:
std::chrono::steady_clock::time_point start_;
std::unique_ptr<Span> span_;
std::string method_ = "_OTHER", route_ = "unmatched", id_;
HttpRequest* previous_;
};
void response(const std::string& bytes) noexcept;
class Job {
public:
Job(const std::string& parent, const std::string& state);
~Job();
void finish(const char* outcome);
private:
Span span_;
std::chrono::steady_clock::time_point start_;
int exceptions_ = std::uncaught_exceptions();
const char* outcome_ = "failed";
};
}
+100
View File
@@ -0,0 +1,100 @@
#include "user_repo.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <random>
#include <sstream>
#include <string>
namespace {
std::string make_uuid() {
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
User row_to_user(const pqxx::row& r) {
User u;
u.id = r["id"].as<std::string>();
u.username = r["username"].as<std::string>();
u.display_name = r["display_name"].as<std::string>();
u.password_hash = r["password_hash"].as<std::string>();
u.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return u;
}
constexpr const char* kUserColumns =
"id, username, display_name, password_hash, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
}
std::optional<User> UserRepo::create(const std::string& username,
const std::string& display_name,
const std::string& password_hash) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
try {
const std::string id = make_uuid();
auto r = tx.exec(
"INSERT INTO users (id, username, display_name, password_hash, created_at) "
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
"RETURNING " + std::string(kUserColumns),
pqxx::params{id, username, display_name, password_hash, now_epoch_str()});
tx.commit();
return row_to_user(r[0]);
} catch (const pqxx::unique_violation&) {
tx.abort();
return std::nullopt;
}
}
std::optional<User> UserRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kUserColumns) + " FROM users WHERE id = $1",
pqxx::params{id});
tx.commit();
if (r.empty()) return std::nullopt;
return row_to_user(r[0]);
}
std::optional<User> UserRepo::find_by_username(const std::string& username) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kUserColumns) + " FROM users WHERE username = $1",
pqxx::params{username});
tx.commit();
if (r.empty()) return std::nullopt;
return row_to_user(r[0]);
}
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
class UserRepo {
public:
explicit UserRepo(PgPool& pool) : pool_(pool) {}
std::optional<User> create(const std::string& username,
const std::string& display_name,
const std::string& password_hash);
std::optional<User> find_by_id(const std::string& id);
std::optional<User> find_by_username(const std::string& username);
private:
PgPool& pool_;
};
+236
View File
@@ -0,0 +1,236 @@
#include "telemetry.hpp"
#include "observability_monitor.hpp"
#include "attachment_queue.hpp"
#include "attachment_repo.hpp"
#include "attachment_storage.hpp"
#include "staging_cleaner.hpp"
#include "image_pipeline.hpp"
#include "job_repo.hpp"
#include "fault_injection.hpp"
#include "s3_client.hpp"
#include <sodium.h>
#include <nlohmann/json.hpp>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <csignal>
#include <cstdlib>
#include <iostream>
#include <regex>
#include <sstream>
#include <thread>
#include <unistd.h>
namespace {
volatile std::sig_atomic_t stop_requested = 0;
void stop(int) { stop_requested = 1; }
class Watchdog {
public:
explicit Watchdog(int seconds) : seconds_(seconds), thread_([this] {
while (!stopping_.load()) {
const auto deadline = deadline_.load();
if (deadline && now_ms() >= deadline) {
std::cerr << "worker job timed out; exiting for durable recovery\n";
std::_Exit(124);
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}) {}
~Watchdog() { stopping_.store(true); thread_.join(); }
void arm() { deadline_.store(now_ms() + seconds_ * 1000LL); }
void clear() { deadline_.store(0); }
private:
static long long now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
int seconds_;
std::atomic<long long> deadline_{0};
std::atomic<bool> stopping_{false};
std::thread thread_;
};
struct Deadline {
Watchdog& timer;
explicit Deadline(Watchdog& timer) : timer(timer) { timer.arm(); }
~Deadline() { timer.clear(); }
};
int setting(const char* name, int fallback, int minimum, int maximum) {
const char* value = std::getenv(name);
if (!value) return fallback;
std::size_t end = 0;
const int result = std::stoi(value, &end);
if (value[end] || result < minimum || result > maximum) {
throw std::runtime_error("invalid worker setting");
}
return result;
}
std::string sha256(const std::string& bytes) {
unsigned char hash[crypto_hash_sha256_BYTES];
crypto_hash_sha256(hash, reinterpret_cast<const unsigned char*>(bytes.data()), bytes.size());
char hex[crypto_hash_sha256_BYTES * 2 + 1];
sodium_bin2hex(hex, sizeof(hex), hash, sizeof(hash));
return hex;
}
bool valid_job(const nlohmann::json& j) {
static const std::regex uuid("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
return j.is_object() && j.value("schema_version", 0) == 1 &&
j.value("type", "") == "attachment.process.v1" && j.value("pipeline_version", 0) >= 1 && j.value("pipeline_version", 0) <= 2 &&
j.value("attempt", 0) >= 1 && j.contains("created_at") && j.at("created_at").is_string() &&
std::regex_match(j.value("job_id", ""), uuid) &&
std::regex_match(j.value("attachment_id", ""), uuid);
}
void verify_ready(S3Client& s3, const Attachment& a) {
std::int64_t size = 0;
std::string etag;
if (!s3.stat_object(a.s3_key, size, etag) || size != a.size_bytes) {
throw std::runtime_error("ready original missing");
}
bool thumbnail = false;
for (const auto& v : a.variants) {
thumbnail = thumbnail || v.name == "thumbnail";
if (!s3.stat_object((v.storage_key.empty() ? attachment_variant_key(a.chat_id, a.id, v.name) : v.storage_key), size, etag) ||
size != v.size_bytes) throw std::runtime_error("ready variant missing");
}
if (!thumbnail) throw std::runtime_error("ready thumbnail missing");
}
} // namespace
int main() {
telemetry::Runtime telemetry_runtime("messenger-worker");
telemetry::gauge("worker.capacity", 1);
std::signal(SIGTERM, stop);
std::signal(SIGINT, stop);
try {
if (sodium_init() < 0) return 1;
const RetryPolicy retry{
setting("JOB_MAX_ATTEMPTS", 6, 1, 100),
setting("JOB_RETRY_BASE_MS", 1000, 1, 3600000),
setting("JOB_RETRY_MAX_MS", 60000, 1, 3600000)};
if (retry.base_delay_ms > retry.max_delay_ms) throw std::runtime_error("invalid retry policy");
const int timeout = setting("JOB_TIMEOUT_SECONDS", 60, 1, 3600);
const int cleanup_interval = setting("STAGING_CLEANUP_INTERVAL_SECONDS", 60, 1, 86400);
const int retention = setting("STAGING_RETENTION_SECONDS", 86400, 1, 31536000);
const int reclaim_idle = setting("JOB_RECLAIM_IDLE_MS", 5000, 1000, 3600000);
const auto db_url = read_secret_file("DATABASE_URL_FILE");
{
auto conn = open_pg_connection(db_url, false);
apply_migrations(*conn, "./migrations");
}
PgPool pg(db_url, 2);
AttachmentRepo attachments(pg);
JobRepo jobs(pg, retry);
const char* broker_file = std::getenv("BROKER_URL_FILE");
RedisPool redis(read_secret_file(
broker_file && *broker_file ? "BROKER_URL_FILE" : "CACHE_URL_FILE"), 2);
AttachmentQueue queue(redis);
S3Client s3(S3Client::from_env());
char random[33];
unsigned char entropy[16];
randombytes_buf(entropy, sizeof(entropy));
sodium_bin2hex(random, sizeof(random), entropy, sizeof(entropy));
const std::string consumer = "worker-" + std::string(random);
std::clog << "worker started consumer=" << consumer << " max_attempts=" << retry.max_attempts << '\n';
StagingCleaner cleaner(pg, s3, retention);
Watchdog watchdog(timeout);
ObservabilityMonitor monitor(db_url, false);
auto next_sweep = std::chrono::steady_clock::now();
while (!stop_requested) {
try {
if (std::chrono::steady_clock::now() >= next_sweep) {
next_sweep = std::chrono::steady_clock::now() + std::chrono::seconds(cleanup_interval);
Deadline deadline(watchdog);
cleaner.sweep();
}
queue.ensure_group();
const auto message = queue.read(consumer, reclaim_idle);
if (!message) continue;
nlohmann::json j;
bool valid = false;
try {
j = nlohmann::json::parse(message->payload);
valid = valid_job(j);
} catch (const nlohmann::json::exception&) {}
if (!valid) {
telemetry::event("worker.job.invalid", true);
queue.reject(message->stream_id);
continue;
}
auto context_field = [&](const char* name) {
auto i = j.find(name);
return i != j.end() && i->is_string() && i->get_ref<const std::string&>().size() <= 512
? i->get<std::string>() : std::string{};
};
telemetry::Job observation(context_field("traceparent"), context_field("tracestate"));
const std::string id = j.at("attachment_id").get<std::string>();
Deadline deadline(watchdog);
const auto result = jobs.run(j.at("job_id").get<std::string>(), id,
j.at("pipeline_version").get<int>(), j.at("attempt").get<int>(), [&](const JobInput& input) {
const auto a = attachments.find_by_id(id);
if (!a) throw std::runtime_error("attachment missing");
std::string original;
original.reserve(a->size_bytes);
s3.get_object(input.source_key, [&](const char* data, std::size_t n) {
if (original.size() > static_cast<std::size_t>(a->size_bytes) ||
n > static_cast<std::size_t>(a->size_bytes) - original.size()) {
throw InvalidImage("source_mismatch");
}
original.append(data, n);
});
if (original.size() != static_cast<std::size_t>(a->size_bytes) ||
sha256(original) != a->sha256) throw InvalidImage("source_mismatch");
const auto images = make_variants(std::vector<std::uint8_t>(original.begin(), original.end()), input.pipeline_version);
const std::string prefix = pipeline_prefix(a->chat_id, a->id, input.pipeline_version);
const std::string key = prefix + "original";
// Publish verified bytes even if a presigned staging key changes.
std::istringstream source(original);
s3.put_object(key, source, original.size(), a->content_type);
nlohmann::json variants = nlohmann::json::array();
for (const auto& v : images) {
std::istringstream body(v.image.bytes);
const std::string variant_key = prefix + v.name + (v.content_type == "image/webp" ? ".webp" : ".jpg");
s3.put_object(variant_key, body,
v.image.bytes.size(), v.content_type);
variants.push_back({{"name", v.name}, {"content_type", v.content_type},
{"size_bytes", v.image.bytes.size()}, {"sha256", sha256(v.image.bytes)},
{"width", v.image.width}, {"height", v.image.height}, {"storage_key", variant_key}});
}
lab_worker_fault("worker_after_effect", j.at("job_id").get<std::string>(), input.pipeline_version);
return JobResult{key, variants.dump()};
});
observation.finish(result == JobRepo::Result::Scheduled ? "retry" :
result == JobRepo::Result::Busy ? "busy" :
result == JobRepo::Result::Missing ? "failed" : "superseded");
if (result == JobRepo::Result::Missing) {
queue.reject(message->stream_id);
} else if (result == JobRepo::Result::Done) {
const auto a = attachments.find_by_id(id);
observation.finish(a && a->status == "ready" ? "success" : "failed");
if (a && a->status == "ready") verify_ready(s3, *a);
lab_worker_fault("worker_before_ack", j.at("job_id").get<std::string>(), j.at("pipeline_version").get<int>());
queue.ack(message->stream_id);
if (a && a->status == "ready") cleaner.clean("staging/" + a->chat_id + "/" + a->id);
} else if (result == JobRepo::Result::Superseded) {
queue.ack(message->stream_id);
} else if (result == JobRepo::Result::Scheduled) {
// ACK only after the next attempt was committed to outbox.
queue.ack(message->stream_id);
}
// Busy remains pending; an active job is protected by its advisory lock.
} catch (const std::exception&) {
telemetry::event("worker.operation.failed", true);
std::this_thread::sleep_for(std::chrono::seconds(1));
}
}
return 0;
} catch (const std::exception&) {
std::cerr << "worker initialization failed\n";
return 1;
}
}
+178
View File
@@ -0,0 +1,178 @@
#!/usr/bin/env python3
"""Controlled lab6 failures, with unconditional service recovery and sanitized evidence."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import shlex
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
sys.path.insert(0, str(ROOT / 'student-tests'))
from observability import get, eventually
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--only', choices=['all', 'retry', 'backend'], default='all')
args = parser.parse_args()
compose = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
def command(*cmd):
subprocess.run(compose + list(cmd), cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def query(expr):
return get(args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr}))['data']['result']
def value(expr):
rows = query(expr)
return float(rows[0]['value'][1]) if rows else 0
def firing(name):
alerts = get(args.prometheus_url + '/api/v1/alerts')['data']['alerts']
return any(a['labels']['alertname'] == name and a['state'] == 'firing' for a in alerts)
c = Client(args.base_url)
alice, bob, _, chat = c.bootstrap()
events = json.loads((ROOT / "evidence/incidents.json").read_text()) if args.only != "all" else []
def record(name, **fields):
events.append({'event': name, 'timestamp': datetime.now(timezone.utc).isoformat(), **fields})
(ROOT / 'evidence/incidents.json').write_text(json.dumps(events, indent=2) + '\n')
print(name, flush=True)
if args.only == "all":
# 1. Durable queue ages while worker is stopped; resume drains the same job.
command('stop', 'worker')
try:
att, _ = c.attachment(alice, chat)
record('worker_stopped_upload_accepted')
eventually(lambda: firing('QueueLagGrowing'), timeout=120)
record('QueueLagGrowing_firing', depth=value('max(queue_depth)'), oldest_seconds=value('max(queue_oldest_age)'))
finally:
command('start', 'worker')
c.wait_ready(bob, att)
eventually(lambda: not firing('QueueLagGrowing') and value('max(queue_depth)') == 0, timeout=90)
record('QueueLagGrowing_resolved_backlog_drained')
# 2. PostgreSQL unavailable: valid authenticated reads must yield 5xx, never successful phantom writes.
command('stop', 'db')
statuses = {}
try:
record('database_stopped')
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
request = urllib.request.Request(args.base_url + '/api/v1/chats/' + chat['id'] + '/messages',
headers={'Authorization': 'Bearer ' + alice['token']})
try:
response = urllib.request.urlopen(request, timeout=12)
except urllib.error.HTTPError as error:
response = error
with response:
code = response.status
statuses[str(code)] = statuses.get(str(code), 0) + 1
if firing('ApiHighErrorRate'):
break
time.sleep(0.5)
check(firing('ApiHighErrorRate'), 'API error alert did not fire')
check(any(int(code) >= 500 for code in statuses), 'No server errors observed')
record('ApiHighErrorRate_firing', response_counts=statuses)
finally:
command('start', 'db')
def recovered():
return c.call('GET', '/health/ready', schema='Health')['status'] == 200
eventually(recovered, timeout=90)
# New traffic moves the one-minute failure window out of the rate calculation.
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage')
if not firing('ApiHighErrorRate'):
break
time.sleep(1)
check(not firing('ApiHighErrorRate'), 'API alert did not resolve')
record('ApiHighErrorRate_resolved_database_recovered')
if args.only != "backend":
# A retriable S3 failure must preserve the upload trace across the next outbox row.
import base64
import uuid
trace = uuid.uuid4().hex
command('pause', 'worker')
paused = True
try:
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
actor=alice, raw=content, headers={'Content-Type': 'image/png',
'traceparent': '00-' + trace + '-0123456789abcdef-01'}, schema='Attachment')['json']
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
grafana = os.environ.get('GRAFANA_URL', 'http://localhost:3000')
def retry_observed():
data = get(grafana + '/api/datasources/proxy/uid/tempo/api/traces/' + trace, auth)
def contains(obj):
if isinstance(obj, dict):
if obj.get('key') == 'job.outcome' and obj.get('value', {}).get('stringValue') == 'retry':
return True
return any(contains(v) for v in obj.values())
return isinstance(obj, list) and any(contains(v) for v in obj)
return contains(data)
command('stop', 'minio')
command('unpause', 'worker')
paused = False
eventually(retry_observed, timeout=45)
record('s3_failure_scheduled_retry', trace_id=trace)
finally:
command('start', 'minio')
if paused:
command('unpause', 'worker')
c.wait_ready(bob, att)
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
grafana = os.environ.get('GRAFANA_URL', 'http://localhost:3000')
def retry_trace():
data = get(grafana + '/api/datasources/proxy/uid/tempo/api/traces/' + trace, auth)
spans = []
def walk(obj):
if isinstance(obj, dict):
if 'spanId' in obj and 'name' in obj:
spans.append(obj)
for value_ in obj.values():
walk(value_)
elif isinstance(obj, list):
for value_ in obj:
walk(value_)
walk(data)
by_id = {s['spanId']: s for s in spans}
# consumer -> enqueue -> publish -> next consumer (all inside the same trace)
for child in spans:
if child['name'] != 'attachment.process':
continue
producer = by_id.get(child.get('parentSpanId'), {})
enqueue = by_id.get(producer.get('parentSpanId'), {})
prior = by_id.get(enqueue.get('parentSpanId'), {})
if producer.get('name') == 'jobs:attachments publish' and enqueue.get('name') == 'attachment.enqueue' and prior.get('name') == 'attachment.process':
return True
return False
eventually(retry_trace)
record('retry_same_trace_ready_verified', trace_id=trace)
# 3. Broken telemetry storage must not break either application process.
before = value('sum(otelcol_exporter_send_failed_spans)')
command('stop', 'tempo', 'loki')
try:
record('tempo_loki_stopped')
att, _ = c.attachment(alice, chat)
c.wait_ready(bob, att)
for _ in range(20):
c.message(alice, chat, 'Synthetic telemetry outage probe')
time.sleep(1)
eventually(lambda: value('sum(otelcol_exporter_send_failed_spans)') > before, timeout=45)
record('telemetry_outage_api_worker_healthy',
failed_span_exports=value('sum(otelcol_exporter_send_failed_spans)')-before,
collector_rss_bytes=value('max(otelcol_process_memory_rss)'))
finally:
command('start', 'tempo', 'loki')
record('tempo_loki_restarted')
print('PASS: worker lag and DB error alerts fired/resolved; API and worker survived telemetry backend outage.')
if __name__ == '__main__':
main()
+133
View File
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Live lab6 checks. No tokens, message contents or backend dumps are printed."""
import argparse
import base64
from datetime import datetime, timezone
import json
import math
import os
from pathlib import Path
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def get(url, auth=None):
headers = {'Accept': 'application/json'}
if auth:
headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode()
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=15) as response:
return json.load(response)
def eventually(fn, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
result = fn()
if result:
return result
except (urllib.error.URLError, AssertionError, KeyError, ValueError):
pass
time.sleep(2)
raise AssertionError('Telemetry check timed out')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--prometheus-url', default='http://localhost:9090')
parser.add_argument('--grafana-url', default='http://localhost:3000')
parser.add_argument('--output', default='evidence/observability.json')
args = parser.parse_args()
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
proxy = args.grafana_url + '/api/datasources/proxy/uid/'
def prom(query):
return get(args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': query}))['data']['result']
c = Client(args.base_url)
alice, bob, _, chat = c.bootstrap()
trace = uuid.uuid4().hex
parent = '00-' + trace + '-0123456789abcdef-01'
secret_marker = 'private-' + uuid.uuid4().hex
c.message(alice, chat, secret_marker)
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
actor=alice, raw=content, headers={'Content-Type': 'image/png', 'traceparent': parent}, schema='Attachment')['json']
c.wait_ready(bob, att)
def trace_complete():
data = get(proxy + 'tempo/api/traces/' + trace, auth)
spans = []
def walk(obj):
if isinstance(obj, dict):
if 'spanId' in obj and 'name' in obj:
spans.append(obj)
for value in obj.values():
walk(value)
elif isinstance(obj, list):
for value in obj:
walk(value)
walk(data)
names = {span['name'] for span in spans}
return (data, spans) if {'attachment.enqueue', 'jobs:attachments publish', 'attachment.process', 's3.put_object', 'postgresql.publish_ready'} <= names else None
trace_data, spans = eventually(trace_complete)
by_id = {span['spanId']: span for span in spans}
consumer = next(span for span in spans if span['name'] == 'attachment.process')
publish = by_id.get(consumer['parentSpanId'])
check(publish and publish['name'] == 'jobs:attachments publish', 'consumer parent must be producer')
enqueue = by_id.get(publish['parentSpanId'])
check(enqueue and enqueue['name'] == 'attachment.enqueue', 'publish must continue durable outbox context')
server = by_id.get(enqueue['parentSpanId'])
check(server and server['name'].startswith('POST '), 'enqueue parent must be HTTP server')
query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace + '"'
def correlated_logs():
result = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'limit': 1000}), auth)['data']['result']
services = {stream['stream'].get('service_name') for stream in result}
return result if {'messenger-api', 'messenger-worker'} <= services else None
logs = eventually(correlated_logs)
required = ['http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'worker_jobs_total',
'worker_job_duration_seconds_bucket', 'attachment_ready_duration_seconds_bucket', 'queue_depth',
'queue_oldest_age', 'db_pool_used', 'db_pool_waiting', 'worker_busy', 'worker_capacity',
'process_memory_usage', 'process_cpu_time']
for name in required:
eventually(lambda: prom(name))
panel_count = 0
dashboard = get(args.grafana_url + '/api/dashboards/uid/messenger-observability', auth)['dashboard']
check(len(dashboard['panels']) == len(json.loads((ROOT / 'infra/grafana/dashboards/messenger.json').read_text())['panels']), 'Provisioned dashboard differs')
for panel in dashboard['panels']:
if panel['datasource']['uid'] != 'prometheus':
continue
for target in panel['targets']:
expr = target['expr'].replace('$__rate_interval', '1m')
result = prom(expr)
# Diagnostics are allowed to be absent when no failure has occurred.
if 'otelcol_' not in expr:
result = eventually(lambda: prom(expr), timeout=45)
panel_count += 1
for item in prom('http_server_requests_total'):
labels = item['metric']
check(not {'user_id','chat_id','message_id','attachment_id','request_id','filename'} & labels.keys(), 'High cardinality metric label')
check(chat['id'] not in json.dumps(labels), 'Raw chat ID in labels')
# Inspect full recent log/trace data in memory, never save private test values.
all_logs = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({
'query': '{service_name=~"messenger-api|messenger-worker"}', 'limit': 5000}), auth)
exported = json.dumps([all_logs, trace_data])
for forbidden in [secret_marker, alice['token'], alice['registration']['password'], base64.b64encode(content).decode()]:
check(forbidden not in exported, 'Private test data leaked to telemetry')
report = {'timestamp': datetime.now(timezone.utc).isoformat(), 'result': 'PASS', 'trace_id': trace,
'span_names': sorted({s['name'] for s in spans}), 'consumer_parent_is_producer': True,
'log_services': sorted({s['stream']['service_name'] for s in logs}),
'metric_names_checked': required, 'dashboard_queries_checked': panel_count,
'privacy_canaries': 'absent in recent logs and upload trace'}
(ROOT / args.output).write_text(json.dumps(report, indent=2) + '\n')
print('PASS: real API -> outbox -> Redis -> worker -> S3/DB trace, correlated logs, metrics and dashboard queries; privacy canaries absent.')
if __name__ == '__main__':
main()
+33
View File
@@ -0,0 +1,33 @@
#include "telemetry.hpp"
#include <cassert>
#include <regex>
int main() {
telemetry::Runtime runtime("messenger-test");
assert(telemetry::route_template("/api/v1/chats/private-id/messages") == "/api/v1/chats/{chat_id}/messages");
assert(telemetry::route_template("/secret-token") == "unmatched");
assert(telemetry::route_template("/api/v1/attachments/private/content") == "/api/v1/attachments/{attachment_id}/content");
const std::string parent = "00-11223344556677889900aabbccddeeff-0123456789abcdef-01";
std::string producer;
{
telemetry::Span request("test.server", opentelemetry::trace::SpanKind::kServer, parent);
assert(telemetry::inject().substr(3,32) == parent.substr(3,32));
{
telemetry::Span publish("test.publish", opentelemetry::trace::SpanKind::kProducer);
producer = telemetry::inject();
assert(producer.substr(3,32) == parent.substr(3,32));
assert(producer != parent);
}
assert(telemetry::inject() != producer);
}
{
telemetry::Span consumer("test.consume", opentelemetry::trace::SpanKind::kConsumer, producer);
assert(telemetry::inject().substr(3,32) == parent.substr(3,32));
}
for (const auto& invalid : {"garbage", "00-00000000000000000000000000000000-0000000000000000-01"}) {
telemetry::Span span("test.invalid", opentelemetry::trace::SpanKind::kConsumer, invalid);
const auto actual = telemetry::inject();
assert(std::regex_match(actual, std::regex("00-[a-f0-9]{32}-[a-f0-9]{16}-01")));
assert(actual.substr(3,32) != parent.substr(3,32));
}
}
+6
View File
@@ -0,0 +1,6 @@
Headers from https://github.com/nothings/stb at the commit in VERSION:
- stb_image.h: PNG/JPEG validation and decoding
- deprecated/stb_image_resize.h: aspect-preserving resize
- stb_image_write.h: JPEG encoding (quality 85)
Each header includes its upstream license.
+1
View File
@@ -0,0 +1 @@
013ac3beddff3dbffafd5177e7972067cd2b5083
+7988
View File
File diff suppressed because it is too large Load Diff
+2634
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More