forked from mtusi2026/mtusi2026lab6
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ccb463aea1 |
@@ -18,3 +18,5 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
build
|
||||
|
||||
+31
-9
@@ -11,20 +11,42 @@ SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
# Same Valkey instance serves sessions and Streams.
|
||||
BROKER_URL_FILE=/run/secrets/cache_url
|
||||
WORKER_CONCURRENCY=1
|
||||
# One initial attempt plus five retries.
|
||||
JOB_MAX_ATTEMPTS=6
|
||||
JOB_RECLAIM_IDLE_MS=5000
|
||||
JOB_RETRY_BASE_MS=1000
|
||||
JOB_RETRY_MAX_MS=60000
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
OTEL_SERVICE_NAME=messenger-api
|
||||
# Worker must set a distinct service.name.
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
|
||||
OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
|
||||
STAGING_CLEANUP_INTERVAL_SECONDS=60
|
||||
|
||||
# Local Compose bootstrap; choose your own values and matching secret files.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change_me_locally
|
||||
POSTGRES_DB=messenger
|
||||
MINIO_ROOT_USER=minioadmin
|
||||
MINIO_ROOT_PASSWORD=change_me_minio_root_locally
|
||||
MINIO_USER_ACCESS_KEY=messenger-s3
|
||||
MINIO_USER_SECRET_KEY=change_me_s3_user_locally
|
||||
MAX_CONCURRENT_UPLOADS=3
|
||||
DIRECT_UPLOAD_TTL_SECONDS=300
|
||||
|
||||
# Observability: distinct service.name is set by each binary.
|
||||
OTEL_SDK_DISABLED=false
|
||||
GRAFANA_ADMIN_USER=admin
|
||||
API_PORT=8080
|
||||
GRAFANA_PORT=3000
|
||||
PROMETHEUS_PORT=9090
|
||||
|
||||
# Optional Collector profile (Track 1 overrides this per trial).
|
||||
OTEL_COLLECTOR_CONFIG=./infra/otel/collector.yaml
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
project(mtusi_messenger CXX)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# Official MinIO C++ SDK (S3-compatible). Pinned tag; not a hand-rolled SigV4.
|
||||
set(MTUSI_MINIO_CPP_VERSION "v0.3.0" CACHE STRING "minio-cpp git tag")
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
find_package(PkgConfig REQUIRED)
|
||||
|
||||
pkg_check_modules(PQXX REQUIRED IMPORTED_TARGET libpqxx>=7.8)
|
||||
pkg_check_modules(HIREDIS REQUIRED IMPORTED_TARGET hiredis)
|
||||
pkg_check_modules(SODIUM REQUIRED IMPORTED_TARGET libsodium)
|
||||
pkg_check_modules(EXIF REQUIRED IMPORTED_TARGET libexif)
|
||||
pkg_check_modules(WEBP REQUIRED IMPORTED_TARGET libwebp)
|
||||
|
||||
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ImportMinioDeps.cmake")
|
||||
|
||||
include(FetchContent)
|
||||
set(MINIO_CPP_TEST OFF CACHE BOOL "" FORCE)
|
||||
set(MINIO_CPP_MAKE_DOC OFF CACHE BOOL "" FORCE)
|
||||
FetchContent_Declare(
|
||||
miniocpp
|
||||
GIT_REPOSITORY https://github.com/minio/minio-cpp.git
|
||||
GIT_TAG ${MTUSI_MINIO_CPP_VERSION}
|
||||
GIT_SHALLOW TRUE
|
||||
)
|
||||
# Inject find_package stubs before miniocpp configures.
|
||||
function(_mtusi_provide_minio_find_packages)
|
||||
# miniocpp calls find_package(unofficial-curlpp CONFIG) etc.
|
||||
# We already created ALIAS targets; create tiny config files in a prefix.
|
||||
set(_cfg_dir "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps")
|
||||
file(MAKE_DIRECTORY "${_cfg_dir}")
|
||||
file(WRITE "${_cfg_dir}/unofficial-curlpp-config.cmake" "")
|
||||
file(WRITE "${_cfg_dir}/unofficial-inih-config.cmake" "")
|
||||
list(PREPEND CMAKE_PREFIX_PATH "${_cfg_dir}")
|
||||
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
_mtusi_provide_minio_find_packages()
|
||||
|
||||
FetchContent_GetProperties(miniocpp)
|
||||
if(NOT miniocpp_POPULATED)
|
||||
FetchContent_Populate(miniocpp)
|
||||
# Neutralize CONFIG find_package calls by providing empty configs + targets.
|
||||
set(CMAKE_FIND_PACKAGE_REDIRECTS_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-find-redirects")
|
||||
file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
|
||||
# CMake 3.24+ redirects; for older, patch via policy: set variables that
|
||||
# make find_package(CONFIG) succeed when targets already exist.
|
||||
set(unofficial-curlpp_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
|
||||
set(unofficial-inih_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
|
||||
# Provide package config bodies that are no-ops (targets already defined).
|
||||
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-curlpp-config.cmake"
|
||||
"# provided by ImportMinioDeps.cmake\n")
|
||||
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-inih-config.cmake"
|
||||
"# provided by ImportMinioDeps.cmake\n")
|
||||
# Skip miniocpp install(EXPORT) — our INTERFACE system deps are not exportable.
|
||||
set(CMAKE_SKIP_INSTALL_RULES ON)
|
||||
add_subdirectory(${miniocpp_SOURCE_DIR} ${miniocpp_BINARY_DIR})
|
||||
set(CMAKE_SKIP_INSTALL_RULES OFF)
|
||||
endif()
|
||||
|
||||
# OpenTelemetry C++ v1.21.0, OTLP HTTP/protobuf only (no gRPC).
|
||||
set(WITH_OTLP_HTTP ON CACHE BOOL "" FORCE)
|
||||
set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE)
|
||||
set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
|
||||
set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE)
|
||||
set(WITH_FUNC_TESTS OFF CACHE BOOL "" FORCE)
|
||||
FetchContent_Declare(opentelemetry_cpp
|
||||
GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git
|
||||
GIT_TAG v1.21.0 # b9cf499ff5715433848b316059714b5c59af1f2c
|
||||
GIT_SHALLOW TRUE
|
||||
GIT_SUBMODULES "")
|
||||
FetchContent_MakeAvailable(opentelemetry_cpp)
|
||||
|
||||
add_library(messenger_core STATIC
|
||||
src/telemetry.cpp
|
||||
src/observability_monitor.cpp
|
||||
src/db.cpp
|
||||
src/auth.cpp
|
||||
src/user_repo.cpp
|
||||
src/chat_repo.cpp
|
||||
src/message_repo.cpp
|
||||
src/redis_pool.cpp
|
||||
src/session_repo.cpp
|
||||
src/pg_session_repo.cpp
|
||||
src/s3_client.cpp
|
||||
src/image_util.cpp
|
||||
src/attachment_repo.cpp
|
||||
src/attachment_queue.cpp
|
||||
src/outbox.cpp
|
||||
src/staging_cleaner.cpp
|
||||
src/job_repo.cpp
|
||||
src/image_pipeline.cpp
|
||||
src/direct_upload_repo.cpp
|
||||
)
|
||||
|
||||
target_include_directories(messenger_core PUBLIC
|
||||
src
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
|
||||
)
|
||||
target_compile_definitions(messenger_core PUBLIC
|
||||
MTUSI_MINIO_CPP_VERSION="${MTUSI_MINIO_CPP_VERSION}"
|
||||
)
|
||||
|
||||
target_link_libraries(messenger_core PUBLIC
|
||||
Threads::Threads
|
||||
PkgConfig::PQXX
|
||||
PkgConfig::HIREDIS
|
||||
PkgConfig::SODIUM
|
||||
PkgConfig::EXIF
|
||||
PkgConfig::WEBP
|
||||
opentelemetry_exporter_otlp_http
|
||||
opentelemetry_exporter_otlp_http_metric
|
||||
opentelemetry_exporter_otlp_http_log
|
||||
miniocpp::miniocpp
|
||||
)
|
||||
|
||||
add_executable(messenger src/main.cpp)
|
||||
target_link_libraries(messenger PRIVATE messenger_core)
|
||||
|
||||
add_executable(worker src/worker_main.cpp)
|
||||
target_link_libraries(worker PRIVATE messenger_core)
|
||||
|
||||
add_executable(s3_smoke tools/s3_smoke.cpp)
|
||||
target_link_libraries(s3_smoke PRIVATE messenger_core)
|
||||
|
||||
add_executable(image_smoke tools/image_smoke.cpp)
|
||||
target_link_libraries(image_smoke PRIVATE messenger_core)
|
||||
target_compile_definitions(image_smoke PRIVATE
|
||||
IMAGE_SMOKE_FIXTURES="${CMAKE_SOURCE_DIR}/tools/fixtures"
|
||||
)
|
||||
|
||||
add_executable(thumbnail_smoke tools/thumbnail_smoke.cpp)
|
||||
target_link_libraries(thumbnail_smoke PRIVATE messenger_core)
|
||||
|
||||
add_executable(variants_smoke tools/variants_smoke.cpp)
|
||||
target_link_libraries(variants_smoke PRIVATE messenger_core)
|
||||
|
||||
add_executable(job_admin src/job_admin_main.cpp)
|
||||
target_link_libraries(job_admin PRIVATE messenger_core)
|
||||
|
||||
add_executable(telemetry_test student-tests/telemetry_test.cpp)
|
||||
target_link_libraries(telemetry_test PRIVATE messenger_core)
|
||||
target_compile_options(telemetry_test PRIVATE -UNDEBUG)
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"version": 6,
|
||||
"cmakeMinimumRequired": {
|
||||
"major": 3,
|
||||
"minor": 16,
|
||||
"patch": 0
|
||||
},
|
||||
"configurePresets": [
|
||||
{
|
||||
"name": "release",
|
||||
"displayName": "Release with minio-cpp",
|
||||
"generator": "Unix Makefiles",
|
||||
"binaryDir": "${sourceDir}/build",
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Release",
|
||||
"MTUSI_MINIO_CPP_VERSION": "v0.3.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"buildPresets": [
|
||||
{
|
||||
"name": "release",
|
||||
"configurePreset": "release"
|
||||
}
|
||||
]
|
||||
}
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
# Clean-clone build: Ubuntu 24.04 + libpqxx from a pinned git tag.
|
||||
# S3 client: official minio-cpp v0.3.0 via FetchContent (library SigV4).
|
||||
# No dependency on mtusi2026lab3-api or other prior-lab images.
|
||||
|
||||
ARG PQXX_VERSION=7.10.0
|
||||
ARG MTUSI_MINIO_CPP_VERSION=v0.3.0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage: build libpqxx into /opt/libpqxx
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:24.04 AS pqxx
|
||||
ARG PQXX_VERSION
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates pkg-config \
|
||||
libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /tmp
|
||||
RUN git clone --depth 1 --branch "${PQXX_VERSION}" \
|
||||
https://github.com/jtv/libpqxx.git libpqxx-src \
|
||||
&& cmake -S libpqxx-src -B libpqxx-build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_INSTALL_PREFIX=/opt/libpqxx \
|
||||
-DBUILD_SHARED_LIBS=ON \
|
||||
-DSKIP_BUILD_TEST=ON \
|
||||
&& cmake --build libpqxx-build -j"$(nproc)" \
|
||||
&& cmake --install libpqxx-build \
|
||||
&& rm -rf /tmp/libpqxx-src /tmp/libpqxx-build
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage: build messenger + worker + smokes
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:24.04 AS builder
|
||||
ARG MTUSI_MINIO_CPP_VERSION
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential cmake pkg-config git ca-certificates \
|
||||
libpq-dev \
|
||||
libhiredis-dev \
|
||||
libsodium-dev libexif-dev libwebp-dev \
|
||||
libprotobuf-dev protobuf-compiler \
|
||||
nlohmann-json3-dev \
|
||||
libcurl4-openssl-dev libssl-dev zlib1g-dev \
|
||||
libpugixml-dev \
|
||||
libcurlpp-dev \
|
||||
libinih-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
|
||||
ENV PKG_CONFIG_PATH=/opt/libpqxx/lib/pkgconfig:/opt/libpqxx/lib/x86_64-linux-gnu/pkgconfig
|
||||
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
|
||||
ENV CMAKE_PREFIX_PATH=/opt/libpqxx
|
||||
|
||||
WORKDIR /src
|
||||
COPY CMakeLists.txt CMakePresets.json ./
|
||||
COPY cmake ./cmake
|
||||
COPY src ./src
|
||||
COPY tools ./tools
|
||||
COPY student-tests/telemetry_test.cpp ./student-tests/telemetry_test.cpp
|
||||
COPY migrations ./migrations
|
||||
COPY third_party ./third_party
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \
|
||||
-DMTUSI_MINIO_CPP_VERSION=${MTUSI_MINIO_CPP_VERSION} \
|
||||
&& cmake --build build --target messenger worker job_admin s3_smoke image_smoke thumbnail_smoke variants_smoke telemetry_test -j2
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runtime
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:24.04 AS runtime
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
libprotobuf32t64 \
|
||||
libpq5 \
|
||||
libhiredis1.1.0 \
|
||||
libsodium23 libexif12 libwebp7 \
|
||||
libssl3t64 \
|
||||
libcurl4t64 \
|
||||
zlib1g \
|
||||
libpugixml1v5 \
|
||||
libcurlpp0t64 \
|
||||
libinih1 \
|
||||
libinireader0 \
|
||||
util-linux \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid app --shell /usr/sbin/nologin --no-create-home app
|
||||
|
||||
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
|
||||
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /src/build/_deps/opentelemetry_cpp-build/libopentelemetry_proto.so /usr/local/lib/libopentelemetry_proto.so
|
||||
COPY --from=builder /src/build/messenger /app/messenger
|
||||
COPY --from=builder /src/build/worker /app/worker
|
||||
COPY --from=builder /src/build/telemetry_test /app/telemetry_test
|
||||
COPY --from=builder /src/build/job_admin /app/job_admin
|
||||
COPY --from=builder /src/build/s3_smoke /app/s3_smoke
|
||||
COPY --from=builder /src/build/image_smoke /app/image_smoke
|
||||
COPY --from=builder /src/build/thumbnail_smoke /app/thumbnail_smoke
|
||||
COPY --from=builder /src/build/variants_smoke /app/variants_smoke
|
||||
COPY --from=builder /src/migrations /app/migrations
|
||||
COPY tools/fixtures /app/fixtures
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
|
||||
RUN chmod 0755 /docker-entrypoint.sh \
|
||||
&& chown -R app:app /app \
|
||||
&& ldconfig
|
||||
|
||||
# Entrypoint starts as root to stage S3 secret files for uid 10001, then drops.
|
||||
USER root
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=3 \
|
||||
CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1
|
||||
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["/app/messenger"]
|
||||
@@ -28,3 +28,16 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: test-observability test-incidents
|
||||
COMPOSE ?= docker compose
|
||||
test-observability:
|
||||
$(COMPOSE) exec -T api /app/telemetry_test
|
||||
$(PYTHON) student-tests/observability.py --base-url "$(BASE_URL)"
|
||||
|
||||
test-incidents:
|
||||
COMPOSE="$(COMPOSE)" $(PYTHON) student-tests/incidents.py --base-url "$(BASE_URL)"
|
||||
|
||||
.PHONY: benchmark-telemetry
|
||||
benchmark-telemetry:
|
||||
COMPOSE="$(COMPOSE)" $(PYTHON) scripts/benchmark_telemetry.py
|
||||
|
||||
@@ -2,7 +2,30 @@
|
||||
|
||||
Научиться находить причину деградации через метрики, логи и трассы всей системы, включая фоновую обработку.
|
||||
|
||||
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
|
||||
В репозитории реализован C++ сервер на основе lab5 и стек наблюдаемости уровня 4:
|
||||
OpenTelemetry C++ → Collector → Prometheus / Loki / Tempo → Grafana.
|
||||
Бизнес-API сохранён. Архитектура, метрики, ограничения и результаты проверок — в [REPORT.md](REPORT.md).
|
||||
|
||||
```sh
|
||||
python3 scripts/bootstrap-local.py
|
||||
docker compose up -d --build
|
||||
# После готовности http://localhost:8080/health/ready:
|
||||
make test
|
||||
make test-observability
|
||||
```
|
||||
|
||||
Grafana: [localhost:3000](http://localhost:3000/d/messenger-observability), login `admin`,
|
||||
пароль — локальный `secrets/grafana_admin_password`. Bootstrap не перезаписывает существующие секреты.
|
||||
Контролируемые отказы с автоматическим восстановлением: `make test-incidents`.
|
||||
|
||||
Track 1 (9 прогонов по 1000 uploads, затем автоматическое восстановление telemetry):
|
||||
|
||||
```sh
|
||||
python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3
|
||||
```
|
||||
|
||||
Результаты: `evidence/track1-benchmark.md` и подробные обезличенные samples в `evidence/track1-benchmark.json`.
|
||||
Ниже сохранены исходные требования и критерии лабораторной.
|
||||
|
||||
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
|
||||
|
||||
|
||||
@@ -1,37 +1,564 @@
|
||||
# Отчёт — лабораторная 6
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Никитин Андрей Александрович, БВТ2301
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 5; Track 1 — цена наблюдаемости.
|
||||
- Commit/PR: реализация находится в рабочем дереве; отдельный commit/PR ещё не опубликован. Baseline шаблона: `470d9d6f1e4fe4f594e37c2ba59550ee5ca54fc1`.
|
||||
- Использованные источники, библиотеки и помощь ИИ: README и контракты курса; первичная документация OpenTelemetry, W3C Trace Context и Grafana; opentelemetry-cpp, minio-cpp, libpqxx, hiredis, libsodium, nlohmann/json, libexif, libwebp и stb. Помощь ИИ: Codex — реализация инструментации, конфигурации, скрипты проверок/бенчмарка и оформление отчёта. Версии и ссылки приведены ниже.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Linux amd64, Docker Engine с Compose v2+, Python 3.9+ и доступ к
|
||||
Ubuntu/GitHub/registry при первой сборке. Приложения собираются как C++17;
|
||||
компилятор, CMake и native-зависимости устанавливаются внутри Dockerfile.
|
||||
Прежние готовые API-образы lab5 не требуются.
|
||||
|
||||
| Компонент | Закреплённая версия |
|
||||
| --- | --- |
|
||||
| Build/runtime OS | Ubuntu 24.04 |
|
||||
| OpenTelemetry C++ / MinIO C++ / libpqxx | v1.21.0 / v0.3.0 / 7.10.0 |
|
||||
| PostgreSQL / Valkey / Nginx | 16.15-alpine / 8.1.10-alpine / 1.28.0-alpine |
|
||||
| MinIO / mc | RELEASE.2025-04-22T22-12-26Z / RELEASE.2025-04-16T18-13-26Z |
|
||||
| Collector Contrib / Prometheus | 0.119.0 / 3.2.1 |
|
||||
| Loki / Tempo / Grafana | 3.4.2 / 2.7.2 / 11.6.0 |
|
||||
|
||||
```sh
|
||||
python3 scripts/bootstrap-local.py
|
||||
docker compose config --quiet
|
||||
docker compose up -d --build
|
||||
# Дождаться HTTP 200:
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
make check
|
||||
make test
|
||||
make test-observability
|
||||
make test-incidents
|
||||
```
|
||||
|
||||
Grafana: http://localhost:3000/d/messenger-observability, пользователь `admin`,
|
||||
пароль в локальном `secrets/grafana_admin_password`. Dashboard: папка Messenger.
|
||||
Prometheus: http://localhost:9090, alerts http://localhost:9090/alerts.
|
||||
В Grafana Explore выбрать Tempo, найти trace, перейти к Loki logs; либо выбрать
|
||||
Loki `{service_name="messenger-worker"}` и фильтровать по `trace_id`.
|
||||
API → HTTP → worker → S3/DB проверяется `student-tests/observability.py`.
|
||||
|
||||
`API_PORT`, `GRAFANA_PORT`, `PROMETHEUS_PORT` настраиваются в `.env`; при их
|
||||
изменении URL тестов задаются параметрами соответствующих Python scripts.
|
||||
`OTEL_SDK_DISABLED=true docker compose up -d --force-recreate api worker` отключает
|
||||
SDK для сравнения; бизнес-процессы продолжают работать.
|
||||
|
||||
Если в окружении Docker Snap команда `docker compose` не найдена, используйте
|
||||
установленный CLI plugin напрямую: `make test-observability COMPOSE=/path/to/docker-compose`.
|
||||
|
||||
Первый запуск требует действий оператора: запустить bootstrap, выполнить build/up,
|
||||
дождаться HTTP 200 readiness и затем запустить проверки. Bootstrap создаёт `.env`
|
||||
и `secrets/` со случайными учебными credentials без вывода значений. Если они уже
|
||||
существуют, скрипт сохраняет их и завершает работу: для повторного запуска достаточно
|
||||
`docker compose up -d`. Секреты в отчёт и evidence не включаются.
|
||||
|
||||
API и Worker применяют SQL-миграции автоматически под PostgreSQL advisory lock.
|
||||
Сервисы `minio-init` и `minio-user-init` идемпотентно создают приватный bucket,
|
||||
policy и ограниченного S3 пользователя; Worker создаёт Redis consumer group.
|
||||
Grafana автоматически загружает datasources/dashboard. Ручное выполнение SQL
|
||||
и ручной импорт dashboard не требуются.
|
||||
|
||||
Безопасная остановка с сохранением named volumes:
|
||||
|
||||
```sh
|
||||
docker compose down
|
||||
```
|
||||
|
||||
Намеренное удаление **всех учебных данных этого Compose-проекта**: БД, Redis,
|
||||
MinIO, Prometheus, Loki, Tempo и состояния Grafana. Ниже приведена только инструкция;
|
||||
в рамках работы эта команда не выполнялась:
|
||||
|
||||
```sh
|
||||
# Только для осознанного сброса собственного учебного стенда:
|
||||
docker compose down --volumes
|
||||
```
|
||||
|
||||
Локальные `.env` и `secrets/` эта команда не удаляет.
|
||||
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
### Основа и границы изменений
|
||||
|
||||
В исходном lab6 был skeleton без C++ приложения, CMakeLists.txt и рабочего compose.yaml.
|
||||
В этот репозиторий перенесены исходники lab5 из соседнего checkout
|
||||
(commit `580b33efe867d348285785bcccd63681ec7ae962`), миграции, MinIO bootstrap,
|
||||
Nginx, инструменты проверки изображений и vendored stb. Соседний lab5 не изменён.
|
||||
|
||||
Сохранены API, авторизация/сессии, схемы БД, Redis Streams, transactional outbox,
|
||||
retry/backoff, дедупликация, ACK после durable результата, проверка изображений,
|
||||
MinIO original/variants и direct-upload расширение lab5. Изменения в job payload:
|
||||
только предусмотренные контрактом optional поля `traceparent`/`tracestate`.
|
||||
Когда SDK отключён, отсутствующий traceparent не записывается как пустая строка.
|
||||
|
||||
### Процессы, хранилища и версии SDK
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
API[API x2 / opentelemetry-cpp] -->|OTLP HTTP protobuf :4318| C[OTel Collector]
|
||||
W[Worker / opentelemetry-cpp] -->|OTLP HTTP protobuf :4318| C
|
||||
API --> O[PostgreSQL outbox + W3C context]
|
||||
O --> P[Publisher producer span]
|
||||
P --> R[Redis Stream / JSON + traceparent]
|
||||
R --> W
|
||||
C -->|Prometheus exporter :9464| M[Prometheus / 7 days]
|
||||
C -->|OTLP HTTP /otlp/v1/logs| L[Loki / 7 days]
|
||||
C -->|OTLP HTTP /v1/traces| T[Tempo / 7 days]
|
||||
G[Grafana] --> M
|
||||
G --> L
|
||||
G --> T
|
||||
```
|
||||
|
||||
API и worker инициализируют SDK в своих `main()`. В каждом процессе отдельные
|
||||
TracerProvider, MeterProvider, LoggerProvider, resource attributes:
|
||||
`service.name=messenger-api|messenger-worker`, `service.version=lab6-1.0`,
|
||||
`service.instance.id=<hostname контейнера>`, `deployment.environment.name=lab`.
|
||||
Instrumentation scope `messenger`, version `1.0`. Используются имена OTel HTTP/messaging
|
||||
и локальные документированные метрики worker/queue/pool; это не заявление о полном
|
||||
соответствии всем экспериментальным semantic conventions конкретного schema URL.
|
||||
|
||||
SDK закреплён на `opentelemetry-cpp v1.21.0` (commit
|
||||
`b9cf499ff5715433848b316059714b5c59af1f2c`), ABI 1. Экспортер только OTLP
|
||||
HTTP/protobuf: `WITH_OTLP_HTTP=ON`, `WITH_OTLP_GRPC=OFF`.
|
||||
Collector Contrib 0.119.0, Prometheus 3.2.1, Loki 3.4.2, Tempo 2.7.2,
|
||||
Grafana 11.6.0. PostgreSQL 16.15, Valkey 8.1.10, Nginx 1.28.0;
|
||||
MinIO RELEASE.2025-04-22T22-12-26Z, mc RELEASE.2025-04-16T18-13-26Z.
|
||||
Теги и конфиги закреплены в Compose. Loki использует native OTLP endpoint,
|
||||
а не удалённый Loki exporter. Collector не является долговременным хранилищем;
|
||||
Grafana строит запросы и визуализирует данные хранилищ. Все четыре хранилища
|
||||
состояния telemetry имеют отдельные named volumes.
|
||||
|
||||
### Источник истины и транзакционные границы
|
||||
|
||||
PostgreSQL — источник истины для пользователей, чатов, сообщений, вложений,
|
||||
состояний jobs и durable outbox; MinIO хранит бинарные оригиналы/варианты.
|
||||
Valkey используется для Redis Streams и, при `SESSION_STORE=redis`, сессий;
|
||||
в проверенной `.env` выбран `SESSION_STORE=postgres`. Брокер доставляет задания,
|
||||
а не заменяет durable состояние job в PostgreSQL.
|
||||
|
||||
Создание job и outbox записи выполняется в одной PostgreSQL-транзакции.
|
||||
Publisher отмечает outbox published после XADD; повторная доставка допускается.
|
||||
Worker сохраняет durable результат/следующую попытку до ACK. Публикация ссылки на
|
||||
готовые S3-объекты и статуса ready происходит транзакционно в БД. Эти правила lab5
|
||||
сохранены; telemetry не участвует в принятии бизнес-решений.
|
||||
|
||||
### Соответствие `.env.example` конфигурации
|
||||
|
||||
| Переменные | Реальное применение |
|
||||
| --- | --- |
|
||||
| `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` | Bootstrap PostgreSQL; `secrets/database_url` должен содержать согласованный connection string. Генератор создаёт согласованные значения. |
|
||||
| `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `BROKER_URL_FILE`, S3 `*_FILE` | Compose монтирует file secrets в `/run/secrets`; entrypoint переносит их в `/tmp/app-secrets` для uid 10001. Worker и API используют один Redis/Valkey endpoint. |
|
||||
| `SESSION_STORE`, `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` | Выбор хранилища/TTL сессий и идемпотентности API. |
|
||||
| `API_REPLICAS`, `API_PORT` | Число API-реплик и внешний порт lb; `APP_HOST`/`APP_PORT` внутри Compose фиксированы как `0.0.0.0:8080`. |
|
||||
| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`, MinIO credentials | Endpoint/bucket приложения и идемпотентный S3 bootstrap; приложения используют отдельного ограниченного пользователя. |
|
||||
| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS`, `MAX_CONCURRENT_UPLOADS`, `DIRECT_UPLOAD_TTL_SECONDS` | Ограничения upload/image/direct-upload из lab5. |
|
||||
| `JOB_MAX_ATTEMPTS`, `JOB_RECLAIM_IDLE_MS`, `JOB_RETRY_BASE_MS`, `JOB_RETRY_MAX_MS`, `JOB_TIMEOUT_SECONDS` | Retry/reclaim/backoff и watchdog Worker. |
|
||||
| `STAGING_RETENTION_SECONDS`, `STAGING_CLEANUP_INTERVAL_SECONDS` | Очистка staging. |
|
||||
| `WORKER_CONCURRENCY=1` | Описывает реализованный последовательный Worker; изменение этой переменной не включает дополнительные потоки. Метрика capacity равна 1. |
|
||||
| `OTEL_SDK_DISABLED` | Runtime on/off SDK в обоих приложениях. |
|
||||
| `OTEL_COLLECTOR_CONFIG` | Путь к Collector profile: по умолчанию 100%; для Track 1 используется отдельный 10% traces profile. |
|
||||
| `GRAFANA_ADMIN_USER`, `GRAFANA_PORT`, `PROMETHEUS_PORT` | Учётная запись Grafana и loopback-порты UI. Пароль Grafana читается из отдельного file secret. |
|
||||
| `APP_ENV` | В этом стенде окружение фиксировано как `deployment.environment.name=lab`; переменная не переключает ресурсные атрибуты. |
|
||||
|
||||
OTLP endpoint задаётся Compose как `http://otel-collector:4318`; service.name и
|
||||
service.version — кодом инициализации SDK. Настройки HTTP/DB pool и CPU/RAM имеют
|
||||
дополнительные defaults в Compose; проверенные лимиты приведены в экспериментах.
|
||||
|
||||
### Сквозная передача контекста
|
||||
|
||||
1. После разбора заголовков API извлекает только W3C `traceparent`/`tracestate`
|
||||
через `HttpTraceContext::Extract`, создаёт SERVER span, активирует Scope.
|
||||
Невалидный traceparent не становится родителем. Baggage не переносится.
|
||||
2. `enqueue_attachment_job()` создаёт PRODUCER span `attachment.enqueue` и
|
||||
в той же транзакции, что и job, записывает текущие W3C поля в JSON outbox.
|
||||
Это существенно: позднее publisher работает в другом потоке и не имеет HTTP Scope.
|
||||
3. Publisher извлекает durable контекст из JSON, создаёт PRODUCER span
|
||||
`jobs:attachments publish`, подставляет **его** traceparent в публикуемый Redis JSON.
|
||||
Исходный durable payload остаётся неизменным, повторная публикация получает
|
||||
новый producer span с прежним причинным родителем.
|
||||
4. Worker извлекает контекст из полученного JSON и создаёт CONSUMER span
|
||||
`attachment.process` с родителем Redis producer. S3 операции и DB публикация
|
||||
ready — дочерние CLIENT spans. В спаны не записываются object keys и SQL-параметры.
|
||||
5. Если JobRepo планирует retry, `enqueue_attachment_job()` вызывается под текущим
|
||||
consumer Scope. Следующая попытка продолжает тот же trace через новый outbox
|
||||
producer; исходные правила retry/ACK не меняются.
|
||||
|
||||
Root sampling — always on, дочерние spans используют ParentBasedSampler и
|
||||
уважают sampled flag родителя. API и worker ведут реальные spans операций,
|
||||
а не фиксированные демонстрационные traces. Логи завершения HTTP и job получают
|
||||
тот же trace_id/span_id; HTTP `request_id` сохраняется отдельно и соответствует
|
||||
`X-Request-Id`, уникальность между репликами задаёт `service.instance.id`.
|
||||
|
||||
### Сигналы и имена в Prometheus
|
||||
|
||||
| SDK instrument | Prometheus | Единицы / смысл |
|
||||
|---|---|---|
|
||||
| http.server.requests | http_server_requests_total | бизнес-запросы; method, route template, status class |
|
||||
| http.server.request.duration | http_server_request_duration_seconds_bucket/count/sum | секунды, включая чтение запроса |
|
||||
| worker.jobs | worker_jobs_total | доставки; job.type, outcome |
|
||||
| worker.job.duration | worker_job_duration_seconds_bucket/count/sum | секунды от consume до завершения обработки/ACK |
|
||||
| attachment.ready.duration | attachment_ready_duration_seconds_bucket/count/sum | durable job creation → транзакция публикации ready |
|
||||
| queue.depth | queue_depth | незавершённые durable jobs |
|
||||
| queue.oldest.age | queue_oldest_age | возраст старейшей незавершённой job, секунды |
|
||||
| queue.observation.timestamp | queue_observation_timestamp | Unix seconds последнего успешного наблюдения |
|
||||
| db.pool.used/waiting/capacity | db_pool_used/waiting/capacity | занятые, ожидающие и все соединения PgPool процесса |
|
||||
| worker.busy/capacity | worker_busy/capacity | активные задачи / 1 последовательный worker |
|
||||
| process.cpu.time | process_cpu_time | user+system CPU seconds; rate = использованные ядра |
|
||||
| process.memory.usage | process_memory_usage | текущий RSS bytes из /proc/self/statm |
|
||||
| telemetry.sdk.errors | telemetry_sdk_errors | накопленные внутренние SDK warning/error diagnostics |
|
||||
| telemetry.sdk.dropped.spans | telemetry_sdk_dropped_spans | накопленные предупреждения переполнения span batch queue |
|
||||
|
||||
Health endpoints исключены из RPS/errors/duration и логов завершения запроса.
|
||||
Duration записывается и на ранних HTTP-ошибках; неподдержанные/неизвестные маршруты
|
||||
получают `unmatched`, неизвестные методы `_OTHER`. Идентификаторы и query strings
|
||||
не попадают в metric labels. Границы histogram в секундах:
|
||||
`0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120`.
|
||||
|
||||
Worker outcomes: `success`, `failed`, `retry`, `busy`, `superseded`.
|
||||
Это счётчик **попыток доставки**, а не unique jobs: повторное чтение ready job
|
||||
учитывается как ещё одна успешная доставка. `retry` означает неудачную попытку
|
||||
с durable перепланированием, `failed` — terminal failure/невалидная ссылка либо
|
||||
исключение, оставившее доставку pending. Не смешивайте эти категории с числом
|
||||
уникально опубликованных attachments.
|
||||
|
||||
Queue observer раз в 5 секунд читает `jobs WHERE status IN ('queued','processing')`
|
||||
по отдельному соединению PostgreSQL, только в API. `max` объединяет одинаковое
|
||||
наблюдение от двух реплик. Метрика включает отложенные retry и непубликованный
|
||||
outbox, не включает terminal jobs и историю ACKed stream entries. Это **логический
|
||||
backlog**, а не XLEN и не время обработки. При недоступной БД последнее значение
|
||||
может устареть — для этого есть timestamp, панель возраста наблюдения и alert.
|
||||
Соединение observer не входит в PgPool; фоновые outbox/cleaner PgPool входят.
|
||||
|
||||
Dashboard `Messenger — Observability`, UID `messenger-observability`, импортируется
|
||||
автоматически из `infra/grafana/dashboards/messenger.json`. Есть RPS, p50/p95/p99
|
||||
HTTP/worker/time-to-ready, error ratio, queue depth/age, worker saturation,
|
||||
DB pools, CPU/RSS, диагностика Collector и связанные логи.
|
||||
|
||||
Пример percentile всех реплик:
|
||||
|
||||
```promql
|
||||
histogram_quantile(0.95,
|
||||
sum by(le)(rate(http_server_request_duration_seconds_bucket[5m])))
|
||||
```
|
||||
|
||||
Error ratio: `5xx / все бизнес-запросы`. Ответы 4xx в numerator не входят.
|
||||
При отсутствии серии 5xx numerator равен нулю (`or vector(0)`).
|
||||
Worker saturation: `sum(worker_busy) / clamp_min(sum(worker_capacity), 1)`.
|
||||
Histogram quantiles при отсутствии наблюдений дают NaN; dashboard не подменяет
|
||||
его средним или выдуманным percentile. OTLP Prometheus exporter может ещё хранить
|
||||
последние серии остановленного процесса до истечения настроенного expiration interval
|
||||
20s (затем требуется ещё один scrape Prometheus).
|
||||
|
||||
### SLI/SLO и alerts
|
||||
|
||||
Учебный availability SLI = `1 - 5xx / все бизнес-запросы`, health исключён.
|
||||
Цель: ≥99% за скользящие 30 минут при ненулевом трафике. Практический сигнал
|
||||
быстрой диагностики: error ratio >5% за 1 минуту, `for: 30s` (`ApiHighErrorRate`).
|
||||
Это локальный диагностический alert, а не burn-rate модель production SLO.
|
||||
|
||||
`QueueLagGrowing`: oldest age >30s, depth >0, delta(age[1m]) >0, `for: 30s`.
|
||||
`QueueObservationStale`: наблюдение старше 30s или отсутствует, `for: 30s`.
|
||||
Все правила, пороги, окна и пояснения сохранены в `infra/prometheus/alerts.yaml`.
|
||||
Локальное состояние доступно в Prometheus `/alerts`; внешних уведомлений нет.
|
||||
|
||||
### Отказоустойчивость и безопасность
|
||||
|
||||
SDK batch queues: максимум 1024 span records и 1024 log records на процесс,
|
||||
пачка 128, интервал 500ms, timeout HTTP 500ms, одна попытка экспорта.
|
||||
Экспорт выполняется в фоновых потоках SDK; периодический reader метрик — 5s,
|
||||
export timeout 1s. При недоступном Collector приложения продолжают работу,
|
||||
возможна потеря telemetry. Graceful shutdown ограничивает ожидание каждой
|
||||
из трёх providers двумя секундами.
|
||||
|
||||
Collector: memory_limiter первым процессором (192 MiB, spike 32 MiB, 1s),
|
||||
batch 256, max 512, timeout 1s. Очередь каждого backend exporter ограничена
|
||||
256 batches, 2 consumers, HTTP timeout 2s, retry с backoff 1–5s, общий лимит
|
||||
15s. Docker memory limit 256 MiB и GOMEMLIMIT 200MiB ограничивают процесс.
|
||||
API/worker не зависят через depends_on от Collector/Loki/Tempo/Grafana.
|
||||
При отказе backend сохраняются работа API/worker и экспорт метрик в Prometheus;
|
||||
Collector регистрирует ошибки/потери и после восстановления возобновляет экспорт.
|
||||
|
||||
Диагностика потерь: `otelcol_exporter_send_failed_spans`,
|
||||
`otelcol_exporter_send_failed_log_records`, `otelcol_exporter_enqueue_failed_spans`,
|
||||
`otelcol_processor_refused_spans`, `telemetry_sdk_errors`,
|
||||
`telemetry_sdk_dropped_spans`. Не складывайте эти значения как точное количество
|
||||
уникально потерянных spans: неуспешный экспорт может повторяться при retry.
|
||||
SDK 1.21 не сообщает отдельный точный счётчик переполнения log batch queue;
|
||||
он может терять логи молча. Точный аудит всех потерь потребовал бы отдельного
|
||||
processor/exporter wrapper. Падение/kill процесса до flush также теряет буферы.
|
||||
|
||||
Телеметрия строится по allowlist: статические имена событий/операций, route
|
||||
шаблоны, метод, status class/code, job.type/outcome и resource attributes.
|
||||
Не экспортируются Authorization/Basic/Bearer, пароли, session tokens, тела
|
||||
сообщений/запросов, filename, binary image, SQL-параметры, object keys,
|
||||
signed URL и тексты исключений. SDK diagnostics считаются без вывода текста.
|
||||
Отдельный privacy test ищет canary сообщения, пароль, token и base64 картинки
|
||||
в свежих логах и upload trace; это проверка выбранного сценария, не доказательство
|
||||
отсутствия утечек для всех возможных входов.
|
||||
|
||||
Пароли генерирует bootstrap, `.env` и `secrets/` исключены из Git и Docker context.
|
||||
Bootstrap сохраняет существующие credentials, а не перезаписывает их.
|
||||
Secret directory 0700, app secrets 0600; entrypoint копирует их в 0400 файлы
|
||||
для uid 10001 и сбрасывает root. Grafana secret 0444 внутри защищённого 0700
|
||||
каталога нужен для bind mount в Grafana uid 472. Telemetry backends и OTLP
|
||||
доступны только в Compose network; Grafana/Prometheus/API опубликованы на loopback.
|
||||
|
||||
### Ключевые файлы
|
||||
|
||||
- `src/telemetry.hpp`, `src/telemetry.cpp`: SDK lifecycle, RAII spans, propagator,
|
||||
HTTP/jobs metrics и структурные логи.
|
||||
- `src/main.cpp`, `src/worker_main.cpp`: инициализация SDK и границы операций.
|
||||
- `src/outbox.cpp`: durable W3C context, producer publication и retry propagation.
|
||||
- `src/db.cpp`, `src/s3_client.cpp`, `src/job_repo.cpp`: pool и dependency spans,
|
||||
публикация ready и end-to-end histogram.
|
||||
- `src/observability_monitor.*`: отдельное наблюдение очереди, process CPU/RSS.
|
||||
- `CMakeLists.txt`, `Dockerfile`, `docker-entrypoint.sh`: SDK, зависимости,
|
||||
сборка и безопасное чтение Compose secrets непривилегированными процессами.
|
||||
- `compose.yaml`, `infra/{otel,prometheus,loki,tempo,grafana}`: весь telemetry stack,
|
||||
persistent volumes, provisioning, dashboard и alert rules.
|
||||
- `scripts/bootstrap-local.py`, `student-tests/*`, `Makefile`: bootstrap и проверки.
|
||||
|
||||
Первичные справочные материалы:
|
||||
[OTel C++ exporters](https://opentelemetry.io/docs/languages/cpp/exporters/),
|
||||
[Loki native OTLP](https://grafana.com/docs/loki/latest/send-data/otel/),
|
||||
[W3C Trace Context](https://www.w3.org/TR/trace-context/).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Проверки реализации выполнены 16.11.2026. Таблица связывает отдельные заявленные
|
||||
критерии с воспроизводимыми командами, наблюдениями и сохранёнными файлами.
|
||||
Офлайн `make check` проверяет структуру, но не доказывает работу приложения.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke; сохранение бизнес-API lab5 | `make test` | PASS: пользователи/сессии, чат, сообщения, идемпотентность, upload → ready, original/thumbnail и доступы | [evidence/verification.md](evidence/verification.md), публичный `tests/smoke.py` |
|
||||
| Уровень 3: API-реплики, БД, S3, broker и Worker | `docker compose up -d --build`; `docker compose ps` | Две API-реплики, Worker и бизнес-хранилища запущены; API readiness healthy; upload обрабатывается | `compose.yaml`, [evidence/verification.md](evidence/verification.md) |
|
||||
| Уровень 3: Collector и хранилища трёх сигналов в общем стенде | `docker compose config --quiet`; Collector `validate`; запуск Compose | OTLP HTTP receiver и три pipelines; Prometheus/Loki/Tempo/Grafana работают, конфиги и named volumes заданы | `compose.yaml`, `infra/otel/collector.yaml`, `infra/{prometheus,loki,tempo,grafana}`, [evidence/verification.md](evidence/verification.md) |
|
||||
| Уровень 3: SDK API/Worker, HTTP/job counters и histogram | `make test-observability` | Реальные метрики обоих приложений; проверены имена и запросы dashboard | [evidence/observability.json](evidence/observability.json), `src/telemetry.cpp` |
|
||||
| Уровень 3: реальные traces и структурные логи с корреляцией | `make test-observability` | Upload trace содержит API/Worker и S3/DB spans; Loki events обоих service.name найдены по trace ID | [evidence/observability.json](evidence/observability.json) |
|
||||
| Уровень 3: низкая кардинальность и отсутствие приватных payloads в проверенном сценарии | Privacy checks внутри `make test-observability` | Нет raw chat ID/запрещённых labels, canary сообщения, пароля, token и base64 изображения в проверенных logs/trace | [evidence/observability.json](evidence/observability.json), allowlist в `src/telemetry.cpp` |
|
||||
| Уровень 3: импортируемые dashboard/datasources | `make test-observability` | Provisioned dashboard найден; 25 PromQL queries проверены | `infra/grafana/dashboards/messenger.json`, `infra/grafana/provisioning`, [evidence/observability.json](evidence/observability.json) |
|
||||
| Уровень 4: W3C API → outbox → Redis → Worker | `make test-observability`; `/app/telemetry_test` | Consumer parent — producer, producer продолжает durable outbox context; invalid context и route privacy self-test PASS | [evidence/observability.json](evidence/observability.json), `student-tests/telemetry_test.cpp` |
|
||||
| Уровень 4: цепочка до S3/DB, включая retry | `make test-incidents`, ограниченный отказ S3 | Retry сохранил trace ID, следующая попытка дошла до ready; parent-child цепочка проверена | [evidence/incidents.json](evidence/incidents.json), событие `retry_same_trace_ready_verified` |
|
||||
| Уровень 4: p50/p95/p99 из histogram всех реплик | `make test-observability` | Запросы с `histogram_quantile` и `sum by(le)` выполняются; единицы seconds документированы | `infra/grafana/dashboards/messenger.json`, [evidence/observability.json](evidence/observability.json) |
|
||||
| Уровень 4: queue depth/age, DB pool, Worker saturation, process CPU/RAM и time-to-ready | `make test-observability`; остановка Worker | Метрики присутствуют; при остановке Worker depth=1, oldest age=61.10s, после запуска backlog=0 | [evidence/observability.json](evidence/observability.json), [evidence/incidents.json](evidence/incidents.json) |
|
||||
| Уровень 4: проверяемый SLI/SLO и alert rules | `promtool check config /etc/prometheus/prometheus.yaml`; `promtool check rules /etc/prometheus/alerts.yaml` в контейнере Prometheus | Конфиг и 3 правила валидны; задан availability ≥99% / 30min; достижение SLO за полное окно отдельно не доказывалось | `infra/prometheus/alerts.yaml`, [evidence/verification.md](evidence/verification.md), подраздел SLI/SLO |
|
||||
| Уровень 4: alerts при росте queue lag и DB outage | `make test-incidents` | `QueueLagGrowing` и `ApiHighErrorRate` прошли firing → resolved | [evidence/incidents.json](evidence/incidents.json), [evidence/verification.md](evidence/verification.md) |
|
||||
| Уровень 4: ограниченные очереди и отказ telemetry backend | `make test-incidents`, остановка Tempo/Loki | Upload дошёл до ready, сообщения записывались; +46 failed span export attempts, RSS Collector 179634176 bytes; сервисы восстановлены | [evidence/incidents.json](evidence/incidents.json), `infra/otel/collector.yaml`, `src/telemetry.cpp` |
|
||||
| Уровень 4: локальные runbooks двух инцидентов | Диагностика Worker stopped / DB unavailable по подразделу ниже | Описаны сигналы, различение причин, команды восстановления и проверка resolved | Подраздел «Диагностика контролируемых инцидентов», [evidence/verification.md](evidence/verification.md) |
|
||||
| Уровень 5, Track 1: одинаковая нагрузка Off / On 100% / Collector 10% | `python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3` | 9 прогонов ×1000 uploads, concurrency 4; все 9000 HTTP 202/jobs ready, docker stats и ingest counters сохранены | [evidence/track1-benchmark.md](evidence/track1-benchmark.md), [evidence/track1-benchmark.json](evidence/track1-benchmark.json) |
|
||||
| Уровень 5, Track 1: цена сигналов, потери редких ошибок, выбор конфигурации | Тот же benchmark и отдельный ERROR probe | API CPU +22.5% по медианам средних; C сохранил 10.92% spans; в Tempo найдены 9 из 100 synthetic ERROR traces; компромиссы объяснены | Подраздел «Track 1: Цена наблюдаемости», [evidence/track1-benchmark.md](evidence/track1-benchmark.md) |
|
||||
| Восстановление после эксперимента и проверки инструмента | `python3 scripts/benchmark_telemetry.py --self-test`; затем `make test`, `make test-observability` | Парсеры/trace-only config self-test PASS; SDK on и Collector 100% восстановлены; повторные smoke PASS | [evidence/track1-benchmark.json](evidence/track1-benchmark.json), скрипт benchmark, [evidence/observability.json](evidence/observability.json) |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Сохранённые результаты проверок
|
||||
|
||||
Track 1: методика и итоговая таблица — [evidence/track1-benchmark.md](evidence/track1-benchmark.md);
|
||||
исходные latency/Docker stats/counters — [evidence/track1-benchmark.json](evidence/track1-benchmark.json).
|
||||
|
||||
Проверено 16.11.2026 на финальном образе: Docker build, Compose config,
|
||||
Collector validate, promtool (конфиг и 3 правила), C++ self-test, публичный
|
||||
`make test` с telemetry on/off — PASS. Сквозная проверка реального upload,
|
||||
связанных логов, privacy canaries и 25 запросов provisioned dashboard — PASS.
|
||||
Полный поставляемый сценарий инцидентов на финальном образе — PASS: оба alerts
|
||||
firing/resolved, retry в том же trace до ready, работа API/worker при остановленных
|
||||
Loki/Tempo и последующее восстановление сервисов.
|
||||
|
||||
Нагрузка: 10338 успешных POST за 30.026s, 0 ошибок,
|
||||
344.305 successful RPS, клиентские p50/p95/p99 =
|
||||
5.666/52.505/61.702 ms.
|
||||
Это closed-loop тест, concurrency 4, percentile только успешных запросов;
|
||||
он не доказывает предельную производительность или соблюдение 30-минутного SLO.
|
||||
|
||||
Даты/времена инцидентов и итоговые проверки: `evidence/verification.md`.
|
||||
Длительный memory soak и исчерпывающий privacy-аудит не проводились.
|
||||
Автоматические evidence файлы не содержат токены, пароли или тела сообщений:
|
||||
`evidence/observability.json` — реальный trace ID, имена spans, проверенные метрики,
|
||||
связь producer/consumer и query checks; `evidence/incidents.json` — времена
|
||||
firing/resolved, агрегированные response codes и RSS Collector.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Нагрузочный smoke
|
||||
|
||||
Хост: Intel Core i5-12450H, 12 logical CPU, 14.85 GiB RAM. Лимиты API —
|
||||
2 ×0.5 CPU/256 MiB, Worker — 0.5 CPU/512 MiB; образы и SDK закреплены выше.
|
||||
Сценарий `python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json`:
|
||||
closed-loop POST сообщений без автоматических retry. Получено 10338 успешных
|
||||
запросов за 30.026s, ошибок 0, 344.305 RPS, p50/p95/p99 успешных запросов
|
||||
5.666/52.505/61.702ms. Это один smoke-прогон, не измерение предельной мощности;
|
||||
CPU/RAM этим генератором не снимались. Полный сравнительный эксперимент вложений,
|
||||
включая CPU/RAM и прогрев/повторы, приведён в Track 1.
|
||||
|
||||
### Контролируемые отказы
|
||||
|
||||
Команда `make test-incidents` останавливает/восстанавливает реальные Compose-сервисы
|
||||
`worker`, `db`, `minio`, `tempo`, `loki`; volumes не удаляет. Для S3 retry применяется
|
||||
pause/unpause уже запущенного Worker. Это прямые действия скрипта, а не
|
||||
проверка restart/failover hook; отдельные hooks в этой работе не заявляются.
|
||||
Времена ниже рассчитаны по `evidence/incidents.json`, дата — 16.11.2026.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка `worker`, затем upload | HTTP 202/queued, рост backlog/age, alert; после запуска job ready | `QueueLagGrowing` firing через 74.1s после принятия upload; depth=1, oldest age=61.10s; затем backlog=0 и resolved | От firing до drained/resolved 20.6s; та же attachment дошла до ready |
|
||||
| Остановка `db`, валидные reads | 5xx, ошибки DB dependency, `ApiHighErrorRate` firing | Зафиксированы 1×503 и 87×502 на внешнем API; alert firing через 44.0s, после запуска БД чтение восстановлено | Firing → resolved за 60.1s, включая вымывание ошибок из минутного окна; отдельный аудит всех COMMIT не проводился |
|
||||
| Остановка `minio` при существующем Worker | Неудачная попытка → durable retry → ready в прежнем trace | `s3_failure_scheduled_retry` → `retry_same_trace_ready_verified`, trace `997e005d6cc34eaa869fc3ced3e479ee` | Между этими событиями 1.23s; job дошёл до ready без смены trace |
|
||||
| Остановка `tempo` и `loki` | API/Worker работают, экспорт ошибается, память/очереди ограничены | Во время outage upload ready, сообщения сохранялись; +46 failed span export attempts, RSS Collector 171.31 MiB при Docker limit 256 MiB | Работа приложения подтверждена через 21.3s намеренного outage; backends запущены, последующие smoke PASS. Потеря части telemetry допустима; failed attempts не равны уникальным потерям spans |
|
||||
|
||||
Точное время команд restart/start не измерялось отдельно: указанные интервалы
|
||||
связаны с сохранёнными событиями наблюдения, а не обещанием recovery SLA.
|
||||
|
||||
### Диагностика контролируемых инцидентов
|
||||
|
||||
`make test-incidents` создаёт только синтетические данные. Скрипт временно
|
||||
останавливает worker, БД и затем Loki/Tempo по очереди; каждый остановленный
|
||||
сервис запускается обратно в `finally`. Volumes не удаляются.
|
||||
Не запускайте сценарий на общем/боевом стенде с реальными пользователями.
|
||||
|
||||
1. **Worker stopped.** Upload остаётся 202/queued, depth и oldest age растут,
|
||||
API продолжает отвечать, нет свежих consumer spans после producer.
|
||||
Проверить QueueLagGrowing firing в Prometheus, worker container state и logs.
|
||||
`docker compose start worker`; та же attachment становится ready, backlog
|
||||
уменьшается до 0, alert resolved. Низкий busy сам по себе не доказывает
|
||||
отсутствие очереди: проверять также age и свежесть наблюдения.
|
||||
2. **DB unavailable.** Валидные аутентифицированные reads возвращают 5xx,
|
||||
ApiHighErrorRate firing. В Tempo SERVER span error, задержка/ошибка
|
||||
`postgresql.pool.acquire`; `db_pool_waiting` позволяет отличить ожидание
|
||||
соединения от медленного S3. Queue observation становится stale; это не
|
||||
означает нулевой backlog. После `docker compose start db` pool переподключается,
|
||||
запросы снова успешны, alert resolved после вымывания ошибок из минутного окна.
|
||||
3. **Tempo/Loki unavailable.** Upload проходит до ready, сообщения сохраняются.
|
||||
Collector exporter failures растут; очередь ограничена, RSS доступен в
|
||||
`otelcol_process_memory_rss`. После возврата backend exporter восстанавливается;
|
||||
пакеты, истёкшие по retry/queue limit, не восстанавливаются из application data.
|
||||
|
||||
### Track 1: Цена наблюдаемости
|
||||
|
||||
**Гипотеза.** Telemetry 100% увеличивает CPU/RAM API и Worker. Sampling 10% в
|
||||
Collector уменьшает экспорт в Tempo примерно на 90%, но не стоимость SDK и не
|
||||
входящий OTLP ingest. Ожидание «CPU +20–40%» проверяется, а не подставляется в результаты.
|
||||
|
||||
**Методика.** 16.11.2026, Intel Core i5-12450H, 12 logical CPU, 14.85 GiB RAM.
|
||||
Две API-реплики по 0.5 CPU/256 MiB, Worker 0.5 CPU/512 MiB; одинаковый binary
|
||||
и бизнес-зависимости. По три прогона каждого варианта, порядок ABC / BCA / CAB.
|
||||
Перед каждым — 20 warm-up uploads, затем **1000 POST вложений, concurrency 4**:
|
||||
PNG 320×200, 111817 bytes. Все 9000 измеренных uploads вернули 202 и дошли до ready;
|
||||
очередь между прогонами полностью обработана, volumes не удаляются.
|
||||
|
||||
A: SDK disabled; B: SDK ParentBased(AlwaysOn), Collector 100%; C: SDK остаётся
|
||||
100%, только traces pipeline получает `probabilistic_sampler/track1`
|
||||
(`sampling_percentage: 10`, `hash_seed: 22`) после memory_limiter и перед batch.
|
||||
Внешний traceparent не задаётся. Логи и метрики в C сохраняются на 100%.
|
||||
Off оставляет no-op обёртки и background observer: это runtime SDK on/off,
|
||||
а не сравнение со сборкой без инструментации.
|
||||
|
||||
Latency/mean/RPS измерены клиентом до HTTP 202, не до ready; p50/p99 — nearest rank.
|
||||
CPU/RAM — средние `docker stats` за POST + полный drain Worker; API = сумма двух
|
||||
реплик, 100% CPU = одно logical core, RAM = Docker working set в MiB, не peak RSS.
|
||||
Spans/s = delta `otelcol_receiver_accepted_spans` / окно POST + drain + финальный flush.
|
||||
Collector пересоздаётся перед прогоном; delta берётся внутри одного его запуска.
|
||||
Ниже медианы трёх прогонов, включая медиану индивидуальных p50/p99, не pooled percentile.
|
||||
|
||||
| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| A: Off | 84.37 | 27.30 | 202.70 | 10.32% | 14.22 | 34.79% | 6.88 | 0.00 |
|
||||
| B: On, SDK 100% / Collector 100% | 75.44 | 30.46 | 288.30 | 12.64% | 17.05 | 38.48% | 7.81 | 391.01 |
|
||||
| C: On, SDK 100% / Collector 10% | 75.28 | 30.48 | 204.05 | 13.31% | 16.58 | 39.12% | 7.88 | 412.19 |
|
||||
|
||||
Медианное **среднее HTTP-время** A/B/C: **46.89 / 52.56 / 52.82 ms**.
|
||||
Отдельные прогоны, дополнительные counters и длительности —
|
||||
[evidence/track1-benchmark.md](evidence/track1-benchmark.md);
|
||||
исходные latency samples, Docker stats, SHA256 конфигов и hardware/limits —
|
||||
[evidence/track1-benchmark.json](evidence/track1-benchmark.json).
|
||||
|
||||
**Цена CPU/RAM.** B против A: средний CPU API **+22.5%**, Worker **+10.6%**;
|
||||
RAM API **+19.9%** (+2.83 MiB на две реплики), Worker **+13.5%** (+0.93 MiB).
|
||||
Но CPU% усредняется за окна разной длины. Для одинакового объёма работы оценка
|
||||
CPU·мс/job = (CPU% / 100) × pipeline_seconds × 1000 / jobs_count; медиана её
|
||||
парного прироста по повторам — **+9.9% API, +2.2% Worker**.
|
||||
Диапазон +20–40% наблюдался у среднего CPU API, но не является универсальной
|
||||
надбавкой SDK и не подтвердился для нормированного CPU Worker.
|
||||
|
||||
**Ingest и экспорт.** В B и C принято примерно 28.2 тыс. spans на 1000 uploads.
|
||||
Медианные accepted/sent: B **28231 / 28230**, C **28218 / 3083**.
|
||||
C сохраняет медианно **10.92%** spans, отбрасывая около **89.08%** перед Tempo.
|
||||
Это вероятностный отбор по trace ID, а не точная квота. Исходящий поток в Tempo
|
||||
снизился с **391.00 до 44.78 spans/s**. Входящий ingest не уменьшился: SDK всё
|
||||
создаёт и отправляет; разница его rate связана с длительностью окна.
|
||||
Это количества spans, **не bytes**. В каждом on-прогоне принято 3000 log records:
|
||||
экономия относится к traces, а не ко всем трём сигналам. Разгрузка CPU/RAM
|
||||
приложений вариантом C не доказана.
|
||||
|
||||
**Какие редкие ошибки теряются.** Отдельно от замеров API отправлены 100
|
||||
синтетических ERROR-traces со status 503: в Tempo найдены **9**, отсутствуют **91**.
|
||||
Это проверка sampler, не 100 реальных отказов API. Чистый probabilistic sampler
|
||||
не учитывает тяжесть ошибки: единичный 5xx, неудачный Worker/S3 retry или редкий
|
||||
медленный запрос теряет подробный trace примерно с вероятностью 90%.
|
||||
Вероятность потерять все пять независимых редких error traces — 0.9⁵ ≈ 59%.
|
||||
Ошибки в метриках Prometheus и ERROR logs Loki остаются; ссылка из сохранённого
|
||||
лога может не найти trace в Tempo. Поведение соответствует
|
||||
[probabilistic sampler 0.119.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.119.0/processor/probabilisticsamplerprocessor/README.md).
|
||||
|
||||
**Выбор для production.** Из A/B/C — **C при ограниченном бюджете хранения**:
|
||||
полные метрики/логи и около десятой части traces. A лишает систему наблюдаемости;
|
||||
B полезен при малом трафике и диагностике, но сохраняет все traces. Для критичных
|
||||
потоков чистого C недостаточно: предпочтительнее error/slow-aware tail sampling,
|
||||
сохраняющий ERROR и медленные traces плюс 10% обычных. Это рекомендация, не
|
||||
дополнительная реализованная политика. Для async Worker decision window должен
|
||||
учитывать очередь/retry и иметь ограниченную память. Для снижения именно CPU SDK
|
||||
или OTLP ingress sampling потребуется в SDK, с риском потерять ошибки ещё до Collector.
|
||||
|
||||
**Качество и ограничения.** Во всех прогонах HTTP errors, refused spans и failed
|
||||
span exports = 0; внутри on-окон delta SDK diagnostics = 0, максимум счётчика
|
||||
переполнения span batch queue = 0. До начала некоторых окон SDK diagnostics уже
|
||||
равнялся 4: startup/reconfiguration исключены из steady-state замеров. Разница
|
||||
accepted/sent в B до нескольких spans объяснима асинхронными границами и фоновыми
|
||||
health/outbox spans; counters не являются строгим аудитом всех возможных потерь SDK.
|
||||
|
||||
Closed-loop, один хост, три повтора, без CPU pinning и изоляции фоновой нагрузки.
|
||||
БД/S3 растут; порядок вариантов ротируется. В третьем повторе pipeline A/B занял
|
||||
140.02/137.77s вместо примерно 64–80s в других прогонах; причина отдельно не
|
||||
профилировалась, результаты не удалялись. p99 отдельных прогонов:
|
||||
A 201.98–413.17ms, B 200.24–437.94ms, C 202.22–212.64ms. По медианному p99 нельзя
|
||||
доказывать, что C быстрее B; весь рост CPU% нельзя приписать SDK. Startup cost,
|
||||
точные OTLP bytes и backend CPU/RAM не измерялись. После завершения восстановлены
|
||||
SDK on и Collector 100%; API/observability smoke повторно прошли.
|
||||
|
||||
Воспроизведение:
|
||||
|
||||
```sh
|
||||
python3 scripts/benchmark_telemetry.py --self-test
|
||||
python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3
|
||||
```
|
||||
|
||||
Для Docker Snap при необходимости передать `--compose /path/to/docker-compose`
|
||||
и `--docker /path/to/docker`. Скрипт восстанавливает baseline в `finally`.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- API/Worker выдержали ограниченные остановки Worker, БД, S3 и telemetry backends
|
||||
с описанным восстановлением; это не HA-гарантия для БД, lb, MinIO или всего хоста.
|
||||
Persistent volumes заданы, но отдельный полный persistence/failover hook в lab6
|
||||
не проверялся. Достижение availability SLO ≥99% за 30 минут отдельно не измерялось.
|
||||
- Временные ошибки экспорта, переполнение bounded buffers и аварийный kill могут
|
||||
терять telemetry. SDK 1.21 не даёт точного счётчика потерь log batch queue;
|
||||
export failures/SDK diagnostics не являются строгим аудитом уникальных потерь.
|
||||
- Queue depth — логические незавершённые jobs из PostgreSQL, не Redis XLEN.
|
||||
При отказе БД значения устаревают, что отмечается отдельным alert. PgPool metrics
|
||||
не включают dedicated connection observer. Worker последовательный, capacity=1.
|
||||
- Low-cardinality allowlist и privacy canaries проверены для синтетического сценария;
|
||||
исчерпывающий privacy/security audit и длительный memory soak не проводились.
|
||||
Наблюдение RSS за 21.3s outage не заменяет длительный тест ограниченности памяти.
|
||||
- Track 1 — closed-loop, один хост, 3 повтора без CPU pinning/изоляции фоновой нагрузки,
|
||||
с растущей БД/S3. Все результаты, включая медленные третьи прогоны, сохранены.
|
||||
Выводы относятся к этим лимитам/PNG/нагрузке, не к произвольному production.
|
||||
- Подтверждено: Collector sampling уменьшает число отправленных в Tempo spans
|
||||
примерно на 89%, сохраняя metrics/logs; не подтверждено снижение CPU SDK,
|
||||
OTLP bytes или backend CPU/стоимости хранения в деньгах. Сохранение только 9 из
|
||||
100 synthetic ERROR traces показывает отсутствие error-aware гарантии.
|
||||
- Следующие шаги: отдельный хост/CPU pinning и более длинные повторы для устойчивой
|
||||
оценки overhead; измерение OTLP bytes и ingest/storage CPU; эксперимент с SDK
|
||||
sampling, если нужна экономия приложения. Error/slow-aware tail sampling с bounded
|
||||
memory и окном, учитывающим async queue/retry, остаётся рекомендацией и требует
|
||||
отдельной проверки. Также нужны длительный memory soak и полный аудит потерь.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Provide the imported target names expected by minio-cpp's CMakeLists
|
||||
# (unofficial-curlpp / unofficial-inih) using system packages from apt.
|
||||
|
||||
find_package(OpenSSL REQUIRED)
|
||||
find_package(ZLIB REQUIRED)
|
||||
find_package(CURL REQUIRED)
|
||||
|
||||
# nlohmann_json (header-only; Ubuntu ships /usr/include/nlohmann/json.hpp)
|
||||
if(NOT TARGET nlohmann_json::nlohmann_json)
|
||||
find_package(nlohmann_json QUIET CONFIG)
|
||||
endif()
|
||||
if(NOT TARGET nlohmann_json::nlohmann_json)
|
||||
add_library(nlohmann_json INTERFACE)
|
||||
add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json)
|
||||
find_path(NLOHMANN_JSON_INCLUDE_DIR nlohmann/json.hpp)
|
||||
if(NOT NLOHMANN_JSON_INCLUDE_DIR)
|
||||
message(FATAL_ERROR "nlohmann/json.hpp not found (install nlohmann-json3-dev)")
|
||||
endif()
|
||||
target_include_directories(nlohmann_json INTERFACE "${NLOHMANN_JSON_INCLUDE_DIR}")
|
||||
endif()
|
||||
|
||||
# pugixml
|
||||
if(NOT TARGET pugixml)
|
||||
find_package(pugixml QUIET CONFIG)
|
||||
endif()
|
||||
if(NOT TARGET pugixml AND NOT TARGET pugixml::pugixml)
|
||||
pkg_check_modules(PUGIXML REQUIRED IMPORTED_TARGET pugixml)
|
||||
add_library(pugixml INTERFACE)
|
||||
target_link_libraries(pugixml INTERFACE PkgConfig::PUGIXML)
|
||||
endif()
|
||||
if(TARGET pugixml::pugixml AND NOT TARGET pugixml)
|
||||
add_library(pugixml ALIAS pugixml::pugixml)
|
||||
endif()
|
||||
|
||||
# curlpp → unofficial::curlpp::curlpp
|
||||
if(NOT TARGET unofficial::curlpp::curlpp)
|
||||
find_library(CURLPP_LIBRARY NAMES curlpp REQUIRED)
|
||||
find_path(CURLPP_INCLUDE_DIR curlpp/cURLpp.hpp REQUIRED)
|
||||
add_library(unofficial_curlpp INTERFACE)
|
||||
target_include_directories(unofficial_curlpp INTERFACE "${CURLPP_INCLUDE_DIR}")
|
||||
target_link_libraries(unofficial_curlpp INTERFACE "${CURLPP_LIBRARY}" CURL::libcurl)
|
||||
add_library(unofficial::curlpp::curlpp ALIAS unofficial_curlpp)
|
||||
endif()
|
||||
|
||||
# inih INIReader → unofficial::inih::inireader
|
||||
if(NOT TARGET unofficial::inih::inireader)
|
||||
find_library(INIREADER_LIBRARY NAMES INIReader REQUIRED)
|
||||
find_library(INIH_LIBRARY NAMES inih REQUIRED)
|
||||
find_path(INIREADER_INCLUDE_DIR INIReader.h REQUIRED)
|
||||
add_library(unofficial_inih_inireader INTERFACE)
|
||||
target_include_directories(unofficial_inih_inireader INTERFACE "${INIREADER_INCLUDE_DIR}")
|
||||
target_link_libraries(unofficial_inih_inireader INTERFACE
|
||||
"${INIREADER_LIBRARY}" "${INIH_LIBRARY}")
|
||||
add_library(unofficial::inih::inireader ALIAS unofficial_inih_inireader)
|
||||
endif()
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.15-alpine
|
||||
command: postgres -c max_connections=200 -c shared_buffers=256MB
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
cpus: ${DB_CPU_LIMIT:-1.0}
|
||||
mem_limit: 512m
|
||||
stop_grace_period: 20s
|
||||
|
||||
cache:
|
||||
image: valkey/valkey:8.1.10-alpine
|
||||
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "always"]
|
||||
volumes:
|
||||
- valkeydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
cpus: "0.50"
|
||||
mem_limit: 256m
|
||||
stop_grace_period: 15s
|
||||
|
||||
api: &api
|
||||
build: .
|
||||
image: mtusi2026lab6-api:local
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
cache:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
minio-user-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: ${SESSION_STORE:-redis}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
BROKER_URL_FILE: /run/secrets/cache_url
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
|
||||
HTTP_WORKERS: ${HTTP_WORKERS:-8}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
|
||||
DIRECT_UPLOAD_TTL_SECONDS: ${DIRECT_UPLOAD_TTL_SECONDS:-300}
|
||||
# Optional: sign URLs for a host-reachable MinIO (compose.presign.yaml).
|
||||
S3_PRESIGN_ENDPOINT: ${S3_PRESIGN_ENDPOINT:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: ${API_CPU_LIMIT:-0.50}
|
||||
memory: ${API_MEM_LIMIT:-256m}
|
||||
cpus: ${API_CPU_LIMIT:-0.50} # для совместимости с docker compose up
|
||||
mem_limit: ${API_MEM_LIMIT:-256m}
|
||||
stop_grace_period: 15s
|
||||
|
||||
worker:
|
||||
<<: *api
|
||||
# Keep the shared entrypoint: it stages secrets and drops root privileges.
|
||||
entrypoint: ["/docker-entrypoint.sh", "/app/worker"]
|
||||
command: []
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
||||
OTEL_SDK_DISABLED: ${OTEL_SDK_DISABLED:-false}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-6}
|
||||
JOB_RECLAIM_IDLE_MS: ${JOB_RECLAIM_IDLE_MS:-5000}
|
||||
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000}
|
||||
JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-60000}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60}
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512m
|
||||
cpus: "0.50"
|
||||
mem_limit: 512m
|
||||
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "127.0.0.1:${API_PORT:-8080}:8080"
|
||||
environment:
|
||||
NGINX_WORKERS: ${NGINX_WORKERS:-1}
|
||||
NGINX_ENVSUBST_TEMPLATE_DIR: /etc/nginx/templates
|
||||
NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx
|
||||
NGINX_ENVSUBST_FILTER: "^NGINX_"
|
||||
volumes:
|
||||
- ./lb/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
cpus: ${LB_CPU_LIMIT:-0.25}
|
||||
mem_limit: 128m
|
||||
stop_grace_period: 10s
|
||||
|
||||
# Private S3-compatible store. Ports 9000/9001 are NOT published to the host.
|
||||
# Image: pinned MinIO RELEASE.2025-04-22T22-12-26Z via local Dockerfile
|
||||
# (official binary from GitHub; docker.io hub may be unreachable).
|
||||
minio:
|
||||
build:
|
||||
context: ./infra/minio
|
||||
args:
|
||||
MINIO_VERSION: RELEASE.2025-04-22T22-12-26Z
|
||||
image: mtusi2026lab5-minio:RELEASE.2025-04-22T22-12-26Z
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
cpus: ${MINIO_CPU_LIMIT:-0.50}
|
||||
mem_limit: ${MINIO_MEM_LIMIT:-512m}
|
||||
stop_grace_period: 15s
|
||||
|
||||
# Idempotent bucket bootstrap: create if missing, force private ACL.
|
||||
minio-init:
|
||||
build:
|
||||
context: ./infra/mc
|
||||
args:
|
||||
MC_VERSION: RELEASE.2025-04-16T18-13-26Z
|
||||
image: mtusi2026lab5-mc:RELEASE.2025-04-16T18-13-26Z
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
echo "minio-init: waiting for endpoint $$S3_ENDPOINT"
|
||||
i=0
|
||||
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
i=$$((i + 1))
|
||||
if [ "$$i" -ge 30 ]; then
|
||||
echo "minio-init: cannot reach MinIO" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
mc mb --ignore-existing "local/$$S3_BUCKET"
|
||||
mc anonymous set none "local/$$S3_BUCKET"
|
||||
echo "minio-init: bucket listing:"
|
||||
mc ls "local/$$S3_BUCKET" || true
|
||||
echo "minio-init: bucket $$S3_BUCKET ready (anonymous=none)"
|
||||
restart: "no"
|
||||
|
||||
# Idempotent limited S3 user for the API (not MinIO root).
|
||||
minio-user-init:
|
||||
image: mtusi2026lab5-mc:RELEASE.2025-04-16T18-13-26Z
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_USER_ACCESS_KEY: ${MINIO_USER_ACCESS_KEY}
|
||||
MINIO_USER_SECRET_KEY: ${MINIO_USER_SECRET_KEY}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
POLICY_NAME: messenger-attachments-rw
|
||||
volumes:
|
||||
- ./infra/minio/policy-attachments.json:/policy.template.json:ro
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
echo "minio-user-init: alias $$S3_ENDPOINT"
|
||||
i=0
|
||||
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
i=$$((i + 1))
|
||||
if [ "$$i" -ge 30 ]; then
|
||||
echo "minio-user-init: cannot reach MinIO" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy.template.json > /tmp/policy.json
|
||||
|
||||
if mc admin policy info local "$$POLICY_NAME" >/dev/null 2>&1; then
|
||||
mc admin policy remove local "$$POLICY_NAME" >/dev/null 2>&1 || true
|
||||
fi
|
||||
mc admin policy create local "$$POLICY_NAME" /tmp/policy.json
|
||||
|
||||
if mc admin user info local "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1; then
|
||||
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY already exists"
|
||||
else
|
||||
mc admin user add local "$$MINIO_USER_ACCESS_KEY" "$$MINIO_USER_SECRET_KEY"
|
||||
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY created"
|
||||
fi
|
||||
|
||||
mc admin policy attach local "$$POLICY_NAME" --user "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1 || true
|
||||
echo "minio-user-init: policy $$POLICY_NAME attached to $$MINIO_USER_ACCESS_KEY"
|
||||
restart: "no"
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.119.0
|
||||
command: [--config=/etc/otelcol/config.yaml]
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
environment:
|
||||
GOMEMLIMIT: 200MiB
|
||||
volumes:
|
||||
- ${OTEL_COLLECTOR_CONFIG:-./infra/otel/collector.yaml}:/etc/otelcol/config.yaml:ro
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.2.1
|
||||
command: [--config.file=/etc/prometheus/prometheus.yaml, --storage.tsdb.retention.time=7d]
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
ports: ["127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"]
|
||||
volumes:
|
||||
- ./infra/prometheus:/etc/prometheus:ro
|
||||
- prometheusdata:/prometheus
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.4.2
|
||||
command: [-config.file=/etc/loki/config.yaml]
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/config.yaml:ro
|
||||
- lokidata:/loki
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.7.2
|
||||
user: "0:0"
|
||||
command: [-config.file=/etc/tempo/config.yaml]
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/config.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.6.0
|
||||
restart: unless-stopped
|
||||
mem_limit: 256m
|
||||
ports: ["127.0.0.1:${GRAFANA_PORT:-3000}:3000"]
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_PLUGINS_PREINSTALL_DISABLED: "true"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
secrets: [grafana_admin_password]
|
||||
volumes:
|
||||
- grafanadata:/var/lib/grafana
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
|
||||
secrets:
|
||||
grafana_admin_password:
|
||||
file: ./secrets/grafana_admin_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
valkeydata: {}
|
||||
miniodata: {}
|
||||
prometheusdata: {}
|
||||
lokidata: {}
|
||||
tempodata: {}
|
||||
grafanadata: {}
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Compose bind-backed secrets retain host permissions. Stage for application uid.
|
||||
if [ "$(id -u)" = "0" ]; then
|
||||
mkdir -p /tmp/app-secrets
|
||||
chmod 0700 /tmp/app-secrets
|
||||
for name in database_url cache_url s3_access_key_id s3_secret_access_key; do
|
||||
if [ -r "/run/secrets/$name" ]; then
|
||||
cp "/run/secrets/$name" "/tmp/app-secrets/$name"
|
||||
chown 10001:10001 "/tmp/app-secrets/$name"
|
||||
chmod 0400 "/tmp/app-secrets/$name"
|
||||
fi
|
||||
done
|
||||
chown 10001:10001 /tmp/app-secrets
|
||||
export DATABASE_URL_FILE=/tmp/app-secrets/database_url
|
||||
export CACHE_URL_FILE=/tmp/app-secrets/cache_url
|
||||
export BROKER_URL_FILE=/tmp/app-secrets/cache_url
|
||||
export S3_ACCESS_KEY_ID_FILE=/tmp/app-secrets/s3_access_key_id
|
||||
export S3_SECRET_ACCESS_KEY_FILE=/tmp/app-secrets/s3_secret_access_key
|
||||
exec setpriv --reuid=10001 --regid=10001 --clear-groups -- "$@"
|
||||
fi
|
||||
exec "$@"
|
||||
@@ -3,3 +3,16 @@
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
|
||||
Результаты реализации lab6:
|
||||
|
||||
- `verification.md` — итоговые проверки и время инцидентов (Москва).
|
||||
- `observability.json` — реальный trace, parent-child связь, проверенные метрики/PromQL.
|
||||
- `incidents.json` — firing/resolved, retry trace и ограниченный отказ telemetry backend.
|
||||
- `load.json` — 30-секундный closed-loop прогон синтетических POST сообщений.
|
||||
|
||||
Запросы Grafana находятся в `infra/grafana/dashboards/messenger.json`, методика — в `REPORT.md`.
|
||||
|
||||
Track 1: `track1-benchmark.md` содержит сравнение A/B/C, `track1-benchmark.json` —
|
||||
все отдельные прогоны, latency samples, Docker stats, counters Collector и параметры хоста.
|
||||
Методика: `python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3`.
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
[
|
||||
{
|
||||
"event": "worker_stopped_upload_accepted",
|
||||
"timestamp": "2026-11-16T07:30:35.969738+00:00"
|
||||
},
|
||||
{
|
||||
"event": "QueueLagGrowing_firing",
|
||||
"timestamp": "2026-11-16T07:31:50.091878+00:00",
|
||||
"depth": 1.0,
|
||||
"oldest_seconds": 61.100729
|
||||
},
|
||||
{
|
||||
"event": "QueueLagGrowing_resolved_backlog_drained",
|
||||
"timestamp": "2026-11-16T07:32:10.680368+00:00"
|
||||
},
|
||||
{
|
||||
"event": "database_stopped",
|
||||
"timestamp": "2026-11-16T07:32:10.928124+00:00"
|
||||
},
|
||||
{
|
||||
"event": "ApiHighErrorRate_firing",
|
||||
"timestamp": "2026-11-16T07:32:54.959429+00:00",
|
||||
"response_counts": {
|
||||
"503": 1,
|
||||
"502": 87
|
||||
}
|
||||
},
|
||||
{
|
||||
"event": "ApiHighErrorRate_resolved_database_recovered",
|
||||
"timestamp": "2026-11-16T07:33:55.054932+00:00"
|
||||
},
|
||||
{
|
||||
"event": "s3_failure_scheduled_retry",
|
||||
"timestamp": "2026-11-16T07:33:57.449574+00:00",
|
||||
"trace_id": "997e005d6cc34eaa869fc3ced3e479ee"
|
||||
},
|
||||
{
|
||||
"event": "retry_same_trace_ready_verified",
|
||||
"timestamp": "2026-11-16T07:33:58.678216+00:00",
|
||||
"trace_id": "997e005d6cc34eaa869fc3ced3e479ee"
|
||||
},
|
||||
{
|
||||
"event": "tempo_loki_stopped",
|
||||
"timestamp": "2026-11-16T07:34:08.938180+00:00"
|
||||
},
|
||||
{
|
||||
"event": "telemetry_outage_api_worker_healthy",
|
||||
"timestamp": "2026-11-16T07:34:30.248711+00:00",
|
||||
"failed_span_exports": 46.0,
|
||||
"collector_rss_bytes": 179634176.0
|
||||
},
|
||||
{
|
||||
"event": "tempo_loki_restarted",
|
||||
"timestamp": "2026-11-16T07:34:30.765794+00:00"
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 6,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.026,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 10338,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 344.305,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.666,
|
||||
"p95": 52.505,
|
||||
"p99": 61.702
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"timestamp": "2026-11-16T15:21:12.712973+00:00",
|
||||
"result": "PASS",
|
||||
"trace_id": "c3280a5a6f0d494f8f62834603b982a0",
|
||||
"span_names": [
|
||||
"POST /api/v1/chats/{chat_id}/attachments",
|
||||
"attachment.enqueue",
|
||||
"attachment.process",
|
||||
"jobs:attachments publish",
|
||||
"postgresql.pool.acquire",
|
||||
"postgresql.publish_ready",
|
||||
"s3.delete_object",
|
||||
"s3.get_object",
|
||||
"s3.put_object",
|
||||
"s3.stat_object"
|
||||
],
|
||||
"consumer_parent_is_producer": true,
|
||||
"log_services": [
|
||||
"messenger-api",
|
||||
"messenger-worker"
|
||||
],
|
||||
"metric_names_checked": [
|
||||
"http_server_requests_total",
|
||||
"http_server_request_duration_seconds_bucket",
|
||||
"worker_jobs_total",
|
||||
"worker_job_duration_seconds_bucket",
|
||||
"attachment_ready_duration_seconds_bucket",
|
||||
"queue_depth",
|
||||
"queue_oldest_age",
|
||||
"db_pool_used",
|
||||
"db_pool_waiting",
|
||||
"worker_busy",
|
||||
"worker_capacity",
|
||||
"process_memory_usage",
|
||||
"process_cpu_time"
|
||||
],
|
||||
"dashboard_queries_checked": 25,
|
||||
"privacy_canaries": "absent in recent logs and upload trace"
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,59 @@
|
||||
# Track 1: Цена наблюдаемости
|
||||
|
||||
Результат: **PASS**. Начало UTC: 2026-11-16T14:52:30.645168+00:00.
|
||||
|
||||
По 3 прогона каждого варианта, 1000 POST uploads, concurrency 4, warm-up 20 uploads.
|
||||
Порядок ABC / BCA / CAB. PNG: 111817 bytes. Каждый job дошёл до ready; очередь между прогонами пуста.
|
||||
Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).
|
||||
RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.
|
||||
CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.
|
||||
|
||||
| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|---:|
|
||||
| A: Off | 84.37 | 27.30 | 202.70 | 10.32% | 14.22 | 34.79% | 6.88 | 0.00 |
|
||||
| B: On, SDK 100% / Collector 100% | 75.44 | 30.46 | 288.30 | 12.64% | 17.05 | 38.48% | 7.81 | 391.01 |
|
||||
| C: On, SDK 100% / Collector 10% | 75.28 | 30.48 | 204.05 | 13.31% | 16.58 | 39.12% | 7.88 | 412.19 |
|
||||
|
||||
| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |
|
||||
|---|---:|---:|---:|---:|---:|---:|---:|
|
||||
| A | 46.89 | 79.95 | 0 | 0 | 0.00 | 7.80 | 27.81 |
|
||||
| B | 52.56 | 69.12 | 28231 | 28230 | 391.00 | 8.57 | 26.60 |
|
||||
| C | 52.82 | 65.37 | 28218 | 3083 | 44.78 | 8.75 | 25.82 |
|
||||
|
||||
*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.
|
||||
Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.
|
||||
В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.
|
||||
|
||||
| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |
|
||||
|---|---:|---:|---:|---:|---:|---:|
|
||||
| A/1 | 79.03 | 50.13 | 201.98 | 79.95 | 320 / 160 | 0 / 0 |
|
||||
| B/1 | 78.94 | 50.20 | 200.24 | 69.12 | 276 / 138 | 0 / 0 |
|
||||
| C/1 | 75.28 | 52.82 | 204.05 | 65.37 | 260 / 130 | 0 / 0 |
|
||||
| B/2 | 75.44 | 52.56 | 288.30 | 65.36 | 260 / 130 | 0 / 0 |
|
||||
| C/2 | 77.95 | 50.86 | 202.22 | 64.91 | 258 / 129 | 0 / 0 |
|
||||
| A/2 | 85.62 | 46.41 | 202.70 | 63.87 | 254 / 127 | 0 / 0 |
|
||||
| C/3 | 68.84 | 57.60 | 212.64 | 73.28 | 292 / 146 | 0 / 0 |
|
||||
| A/3 | 84.37 | 46.89 | 413.17 | 140.02 | 560 / 280 | 0 / 0 |
|
||||
| B/3 | 69.37 | 57.14 | 437.94 | 137.77 | 550 / 275 | 0 / 0 |
|
||||
|
||||
SDK audit: delta warning/error diagnostics в on-окнах: [0.0]; значения перед началом окон: [0.0, 4.0]. Startup/reconfiguration не входит в steady-state замер.
|
||||
Максимум telemetry_sdk_dropped_spans в интервале эксперимента: 0.0. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.
|
||||
|
||||
Отдельный synthetic ERROR probe (вне замеров API): {"type": "synthetic ERROR spans, separate from upload benchmark", "sent": 100, "found_in_tempo": 9, "missing_in_tempo": 91}.
|
||||
|
||||
## Воспроизведение
|
||||
|
||||
```sh
|
||||
python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3
|
||||
```
|
||||
|
||||
Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.
|
||||
Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.
|
||||
В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.
|
||||
Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.
|
||||
Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.
|
||||
Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Фактическая проверка — 16.11.2026
|
||||
|
||||
- Clean Dockerfile build: PASS, runtime loader dependency included.
|
||||
- Compose config and Collector validate: PASS.
|
||||
- Prometheus config + 3 alert rules (promtool): PASS.
|
||||
- C++ W3C context/invalid-context/route-privacy self-test: PASS.
|
||||
- Public lab6 API smoke with telemetry on and off: PASS.
|
||||
- Disabled SDK durable payload: traceparent absent, not an empty invalid field.
|
||||
- Real upload trace parent-child chain API/outbox/Redis/worker/S3/DB: PASS.
|
||||
- Loki events from both services correlated by trace ID: PASS.
|
||||
- Provisioned dashboard: 25 PromQL queries checked.
|
||||
- Privacy canaries absent in recent Loki events and selected upload trace.
|
||||
- Whole delivered incidents.py run against final image: PASS.
|
||||
- All affected services restored; SDK enabled.
|
||||
|
||||
Load: 10338 successes / 0 errors, 344.305 RPS; client p50/p95/p99 5.666/52.505/61.702 ms.
|
||||
Closed-loop synthetic message load, concurrency 4, 30 seconds; not a capacity or 30-minute SLO proof.
|
||||
|
||||
| Событие | Москва, UTC+3 |
|
||||
|---|---|
|
||||
| worker_stopped_upload_accepted | 10:30:35 |
|
||||
| QueueLagGrowing_firing | 10:31:50 |
|
||||
| QueueLagGrowing_resolved_backlog_drained | 10:32:10 |
|
||||
| database_stopped | 10:32:10 |
|
||||
| ApiHighErrorRate_firing | 10:32:54 |
|
||||
| ApiHighErrorRate_resolved_database_recovered | 10:33:55 |
|
||||
| s3_failure_scheduled_retry | 10:33:57 |
|
||||
| retry_same_trace_ready_verified | 10:33:58 |
|
||||
| tempo_loki_stopped | 10:34:08 |
|
||||
| telemetry_outage_api_worker_healthy | 10:34:30 |
|
||||
| tempo_loki_restarted | 10:34:30 |
|
||||
|
||||
Evidence contains only synthetic technical IDs, timestamps and aggregate metrics. Full backend payloads, credentials and message bodies were not saved.
|
||||
Limits: no prolonged memory soak or exhaustive privacy audit; SDK log-queue drops are not counted exactly (see REPORT.md).
|
||||
+30
-4
@@ -1,7 +1,33 @@
|
||||
# Инфраструктура студента
|
||||
# Инфраструктура lab6
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Рабочая топология — в корневом `compose.yaml`. `compose.example.yaml` остаётся
|
||||
учебным worksheet, его запускать не нужно. Bootstrap: `python3 scripts/bootstrap-local.py`,
|
||||
затем `docker compose up -d --build`. Повторный запуск сохраняет volumes и данные.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
| Роль | Compose service | Persistent volume | Host port |
|
||||
|---|---|---|---|
|
||||
| C++ API (2 реплики) | api | — | через lb |
|
||||
| Reverse proxy | lb | — | 127.0.0.1:8080 |
|
||||
| PostgreSQL 16.15 | db | pgdata | нет |
|
||||
| Valkey 8.1.10 / Redis Streams | cache | valkeydata | нет |
|
||||
| MinIO | minio | miniodata | нет |
|
||||
| Bucket / least-privilege user bootstrap | minio-init, minio-user-init | — | нет |
|
||||
| C++ последовательный worker | worker | — | нет |
|
||||
| Collector 0.119.0 | otel-collector | — | нет |
|
||||
| Prometheus 3.2.1 | prometheus | prometheusdata | 127.0.0.1:9090 |
|
||||
| Loki 3.4.2 | loki | lokidata | нет |
|
||||
| Tempo 2.7.2 | tempo | tempodata | нет |
|
||||
| Grafana 11.6.0 | grafana | grafanadata | 127.0.0.1:3000 |
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
API/worker строятся из исходников этого репозитория; MinIO/mc — из закреплённых
|
||||
официальных release binaries с проверкой SHA256. PostgreSQL/Valkey/MinIO имеют
|
||||
healthchecks; API ждёт готовности бизнес-зависимостей и bootstrap S3, lb ждёт API.
|
||||
Telemetry backends не блокируют запуск или работу бизнес-процессов.
|
||||
|
||||
Credentials хранятся в локальных `.env` и `secrets/`, исключённых из Git/context.
|
||||
API/worker читают Docker file secrets после staging для uid 10001; Grafana читает
|
||||
пароль из отдельного file secret. Корневые MinIO credentials нужны только store/bootstrap;
|
||||
приложения используют ограниченного S3 пользователя. Все сервисы находятся в default
|
||||
Compose network. Метрики, логи, трассы и состояние Grafana переживают restart.
|
||||
|
||||
Provisioning, версии, SLO, runbooks и реальные проверки: [REPORT.md](../REPORT.md).
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
{
|
||||
"uid": "messenger-observability",
|
||||
"title": "Messenger — Observability",
|
||||
"schemaVersion": 40,
|
||||
"version": 1,
|
||||
"refresh": "5s",
|
||||
"time": {
|
||||
"from": "now-15m",
|
||||
"to": "now"
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "API RPS (business requests)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(http_route)(rate(http_server_requests_total[$__rate_interval]))",
|
||||
"legendFormat": "{{http_route}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "API p50 / p95 / p99 (all replicas)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by(le)(rate(http_server_request_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95, sum by(le)(rate(http_server_request_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99, sum by(le)(rate(http_server_request_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "API error rate (5xx / all business requests)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(http_server_requests_total{http_response_status_class=\"5xx\"}[$__rate_interval])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[$__rate_interval])),0.001)",
|
||||
"legendFormat": "5xx ratio"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Queue depth (durable outstanding jobs)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_depth)",
|
||||
"legendFormat": "jobs"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "Oldest outstanding job / observation age",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(queue_oldest_age)",
|
||||
"legendFormat": "oldest"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "time()-max(queue_observation_timestamp)",
|
||||
"legendFormat": "observation age"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Worker saturation (busy / capacity)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 16,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(worker_busy) / clamp_min(sum(worker_capacity),1)",
|
||||
"legendFormat": "saturation"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Worker outcomes per second",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by(outcome)(rate(worker_jobs_total[$__rate_interval]))",
|
||||
"legendFormat": "{{outcome}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Worker p50 / p95 / p99",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5,sum by(le)(rate(worker_job_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(worker_job_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(worker_job_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "DB pools (application + outbox)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(db_pool_used)",
|
||||
"legendFormat": "used"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(db_pool_waiting)",
|
||||
"legendFormat": "waiting"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "sum(db_pool_capacity)",
|
||||
"legendFormat": "capacity"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Acceptance to ready p50 / p95 / p99",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5,sum by(le)(rate(attachment_ready_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "histogram_quantile(0.95,sum by(le)(rate(attachment_ready_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "histogram_quantile(0.99,sum by(le)(rate(attachment_ready_duration_seconds_bucket[$__rate_interval])))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Process resident memory",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "process_memory_usage",
|
||||
"legendFormat": "{{service_name}} / {{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Process CPU (cores)",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "rate(process_cpu_time[$__rate_interval])",
|
||||
"legendFormat": "{{service_name}} / {{service_instance_id}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "Collector rejected/failed signal export",
|
||||
"type": "timeseries",
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_spans[$__rate_interval]))",
|
||||
"legendFormat": "exporter_send_failed_spans"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"expr": "sum(rate(otelcol_exporter_send_failed_log_records[$__rate_interval]))",
|
||||
"legendFormat": "exporter_send_failed_log_records"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"expr": "sum(rate(otelcol_processor_refused_spans[$__rate_interval]))",
|
||||
"legendFormat": "processor_refused_spans"
|
||||
},
|
||||
{
|
||||
"refId": "D",
|
||||
"expr": "sum(rate(otelcol_exporter_enqueue_failed_spans[$__rate_interval]))",
|
||||
"legendFormat": "exporter_enqueue_failed_spans"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Correlated application events",
|
||||
"type": "logs",
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
|
||||
"legendFormat": "logs"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {}
|
||||
}
|
||||
],
|
||||
"tags": [
|
||||
"lab6"
|
||||
],
|
||||
"annotations": {
|
||||
"list": []
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
apiVersion: 1
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,33 @@
|
||||
apiVersion: 1
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: TraceID
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
datasourceUid: tempo
|
||||
url: '$${__value.raw}'
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: '-1m'
|
||||
spanEndTimeShift: '1m'
|
||||
tags: [{key: service.name, value: service_name}]
|
||||
filterByTraceID: true
|
||||
filterBySpanID: true
|
||||
@@ -0,0 +1,31 @@
|
||||
auth_enabled: false
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
common:
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2024-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,19 @@
|
||||
# Local MinIO Client (mc) from the official GitHub release binary (pinned).
|
||||
# Fallback when docker.io/minio/mc cannot be pulled.
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG MC_VERSION=RELEASE.2025-04-16T18-13-26Z
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
&& curl -fsSL -o /tmp/mc.bin \
|
||||
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}" \
|
||||
&& curl -fsSL -o /tmp/mc.sha256sum \
|
||||
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}.sha256sum" \
|
||||
&& expected="$(awk '{print $1}' /tmp/mc.sha256sum)" \
|
||||
&& echo "${expected} /tmp/mc.bin" | sha256sum -c - \
|
||||
&& install -m 0755 /tmp/mc.bin /usr/local/bin/mc \
|
||||
&& rm -f /tmp/mc.bin /tmp/mc.sha256sum \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENTRYPOINT ["mc"]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Local MinIO image from the official GitHub release binary (pinned).
|
||||
# Fallback when docker.io/minio/minio cannot be pulled.
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG MINIO_VERSION=RELEASE.2025-04-22T22-12-26Z
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
&& curl -fsSL -o /tmp/minio.bin \
|
||||
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}" \
|
||||
&& curl -fsSL -o /tmp/minio.sha256sum \
|
||||
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}.sha256sum" \
|
||||
&& expected="$(awk '{print $1}' /tmp/minio.sha256sum)" \
|
||||
&& echo "${expected} /tmp/minio.bin" | sha256sum -c - \
|
||||
&& install -m 0755 /tmp/minio.bin /usr/local/bin/minio \
|
||||
&& rm -f /tmp/minio.bin /tmp/minio.sha256sum \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 9000 9001
|
||||
ENTRYPOINT ["minio"]
|
||||
CMD ["server", "/data", "--console-address", ":9001"]
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+36
-4
@@ -1,7 +1,39 @@
|
||||
# Конфигурация наблюдаемости — реализовать
|
||||
# OpenTelemetry Collector
|
||||
|
||||
Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
|
||||
`collector.yaml` рассчитан на Collector Contrib **0.119.0**.
|
||||
|
||||
Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
|
||||
- Receiver: OTLP HTTP/protobuf, `0.0.0.0:4318`, только Compose network.
|
||||
- Все pipelines: `memory_limiter` → `batch`.
|
||||
- Metrics → Prometheus exporter `:9464`; process diagnostics `:8888`.
|
||||
- Traces → Tempo `http://tempo:4318/v1/traces`.
|
||||
- Logs → Loki native OTLP `http://loki:3100/otlp/v1/logs`.
|
||||
- Ограниченные очереди 256 batches, retry не более 15s; Docker limit 256 MiB.
|
||||
- Метрики остановленных SDK instances истекают через 20s, чтобы pool/capacity
|
||||
не считались повторно после смены контейнеров.
|
||||
|
||||
Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
|
||||
API/worker экспортируют в Collector асинхронно и не зависят от его readiness.
|
||||
Payloads не печатаются debug exporter-ом. Подробности и ограничения потерь —
|
||||
в [REPORT.md](../../REPORT.md).
|
||||
|
||||
Проверка:
|
||||
|
||||
```sh
|
||||
docker compose exec -T otel-collector /otelcol-contrib validate --config=/etc/otelcol/config.yaml
|
||||
```
|
||||
|
||||
## Track 1
|
||||
|
||||
`collector-sampling10.yaml` отличается только traces processor:
|
||||
`memory_limiter → probabilistic_sampler/track1 (10%, hash_seed=22) → batch`.
|
||||
Логи и метрики сохраняются без sampling. Переключение:
|
||||
|
||||
```sh
|
||||
OTEL_COLLECTOR_CONFIG=./infra/otel/collector-sampling10.yaml docker compose up -d otel-collector
|
||||
# Вернуть исходную конфигурацию:
|
||||
OTEL_COLLECTOR_CONFIG=./infra/otel/collector.yaml docker compose up -d otel-collector
|
||||
```
|
||||
|
||||
Пересоздание Collector сбрасывает его собственные counters: измерять delta только
|
||||
внутри одного прогона. `otelcol_receiver_accepted_spans` отражает вход до sampling;
|
||||
эффект 10% проверяется также через `otelcol_exporter_sent_spans`.
|
||||
Полный эксперимент и автоматическое восстановление: `scripts/benchmark_telemetry.py`.
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# Track 1: sample traces only; metrics and logs remain at 100%.
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
probabilistic_sampler/track1:
|
||||
hash_seed: 22
|
||||
sampling_percentage: 10
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
metric_expiration: 20s
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 256
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 15s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 256
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 15s
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
address: 0.0.0.0:8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, probabilistic_sampler/track1, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,63 @@
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 192
|
||||
spike_limit_mib: 32
|
||||
batch:
|
||||
timeout: 1s
|
||||
send_batch_size: 256
|
||||
send_batch_max_size: 512
|
||||
exporters:
|
||||
prometheus:
|
||||
endpoint: 0.0.0.0:9464
|
||||
metric_expiration: 20s
|
||||
resource_to_telemetry_conversion:
|
||||
enabled: true
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 256
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 15s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 2s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 256
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 5s
|
||||
max_elapsed_time: 15s
|
||||
service:
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
address: 0.0.0.0:8888
|
||||
pipelines:
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
@@ -0,0 +1,27 @@
|
||||
groups:
|
||||
- name: messenger-slo
|
||||
rules:
|
||||
- alert: ApiHighErrorRate
|
||||
expr: (sum(rate(http_server_requests_total{http_response_status_class="5xx"}[1m])) or vector(0)) / clamp_min(sum(rate(http_server_requests_total[1m])), 0.001) > 0.05
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: API 5xx ratio above 5% for 30 seconds (1 minute window)
|
||||
description: Check server spans, postgresql.pool.acquire and DB pool metrics. Runbook in REPORT.md.
|
||||
- alert: QueueLagGrowing
|
||||
expr: max(queue_oldest_age) > 30 and max(queue_depth) > 0 and max(delta(queue_oldest_age[1m])) > 0
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Durable backlog is ageing beyond 30 seconds
|
||||
description: Check worker capacity/busy, worker logs and Redis/DB/S3 spans. Runbook in REPORT.md.
|
||||
- alert: QueueObservationStale
|
||||
expr: time() - max(queue_observation_timestamp) > 30 or absent(queue_observation_timestamp)
|
||||
for: 30s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Queue observer unavailable
|
||||
description: Queue values may be stale; check PostgreSQL and API observer logs.
|
||||
@@ -0,0 +1,11 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files: [/etc/prometheus/alerts.yaml]
|
||||
scrape_configs:
|
||||
- job_name: messenger
|
||||
static_configs:
|
||||
- targets: [otel-collector:9464]
|
||||
- job_name: collector
|
||||
static_configs:
|
||||
- targets: [otel-collector:8888]
|
||||
@@ -0,0 +1,22 @@
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,44 @@
|
||||
worker_processes ${NGINX_WORKERS};
|
||||
error_log /dev/stderr warn;
|
||||
pid /tmp/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log off;
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 2s;
|
||||
|
||||
upstream api_upstream {
|
||||
zone api_upstream 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=5s;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
# Lab4 attachments ≤10 MiB; allow a little headroom for multipart/headers.
|
||||
client_max_body_size 12m;
|
||||
client_body_timeout 60s;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 10s;
|
||||
proxy_connect_timeout 2s;
|
||||
# Uploads/downloads of ≤10 MiB need longer than JSON chat traffic.
|
||||
proxy_read_timeout 120s;
|
||||
proxy_send_timeout 120s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_keyset_idx
|
||||
ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE TABLE sessions (
|
||||
token text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,22 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
filename text NOT NULL
|
||||
CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL
|
||||
CHECK (size_bytes >= 1 AND size_bytes <= 10485760),
|
||||
sha256 text NOT NULL
|
||||
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
status text NOT NULL
|
||||
CHECK (status IN ('ready')),
|
||||
s3_key text NOT NULL UNIQUE,
|
||||
error_code text NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
|
||||
CREATE INDEX attachments_created_at_idx ON attachments (created_at);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position integer NOT NULL CHECK (position >= 0 AND position < 10),
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx
|
||||
ON message_attachments (attachment_id);
|
||||
|
||||
CREATE UNIQUE INDEX message_attachments_order_idx
|
||||
ON message_attachments (message_id, position);
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Track 1 (grade 5): pending direct/presigned upload sessions.
|
||||
-- Attachment rows stay status=ready only; incomplete uploads live here.
|
||||
CREATE TABLE direct_uploads (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
filename text NOT NULL
|
||||
CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
expected_size integer NOT NULL
|
||||
CHECK (expected_size >= 1 AND expected_size <= 10485760),
|
||||
expected_sha256 text NOT NULL
|
||||
CHECK (expected_sha256 ~ '^[a-f0-9]{64}$'),
|
||||
s3_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL
|
||||
CHECK (status IN ('pending', 'completed', 'aborted', 'expired')),
|
||||
attachment_id uuid NULL REFERENCES attachments(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX direct_uploads_status_expires_idx
|
||||
ON direct_uploads (status, expires_at);
|
||||
CREATE INDEX direct_uploads_owner_idx ON direct_uploads (owner_id);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Track 1: presigned PUT lands in staging/; complete copies to attachments/.
|
||||
-- Existing rows keep the old key as staging so in-flight sessions still resolve.
|
||||
ALTER TABLE direct_uploads ADD COLUMN staging_key text;
|
||||
|
||||
UPDATE direct_uploads
|
||||
SET staging_key = s3_key
|
||||
WHERE staging_key IS NULL;
|
||||
|
||||
ALTER TABLE direct_uploads ALTER COLUMN staging_key SET NOT NULL;
|
||||
|
||||
ALTER TABLE direct_uploads
|
||||
ADD CONSTRAINT direct_uploads_staging_key_key UNIQUE (staging_key);
|
||||
@@ -0,0 +1,21 @@
|
||||
-- Keep existing lab4 originals ready; new jobs set queued explicitly.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'::jsonb
|
||||
CHECK (jsonb_typeof(variants) = 'array');
|
||||
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE
|
||||
REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
status text NOT NULL DEFAULT 'queued'
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
retries integer NOT NULL DEFAULT 0 CHECK (retries >= 0),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX jobs_status_created_at_idx ON jobs (status, created_at);
|
||||
@@ -0,0 +1,20 @@
|
||||
ALTER TABLE jobs ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now();
|
||||
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
payload jsonb NOT NULL CHECK (jsonb_typeof(payload) = 'object'),
|
||||
published boolean NOT NULL DEFAULT false,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
available_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE NOT published;
|
||||
|
||||
-- Upgrade jobs accepted before outbox existed; duplicates are safe.
|
||||
INSERT INTO outbox (payload, available_at)
|
||||
SELECT jsonb_build_object(
|
||||
'schema_version', 1, 'type', 'attachment.process.v1',
|
||||
'job_id', id::text, 'attachment_id', attachment_id::text,
|
||||
'pipeline_version', 1, 'attempt', retries + 1,
|
||||
'created_at', to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')
|
||||
), next_attempt_at FROM jobs WHERE status IN ('queued', 'processing');
|
||||
@@ -0,0 +1,22 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key text,
|
||||
ADD COLUMN staging_cleaned_at timestamptz;
|
||||
|
||||
UPDATE attachments a SET staging_key='staging/' || a.chat_id::text || '/' || a.id::text
|
||||
WHERE EXISTS (SELECT 1 FROM jobs j WHERE j.attachment_id=a.id);
|
||||
|
||||
CREATE INDEX attachments_staging_cleanup_idx ON attachments(status, updated_at)
|
||||
WHERE staging_key IS NOT NULL AND staging_cleaned_at IS NULL;
|
||||
|
||||
ALTER TABLE jobs ADD COLUMN retry_base integer NOT NULL DEFAULT 0
|
||||
CHECK (retry_base >= 0 AND retry_base <= retries);
|
||||
|
||||
CREATE TABLE job_replays (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs(id),
|
||||
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100),
|
||||
reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500),
|
||||
previous_error_code text,
|
||||
retries_before integer NOT NULL CHECK (retries_before >= 0),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,41 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
|
||||
ADD COLUMN published_version integer CHECK (published_version > 0),
|
||||
ADD CONSTRAINT attachments_version_order CHECK (published_version <= pipeline_version);
|
||||
UPDATE attachments SET published_version=1 WHERE status='ready';
|
||||
|
||||
ALTER TABLE jobs
|
||||
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
|
||||
ADD COLUMN source_key text;
|
||||
UPDATE jobs j SET source_key=a.s3_key FROM attachments a WHERE a.id=j.attachment_id;
|
||||
ALTER TABLE jobs ALTER COLUMN source_key SET NOT NULL;
|
||||
ALTER TABLE jobs DROP CONSTRAINT jobs_attachment_id_key;
|
||||
ALTER TABLE jobs ADD CONSTRAINT jobs_attachment_version_key UNIQUE (attachment_id, pipeline_version);
|
||||
ALTER TABLE jobs DROP CONSTRAINT jobs_status_check;
|
||||
ALTER TABLE jobs ADD CONSTRAINT jobs_status_check
|
||||
CHECK (status IN ('queued','processing','ready','failed','superseded'));
|
||||
|
||||
-- One immutable logical result per (attachment, pipeline version).
|
||||
CREATE TABLE attachment_results (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
job_id uuid UNIQUE REFERENCES jobs(id),
|
||||
original_key text NOT NULL,
|
||||
variants jsonb NOT NULL CHECK (jsonb_typeof(variants)='array'),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version)
|
||||
);
|
||||
INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants)
|
||||
SELECT a.id,1,j.id,a.s3_key,a.variants FROM attachments a
|
||||
LEFT JOIN jobs j ON j.attachment_id=a.id AND j.pipeline_version=1 WHERE a.status='ready';
|
||||
|
||||
CREATE TABLE pipeline_requests (
|
||||
id bigserial PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
|
||||
job_id uuid NOT NULL UNIQUE REFERENCES jobs(id),
|
||||
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100),
|
||||
reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE(attachment_id,pipeline_version)
|
||||
);
|
||||
@@ -0,0 +1,461 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 1: fixed-count attachment benchmark; stdlib only, no credentials in evidence.
|
||||
|
||||
Creates synthetic durable data. Never deletes volumes or changes business code.
|
||||
Always restores SDK on / Collector 100% when the experiment finishes or fails.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import signal
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
VARIANTS = {'A': ('Off', True, 'collector.yaml'),
|
||||
'B': ('On, SDK 100% / Collector 100%', False, 'collector.yaml'),
|
||||
'C': ('On, SDK 100% / Collector 10%', False, 'collector-sampling10.yaml')}
|
||||
|
||||
|
||||
def percentile(values, quantile):
|
||||
return sorted(values)[max(0, math.ceil(len(values) * quantile) - 1)]
|
||||
|
||||
|
||||
def memory_mib(text):
|
||||
number, unit = re.fullmatch(r'([\d.]+)\s*([kKMGT]?i?B)', text.strip()).groups()
|
||||
factors = {'B': 1, 'kB': 1000, 'MB': 1000**2, 'GB': 1000**3, 'TB': 1000**4,
|
||||
'KiB': 1024, 'MiB': 1024**2, 'GiB': 1024**3, 'TiB': 1024**4}
|
||||
return float(number) * factors[unit] / 1024**2
|
||||
|
||||
|
||||
def parse_metrics(text):
|
||||
result = {}
|
||||
for line in text.splitlines():
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
name = line.split('{', 1)[0].split(' ', 1)[0]
|
||||
result[name] = result.get(name, 0) + float(line.rsplit(' ', 1)[1])
|
||||
return result
|
||||
|
||||
|
||||
class Bench:
|
||||
def __init__(self, args):
|
||||
self.args = args
|
||||
self.compose = shlex.split(args.compose)
|
||||
self.docker = shlex.split(args.docker)
|
||||
self.output = ROOT / args.output
|
||||
self.output.mkdir(parents=True, exist_ok=True)
|
||||
self.content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
self.report = {'started_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'method': {'requests': args.requests, 'concurrency': args.concurrency,
|
||||
'repetitions': args.repetitions, 'warmup': args.warmup,
|
||||
'fixture_bytes': len(self.content),
|
||||
'fixture_sha256': hashlib.sha256(self.content).hexdigest(),
|
||||
'stats_source': 'docker stats stream; 100% CPU = one logical core',
|
||||
'cpu_ram_window': 'uploads plus complete worker drain, excluding final export wait',
|
||||
'percentile': 'nearest rank; HTTP upload acceptance, client wall clock',
|
||||
'config_sha256': {name: hashlib.sha256((ROOT/'infra/otel'/name).read_bytes()).hexdigest() for name in ('collector.yaml', 'collector-sampling10.yaml')},
|
||||
'order': 'ABC / BCA / CAB (rotated each repetition)'},
|
||||
'host': {'machine': platform.machine(), 'logical_cpus': os.cpu_count(),
|
||||
'cpu_model': next((line.split(':',1)[1].strip() for line in Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'unknown'),
|
||||
'memory_kib': int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')))},
|
||||
'trials': []}
|
||||
|
||||
def run(self, cmd, *, env=None, data=None):
|
||||
completed = subprocess.run(cmd, cwd=ROOT, env=env, input=data, text=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if completed.returncode:
|
||||
# Subprocess output can contain connection credentials; do not forward it.
|
||||
raise RuntimeError('Command failed: ' + Path(cmd[0]).name + ' ' + cmd[1] +
|
||||
' (exit ' + str(completed.returncode) + ')')
|
||||
return completed.stdout
|
||||
|
||||
def compose_run(self, *args, **kwargs):
|
||||
return self.run(self.compose + list(args), **kwargs)
|
||||
|
||||
def sql(self, sql):
|
||||
return self.compose_run('exec', '-T', 'db', 'sh', '-c',
|
||||
'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tA', data=sql+'\n').strip()
|
||||
|
||||
def wait_jobs(self, chat=None, expected=None):
|
||||
deadline = time.monotonic() + self.args.drain_timeout
|
||||
while time.monotonic() < deadline:
|
||||
if chat:
|
||||
uuid.UUID(chat) # Only synthetic UUIDs may enter the query.
|
||||
row = self.sql("SELECT count(*),count(*) FILTER (WHERE j.status='ready'),"
|
||||
"count(*) FILTER (WHERE j.status='failed') FROM jobs j JOIN attachments a "
|
||||
"ON a.id=j.attachment_id WHERE a.chat_id='" + chat + "'::uuid;")
|
||||
total, ready, failed = map(int, row.split('|'))
|
||||
if failed:
|
||||
raise RuntimeError('Worker produced failed benchmark jobs')
|
||||
if total == expected and ready == expected:
|
||||
return
|
||||
elif int(self.sql("SELECT count(*) FROM jobs WHERE status IN ('queued','processing');")) == 0:
|
||||
return
|
||||
time.sleep(1)
|
||||
raise RuntimeError('Worker backlog did not drain; evidence is incomplete')
|
||||
|
||||
def configure(self, variant):
|
||||
_, disabled, filename = VARIANTS[variant]
|
||||
env = dict(os.environ, OTEL_SDK_DISABLED=str(disabled).lower(),
|
||||
OTEL_COLLECTOR_CONFIG=str(ROOT / 'infra/otel' / filename))
|
||||
self.compose_run('config', '--quiet', env=env)
|
||||
self.compose_run('up', '-d', '--no-deps', '--force-recreate', '--wait',
|
||||
'--wait-timeout', '120', 'otel-collector', 'api', 'worker', env=env)
|
||||
self.compose_run('exec', '-T', 'otel-collector', '/otelcol-contrib', 'validate',
|
||||
'--config=/etc/otelcol/config.yaml', env=env)
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
Client(self.args.base_url).call('GET', '/health/ready', schema='Health')
|
||||
self.collector()
|
||||
return
|
||||
except (Failure, RuntimeError):
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('API/Collector did not become ready') from None
|
||||
time.sleep(1)
|
||||
|
||||
def collector(self):
|
||||
return parse_metrics(self.compose_run('exec', '-T', '--index', '1', 'api',
|
||||
'curl', '-fsS', '--max-time', '5', 'http://otel-collector:8888/metrics'))
|
||||
|
||||
def upload(self, count, actor, chat):
|
||||
barrier = threading.Barrier(self.args.concurrency)
|
||||
def work(slot):
|
||||
client = Client(self.args.base_url, timeout=60)
|
||||
results = []
|
||||
barrier.wait()
|
||||
for index in range(slot, count, self.args.concurrency):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
# No supplied traceparent: the SDK's AlwaysOn root policy is exercised.
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
||||
expected=202, actor=actor, raw=self.content,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
results.append((time.monotonic()-started, True))
|
||||
except Failure:
|
||||
results.append((time.monotonic()-started, False))
|
||||
return results
|
||||
start = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=self.args.concurrency) as pool:
|
||||
rows = [r for batch in pool.map(work, range(self.args.concurrency)) for r in batch]
|
||||
return rows, time.monotonic()-start
|
||||
|
||||
def sample_stats(self, ids, stop, rows, errors):
|
||||
process = subprocess.Popen(self.docker + ['stats', '--format', '{{json .}}'] + list(ids),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
|
||||
def terminate():
|
||||
stop.wait()
|
||||
process.terminate()
|
||||
closer = threading.Thread(target=terminate, daemon=True)
|
||||
closer.start()
|
||||
try:
|
||||
for line in process.stdout:
|
||||
clean = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', line).strip()
|
||||
if not clean:
|
||||
continue
|
||||
if stop.is_set() and not clean.endswith('}'):
|
||||
continue
|
||||
obj = json.loads(clean)
|
||||
ident = obj['ID']
|
||||
if ident not in ids:
|
||||
ident = next((key for key in ids if key.startswith(ident)), '')
|
||||
if ident:
|
||||
rows.append({'t': time.monotonic(), 'container': ident, 'service': ids[ident],
|
||||
'cpu_percent': float(obj['CPUPerc'].rstrip('%')),
|
||||
'ram_mib': memory_mib(obj['MemUsage'].split('/')[0])})
|
||||
if not stop.is_set():
|
||||
errors.append('docker stats stopped unexpectedly')
|
||||
except Exception as error:
|
||||
errors.append(type(error).__name__ + ': ' + str(error))
|
||||
finally:
|
||||
stop.set()
|
||||
process.terminate()
|
||||
process.wait(timeout=10)
|
||||
closer.join(timeout=10)
|
||||
|
||||
def trial(self, variant, repetition):
|
||||
print(f'Preparing {variant}, repetition {repetition}', flush=True)
|
||||
self.wait_jobs()
|
||||
self.configure(variant)
|
||||
client = Client(self.args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
warm, _ = self.upload(self.args.warmup, alice, chat)
|
||||
if not all(ok for _, ok in warm):
|
||||
raise RuntimeError('Warm-up HTTP errors')
|
||||
self.wait_jobs(chat['id'], self.args.warmup)
|
||||
time.sleep(8) # SDK reader/processor + Collector batch settle before baseline.
|
||||
ids = {}
|
||||
for service in ('api', 'worker'):
|
||||
for ident in self.compose_run('ps', '-q', service).split():
|
||||
ids[ident] = service
|
||||
self.report['container_limits'] = {service: self.run(self.docker + ['inspect', '--format', '{{.Config.Image}}|{{.Image}}|{{.HostConfig.NanoCpus}}|{{.HostConfig.Memory}}', ident]).strip() for ident, service in ids.items()}
|
||||
if sum(s == 'api' for s in ids.values()) != 2 or sum(s == 'worker' for s in ids.values()) != 1:
|
||||
raise RuntimeError('Benchmark requires two API replicas and one worker')
|
||||
rows, errors, stop = [], [], threading.Event()
|
||||
sampler = threading.Thread(target=self.sample_stats, args=(ids, stop, rows, errors))
|
||||
sampler.start()
|
||||
try:
|
||||
time.sleep(3)
|
||||
before = self.collector()
|
||||
start = time.monotonic()
|
||||
measurements, http_seconds = self.upload(self.args.requests, alice, chat)
|
||||
if any(not ok for _, ok in measurements):
|
||||
raise RuntimeError('HTTP upload errors invalidate the comparison')
|
||||
self.wait_jobs(chat['id'], self.args.warmup+self.args.requests)
|
||||
end = time.monotonic()
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join(timeout=15)
|
||||
if sampler.is_alive() or errors:
|
||||
raise RuntimeError('docker stats sampling failed: ' + ', '.join(errors))
|
||||
# Export is asynchronous. Wait for SDK batches and for backend queues to empty.
|
||||
time.sleep(3)
|
||||
after = self.collector()
|
||||
for _ in range(10):
|
||||
if after.get('otelcol_exporter_queue_size', 0) == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
after = self.collector()
|
||||
metric_seconds = time.monotonic()-start
|
||||
def delta(name):
|
||||
return after.get(name, 0)-before.get(name, 0)
|
||||
samples = [r for r in rows if start <= r['t'] <= end]
|
||||
resource = {}
|
||||
for service in ('api', 'worker'):
|
||||
per_container = []
|
||||
for ident, kind in ids.items():
|
||||
if kind != service:
|
||||
continue
|
||||
series = [r for r in samples if r['container'] == ident]
|
||||
if len(series) < 3:
|
||||
raise RuntimeError('Too few docker stats samples; use more requests')
|
||||
per_container.append({'cpu_percent_mean': statistics.mean(r['cpu_percent'] for r in series),
|
||||
'ram_mib_mean': statistics.mean(r['ram_mib'] for r in series),
|
||||
'ram_mib_max': max(r['ram_mib'] for r in series), 'samples': len(series)})
|
||||
resource[service] = {key: sum(r[key] for r in per_container)
|
||||
for key in ('cpu_percent_mean', 'ram_mib_mean', 'ram_mib_max', 'samples')}
|
||||
resource[service]['cpu_seconds_per_job_estimate'] = resource[service]['cpu_percent_mean']/100*(end-start)/self.args.requests
|
||||
good = [seconds*1000 for seconds, ok in measurements if ok]
|
||||
failures = sum(not ok for _, ok in measurements)
|
||||
counters = {name: delta(name) for name in (
|
||||
'otelcol_receiver_accepted_spans', 'otelcol_exporter_sent_spans',
|
||||
'otelcol_receiver_refused_spans', 'otelcol_exporter_send_failed_spans',
|
||||
'otelcol_receiver_accepted_log_records', 'otelcol_receiver_accepted_metric_points')}
|
||||
if failures or not good or counters['otelcol_receiver_refused_spans'] or counters['otelcol_exporter_send_failed_spans']:
|
||||
raise RuntimeError('HTTP or Collector failures invalidate the comparison')
|
||||
if variant == 'A' and counters['otelcol_receiver_accepted_spans'] != 0:
|
||||
raise RuntimeError('Telemetry Off still exported spans')
|
||||
retention = counters['otelcol_exporter_sent_spans']/max(counters['otelcol_receiver_accepted_spans'], 1)
|
||||
if variant == 'B' and not 0.95 <= retention <= 1.05:
|
||||
raise RuntimeError('100% trace export did not match receiver ingest')
|
||||
if variant == 'C' and not 0.04 <= retention <= 0.18:
|
||||
raise RuntimeError('Collector sampling did not reduce export to approximately 10%')
|
||||
result = {'variant': variant, 'repetition': repetition, 'completed_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'successful_uploads': len(good), 'http_errors': failures, 'http_seconds': http_seconds,
|
||||
'pipeline_seconds': end-start, 'collector_window_seconds': metric_seconds,
|
||||
'rps': len(good)/http_seconds, 'latency_mean_ms': statistics.mean(good),
|
||||
'latency_p50_ms': percentile(good, .5), 'latency_p99_ms': percentile(good, .99),
|
||||
'latencies_ms': good,
|
||||
'resources': resource, 'collector_deltas': counters,
|
||||
'ingest_spans_per_second': counters['otelcol_receiver_accepted_spans']/metric_seconds,
|
||||
'tempo_spans_per_second': counters['otelcol_exporter_sent_spans']/metric_seconds,
|
||||
'trace_retention': retention, 'stats_samples': [{**r, 't': r['t']-start} for r in samples]}
|
||||
self.report['trials'].append(result)
|
||||
self.save()
|
||||
print(f'{variant}/{repetition}: {result["rps"]:.2f} RPS, p99 {result["latency_p99_ms"]:.2f} ms, '
|
||||
f'accepted {counters["otelcol_receiver_accepted_spans"]:.0f}, exported {counters["otelcol_exporter_sent_spans"]:.0f}', flush=True)
|
||||
|
||||
def save(self):
|
||||
(self.output / 'track1-benchmark.json').write_text(json.dumps(self.report, indent=2)+'\n')
|
||||
|
||||
def write_summary(self):
|
||||
def med(variant, getter):
|
||||
return statistics.median(getter(t) for t in self.report['trials'] if t['variant'] == variant)
|
||||
lines = ['# Track 1: Цена наблюдаемости', '',
|
||||
'Результат: **' + self.report.get('result', 'INCOMPLETE') + '**. Начало UTC: ' + self.report['started_utc'] + '.', '',
|
||||
f"По {self.args.repetitions} прогона каждого варианта, {self.args.requests} POST uploads, concurrency {self.args.concurrency}, warm-up {self.args.warmup} uploads.",
|
||||
'Порядок ABC / BCA / CAB. PNG: ' + str(len(self.content)) + ' bytes. Каждый job дошёл до ready; очередь между прогонами пуста.',
|
||||
'Таблица содержит медианы результатов прогонов, включая медиану индивидуальных p50/p99 (не pooled percentile).',
|
||||
'RPS и latency — HTTP 202 acceptance. CPU/RAM — средние docker stats за upload + полный drain worker; API = сумма двух реплик.',
|
||||
'CPU: 100% = одно логическое ядро. RAM: MiB; это Docker working set, не peak RSS. Spans/s = delta accepted_spans / окно ingest, включающее финальный flush.', '',
|
||||
'| Настройка | RPS | p50 latency, ms | p99 latency, ms | CPU API | RAM API, MiB | CPU Worker | RAM Worker, MiB | Spans/sec (ingest) |',
|
||||
'|---|---:|---:|---:|---:|---:|---:|---:|---:|']
|
||||
for v in 'ABC':
|
||||
values = [med(v, lambda t: t[k]) for k in ('rps', 'latency_p50_ms', 'latency_p99_ms')]
|
||||
api = [med(v, lambda t: t['resources']['api'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
|
||||
worker = [med(v, lambda t: t['resources']['worker'][k]) for k in ('cpu_percent_mean', 'ram_mib_mean')]
|
||||
lines.append(f'| {v}: {VARIANTS[v][0]} | {values[0]:.2f} | {values[1]:.2f} | {values[2]:.2f} | {api[0]:.2f}% | {api[1]:.2f} | {worker[0]:.2f}% | {worker[1]:.2f} | {med(v, lambda t:t["ingest_spans_per_second"]):.2f} |')
|
||||
lines += ['', '| Настройка | Mean HTTP, ms | Pipeline, s | Accepted spans | Tempo sent spans | Tempo spans/s | CPU API, ms/job* | CPU Worker, ms/job* |',
|
||||
'|---|---:|---:|---:|---:|---:|---:|---:|']
|
||||
for v in 'ABC':
|
||||
lines.append(f'| {v} | {med(v, lambda t:t["latency_mean_ms"]):.2f} | {med(v, lambda t:t["pipeline_seconds"]):.2f} | {med(v, lambda t:t["collector_deltas"]["otelcol_receiver_accepted_spans"]):.0f} | {med(v, lambda t:t["collector_deltas"]["otelcol_exporter_sent_spans"]):.0f} | {med(v, lambda t:t["tempo_spans_per_second"]):.2f} | {1000*med(v, lambda t:t["resources"]["api"]["cpu_seconds_per_job_estimate"]):.2f} | {1000*med(v, lambda t:t["resources"]["worker"]["cpu_seconds_per_job_estimate"]):.2f} |')
|
||||
lines += ['', '*CPU ms/job — оценка по среднему docker stats CPU% и длительности pipeline, не аппаратный профиль CPU.',
|
||||
'Accepted spans — вход в Collector; sampling в C выполняется после receiver и не уменьшает стоимость SDK/OTLP ingest. Это количества spans, не bytes.',
|
||||
'В C логи и метрики остаются на 100%; отношение sent/accepted измеряет намеренное сокращение trace export.', '',
|
||||
'| Прогон | RPS | mean HTTP, ms | p99, ms | Pipeline, s | Stats samples API / Worker | Refused / failed export spans |',
|
||||
'|---|---:|---:|---:|---:|---:|---:|']
|
||||
for t in self.report['trials']:
|
||||
lines.append(f'| {t["variant"]}/{t["repetition"]} | {t["rps"]:.2f} | {t["latency_mean_ms"]:.2f} | {t["latency_p99_ms"]:.2f} | {t["pipeline_seconds"]:.2f} | {t["resources"]["api"]["samples"]} / {t["resources"]["worker"]["samples"]} | {t["collector_deltas"]["otelcol_receiver_refused_spans"]:.0f} / {t["collector_deltas"]["otelcol_exporter_send_failed_spans"]:.0f} |')
|
||||
if all('sdk_diagnostic_audit' in t for t in self.report['trials']):
|
||||
starts = sorted({t['sdk_diagnostic_audit']['warning_error_start'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_start'] is not None})
|
||||
deltas = sorted({t['sdk_diagnostic_audit']['warning_error_delta'] for t in self.report['trials'] if t['sdk_diagnostic_audit']['warning_error_delta'] is not None})
|
||||
drops = self.report.get('sdk_diagnostic_audit', {}).get('span_queue_drop_max_during_experiment')
|
||||
lines += ['', f'SDK audit: delta warning/error diagnostics в on-окнах: {deltas}; значения перед началом окон: {starts}. Startup/reconfiguration не входит в steady-state замер.',
|
||||
f'Максимум telemetry_sdk_dropped_spans в интервале эксперимента: {drops}. Это счётчик переполнения span batch queue, не точный аудит всех возможных потерь SDK.']
|
||||
probe = self.report.get('error_probe', {})
|
||||
lines += ['', 'Отдельный synthetic ERROR probe (вне замеров API): ' + json.dumps(probe, ensure_ascii=False) + '.', '',
|
||||
'## Воспроизведение', '', '```sh',
|
||||
'python3 scripts/benchmark_telemetry.py --requests 1000 --concurrency 4 --repetitions 3', '```', '',
|
||||
'Если Docker Snap не находит Compose plugin, передать --compose /path/to/docker-compose --docker /path/to/docker.',
|
||||
'Скрипт проверяет 2 API + 1 worker, завершение всех jobs, отсутствие HTTP/Collector ошибок и фактический процент экспорта.',
|
||||
'В finally восстанавливаются SDK on и исходный Collector 100%. Данные/volumes не удаляются.', '',
|
||||
'## Ограничения', '',
|
||||
'Closed-loop, один хост, 3 повтора, фоновые health/outbox spans включены. Размер БД/S3 растёт; порядок вариантов ротируется, новых чатов и файлов поровну.',
|
||||
'Warm-up не устраняет весь cache/thermal/I/O шум. Относительные CPU/RAM различия — наблюдения этой конфигурации, не универсальная надбавка 20–40%.',
|
||||
'Docker CPU/RAM и клиентская latency не изолируют стоимость SDK от общего давления Collector/backends на тот же хост.',
|
||||
'Приватные payloads, пароли и токены в evidence не записываются. Полные sanitized samples/latencies и hardware/limits: track1-benchmark.json.']
|
||||
(self.output/'track1-benchmark.md').write_text('\n'.join(lines)+'\n')
|
||||
|
||||
def audit_sdk(self):
|
||||
# Read the same externally scraped diagnostics for each trial's own SDK instances.
|
||||
def query(expr, timestamp):
|
||||
url = self.args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr, 'time': timestamp})
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
rows = json.load(response)['data']['result']
|
||||
return float(rows[0]['value'][1]) if rows else None
|
||||
end = 0
|
||||
for trial in self.report['trials']:
|
||||
ids = '|'.join(sorted({s['container'][:12] for s in trial['stats_samples']}))
|
||||
end = datetime.fromisoformat(trial['completed_utc']).timestamp()
|
||||
start = end-trial['collector_window_seconds']
|
||||
expr = 'sum(telemetry_sdk_errors{service_instance_id=~"' + ids + '"})'
|
||||
a, b = query(expr, start), query(expr, end)
|
||||
trial['sdk_diagnostic_audit'] = {'source': 'Prometheus instant queries at trial start/end, restricted to trial container hostnames',
|
||||
'warning_error_start': a, 'warning_error_end': b,
|
||||
'warning_error_delta': b-a if a is not None and b is not None else None}
|
||||
seconds = math.ceil(end-datetime.fromisoformat(self.report['started_utc']).timestamp())
|
||||
self.report['sdk_diagnostic_audit'] = {
|
||||
'span_queue_drop_max_during_experiment': query(f'max(max_over_time(telemetry_sdk_dropped_spans[{seconds}s]))', end),
|
||||
'note': 'Warning/error diagnostics are not counts of lost spans. Compare per-trial deltas; sampling discard is sent/accepted.'}
|
||||
self.save()
|
||||
if any(t['sdk_diagnostic_audit']['warning_error_delta'] not in (None, 0) for t in self.report['trials']):
|
||||
raise RuntimeError('SDK diagnostics increased during measured windows')
|
||||
|
||||
def probe_errors(self):
|
||||
# Separate from measured trials: synthetic ERROR spans verify that probability sampling is not error-aware.
|
||||
self.configure('C')
|
||||
traces = [uuid.uuid4().hex for _ in range(100)]
|
||||
now = time.time_ns()
|
||||
spans = [{'traceId': ident, 'spanId': '0123456789abcdef', 'name': 'track1.synthetic-error-probe',
|
||||
'kind': 2, 'startTimeUnixNano': str(now), 'endTimeUnixNano': str(now+1000000),
|
||||
'status': {'code': 2}, 'attributes': [{'key': 'http.response.status_code', 'value': {'intValue': '503'}}]}
|
||||
for ident in traces]
|
||||
data = json.dumps({'resourceSpans': [{'resource': {'attributes': [{'key': 'service.name',
|
||||
'value': {'stringValue': 'track1-sampling-probe'}}]}, 'scopeSpans': [{'spans': spans}]}]})
|
||||
self.compose_run('exec', '-T', '--index', '1', 'api', 'curl', '-fsS', '--max-time', '10',
|
||||
'-H', 'Content-Type: application/json', '--data-binary', '@-',
|
||||
'http://otel-collector:4318/v1/traces', data=data)
|
||||
time.sleep(5)
|
||||
auth = base64.b64encode((os.environ.get('GRAFANA_ADMIN_USER', 'admin')+':'+(ROOT/'secrets/grafana_admin_password').read_text().strip()).encode()).decode()
|
||||
def retained(ident):
|
||||
request = urllib.request.Request(self.args.grafana_url+'/api/datasources/proxy/uid/tempo/api/traces/'+ident,
|
||||
headers={'Authorization': 'Basic '+auth})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=15) as response:
|
||||
return response.status == 200
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code == 404:
|
||||
return False
|
||||
raise RuntimeError('Tempo probe query failed') from None
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
found = sum(pool.map(retained, traces))
|
||||
if not 0 < found < len(traces):
|
||||
raise RuntimeError('Error sampling probe failed')
|
||||
self.report['error_probe'] = {'type': 'synthetic ERROR spans, separate from upload benchmark',
|
||||
'sent': len(traces), 'found_in_tempo': found, 'missing_in_tempo': len(traces)-found}
|
||||
self.save()
|
||||
print(f'Error probe: {found}/100 ERROR traces retained', flush=True)
|
||||
|
||||
|
||||
def self_test():
|
||||
assert re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', '\x1b[?25l\x1b[H{}\x1b[?25h') == '{}'
|
||||
assert percentile([1, 2, 3, 4], .5) == 2
|
||||
assert percentile(list(range(1, 101)), .99) == 99
|
||||
assert memory_mib('1024KiB') == 1
|
||||
assert memory_mib('1GiB') == 1024
|
||||
assert abs(memory_mib('1048576B')-1) < 1e-9
|
||||
assert parse_metrics('# comment\nx{a="b"} 2\nx{a="c"} 3\ny 4\n') == {'x': 5, 'y': 4}
|
||||
base = (ROOT/'infra/otel/collector.yaml').read_text()
|
||||
sampled = (ROOT/'infra/otel/collector-sampling10.yaml').read_text()
|
||||
expected = base.replace(' batch:\n', ' probabilistic_sampler/track1:\n hash_seed: 22\n sampling_percentage: 10\n batch:\n', 1)
|
||||
expected = expected.replace('processors: [memory_limiter, batch]\n exporters: [otlphttp/tempo]',
|
||||
'processors: [memory_limiter, probabilistic_sampler/track1, batch]\n exporters: [otlphttp/tempo]')
|
||||
assert sampled.split('\n', 1)[1] == expected, 'Sampling must be the only Collector configuration difference'
|
||||
assert 'sampling_percentage: 10' in sampled
|
||||
assert sampled.count('processors: [memory_limiter, probabilistic_sampler/track1, batch]') == 1
|
||||
assert sampled.count('processors: [memory_limiter, batch]') == base.count('processors: [memory_limiter, batch]')-1
|
||||
print('PASS: percentile, docker memory units, metric sums, trace-only sampler configuration')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--requests', type=int, default=1000)
|
||||
parser.add_argument('--concurrency', type=int, default=4)
|
||||
parser.add_argument('--repetitions', type=int, default=3)
|
||||
parser.add_argument('--warmup', type=int, default=20)
|
||||
parser.add_argument('--drain-timeout', type=int, default=600)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||||
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
||||
parser.add_argument('--compose', default=os.environ.get('COMPOSE', 'docker compose'))
|
||||
parser.add_argument('--docker', default=os.environ.get('DOCKER', 'docker'))
|
||||
parser.add_argument('--output', default='evidence')
|
||||
parser.add_argument('--self-test', action='store_true')
|
||||
args = parser.parse_args()
|
||||
if args.self_test:
|
||||
return self_test()
|
||||
if not (args.requests >= 100 and 1 <= args.concurrency <= 16 and 1 <= args.repetitions <= 10 and args.warmup >= args.concurrency):
|
||||
parser.error('requests >=100, concurrency 1..16, repetitions 1..10, warmup >= concurrency required')
|
||||
def interrupted(signum, frame):
|
||||
raise KeyboardInterrupt('Benchmark interrupted; restoring baseline')
|
||||
signal.signal(signal.SIGTERM, interrupted)
|
||||
bench = Bench(args)
|
||||
try:
|
||||
for repetition in range(args.repetitions):
|
||||
order = 'ABC'[repetition % 3:] + 'ABC'[:repetition % 3]
|
||||
for variant in order:
|
||||
bench.trial(variant, repetition+1)
|
||||
bench.audit_sdk()
|
||||
bench.probe_errors()
|
||||
bench.report['result'] = 'PASS'
|
||||
finally:
|
||||
print('Restoring SDK on / Collector 100%', flush=True)
|
||||
bench.configure('B')
|
||||
bench.report['restored'] = 'SDK enabled, default Collector configuration'
|
||||
bench.save()
|
||||
bench.write_summary()
|
||||
print('PASS: all trials complete; raw sanitized evidence saved', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate isolated local lab credentials without printing secret values."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.umask(0o077)
|
||||
if (root / '.env').exists() or (root / 'secrets').exists():
|
||||
raise SystemExit('Existing .env/secrets preserved; use the existing credentials.')
|
||||
password, minio_root, s3, grafana = [secrets.token_hex(24) for _ in range(4)]
|
||||
text = (root / '.env.example').read_text()
|
||||
for key, value in {'POSTGRES_PASSWORD': password, 'MINIO_ROOT_PASSWORD': minio_root,
|
||||
'MINIO_USER_SECRET_KEY': s3}.items():
|
||||
import re
|
||||
text = re.sub(r'^' + key + r'=.*$', key + '=' + value, text, flags=re.M)
|
||||
(root / '.env').write_text(text)
|
||||
(root / 'secrets').mkdir(mode=0o700)
|
||||
for name, value in {'database_url': f'postgresql://messenger:{password}@db:5432/messenger',
|
||||
'cache_url': 'redis://cache:6379', 's3_access_key_id': 'messenger-s3',
|
||||
's3_secret_access_key': s3, 'grafana_admin_password': grafana}.items():
|
||||
(root / 'secrets' / name).write_text(value + '\n')
|
||||
# Parent directory stays 0700; bind-mounted file must be readable by Grafana uid 472.
|
||||
(root / 'secrets' / 'grafana_admin_password').chmod(0o444)
|
||||
print('Created .env and secrets/. Grafana user: admin; password: secrets/grafana_admin_password')
|
||||
@@ -0,0 +1,94 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "attachment_queue.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <memory>
|
||||
|
||||
void AttachmentQueue::publish(const std::string& payload) {
|
||||
auto lease = pool_.acquire();
|
||||
std::unique_ptr<redisReply, decltype(&freeReplyObject)> reply(
|
||||
static_cast<redisReply*>(redisCommand(
|
||||
lease.ctx(), "XADD jobs:attachments * payload %b",
|
||||
payload.data(), payload.size())), &freeReplyObject);
|
||||
if (!reply || reply->type != REDIS_REPLY_STRING || reply->len == 0) {
|
||||
throw DependencyUnavailable("attachment queue publish failed");
|
||||
}
|
||||
}
|
||||
|
||||
namespace {
|
||||
using Reply = std::unique_ptr<redisReply, decltype(&freeReplyObject)>;
|
||||
Reply checked(redisReply* reply) {
|
||||
Reply owned(reply, &freeReplyObject);
|
||||
if (!owned || owned->type == REDIS_REPLY_ERROR) {
|
||||
throw DependencyUnavailable("attachment stream command failed");
|
||||
}
|
||||
return owned;
|
||||
}
|
||||
JobDelivery delivery(const redisReply* entry) {
|
||||
if (entry->type != REDIS_REPLY_ARRAY || entry->elements != 2 ||
|
||||
entry->element[0]->type != REDIS_REPLY_STRING ||
|
||||
entry->element[1]->type != REDIS_REPLY_ARRAY) {
|
||||
throw DependencyUnavailable("invalid stream response");
|
||||
}
|
||||
JobDelivery result{std::string(entry->element[0]->str, entry->element[0]->len), {}};
|
||||
const auto* fields = entry->element[1];
|
||||
for (std::size_t i = 0; i + 1 < fields->elements; i += 2) {
|
||||
if (fields->element[i]->type == REDIS_REPLY_STRING &&
|
||||
std::string(fields->element[i]->str, fields->element[i]->len) == "payload" &&
|
||||
fields->element[i + 1]->type == REDIS_REPLY_STRING && fields->element[i + 1]->len <= 4096) {
|
||||
result.payload.assign(fields->element[i + 1]->str, fields->element[i + 1]->len);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void AttachmentQueue::ensure_group() {
|
||||
auto lease = pool_.acquire();
|
||||
Reply reply(static_cast<redisReply*>(redisCommand(lease.ctx(),
|
||||
"XGROUP CREATE jobs:attachments attachment-workers 0 MKSTREAM")), &freeReplyObject);
|
||||
if (!reply || (reply->type == REDIS_REPLY_ERROR &&
|
||||
std::string(reply->str, reply->len).rfind("BUSYGROUP", 0) != 0)) {
|
||||
throw DependencyUnavailable("attachment consumer group unavailable");
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<JobDelivery> AttachmentQueue::read(const std::string& consumer, int reclaim_idle_ms) {
|
||||
auto lease = pool_.acquire();
|
||||
auto pending = checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
|
||||
"XAUTOCLAIM jobs:attachments attachment-workers %s %d %s COUNT 1",
|
||||
consumer.c_str(), reclaim_idle_ms, claim_cursor_.c_str())));
|
||||
if (pending->type != REDIS_REPLY_ARRAY || pending->elements < 2 ||
|
||||
pending->element[0]->type != REDIS_REPLY_STRING ||
|
||||
pending->element[1]->type != REDIS_REPLY_ARRAY) {
|
||||
throw DependencyUnavailable("invalid pending response");
|
||||
}
|
||||
claim_cursor_.assign(pending->element[0]->str, pending->element[0]->len);
|
||||
if (pending->element[1]->elements) return delivery(pending->element[1]->element[0]);
|
||||
auto fresh = checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
|
||||
"XREADGROUP GROUP attachment-workers %s COUNT 1 BLOCK 1000 STREAMS jobs:attachments >",
|
||||
consumer.c_str())));
|
||||
if (fresh->type == REDIS_REPLY_NIL) return std::nullopt;
|
||||
if (fresh->type != REDIS_REPLY_ARRAY || fresh->elements != 1 ||
|
||||
fresh->element[0]->type != REDIS_REPLY_ARRAY || fresh->element[0]->elements != 2 ||
|
||||
fresh->element[0]->element[1]->type != REDIS_REPLY_ARRAY ||
|
||||
fresh->element[0]->element[1]->elements != 1) {
|
||||
throw DependencyUnavailable("invalid stream read response");
|
||||
}
|
||||
return delivery(fresh->element[0]->element[1]->element[0]);
|
||||
}
|
||||
|
||||
void AttachmentQueue::ack(const std::string& stream_id) {
|
||||
auto lease = pool_.acquire();
|
||||
checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
|
||||
"XACK jobs:attachments attachment-workers %s", stream_id.c_str())));
|
||||
}
|
||||
|
||||
void AttachmentQueue::reject(const std::string& stream_id) {
|
||||
{
|
||||
auto lease = pool_.acquire();
|
||||
checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
|
||||
"XADD jobs:attachments:invalid * stream_id %s error_code invalid_job", stream_id.c_str())));
|
||||
}
|
||||
ack(stream_id);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include "redis_pool.hpp"
|
||||
|
||||
#include <string>
|
||||
#include <optional>
|
||||
|
||||
struct JobDelivery {
|
||||
std::string stream_id;
|
||||
std::string payload;
|
||||
};
|
||||
|
||||
class AttachmentQueue {
|
||||
public:
|
||||
explicit AttachmentQueue(RedisPool& pool) : pool_(pool) {}
|
||||
|
||||
void publish(const std::string& payload);
|
||||
void ensure_group();
|
||||
std::optional<JobDelivery> read(const std::string& consumer, int reclaim_idle_ms);
|
||||
void ack(const std::string& stream_id);
|
||||
void reject(const std::string& stream_id);
|
||||
bool ping() { return pool_.ping(); }
|
||||
|
||||
private:
|
||||
RedisPool& pool_;
|
||||
std::string claim_cursor_ = "0-0";
|
||||
};
|
||||
@@ -0,0 +1,156 @@
|
||||
#include "attachment_repo.hpp"
|
||||
#include "outbox.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch())
|
||||
.count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
Attachment row_to_attachment(const pqxx::row& r) {
|
||||
Attachment a;
|
||||
a.id = r["id"].as<std::string>();
|
||||
a.chat_id = r["chat_id"].as<std::string>();
|
||||
a.owner_id = r["owner_id"].as<std::string>();
|
||||
a.filename = r["filename"].as<std::string>();
|
||||
a.content_type = r["content_type"].as<std::string>();
|
||||
a.size_bytes = r["size_bytes"].as<int>();
|
||||
a.sha256 = r["sha256"].as<std::string>();
|
||||
a.status = r["status"].as<std::string>();
|
||||
a.s3_key = r["s3_key"].as<std::string>();
|
||||
if (r["error_code"].is_null()) {
|
||||
a.error_code = std::nullopt;
|
||||
} else {
|
||||
a.error_code = r["error_code"].as<std::string>();
|
||||
}
|
||||
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
|
||||
for (const auto& v : nlohmann::json::parse(r["variants"].as<std::string>())) {
|
||||
a.variants.push_back(Variant{v.at("name").get<std::string>(),
|
||||
v.at("content_type").get<std::string>(), v.at("size_bytes").get<int>(),
|
||||
v.at("sha256").get<std::string>(), v.at("width").get<int>(),
|
||||
v.at("height").get<int>(), v.value("storage_key", "")});
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
constexpr const char* kAttachmentColumns =
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, s3_key, error_code, variants, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
|
||||
|
||||
} // namespace
|
||||
|
||||
Attachment AttachmentRepo::insert(const Attachment& in) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
pqxx::result r;
|
||||
if (in.error_code) {
|
||||
r = tx.exec(
|
||||
"INSERT INTO attachments ("
|
||||
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
" status, s3_key, error_code, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, "
|
||||
" to_timestamp($11::bigint), to_timestamp($12::bigint)"
|
||||
") RETURNING " +
|
||||
std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.status,
|
||||
in.s3_key, *in.error_code, ts, ts});
|
||||
} else {
|
||||
r = tx.exec(
|
||||
"INSERT INTO attachments ("
|
||||
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
" status, s3_key, error_code, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
|
||||
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
|
||||
") RETURNING " +
|
||||
std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.status,
|
||||
in.s3_key, ts, ts});
|
||||
}
|
||||
tx.commit();
|
||||
return row_to_attachment(r[0]);
|
||||
}
|
||||
|
||||
std::optional<Attachment> AttachmentRepo::find_by_id(const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kAttachmentColumns) +
|
||||
" FROM attachments WHERE id = $1::uuid",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return row_to_attachment(r[0]);
|
||||
}
|
||||
|
||||
Attachment AttachmentRepo::insert_queued(const Attachment& in,
|
||||
const std::string& job_id,
|
||||
const std::function<void()>& save_original) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
tx.exec("SELECT pg_advisory_xact_lock(hashtextextended('staging:' || $1::uuid::text, 0))",
|
||||
pqxx::params{in.id});
|
||||
save_original();
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO attachments ("
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, s3_key, staging_key, created_at, updated_at) VALUES ("
|
||||
"$1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, 'queued', $8, $8, now(), now()) "
|
||||
"RETURNING " + std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.s3_key});
|
||||
tx.exec("INSERT INTO jobs (id, attachment_id, source_key) VALUES ($1::uuid, $2::uuid, $3)",
|
||||
pqxx::params{job_id, in.id, in.s3_key});
|
||||
enqueue_attachment_job(tx, job_id);
|
||||
Attachment saved = row_to_attachment(r[0]);
|
||||
tx.commit();
|
||||
return saved;
|
||||
}
|
||||
|
||||
bool AttachmentRepo::all_belong_to_chat(const std::string& chat_id,
|
||||
const std::vector<std::string>& ids) {
|
||||
if (ids.empty()) {
|
||||
return true;
|
||||
}
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
for (const auto& id : ids) {
|
||||
auto r = tx.exec(
|
||||
"SELECT 1 FROM attachments "
|
||||
"WHERE id = $1::uuid AND chat_id = $2::uuid AND status <> 'failed'",
|
||||
pqxx::params{id, chat_id});
|
||||
if (r.empty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class AttachmentRepo {
|
||||
public:
|
||||
explicit AttachmentRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
Attachment insert(const Attachment& a);
|
||||
Attachment insert_queued(const Attachment& a, const std::string& job_id,
|
||||
const std::function<void()>& save_original);
|
||||
|
||||
std::optional<Attachment> find_by_id(const std::string& id);
|
||||
|
||||
// All IDs must exist, belong to chat_id, and not have status=failed.
|
||||
// Returns false if any id is missing/foreign/failed.
|
||||
bool all_belong_to_chat(const std::string& chat_id,
|
||||
const std::vector<std::string>& ids);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
inline std::string attachment_variant_key(const std::string& chat_id,
|
||||
const std::string& id,
|
||||
const std::string& variant) {
|
||||
const char* suffix = variant == "thumbnail" ? "_256.jpg" :
|
||||
variant == "crop" ? "_128.jpg" :
|
||||
variant == "optimized" ? "_optimized.webp" : nullptr;
|
||||
if (!suffix) throw std::invalid_argument("unknown variant");
|
||||
return "attachments/" + chat_id + "/" + id + suffix;
|
||||
}
|
||||
|
||||
inline std::string pipeline_prefix(const std::string& chat_id, const std::string& id,
|
||||
int version) {
|
||||
if (version < 1) throw std::invalid_argument("invalid pipeline version");
|
||||
return "attachments/" + chat_id + "/" + id + "/v" + std::to_string(version) + "/";
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "auth.hpp"
|
||||
|
||||
#include <sodium.h>
|
||||
#include <stdexcept>
|
||||
|
||||
|
||||
namespace {
|
||||
struct SodiumInit {
|
||||
SodiumInit() {
|
||||
if (sodium_init() < 0) {
|
||||
throw std::runtime_error("sodium_init failed");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const SodiumInit g_sodium_init;
|
||||
}
|
||||
|
||||
|
||||
std::string hash_password(const std::string& plain) {
|
||||
char hashed[crypto_pwhash_STRBYTES];
|
||||
if (crypto_pwhash_str(
|
||||
hashed,
|
||||
plain.data(), plain.size(),
|
||||
crypto_pwhash_OPSLIMIT_INTERACTIVE,
|
||||
crypto_pwhash_MEMLIMIT_INTERACTIVE) != 0) {
|
||||
throw std::runtime_error("crypto_pwhash_str failed (out of memory?)");
|
||||
}
|
||||
return std::string(hashed);
|
||||
}
|
||||
|
||||
bool verify_password(const std::string& plain, const std::string& hash) {
|
||||
if (hash.empty() || hash.size() >= crypto_pwhash_STRBYTES) {
|
||||
return false;
|
||||
}
|
||||
return crypto_pwhash_str_verify(
|
||||
hash.c_str(),
|
||||
plain.data(), plain.size()) == 0;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
std::string hash_password(const std::string& plain);
|
||||
|
||||
bool verify_password(const std::string& plain, const std::string& hash);
|
||||
@@ -0,0 +1,186 @@
|
||||
#include "chat_repo.hpp"
|
||||
#include "cursor.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string make_uuid() {
|
||||
static std::mt19937_64 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
|
||||
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
|
||||
std::ostringstream out;
|
||||
out << std::hex << std::setfill('0');
|
||||
out << std::setw(8) << d32(rng);
|
||||
out << '-' << std::setw(4) << d16(rng);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
|
||||
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch()).count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
Chat row_to_chat(const pqxx::row& r) {
|
||||
Chat c;
|
||||
c.id = r["id"].as<std::string>();
|
||||
c.title = r["title"].as<std::string>();
|
||||
c.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
return c;
|
||||
}
|
||||
|
||||
std::vector<std::string> load_member_ids(pqxx::work& tx,
|
||||
const std::string& chat_id) {
|
||||
std::vector<std::string> ids;
|
||||
auto r = tx.exec(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id",
|
||||
pqxx::params{chat_id});
|
||||
ids.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
ids.push_back(row["user_id"].as<std::string>());
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
constexpr const char* kChatSelect =
|
||||
"id, title, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
|
||||
|
||||
}
|
||||
|
||||
std::optional<Chat> ChatRepo::create(const std::string& title,
|
||||
const std::vector<std::string>& member_ids) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
const std::string chat_id = make_uuid();
|
||||
const std::string created = now_epoch_str();
|
||||
|
||||
try {
|
||||
tx.exec(
|
||||
"INSERT INTO chats (id, title, created_at) "
|
||||
"VALUES ($1, $2, to_timestamp($3::bigint))",
|
||||
pqxx::params{chat_id, title, created});
|
||||
|
||||
for (const auto& uid : member_ids) {
|
||||
tx.exec(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
|
||||
pqxx::params{chat_id, uid});
|
||||
}
|
||||
} catch (const pqxx::foreign_key_violation&) {
|
||||
tx.abort();
|
||||
return std::nullopt;
|
||||
} catch (const pqxx::unique_violation&) {
|
||||
tx.abort();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
|
||||
pqxx::params{chat_id});
|
||||
Chat c = row_to_chat(r[0]);
|
||||
c.member_ids = load_member_ids(tx, chat_id);
|
||||
tx.commit();
|
||||
return c;
|
||||
}
|
||||
|
||||
std::optional<Chat> ChatRepo::find_by_id(const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
Chat c = row_to_chat(r[0]);
|
||||
c.member_ids = load_member_ids(tx, id);
|
||||
tx.commit();
|
||||
return c;
|
||||
}
|
||||
|
||||
|
||||
std::optional<ChatPage> ChatRepo::list_for_user(
|
||||
const std::string& user_id,
|
||||
int limit,
|
||||
const std::optional<std::string>& cursor) {
|
||||
|
||||
std::optional<Cursor> cur;
|
||||
if (cursor) {
|
||||
cur = parse_cursor(*cursor);
|
||||
if (!cur) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
if (cur) {
|
||||
auto owner_check = tx.exec(
|
||||
"SELECT 1 FROM chats WHERE id = $1 AND id IN "
|
||||
"(SELECT chat_id FROM chat_members WHERE user_id = $2)",
|
||||
pqxx::params{cur->id, user_id});
|
||||
if (owner_check.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::string sql =
|
||||
"SELECT " + std::string(kChatSelect) + ", " + kCreatedUsSql + " "
|
||||
"FROM chats "
|
||||
"WHERE id IN (SELECT chat_id FROM chat_members WHERE user_id = $1) ";
|
||||
if (cur) {
|
||||
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
|
||||
}
|
||||
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
|
||||
|
||||
pqxx::result r;
|
||||
if (cur) {
|
||||
r = tx.exec(sql, pqxx::params{user_id, limit + 1, cur->created_us, cur->id});
|
||||
} else {
|
||||
r = tx.exec(sql, pqxx::params{user_id, limit + 1});
|
||||
}
|
||||
|
||||
ChatPage page;
|
||||
const bool has_more = (static_cast<int>(r.size()) > limit);
|
||||
const int take = has_more ? limit : static_cast<int>(r.size());
|
||||
page.items.reserve(take);
|
||||
for (int i = 0; i < take; ++i) {
|
||||
Chat c = row_to_chat(r[static_cast<std::size_t>(i)]);
|
||||
c.member_ids = load_member_ids(tx, c.id);
|
||||
page.items.push_back(std::move(c));
|
||||
}
|
||||
|
||||
if (has_more && take > 0) {
|
||||
const auto& last = r[static_cast<std::size_t>(take - 1)];
|
||||
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
|
||||
last["id"].as<std::string>());
|
||||
}
|
||||
|
||||
tx.commit();
|
||||
return page;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class ChatRepo {
|
||||
public:
|
||||
explicit ChatRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
std::optional<Chat> create(const std::string& title,
|
||||
const std::vector<std::string>& member_ids);
|
||||
|
||||
std::optional<Chat> find_by_id(const std::string& id);
|
||||
|
||||
std::optional<ChatPage> list_for_user(const std::string& user_id, int limit, const std::optional<std::string>& cursor);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <optional>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
|
||||
// Непрозрачный для клиента курсор keyset-пагинации: "<created_at в мкс от epoch>|<uuid>".
|
||||
// Микросекунды сохраняют точный created_at строки, даже если в JSON время округлено до секунд.
|
||||
struct Cursor {
|
||||
std::string created_us;
|
||||
std::string id;
|
||||
};
|
||||
|
||||
// Выражение SQL, восстанавливающее timestamptz из created_us без потери точности.
|
||||
constexpr const char* kCursorTimestampSql =
|
||||
"(timestamptz 'epoch' + $3::bigint * interval '1 microsecond')";
|
||||
|
||||
constexpr const char* kCreatedUsSql =
|
||||
"(EXTRACT(EPOCH FROM created_at) * 1000000)::bigint AS created_us";
|
||||
|
||||
inline bool is_uuid(const std::string& s) {
|
||||
static const std::regex re(
|
||||
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
|
||||
"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
||||
return std::regex_match(s, re);
|
||||
}
|
||||
|
||||
inline std::optional<Cursor> parse_cursor(const std::string& raw) {
|
||||
const auto bar = raw.find('|');
|
||||
if (bar == std::string::npos) {
|
||||
return std::nullopt;
|
||||
}
|
||||
Cursor c{raw.substr(0, bar), raw.substr(bar + 1)};
|
||||
const bool digits = !c.created_us.empty() && c.created_us.size() <= 18 &&
|
||||
std::all_of(c.created_us.begin(), c.created_us.end(),
|
||||
[](unsigned char ch) { return std::isdigit(ch); });
|
||||
if (!digits || !is_uuid(c.id)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
inline std::string make_cursor(const std::string& created_us, const std::string& id) {
|
||||
return created_us + "|" + id;
|
||||
}
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "db.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <future>
|
||||
#include <thread>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/time.h>
|
||||
#include <stdexcept>
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <iostream>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int kConnectTimeoutSeconds = 2;
|
||||
constexpr int kTcpUserTimeoutMs = 5000;
|
||||
constexpr int kStatementTimeoutMs = 5000;
|
||||
constexpr int kClientIoTimeoutSeconds = 5;
|
||||
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
|
||||
|
||||
std::string add_conn_param(std::string conninfo, const std::string& key,
|
||||
const std::string& value) {
|
||||
if (conninfo.find(key + "=") != std::string::npos) {
|
||||
return conninfo;
|
||||
}
|
||||
const bool is_url = conninfo.rfind("postgres://", 0) == 0 ||
|
||||
conninfo.rfind("postgresql://", 0) == 0;
|
||||
if (is_url) {
|
||||
conninfo += (conninfo.find('?') == std::string::npos ? "?" : "&");
|
||||
} else {
|
||||
conninfo += " ";
|
||||
}
|
||||
return conninfo + key + "=" + value;
|
||||
}
|
||||
|
||||
bool is_alive(pqxx::connection* conn) {
|
||||
std::promise<void> done;
|
||||
auto fut = done.get_future();
|
||||
std::thread th([conn, &done]() {
|
||||
try {
|
||||
pqxx::nontransaction tx(*conn);
|
||||
tx.exec("SELECT 1");
|
||||
done.set_value();
|
||||
} catch (...) {
|
||||
done.set_exception(std::current_exception());
|
||||
}
|
||||
});
|
||||
if (fut.wait_for(std::chrono::seconds(kClientIoTimeoutSeconds)) !=
|
||||
std::future_status::ready) {
|
||||
const int fd = conn->sock();
|
||||
if (fd >= 0) {
|
||||
::shutdown(fd, SHUT_RDWR);
|
||||
}
|
||||
th.join();
|
||||
return false;
|
||||
}
|
||||
th.join();
|
||||
try {
|
||||
fut.get();
|
||||
return true;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
struct Migration {
|
||||
int version;
|
||||
std::string path;
|
||||
};
|
||||
|
||||
constexpr int kMigrationsAdvisoryLockKey = 0x6d747573;
|
||||
|
||||
|
||||
Migration parse_migration_name(const fs::path& p) {
|
||||
const std::string name = p.filename().string();
|
||||
const auto dot = name.find('.');
|
||||
|
||||
if (dot == std::string::npos || name.substr(dot) != ".sql") {
|
||||
throw std::runtime_error("migration file must end with .sql: " + name);
|
||||
}
|
||||
|
||||
const auto us = name.find('_');
|
||||
if (us == std::string::npos || us == 0) {
|
||||
throw std::runtime_error(
|
||||
"migration file must be named NNN_name.sql: " + name);
|
||||
}
|
||||
|
||||
const std::string num = name.substr(0, us);
|
||||
if (!std::all_of(num.begin(), num.end(),
|
||||
[](unsigned char c){ return std::isdigit(c); })) {
|
||||
throw std::runtime_error(
|
||||
"migration file prefix must be digits: " + name);
|
||||
}
|
||||
|
||||
Migration m;
|
||||
m.version = std::stoi(num);
|
||||
m.path = p.string();
|
||||
return m;
|
||||
}}
|
||||
|
||||
|
||||
void apply_migrations(pqxx::connection& conn, const std::string& dir) {
|
||||
{
|
||||
pqxx::work tx(conn);
|
||||
tx.exec("SELECT pg_advisory_lock($1)",
|
||||
pqxx::params{kMigrationsAdvisoryLockKey});
|
||||
tx.commit();
|
||||
}
|
||||
std::clog << "migration advisory lock acquired\n";
|
||||
struct Unlock {
|
||||
pqxx::connection& c;
|
||||
~Unlock() {
|
||||
try {
|
||||
pqxx::work tx(c);
|
||||
tx.exec("SELECT pg_advisory_unlock($1)",
|
||||
pqxx::params{kMigrationsAdvisoryLockKey});
|
||||
tx.commit();
|
||||
} catch (...) {}
|
||||
}
|
||||
} unlock_guard{conn};
|
||||
|
||||
{
|
||||
pqxx::work tx(conn);
|
||||
tx.exec(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version int PRIMARY KEY,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now()"
|
||||
")");
|
||||
tx.commit();
|
||||
}
|
||||
|
||||
if (!fs::exists(dir) || !fs::is_directory(dir)) {
|
||||
throw std::runtime_error("migrations dir not found: " + dir);
|
||||
}
|
||||
std::vector<Migration> files;
|
||||
for (const auto& entry : fs::directory_iterator(dir)) {
|
||||
if (entry.is_regular_file()) {
|
||||
files.push_back(parse_migration_name(entry.path()));
|
||||
}
|
||||
}
|
||||
std::sort(files.begin(), files.end(),
|
||||
[](const Migration& a, const Migration& b){
|
||||
return a.version < b.version;
|
||||
});
|
||||
|
||||
for (std::size_t i = 1; i < files.size(); ++i) {
|
||||
if (files[i].version == files[i-1].version) {
|
||||
throw std::runtime_error(
|
||||
"duplicate migration version: " + std::to_string(files[i].version));
|
||||
}
|
||||
}
|
||||
|
||||
std::set<int> applied;
|
||||
{
|
||||
pqxx::work tx(conn);
|
||||
for (const auto& row : tx.exec("SELECT version FROM schema_migrations")) {
|
||||
applied.insert(row[0].as<int>());
|
||||
}
|
||||
tx.commit();
|
||||
}
|
||||
|
||||
for (const auto& m : files) {
|
||||
if (applied.count(m.version)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::ifstream f(m.path, std::ios::binary);
|
||||
if (!f) {
|
||||
throw std::runtime_error("cannot open migration: " + m.path);
|
||||
}
|
||||
std::stringstream buf;
|
||||
buf << f.rdbuf();
|
||||
const std::string sql = buf.str();
|
||||
|
||||
pqxx::work tx(conn);
|
||||
tx.exec(sql);
|
||||
tx.exec("INSERT INTO schema_migrations(version) VALUES ($1)",
|
||||
m.version);
|
||||
tx.commit();
|
||||
|
||||
std::clog << "applied migration " << m.version
|
||||
<< " (" << m.path << ")\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::string read_secret_file(const char* env_name) {
|
||||
const char* path = std::getenv(env_name);
|
||||
|
||||
if (path == nullptr || *path == '\0') {
|
||||
throw std::runtime_error(
|
||||
std::string("environment variable not set: ") + env_name);
|
||||
}
|
||||
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f) {
|
||||
throw std::runtime_error(
|
||||
std::string("cannot open secret file: ") + path);
|
||||
}
|
||||
|
||||
std::string s;
|
||||
std::getline(f, s);
|
||||
if (!s.empty() && s.back() == '\r') {
|
||||
s.pop_back();
|
||||
}
|
||||
|
||||
if (s.empty()) {
|
||||
throw std::runtime_error(
|
||||
std::string("secret file is empty: ") + path);
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
bool is_dependency_failure(const std::exception& e) {
|
||||
if (dynamic_cast<const DependencyUnavailable*>(&e) != nullptr ||
|
||||
dynamic_cast<const pqxx::broken_connection*>(&e) != nullptr ||
|
||||
dynamic_cast<const pqxx::in_doubt_error*>(&e) != nullptr) {
|
||||
return true;
|
||||
}
|
||||
if (const auto* sql = dynamic_cast<const pqxx::sql_error*>(&e)) {
|
||||
// 08xxx — ошибки соединения, 57xxx — отмена (statement_timeout) и остановка сервера.
|
||||
const std::string& state = sql->sqlstate();
|
||||
return state.rfind("08", 0) == 0 || state.rfind("57", 0) == 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
|
||||
bool statement_timeout) {
|
||||
std::string full = add_conn_param(conninfo, "connect_timeout",
|
||||
std::to_string(kConnectTimeoutSeconds));
|
||||
full = add_conn_param(full, "tcp_user_timeout", std::to_string(kTcpUserTimeoutMs));
|
||||
auto conn = std::make_unique<pqxx::connection>(full);
|
||||
const int fd = conn->sock();
|
||||
if (fd >= 0) {
|
||||
timeval tv{};
|
||||
tv.tv_sec = kClientIoTimeoutSeconds;
|
||||
tv.tv_usec = 0;
|
||||
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
|
||||
::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
|
||||
}
|
||||
if (statement_timeout) {
|
||||
pqxx::nontransaction tx(*conn);
|
||||
tx.exec("SET statement_timeout = " + std::to_string(kStatementTimeoutMs));
|
||||
}
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
||||
PgPool::PgPool(std::string conninfo, std::size_t n)
|
||||
: conninfo_(std::move(conninfo)) {
|
||||
if (n == 0) {
|
||||
throw std::runtime_error("PgPool: n must be > 0");
|
||||
}
|
||||
all_.reserve(n);
|
||||
idle_.reserve(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
auto conn = open_pg_connection(conninfo_, true);
|
||||
idle_.push_back(conn.get());
|
||||
all_.push_back(std::move(conn));
|
||||
}
|
||||
telemetry::delta("db.pool.capacity", n);
|
||||
}
|
||||
|
||||
|
||||
PgPool::~PgPool() {
|
||||
telemetry::delta("db.pool.capacity", -static_cast<long>(all_.size()));
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
shutting_down_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
|
||||
pqxx::connection* PgPool::take() {
|
||||
telemetry::Span acquire_span("postgresql.pool.acquire", opentelemetry::trace::SpanKind::kClient);
|
||||
struct Waiting {
|
||||
Waiting() { telemetry::delta("db.pool.waiting", 1); }
|
||||
~Waiting() { telemetry::delta("db.pool.waiting", -1); }
|
||||
} waiting;
|
||||
std::unique_lock<std::mutex> lk(mu_);
|
||||
if (!cv_.wait_for(lk, kAcquireTimeout,
|
||||
[&]() { return shutting_down_ || !idle_.empty(); })) {
|
||||
throw DependencyUnavailable("postgres pool: acquire timeout");
|
||||
}
|
||||
if (shutting_down_) {
|
||||
throw DependencyUnavailable("postgres pool is shutting down");
|
||||
}
|
||||
pqxx::connection* conn = idle_.back();
|
||||
idle_.pop_back();
|
||||
telemetry::delta("db.pool.used", 1);
|
||||
lk.unlock();
|
||||
|
||||
if (is_alive(conn)) {
|
||||
return conn;
|
||||
}
|
||||
|
||||
std::unique_ptr<pqxx::connection> fresh;
|
||||
try {
|
||||
fresh = open_pg_connection(conninfo_, true);
|
||||
} catch (const std::exception& e) {
|
||||
// Мёртвое соединение возвращается в пул: следующий take() снова
|
||||
// попробует переподключиться, и пул не теряет слот.
|
||||
give_back(conn);
|
||||
throw DependencyUnavailable(std::string("postgres reconnect failed: ") + e.what());
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lk2(mu_);
|
||||
for (auto& up : all_) {
|
||||
if (up.get() == conn) {
|
||||
up = std::move(fresh);
|
||||
return up.get();
|
||||
}
|
||||
}
|
||||
throw std::logic_error("PgPool: connection not owned by pool");
|
||||
}
|
||||
|
||||
|
||||
void PgPool::give_back(pqxx::connection* conn) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
idle_.push_back(conn);
|
||||
telemetry::delta("db.pool.used", -1);
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
|
||||
PgPool::Lease::Lease(PgPool& pool) : pool_(&pool), conn_(pool.take()) {}
|
||||
|
||||
PgPool::Lease::~Lease() {
|
||||
if (pool_ != nullptr && conn_ != nullptr) {
|
||||
pool_->give_back(conn_);
|
||||
}
|
||||
}
|
||||
|
||||
PgPool::Lease::Lease(Lease&& other) noexcept
|
||||
: pool_(other.pool_), conn_(other.conn_) {
|
||||
other.pool_ = nullptr;
|
||||
other.conn_ = nullptr;
|
||||
}
|
||||
|
||||
PgPool::Lease PgPool::acquire() {
|
||||
return Lease(*this);
|
||||
}
|
||||
|
||||
bool PgPool::ping() {
|
||||
try {
|
||||
auto lease = acquire();
|
||||
pqxx::work tx(*lease);
|
||||
tx.exec("SELECT 1");
|
||||
tx.commit();
|
||||
return true;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
#pragma once
|
||||
#include <condition_variable>
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
std::string read_secret_file(const char* env_name);
|
||||
|
||||
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
|
||||
bool statement_timeout);
|
||||
|
||||
|
||||
void apply_migrations(pqxx::connection& conn, const std::string& dir);
|
||||
|
||||
|
||||
class PgPool {
|
||||
public:
|
||||
bool ping();
|
||||
PgPool(std::string conninfo, std::size_t n);
|
||||
~PgPool();
|
||||
|
||||
PgPool(const PgPool&) = delete;
|
||||
PgPool& operator=(const PgPool&) = delete;
|
||||
|
||||
class Lease {
|
||||
public:
|
||||
explicit Lease(PgPool& pool);
|
||||
~Lease();
|
||||
Lease(Lease&& other) noexcept;
|
||||
Lease& operator=(Lease&&) = delete;
|
||||
Lease(const Lease&) = delete;
|
||||
Lease& operator=(const Lease&) = delete;
|
||||
|
||||
pqxx::connection& operator*() const { return *conn_; }
|
||||
pqxx::connection* operator->() const { return conn_; }
|
||||
|
||||
private:
|
||||
PgPool* pool_;
|
||||
pqxx::connection* conn_;
|
||||
};
|
||||
|
||||
Lease acquire();
|
||||
|
||||
private:
|
||||
friend class Lease;
|
||||
|
||||
pqxx::connection* take();
|
||||
void give_back(pqxx::connection* conn);
|
||||
|
||||
std::string conninfo_;
|
||||
std::vector<std::unique_ptr<pqxx::connection>> all_;
|
||||
std::vector<pqxx::connection*> idle_;
|
||||
std::mutex mu_;
|
||||
std::condition_variable cv_;
|
||||
bool shutting_down_ = false;
|
||||
};
|
||||
@@ -0,0 +1,292 @@
|
||||
#include "direct_upload_repo.hpp"
|
||||
#include "outbox.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch())
|
||||
.count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
DirectUpload row_to(const pqxx::row& r) {
|
||||
DirectUpload d;
|
||||
d.id = r["id"].as<std::string>();
|
||||
d.chat_id = r["chat_id"].as<std::string>();
|
||||
d.owner_id = r["owner_id"].as<std::string>();
|
||||
d.filename = r["filename"].as<std::string>();
|
||||
d.content_type = r["content_type"].as<std::string>();
|
||||
d.expected_size = r["expected_size"].as<int>();
|
||||
d.expected_sha256 = r["expected_sha256"].as<std::string>();
|
||||
d.s3_key = r["s3_key"].as<std::string>();
|
||||
d.staging_key = r["staging_key"].as<std::string>();
|
||||
d.status = r["status"].as<std::string>();
|
||||
d.overdue = r["overdue"].as<bool>();
|
||||
if (r["attachment_id"].is_null()) {
|
||||
d.attachment_id = std::nullopt;
|
||||
} else {
|
||||
d.attachment_id = r["attachment_id"].as<std::string>();
|
||||
}
|
||||
d.expires_at = epoch_to_rfc3339(r["expires_at_epoch"].as<long long>());
|
||||
d.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
d.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
|
||||
return d;
|
||||
}
|
||||
|
||||
constexpr const char* kCols =
|
||||
"id, chat_id, owner_id, filename, content_type, expected_size, "
|
||||
"expected_sha256, s3_key, staging_key, status, attachment_id, "
|
||||
"(status = 'pending' AND expires_at < now()) AS overdue, "
|
||||
"EXTRACT(EPOCH FROM expires_at)::bigint AS expires_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
|
||||
|
||||
constexpr const char* kAttachmentColumns =
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, s3_key, error_code, variants, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
|
||||
|
||||
Attachment attachment_from_row(const pqxx::row& r) {
|
||||
Attachment a;
|
||||
a.id = r["id"].as<std::string>();
|
||||
a.chat_id = r["chat_id"].as<std::string>();
|
||||
a.owner_id = r["owner_id"].as<std::string>();
|
||||
a.filename = r["filename"].as<std::string>();
|
||||
a.content_type = r["content_type"].as<std::string>();
|
||||
a.size_bytes = r["size_bytes"].as<int>();
|
||||
a.sha256 = r["sha256"].as<std::string>();
|
||||
a.status = r["status"].as<std::string>();
|
||||
a.s3_key = r["s3_key"].as<std::string>();
|
||||
if (r["error_code"].is_null()) {
|
||||
a.error_code = std::nullopt;
|
||||
} else {
|
||||
a.error_code = r["error_code"].as<std::string>();
|
||||
}
|
||||
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
|
||||
for (const auto& v : nlohmann::json::parse(r["variants"].as<std::string>())) {
|
||||
a.variants.push_back(Variant{v.at("name").get<std::string>(),
|
||||
v.at("content_type").get<std::string>(), v.at("size_bytes").get<int>(),
|
||||
v.at("sha256").get<std::string>(), v.at("width").get<int>(),
|
||||
v.at("height").get<int>(), v.value("storage_key", "")});
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DirectUpload DirectUploadRepo::insert(const DirectUpload& in) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
// expires_at is already an absolute epoch string from the caller.
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO direct_uploads ("
|
||||
" id, chat_id, owner_id, filename, content_type, expected_size, "
|
||||
" expected_sha256, s3_key, staging_key, status, attachment_id, "
|
||||
" expires_at, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, NULL, "
|
||||
" to_timestamp($11::bigint), to_timestamp($12::bigint), "
|
||||
" to_timestamp($13::bigint)"
|
||||
") RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.expected_size, in.expected_sha256,
|
||||
in.s3_key, in.staging_key, in.status, in.expires_at_epoch,
|
||||
ts, ts});
|
||||
tx.commit();
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> DirectUploadRepo::find_by_id(
|
||||
const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kCols) +
|
||||
" FROM direct_uploads WHERE id = $1::uuid",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> DirectUploadRepo::mark_completed(
|
||||
const std::string& id, const std::string& attachment_id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"UPDATE direct_uploads SET status = 'completed', "
|
||||
" attachment_id = $2::uuid, "
|
||||
" updated_at = to_timestamp($3::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending' "
|
||||
"RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{id, attachment_id, ts});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
tx.commit();
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> DirectUploadRepo::mark_aborted(
|
||||
const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"UPDATE direct_uploads SET status = 'aborted', "
|
||||
" updated_at = to_timestamp($2::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending' "
|
||||
"RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{id, ts});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
tx.commit();
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::vector<DirectUpload> DirectUploadRepo::expire_stale() {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"UPDATE direct_uploads SET status = 'expired', "
|
||||
" updated_at = to_timestamp($1::bigint) "
|
||||
"WHERE status = 'pending' AND expires_at < now() "
|
||||
"RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{ts});
|
||||
tx.commit();
|
||||
std::vector<DirectUpload> out;
|
||||
out.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
out.push_back(row_to(row));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<std::string> DirectUploadRepo::pending_s3_keys() {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT staging_key FROM direct_uploads WHERE status = 'pending'");
|
||||
std::vector<std::string> out;
|
||||
out.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
out.push_back(row["staging_key"].as<std::string>());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void DirectUploadRepo::with_lock(const std::function<void(UploadLock&)>& fn) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
UploadLock lock(tx);
|
||||
fn(lock);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> UploadLock::lock(const std::string& id) {
|
||||
auto r = tx_.exec(
|
||||
"SELECT " + std::string(kCols) +
|
||||
" FROM direct_uploads WHERE id = $1::uuid FOR UPDATE",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<Attachment> UploadLock::find_attachment(const std::string& id) {
|
||||
auto r = tx_.exec(
|
||||
"SELECT " + std::string(kAttachmentColumns) +
|
||||
" FROM attachments WHERE id = $1::uuid",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return attachment_from_row(r[0]);
|
||||
}
|
||||
|
||||
std::optional<Attachment> UploadLock::insert_queued(const Attachment& in,
|
||||
bool& inserted) {
|
||||
inserted = false;
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx_.exec(
|
||||
"INSERT INTO attachments ("
|
||||
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
" status, s3_key, staging_key, error_code, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $9, NULL, "
|
||||
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
|
||||
") ON CONFLICT (id) DO NOTHING RETURNING " +
|
||||
std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.status,
|
||||
in.s3_key, ts, ts});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
tx_.exec("INSERT INTO jobs(id, attachment_id, source_key) VALUES($1::uuid, $1::uuid, $2)",
|
||||
pqxx::params{in.id, in.s3_key});
|
||||
enqueue_attachment_job(tx_, in.id);
|
||||
inserted = true;
|
||||
return attachment_from_row(r[0]);
|
||||
}
|
||||
|
||||
void UploadLock::mark_completed(const std::string& id,
|
||||
const std::string& attachment_id) {
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx_.exec(
|
||||
"UPDATE direct_uploads SET status = 'completed', "
|
||||
" attachment_id = $2::uuid, "
|
||||
" updated_at = to_timestamp($3::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending'",
|
||||
pqxx::params{id, attachment_id, ts});
|
||||
if (r.affected_rows() != 1) {
|
||||
throw std::runtime_error("direct upload complete lost the pending row");
|
||||
}
|
||||
}
|
||||
|
||||
void UploadLock::mark_expired(const std::string& id) {
|
||||
const std::string ts = now_epoch_str();
|
||||
tx_.exec(
|
||||
"UPDATE direct_uploads SET status = 'expired', "
|
||||
" updated_at = to_timestamp($2::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending'",
|
||||
pqxx::params{id, ts});
|
||||
}
|
||||
|
||||
void UploadLock::commit() {
|
||||
if (committed_) {
|
||||
return;
|
||||
}
|
||||
tx_.commit();
|
||||
committed_ = true;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class UploadLock;
|
||||
|
||||
class DirectUploadRepo {
|
||||
public:
|
||||
explicit DirectUploadRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
DirectUpload insert(const DirectUpload& in);
|
||||
|
||||
std::optional<DirectUpload> find_by_id(const std::string& id);
|
||||
|
||||
// Mark completed and link attachment_id. Returns nullopt if not pending
|
||||
// (or already finished / wrong owner handled by caller).
|
||||
std::optional<DirectUpload> mark_completed(const std::string& id,
|
||||
const std::string& attachment_id);
|
||||
|
||||
std::optional<DirectUpload> mark_aborted(const std::string& id);
|
||||
|
||||
// pending → expired where expires_at < now; returns expired rows.
|
||||
std::vector<DirectUpload> expire_stale();
|
||||
|
||||
// Staging keys of still-pending sessions (orphan GC must keep these).
|
||||
std::vector<std::string> pending_s3_keys();
|
||||
|
||||
// One PG transaction. Callback may call S3 while the row lock is held.
|
||||
// Commit explicitly via UploadLock::commit(); otherwise the work rolls back.
|
||||
void with_lock(const std::function<void(UploadLock&)>& fn);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
|
||||
// Row lock for a single complete(). Not safe to use after with_lock returns.
|
||||
class UploadLock {
|
||||
public:
|
||||
std::optional<DirectUpload> lock(const std::string& id);
|
||||
|
||||
std::optional<Attachment> find_attachment(const std::string& id);
|
||||
|
||||
// Inserts attachment and its job in this transaction; job_id equals attachment id.
|
||||
// ON CONFLICT (id) DO NOTHING. inserted=false and nullopt → row already exists.
|
||||
std::optional<Attachment> insert_queued(const Attachment& in, bool& inserted);
|
||||
|
||||
void mark_completed(const std::string& id, const std::string& attachment_id);
|
||||
void mark_expired(const std::string& id);
|
||||
void commit();
|
||||
|
||||
private:
|
||||
friend class DirectUploadRepo;
|
||||
explicit UploadLock(pqxx::work& tx) : tx_(tx) {}
|
||||
|
||||
pqxx::work& tx_;
|
||||
bool committed_ = false;
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <exception>
|
||||
#include <stdexcept>
|
||||
|
||||
// PostgreSQL или Valkey недоступны либо не ответили вовремя: клиент получает 503.
|
||||
struct DependencyUnavailable : std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
struct AttachmentFailed : std::runtime_error {
|
||||
AttachmentFailed() : std::runtime_error("attachment processing failed") {}
|
||||
};
|
||||
|
||||
bool is_dependency_failure(const std::exception& e);
|
||||
@@ -0,0 +1,42 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
// Explicit local-test opt-in; no HTTP parameter can enable fault injection.
|
||||
inline void lab_crash_at(const char* point) {
|
||||
const char* enabled = std::getenv("LAB_ENABLE_FAULT_INJECTION");
|
||||
const char* selected = std::getenv("LAB_FAULT_POINT");
|
||||
if (enabled && std::strcmp(enabled, "1") == 0 && selected &&
|
||||
std::strcmp(selected, point) == 0) std::_Exit(86);
|
||||
}
|
||||
|
||||
#include <chrono>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
// One injection per container/job/point; marker survives a process restart.
|
||||
inline void lab_worker_fault(const char* point, const std::string& job_id, int version) {
|
||||
const char* enabled = std::getenv("LAB_ENABLE_FAULT_INJECTION");
|
||||
const char* selected = std::getenv("LAB_FAULT_POINT");
|
||||
if (!enabled || std::strcmp(enabled, "1") || !selected || std::strcmp(selected, point)) return;
|
||||
const char* job = std::getenv("LAB_FAULT_JOB_ID");
|
||||
const char* target_version = std::getenv("LAB_FAULT_PIPELINE_VERSION");
|
||||
if ((job && job_id != job) || (target_version && std::to_string(version) != target_version)) return;
|
||||
const std::string base = "/tmp/lab-fault/" + std::string(point) + "." + job_id;
|
||||
std::filesystem::create_directories("/tmp/lab-fault");
|
||||
if (std::filesystem::exists(base + ".reached")) return;
|
||||
std::ofstream marker(base + ".reached");
|
||||
marker << version << '\n';
|
||||
marker.close();
|
||||
const char* action = std::getenv("LAB_FAULT_ACTION");
|
||||
if (action && std::strcmp(action, "pause") == 0) {
|
||||
while (!std::filesystem::exists(base + ".release")) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(25));
|
||||
}
|
||||
} else {
|
||||
std::_Exit(86);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
#include "image_pipeline.hpp"
|
||||
#include "image_util.hpp"
|
||||
|
||||
#include <stb_image.h>
|
||||
#define STB_IMAGE_RESIZE_IMPLEMENTATION
|
||||
#include <stb_image_resize.h>
|
||||
#define STB_IMAGE_WRITE_IMPLEMENTATION
|
||||
#include <stb_image_write.h>
|
||||
#include <libexif/exif-data.h>
|
||||
#include <webp/encode.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
#include <memory>
|
||||
|
||||
namespace {
|
||||
int orientation(const std::vector<std::uint8_t>& bytes) {
|
||||
const unsigned char* data = bytes.data();
|
||||
std::size_t size = bytes.size();
|
||||
std::vector<unsigned char> exif;
|
||||
// PNG eXIf stores a TIFF header without the JPEG Exif prefix.
|
||||
if (size >= 8 && std::memcmp(data, "\x89PNG\r\n\x1a\n", 8) == 0) {
|
||||
for (std::size_t pos = 8; pos + 12 <= size;) {
|
||||
const std::uint32_t len = (std::uint32_t(data[pos]) << 24) |
|
||||
(std::uint32_t(data[pos + 1]) << 16) |
|
||||
(std::uint32_t(data[pos + 2]) << 8) | data[pos + 3];
|
||||
if (len > size - pos - 12) break;
|
||||
if (std::memcmp(data + pos + 4, "eXIf", 4) == 0) {
|
||||
exif.assign({'E', 'x', 'i', 'f', 0, 0});
|
||||
exif.insert(exif.end(), data + pos + 8, data + pos + 8 + len);
|
||||
data = exif.data();
|
||||
size = exif.size();
|
||||
break;
|
||||
}
|
||||
pos += std::size_t(len) + 12;
|
||||
}
|
||||
}
|
||||
std::unique_ptr<ExifData, decltype(&exif_data_unref)> metadata(
|
||||
exif_data_new_from_data(data, static_cast<unsigned int>(size)), exif_data_unref);
|
||||
if (!metadata) return 1;
|
||||
const auto* entry = exif_content_get_entry(metadata->ifd[EXIF_IFD_0], EXIF_TAG_ORIENTATION);
|
||||
if (!entry || entry->format != EXIF_FORMAT_SHORT || entry->size < 2) return 1;
|
||||
const int result = exif_get_short(entry->data, exif_data_get_byte_order(metadata.get()));
|
||||
return result >= 1 && result <= 8 ? result : 1;
|
||||
}
|
||||
struct Raster {
|
||||
std::vector<unsigned char> pixels;
|
||||
int width;
|
||||
int height;
|
||||
};
|
||||
|
||||
Raster decode(const std::vector<std::uint8_t>& bytes) {
|
||||
const ImageLimits limits;
|
||||
if (bytes.empty() || bytes.size() > static_cast<std::size_t>(limits.max_bytes) ||
|
||||
!sniff_content_type(bytes.data(), bytes.size())) {
|
||||
throw InvalidImage("invalid_image");
|
||||
}
|
||||
int w = 0, h = 0, channels = 0;
|
||||
if (!stbi_info_from_memory(bytes.data(), static_cast<int>(bytes.size()), &w, &h, &channels) ||
|
||||
w <= 0 || h <= 0) throw InvalidImage("invalid_image");
|
||||
if (std::int64_t(w) * h > limits.max_pixels) throw InvalidImage("image_too_large");
|
||||
std::unique_ptr<stbi_uc, decltype(&stbi_image_free)> rgba(
|
||||
stbi_load_from_memory(bytes.data(), static_cast<int>(bytes.size()), &w, &h, &channels, 4),
|
||||
stbi_image_free);
|
||||
if (!rgba) throw InvalidImage("invalid_image");
|
||||
const int o = orientation(bytes);
|
||||
const int width = o >= 5 ? h : w;
|
||||
const int height = o >= 5 ? w : h;
|
||||
std::vector<unsigned char> rgb(std::size_t(width) * height * 3);
|
||||
for (int y = 0; y < h; ++y) {
|
||||
for (int x = 0; x < w; ++x) {
|
||||
int dx = x, dy = y;
|
||||
switch (o) {
|
||||
case 2: dx = w - 1 - x; break;
|
||||
case 3: dx = w - 1 - x; dy = h - 1 - y; break;
|
||||
case 4: dy = h - 1 - y; break;
|
||||
case 5: dx = y; dy = x; break;
|
||||
case 6: dx = h - 1 - y; dy = x; break;
|
||||
case 7: dx = h - 1 - y; dy = w - 1 - x; break;
|
||||
case 8: dx = y; dy = w - 1 - x; break;
|
||||
}
|
||||
const auto* pixel = rgba.get() + (std::size_t(y) * w + x) * 4;
|
||||
auto* target = rgb.data() + (std::size_t(dy) * width + dx) * 3;
|
||||
for (int c = 0; c < 3; ++c) {
|
||||
target[c] = static_cast<unsigned char>(
|
||||
(pixel[c] * pixel[3] + 255 * (255 - pixel[3]) + 127) / 255);
|
||||
}
|
||||
}
|
||||
}
|
||||
rgba.reset();
|
||||
return Raster{std::move(rgb), width, height};
|
||||
}
|
||||
|
||||
Raster resize(const unsigned char* pixels, int width, int height, int stride,
|
||||
int out_w, int out_h) {
|
||||
Raster result{std::vector<unsigned char>(std::size_t(out_w) * out_h * 3), out_w, out_h};
|
||||
if (!stbir_resize_uint8(pixels, width, height, stride,
|
||||
result.pixels.data(), out_w, out_h, 0, 3)) {
|
||||
throw std::runtime_error("resize failed");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Raster fit(const Raster& input, int limit) {
|
||||
const int extent = std::max(input.width, input.height);
|
||||
const int w = extent > limit ? std::max<std::int64_t>(1, std::int64_t(input.width) * limit / extent) : input.width;
|
||||
const int h = extent > limit ? std::max<std::int64_t>(1, std::int64_t(input.height) * limit / extent) : input.height;
|
||||
return resize(input.pixels.data(), input.width, input.height, 0, w, h);
|
||||
}
|
||||
|
||||
EncodedImage jpeg(const Raster& raster, int quality = 85) {
|
||||
EncodedImage image{{}, raster.width, raster.height};
|
||||
const auto write = [](void* context, void* data, int size) {
|
||||
static_cast<std::string*>(context)->append(static_cast<const char*>(data), size);
|
||||
};
|
||||
if (!stbi_write_jpg_to_func(write, &image.bytes, raster.width, raster.height,
|
||||
3, raster.pixels.data(), quality)) {
|
||||
throw std::runtime_error("JPEG encoding failed");
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
EncodedImage webp(const Raster& raster, float quality) {
|
||||
unsigned char* output = nullptr;
|
||||
const auto size = WebPEncodeRGB(raster.pixels.data(), raster.width, raster.height,
|
||||
raster.width * 3, quality, &output);
|
||||
std::unique_ptr<unsigned char, decltype(&WebPFree)> owned(output, WebPFree);
|
||||
if (!size) throw std::runtime_error("WebP encoding failed");
|
||||
return EncodedImage{std::string(reinterpret_cast<const char*>(output), size),
|
||||
raster.width, raster.height};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
EncodedImage make_thumbnail(const std::vector<std::uint8_t>& bytes) {
|
||||
return jpeg(fit(decode(bytes), 256));
|
||||
}
|
||||
|
||||
std::vector<EncodedVariant> make_variants(const std::vector<std::uint8_t>& bytes, int version) {
|
||||
if (version != 1 && version != 2) throw std::invalid_argument("unsupported pipeline version");
|
||||
const int quality = version == 1 ? 85 : 90;
|
||||
const auto raster = decode(bytes);
|
||||
std::vector<EncodedVariant> result;
|
||||
result.push_back({"thumbnail", "image/jpeg", jpeg(fit(raster, 256), quality)});
|
||||
const int side = std::min(raster.width, raster.height);
|
||||
const int left = (raster.width - side) / 2;
|
||||
const int top = (raster.height - side) / 2;
|
||||
const auto* center = raster.pixels.data() + (std::size_t(top) * raster.width + left) * 3;
|
||||
result.push_back({"crop", "image/jpeg", jpeg(resize(center, side, side, raster.width * 3, 128, 128), quality)});
|
||||
result.push_back({"optimized", "image/webp", webp(fit(raster, 1280), version == 1 ? 80.0f : 90.0f)});
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct InvalidImage : std::runtime_error {
|
||||
explicit InvalidImage(const char* code) : std::runtime_error(code) {}
|
||||
};
|
||||
|
||||
struct EncodedImage {
|
||||
std::string bytes;
|
||||
int width;
|
||||
int height;
|
||||
};
|
||||
|
||||
struct EncodedVariant {
|
||||
std::string name;
|
||||
std::string content_type;
|
||||
EncodedImage image;
|
||||
};
|
||||
|
||||
std::vector<EncodedVariant> make_variants(const std::vector<std::uint8_t>& original, int version = 1);
|
||||
EncodedImage make_thumbnail(const std::vector<std::uint8_t>& original);
|
||||
@@ -0,0 +1,90 @@
|
||||
#include "image_util.hpp"
|
||||
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#define STBI_NO_STDIO
|
||||
#define STBI_ONLY_JPEG
|
||||
#define STBI_ONLY_PNG
|
||||
#include "stb_image.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
|
||||
std::size_t len) {
|
||||
if (!data || len < 3) {
|
||||
return std::nullopt;
|
||||
}
|
||||
// JPEG: FF D8 FF
|
||||
if (data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) {
|
||||
return std::string("image/jpeg");
|
||||
}
|
||||
// PNG: 89 50 4E 47
|
||||
if (len >= 4 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E &&
|
||||
data[3] == 0x47) {
|
||||
return std::string("image/png");
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::string_view image_reject_code(ImageReject r) {
|
||||
switch (r) {
|
||||
case ImageReject::unsupported_media_type:
|
||||
return "unsupported_media_type";
|
||||
case ImageReject::payload_too_large:
|
||||
return "payload_too_large";
|
||||
case ImageReject::invalid_image:
|
||||
return "invalid_image";
|
||||
}
|
||||
return "invalid_image";
|
||||
}
|
||||
|
||||
std::optional<ImageReject> validate_image(const std::uint8_t* data,
|
||||
std::size_t len,
|
||||
const ImageLimits& limits,
|
||||
ImageInfo& out) {
|
||||
if (!data || len == 0) {
|
||||
return ImageReject::unsupported_media_type;
|
||||
}
|
||||
if (static_cast<std::int64_t>(len) > limits.max_bytes) {
|
||||
return ImageReject::payload_too_large;
|
||||
}
|
||||
|
||||
auto sniffed = sniff_content_type(data, len);
|
||||
if (!sniffed) {
|
||||
return ImageReject::unsupported_media_type;
|
||||
}
|
||||
|
||||
int w = 0;
|
||||
int h = 0;
|
||||
int comp = 0;
|
||||
if (!stbi_info_from_memory(reinterpret_cast<const stbi_uc*>(data),
|
||||
static_cast<int>(len), &w, &h, &comp)) {
|
||||
return ImageReject::invalid_image;
|
||||
}
|
||||
if (w <= 0 || h <= 0) {
|
||||
return ImageReject::invalid_image;
|
||||
}
|
||||
|
||||
const std::int64_t pixels =
|
||||
static_cast<std::int64_t>(w) * static_cast<std::int64_t>(h);
|
||||
if (pixels > limits.max_pixels) {
|
||||
return ImageReject::payload_too_large;
|
||||
}
|
||||
|
||||
int load_w = 0;
|
||||
int load_h = 0;
|
||||
int load_comp = 0;
|
||||
stbi_uc* decoded = stbi_load_from_memory(
|
||||
reinterpret_cast<const stbi_uc*>(data), static_cast<int>(len), &load_w,
|
||||
&load_h, &load_comp, 0);
|
||||
if (!decoded) {
|
||||
return ImageReject::invalid_image;
|
||||
}
|
||||
// Integrity only — do not retain decoded raster.
|
||||
stbi_image_free(decoded);
|
||||
|
||||
out.content_type = *sniffed;
|
||||
out.width = w;
|
||||
out.height = h;
|
||||
out.pixel_count = pixels;
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
struct ImageLimits {
|
||||
std::int64_t max_bytes = 10'485'760; // 10 MiB
|
||||
std::int64_t max_pixels = 40'000'000; // decoded width*height
|
||||
};
|
||||
|
||||
enum class ImageReject {
|
||||
unsupported_media_type, // → 415
|
||||
payload_too_large, // → 413 (bytes or pixels)
|
||||
invalid_image, // → 415 (corrupt / undecodable)
|
||||
};
|
||||
|
||||
struct ImageInfo {
|
||||
std::string content_type; // image/png | image/jpeg
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
std::int64_t pixel_count = 0;
|
||||
};
|
||||
|
||||
// Magic sniff only (does not decode). Returns nullopt if neither PNG nor JPEG.
|
||||
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
|
||||
std::size_t len);
|
||||
|
||||
// Full validation pipeline from contracts:
|
||||
// sniff → stbi_info (dims) → pixel limit → stbi_load (integrity) → free.
|
||||
// On success fills `out`. On failure returns ImageReject; decoded pixels are
|
||||
// never retained after return.
|
||||
std::optional<ImageReject> validate_image(const std::uint8_t* data,
|
||||
std::size_t len,
|
||||
const ImageLimits& limits,
|
||||
ImageInfo& out);
|
||||
|
||||
// Human-readable error_code for Attachment failed path / API body.
|
||||
std::string_view image_reject_code(ImageReject r);
|
||||
@@ -0,0 +1,110 @@
|
||||
#include "db.hpp"
|
||||
#include "outbox.hpp"
|
||||
#include "s3_client.hpp"
|
||||
|
||||
#include <sodium.h>
|
||||
#include <iostream>
|
||||
#include <regex>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace {
|
||||
void verify_source(S3Client& s3, const std::string& key, std::size_t expected,
|
||||
const std::string& checksum) {
|
||||
crypto_hash_sha256_state state;
|
||||
crypto_hash_sha256_init(&state);
|
||||
std::size_t received = 0;
|
||||
s3.get_object(key, [&](const char* data, std::size_t n) {
|
||||
if (received > expected || n > expected - received) throw std::runtime_error("source mismatch");
|
||||
crypto_hash_sha256_update(&state, reinterpret_cast<const unsigned char*>(data), n);
|
||||
received += n;
|
||||
});
|
||||
unsigned char digest[crypto_hash_sha256_BYTES];
|
||||
crypto_hash_sha256_final(&state, digest);
|
||||
char hex[crypto_hash_sha256_BYTES * 2 + 1];
|
||||
sodium_bin2hex(hex, sizeof(hex), digest, sizeof(digest));
|
||||
if (received != expected || checksum != hex) throw std::runtime_error("source checksum mismatch");
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const bool replay = argc == 7 && std::string(argv[1]) == "replay" &&
|
||||
std::string(argv[3]) == "--actor" && std::string(argv[5]) == "--reason";
|
||||
const bool upgrade = argc == 9 && std::string(argv[1]) == "upgrade" &&
|
||||
std::string(argv[3]) == "--version" && std::string(argv[5]) == "--actor" &&
|
||||
std::string(argv[7]) == "--reason";
|
||||
if (!replay && !upgrade) {
|
||||
std::cerr << "usage: job_admin replay JOB_UUID --actor NAME --reason REASON\n"
|
||||
" job_admin upgrade ATTACHMENT_UUID --version 2 --actor NAME --reason REASON\n";
|
||||
return 2;
|
||||
}
|
||||
const std::string id = argv[2], actor = argv[replay ? 4 : 6], reason = argv[replay ? 6 : 8];
|
||||
const std::regex uuid("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
||||
if (!std::regex_match(id, uuid) || actor.empty() || actor.size() > 100 || reason.empty() ||
|
||||
reason.size() > 500 || actor.find_first_not_of(" \t\r\n") == std::string::npos ||
|
||||
reason.find_first_not_of(" \t\r\n") == std::string::npos) return 2;
|
||||
if (upgrade && std::string(argv[4]) != "1" && std::string(argv[4]) != "2") return 2;
|
||||
try {
|
||||
if (sodium_init() < 0) return 1;
|
||||
auto conn = open_pg_connection(read_secret_file("DATABASE_URL_FILE"), true);
|
||||
S3Client s3(S3Client::from_env());
|
||||
pqxx::work tx(*conn);
|
||||
if (upgrade) {
|
||||
const int version = std::stoi(argv[4]);
|
||||
tx.exec("SELECT pg_advisory_xact_lock(hashtextextended('pipeline:' || $1::uuid::text,0))", pqxx::params{id});
|
||||
auto rows = tx.exec("SELECT pipeline_version,s3_key,size_bytes,sha256 FROM attachments "
|
||||
"WHERE id=$1::uuid FOR UPDATE", pqxx::params{id});
|
||||
if (rows.empty() || version < rows[0]["pipeline_version"].as<int>()) return 1;
|
||||
if (version == rows[0]["pipeline_version"].as<int>()) {
|
||||
const auto jobs = tx.exec("SELECT id FROM jobs WHERE attachment_id=$1::uuid AND pipeline_version=$2",
|
||||
pqxx::params{id, version});
|
||||
if (jobs.empty()) return 1;
|
||||
std::cout << nlohmann::json{{"job_id", jobs[0][0].as<std::string>()},
|
||||
{"pipeline_version", version}, {"created", false}}.dump() << '\n';
|
||||
return 0;
|
||||
}
|
||||
const auto& row = rows[0];
|
||||
verify_source(s3, row["s3_key"].as<std::string>(), row["size_bytes"].as<std::size_t>(), row["sha256"].as<std::string>());
|
||||
const auto job = tx.exec("INSERT INTO jobs(id,attachment_id,pipeline_version,source_key) "
|
||||
"VALUES(gen_random_uuid(),$1::uuid,$2,$3) RETURNING id",
|
||||
pqxx::params{id, version, row["s3_key"].as<std::string>()})[0][0].as<std::string>();
|
||||
tx.exec("UPDATE attachments SET pipeline_version=$2, "
|
||||
"status=CASE WHEN published_version IS NULL THEN 'queued' ELSE status END, "
|
||||
"error_code=CASE WHEN published_version IS NULL THEN NULL ELSE error_code END, "
|
||||
"updated_at=CASE WHEN published_version IS NULL THEN now() ELSE updated_at END WHERE id=$1::uuid",
|
||||
pqxx::params{id, version});
|
||||
tx.exec("INSERT INTO pipeline_requests(attachment_id,pipeline_version,job_id,actor,reason) "
|
||||
"VALUES($1::uuid,$2,$3::uuid,$4,$5)", pqxx::params{id, version, job, actor, reason});
|
||||
enqueue_attachment_job(tx, job);
|
||||
tx.commit();
|
||||
std::cout << nlohmann::json{{"job_id", job}, {"pipeline_version", version}, {"created", true}}.dump() << '\n';
|
||||
return 0;
|
||||
}
|
||||
if (!tx.exec("SELECT pg_try_advisory_xact_lock(hashtextextended($1::uuid::text,0))",
|
||||
pqxx::params{id})[0][0].as<bool>()) return 1;
|
||||
const auto rows = tx.exec(
|
||||
"SELECT j.attachment_id,j.status,j.retries,j.source_key,j.pipeline_version, "
|
||||
"a.pipeline_version AS desired_version,a.staging_cleaned_at,a.size_bytes,a.sha256 "
|
||||
"FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{id});
|
||||
if (rows.empty() || rows[0]["status"].as<std::string>() != "failed" ||
|
||||
rows[0]["pipeline_version"].as<int>() != rows[0]["desired_version"].as<int>()) return 1;
|
||||
const auto& row = rows[0];
|
||||
const std::string source = row["source_key"].as<std::string>();
|
||||
if (source.rfind("staging/",0) == 0 && !row["staging_cleaned_at"].is_null()) return 1;
|
||||
verify_source(s3, source, row["size_bytes"].as<std::size_t>(), row["sha256"].as<std::string>());
|
||||
tx.exec("INSERT INTO job_replays(job_id,actor,reason,previous_error_code,retries_before) "
|
||||
"SELECT id,$2,$3,error_code,retries FROM jobs WHERE id=$1::uuid", pqxx::params{id, actor, reason});
|
||||
tx.exec("UPDATE jobs SET status='queued',retry_base=retries,error_code=NULL, "
|
||||
"next_attempt_at=now(),updated_at=now() WHERE id=$1::uuid", pqxx::params{id});
|
||||
tx.exec("UPDATE attachments SET status='queued',s3_key=$2,error_code=NULL,variants='[]'::jsonb,updated_at=now() "
|
||||
"WHERE id=$1::uuid AND published_version IS NULL",
|
||||
pqxx::params{row["attachment_id"].as<std::string>(), source});
|
||||
enqueue_attachment_job(tx, id);
|
||||
tx.commit();
|
||||
std::cout << "replayed job=" << id << '\n';
|
||||
return 0;
|
||||
} catch (const std::exception&) {
|
||||
std::cerr << "operator action failed; transaction was not confirmed\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "job_repo.hpp"
|
||||
#include "image_pipeline.hpp"
|
||||
#include "outbox.hpp"
|
||||
#include "fault_injection.hpp"
|
||||
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <algorithm>
|
||||
#include <random>
|
||||
|
||||
JobRepo::Result JobRepo::run(const std::string& job_id, const std::string& attachment_id,
|
||||
int version, int attempt, const std::function<JobResult(const JobInput&)>& process) {
|
||||
auto lease = pool_.acquire();
|
||||
{
|
||||
pqxx::nontransaction tx(*lease);
|
||||
if (!tx.exec("SELECT pg_try_advisory_lock(hashtextextended($1::uuid::text, 0))",
|
||||
pqxx::params{job_id})[0][0].as<bool>()) return Result::Busy;
|
||||
}
|
||||
struct Unlock {
|
||||
pqxx::connection& conn;
|
||||
const std::string& id;
|
||||
~Unlock() {
|
||||
try {
|
||||
pqxx::nontransaction tx(conn);
|
||||
tx.exec("SELECT pg_advisory_unlock(hashtextextended($1::uuid::text, 0))", pqxx::params{id});
|
||||
} catch (...) {}
|
||||
}
|
||||
} unlock{*lease, job_id};
|
||||
JobInput input;
|
||||
{
|
||||
pqxx::work tx(*lease);
|
||||
auto rows = tx.exec(
|
||||
"SELECT j.status, j.retries, j.retry_base, j.source_key, j.pipeline_version AS job_version, "
|
||||
"a.pipeline_version AS desired_version, j.next_attempt_at<=now() AS due, "
|
||||
"a.status AS attachment_status FROM jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.id=$1::uuid AND j.attachment_id=$2::uuid FOR UPDATE OF j, a",
|
||||
pqxx::params{job_id, attachment_id});
|
||||
if (rows.empty()) return Result::Missing;
|
||||
const auto& row = rows[0];
|
||||
if (row["job_version"].as<int>() != version) return Result::Missing;
|
||||
const std::string status = row["status"].as<std::string>();
|
||||
if (version < row["desired_version"].as<int>() || status == "superseded") {
|
||||
tx.exec("UPDATE jobs SET status='superseded', updated_at=now() "
|
||||
"WHERE id=$1::uuid AND status IN ('queued','processing')", pqxx::params{job_id});
|
||||
tx.commit();
|
||||
return Result::Superseded;
|
||||
}
|
||||
if (version > row["desired_version"].as<int>()) return Result::Missing;
|
||||
if (status == "ready" || status == "failed") return Result::Done;
|
||||
input = JobInput{version, row["source_key"].as<std::string>()};
|
||||
const int retries = row["retries"].as<int>();
|
||||
// A previous processing state with no advisory lock means its worker died.
|
||||
if (status == "processing") {
|
||||
return fail(tx, job_id, attachment_id, version, retries + 1, row["retry_base"].as<int>(), "processing_failed", false);
|
||||
}
|
||||
// A newer attempt is already durable in outbox/the stream; discard an old delivery.
|
||||
if (attempt < retries + 1) return Result::Scheduled;
|
||||
if (attempt > retries + 1) return Result::Missing;
|
||||
if (!row["due"].as<bool>()) return Result::Busy;
|
||||
tx.exec("UPDATE jobs SET status='processing', error_code=NULL, updated_at=now() "
|
||||
"WHERE id=$1::uuid", pqxx::params{job_id});
|
||||
tx.exec("UPDATE attachments SET status='processing', error_code=NULL, updated_at=now() "
|
||||
"WHERE id=$1::uuid AND pipeline_version=$2 AND published_version IS NULL",
|
||||
pqxx::params{attachment_id, version});
|
||||
tx.commit();
|
||||
}
|
||||
try {
|
||||
lab_worker_fault("worker_before_effect", job_id, version);
|
||||
const JobResult result = process(input);
|
||||
telemetry::Span commit_span("postgresql.publish_ready", opentelemetry::trace::SpanKind::kClient);
|
||||
pqxx::work tx(*lease);
|
||||
const auto current = tx.exec(
|
||||
"SELECT a.pipeline_version, extract(epoch FROM now()-j.created_at) AS ready_seconds FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{job_id});
|
||||
tx.exec("INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants) "
|
||||
"VALUES($1::uuid,$2,$3::uuid,$4,$5::jsonb) ON CONFLICT DO NOTHING",
|
||||
pqxx::params{attachment_id, version, job_id, result.original_key, result.variants_json});
|
||||
const bool publish = current[0][0].as<int>() == version;
|
||||
if (publish) {
|
||||
// The pointer and complete result become visible in one transaction.
|
||||
tx.exec("UPDATE attachments a SET status='ready', published_version=$2, "
|
||||
"s3_key=r.original_key, variants=r.variants, error_code=NULL, updated_at=now() "
|
||||
"FROM attachment_results r WHERE a.id=$1::uuid AND a.pipeline_version=$2 "
|
||||
"AND r.attachment_id=a.id AND r.pipeline_version=$2",
|
||||
pqxx::params{attachment_id, version});
|
||||
}
|
||||
tx.exec("UPDATE jobs SET status=$2, error_code=NULL, updated_at=now() WHERE id=$1::uuid",
|
||||
pqxx::params{job_id, publish ? "ready" : "superseded"});
|
||||
tx.commit();
|
||||
if (publish) telemetry::duration("attachment.ready.duration", current[0]["ready_seconds"].as<double>());
|
||||
lab_worker_fault("worker_after_commit", job_id, version);
|
||||
return publish ? Result::Done : Result::Superseded;
|
||||
} catch (const std::exception& e) {
|
||||
const auto* invalid = dynamic_cast<const InvalidImage*>(&e);
|
||||
const std::string code = invalid ? invalid->what() : "processing_failed";
|
||||
pqxx::work tx(*lease);
|
||||
auto rows = tx.exec("SELECT j.status,j.retries,j.retry_base,a.pipeline_version FROM jobs j "
|
||||
"JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1::uuid FOR UPDATE OF j,a",
|
||||
pqxx::params{job_id});
|
||||
if (rows.empty()) return Result::Missing;
|
||||
if (rows[0]["pipeline_version"].as<int>() != version) {
|
||||
tx.exec("UPDATE jobs SET status='superseded',updated_at=now() WHERE id=$1::uuid "
|
||||
"AND status IN ('queued','processing')", pqxx::params{job_id});
|
||||
tx.commit();
|
||||
return Result::Superseded;
|
||||
}
|
||||
const std::string status = rows[0]["status"].as<std::string>();
|
||||
// A lost COMMIT reply must not roll back an already published ready result.
|
||||
if (status == "ready" || status == "failed") return Result::Done;
|
||||
return fail(tx, job_id, attachment_id, version, rows[0]["retries"].as<int>() + 1,
|
||||
rows[0]["retry_base"].as<int>(), code, invalid != nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
JobRepo::Result JobRepo::fail(pqxx::work& tx, const std::string& job_id,
|
||||
const std::string& attachment_id, int version, int retries, int retry_base,
|
||||
const std::string& code, bool permanent) {
|
||||
const bool failed = permanent || retries - retry_base >= policy_.max_attempts;
|
||||
long long ceiling = policy_.base_delay_ms;
|
||||
for (int i = 1; i < retries - retry_base; ++i) ceiling = std::min<long long>(policy_.max_delay_ms, ceiling * 2);
|
||||
ceiling = std::min<long long>(policy_.max_delay_ms, ceiling);
|
||||
thread_local std::mt19937 rng(std::random_device{}());
|
||||
const int delay = failed ? 0 : std::uniform_int_distribution<int>(
|
||||
static_cast<int>((ceiling + 1) / 2), static_cast<int>(ceiling))(rng);
|
||||
tx.exec("UPDATE jobs SET status=$2, retries=$3, error_code=$4, "
|
||||
"next_attempt_at=now()+($5::bigint * interval '1 millisecond'), updated_at=now() "
|
||||
"WHERE id=$1::uuid",
|
||||
pqxx::params{job_id, failed ? "failed" : "queued", retries, code, delay});
|
||||
tx.exec("UPDATE attachments SET status=$2, error_code=$3, updated_at=now() "
|
||||
"WHERE id=$1::uuid AND status='processing' AND pipeline_version=$4 AND published_version IS NULL",
|
||||
pqxx::params{attachment_id, failed ? "failed" : "queued",
|
||||
failed ? std::optional<std::string>(code) : std::nullopt, version});
|
||||
if (!failed) enqueue_attachment_job(tx, job_id);
|
||||
tx.commit();
|
||||
telemetry::event(failed ? "worker.attempt.failed" : "worker.attempt.retry", true);
|
||||
return failed ? Result::Done : Result::Scheduled;
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
struct JobInput {
|
||||
int pipeline_version;
|
||||
std::string source_key;
|
||||
};
|
||||
|
||||
struct JobResult {
|
||||
std::string original_key;
|
||||
std::string variants_json;
|
||||
};
|
||||
|
||||
struct RetryPolicy {
|
||||
int max_attempts = 6;
|
||||
int base_delay_ms = 1000;
|
||||
int max_delay_ms = 60000;
|
||||
};
|
||||
|
||||
class JobRepo {
|
||||
public:
|
||||
enum class Result { Done, Scheduled, Busy, Missing, Superseded };
|
||||
JobRepo(PgPool& pool, RetryPolicy policy) : pool_(pool), policy_(policy) {}
|
||||
Result run(const std::string& job_id, const std::string& attachment_id,
|
||||
int version, int attempt, const std::function<JobResult(const JobInput&)>& process);
|
||||
private:
|
||||
Result fail(pqxx::work& tx, const std::string& job_id,
|
||||
const std::string& attachment_id, int version, int retries, int retry_base,
|
||||
const std::string& code, bool permanent);
|
||||
PgPool& pool_;
|
||||
RetryPolicy policy_;
|
||||
};
|
||||
+2361
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,377 @@
|
||||
#include "message_repo.hpp"
|
||||
#include "cursor.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sodium.h>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string make_uuid() {
|
||||
static std::mutex rng_mu;
|
||||
static std::mt19937_64 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
|
||||
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
|
||||
std::lock_guard<std::mutex> lock(rng_mu);
|
||||
std::ostringstream out;
|
||||
out << std::hex << std::setfill('0');
|
||||
out << std::setw(8) << d32(rng);
|
||||
out << '-' << std::setw(4) << d16(rng);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
|
||||
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string sha256_hex(const std::string& s) {
|
||||
unsigned char out[crypto_hash_sha256_BYTES];
|
||||
crypto_hash_sha256(out, reinterpret_cast<const unsigned char*>(s.data()), s.size());
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
std::string r;
|
||||
r.resize(sizeof(out) * 2);
|
||||
for (std::size_t i = 0; i < sizeof(out); ++i) {
|
||||
r[i * 2] = hex[out[i] >> 4];
|
||||
r[i * 2 + 1] = hex[out[i] & 0x0f];
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch()).count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
Message row_to_message(const pqxx::row& r) {
|
||||
Message m;
|
||||
m.id = r["id"].as<std::string>();
|
||||
m.chat_id = r["chat_id"].as<std::string>();
|
||||
m.sender_id = r["sender_id"].as<std::string>();
|
||||
m.text = r["text"].as<std::string>();
|
||||
m.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
return m;
|
||||
}
|
||||
|
||||
constexpr const char* kMessageSelect =
|
||||
"id, chat_id, sender_id, text, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
|
||||
|
||||
std::string canonical_message_body(const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
std::ostringstream out;
|
||||
out << "text=" << text << "\nattachment_ids=";
|
||||
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
|
||||
if (i > 0) {
|
||||
out << ',';
|
||||
}
|
||||
out << attachment_ids[i];
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
|
||||
int idempotency_ttl_seconds() {
|
||||
const char* env = std::getenv("IDEMPOTENCY_TTL_SECONDS");
|
||||
if (env == nullptr || *env == '\0') {
|
||||
return 86400;
|
||||
}
|
||||
try {
|
||||
const int v = std::stoi(env);
|
||||
if (v <= 0) {
|
||||
return 86400;
|
||||
}
|
||||
return v;
|
||||
} catch (...) {
|
||||
return 86400;
|
||||
}
|
||||
}
|
||||
|
||||
bool attachments_belong_to_chat(pqxx::work& tx,
|
||||
const std::string& chat_id,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
bool failed = false;
|
||||
for (const auto& id : attachment_ids) {
|
||||
auto r = tx.exec(
|
||||
"SELECT status FROM attachments "
|
||||
"WHERE id = $1 AND chat_id = $2 FOR SHARE",
|
||||
pqxx::params{id, chat_id});
|
||||
if (r.empty()) {
|
||||
return false;
|
||||
}
|
||||
failed = failed || r[0]["status"].as<std::string>() == "failed";
|
||||
}
|
||||
if (failed) {
|
||||
throw AttachmentFailed();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void link_attachments(pqxx::work& tx,
|
||||
const std::string& message_id,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
|
||||
tx.exec(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) "
|
||||
"VALUES ($1, $2, $3)",
|
||||
pqxx::params{message_id, attachment_ids[i], static_cast<int>(i)});
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> load_attachment_ids(pqxx::work& tx,
|
||||
const std::string& message_id) {
|
||||
auto r = tx.exec(
|
||||
"SELECT attachment_id FROM message_attachments "
|
||||
"WHERE message_id = $1 ORDER BY position ASC",
|
||||
pqxx::params{message_id});
|
||||
std::vector<std::string> ids;
|
||||
ids.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
ids.push_back(row["attachment_id"].as<std::string>());
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
void fill_attachment_ids(pqxx::work& tx, Message& m) {
|
||||
m.attachment_ids = load_attachment_ids(tx, m.id);
|
||||
}
|
||||
|
||||
void fill_attachment_ids(pqxx::work& tx, std::vector<Message>& items) {
|
||||
for (auto& m : items) {
|
||||
fill_attachment_ids(tx, m);
|
||||
}
|
||||
}
|
||||
|
||||
Message insert_message(pqxx::work& tx,
|
||||
const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
const std::string id = make_uuid();
|
||||
const std::string created = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
||||
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
|
||||
"RETURNING " + std::string(kMessageSelect),
|
||||
pqxx::params{id, chat_id, sender_id, text, created});
|
||||
Message m = row_to_message(r[0]);
|
||||
link_attachments(tx, m.id, attachment_ids);
|
||||
m.attachment_ids = attachment_ids;
|
||||
return m;
|
||||
}
|
||||
|
||||
bool insert_idempotency_key(pqxx::work& tx,
|
||||
const std::string& user_id,
|
||||
const std::string& chat_id,
|
||||
const std::string& idem_key,
|
||||
const std::string& request_hash,
|
||||
const std::string& message_id) {
|
||||
auto inserted = tx.exec(
|
||||
"INSERT INTO idempotency_keys "
|
||||
"(user_id, chat_id, key, request_hash, message_id) "
|
||||
"VALUES ($1, $2, $3, $4, $5) "
|
||||
"ON CONFLICT (user_id, chat_id, key) DO NOTHING "
|
||||
"RETURNING message_id",
|
||||
pqxx::params{user_id, chat_id, idem_key, request_hash, message_id});
|
||||
return !inserted.empty();
|
||||
}
|
||||
|
||||
Message load_message(pqxx::work& tx, const std::string& message_id) {
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kMessageSelect) + " FROM messages WHERE id = $1",
|
||||
pqxx::params{message_id});
|
||||
if (r.empty()) {
|
||||
throw std::runtime_error("idempotent message row is missing");
|
||||
}
|
||||
Message m = row_to_message(r[0]);
|
||||
fill_attachment_ids(tx, m);
|
||||
return m;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::optional<Message> MessageRepo::create(const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
auto member_check = tx.exec(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
pqxx::params{chat_id, sender_id});
|
||||
if (member_check.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Message created = insert_message(tx, chat_id, sender_id, text, attachment_ids);
|
||||
tx.commit();
|
||||
return created;
|
||||
}
|
||||
|
||||
std::optional<MessagePage> MessageRepo::list_page(
|
||||
const std::string& chat_id,
|
||||
int limit,
|
||||
const std::optional<std::string>& cursor) {
|
||||
|
||||
std::optional<Cursor> cur;
|
||||
if (cursor) {
|
||||
cur = parse_cursor(*cursor);
|
||||
if (!cur) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
if (cur) {
|
||||
auto owner_check = tx.exec(
|
||||
"SELECT 1 FROM messages WHERE id = $1 AND chat_id = $2",
|
||||
pqxx::params{cur->id, chat_id});
|
||||
if (owner_check.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::string sql =
|
||||
"SELECT " + std::string(kMessageSelect) + ", " + kCreatedUsSql + " "
|
||||
"FROM messages "
|
||||
"WHERE chat_id = $1 ";
|
||||
if (cur) {
|
||||
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
|
||||
}
|
||||
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
|
||||
|
||||
pqxx::result r;
|
||||
if (cur) {
|
||||
r = tx.exec(sql, pqxx::params{chat_id, limit + 1, cur->created_us, cur->id});
|
||||
} else {
|
||||
r = tx.exec(sql, pqxx::params{chat_id, limit + 1});
|
||||
}
|
||||
|
||||
MessagePage page;
|
||||
const bool has_more = (static_cast<int>(r.size()) > limit);
|
||||
const int take = has_more ? limit : static_cast<int>(r.size());
|
||||
page.items.reserve(take);
|
||||
for (int i = 0; i < take; ++i) {
|
||||
page.items.push_back(row_to_message(r[static_cast<std::size_t>(i)]));
|
||||
}
|
||||
fill_attachment_ids(tx, page.items);
|
||||
|
||||
if (has_more && take > 0) {
|
||||
const auto& last = r[static_cast<std::size_t>(take - 1)];
|
||||
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
|
||||
last["id"].as<std::string>());
|
||||
}
|
||||
|
||||
tx.commit();
|
||||
return page;
|
||||
}
|
||||
|
||||
CreateResult MessageRepo::create_idempotent(
|
||||
const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids,
|
||||
const std::string& idem_key) {
|
||||
const int ttl = idempotency_ttl_seconds();
|
||||
const std::string hash = sha256_hex(canonical_message_body(text, attachment_ids));
|
||||
|
||||
for (int attempt = 0; attempt < 3; ++attempt) {
|
||||
{
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
auto member_check = tx.exec(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
pqxx::params{chat_id, sender_id});
|
||||
if (member_check.empty()) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
|
||||
}
|
||||
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
|
||||
}
|
||||
|
||||
const Message created =
|
||||
insert_message(tx, chat_id, sender_id, text, attachment_ids);
|
||||
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
|
||||
created.id)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::Created, created};
|
||||
}
|
||||
}
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto existing = tx.exec(
|
||||
"SELECT request_hash, message_id, "
|
||||
"EXTRACT(EPOCH FROM (now() - created_at))::bigint AS age_seconds "
|
||||
"FROM idempotency_keys "
|
||||
"WHERE user_id = $1 AND chat_id = $2 AND key = $3 "
|
||||
"FOR UPDATE",
|
||||
pqxx::params{sender_id, chat_id, idem_key});
|
||||
if (existing.empty()) {
|
||||
tx.commit();
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string existing_hash = existing[0]["request_hash"].as<std::string>();
|
||||
const long long age = existing[0]["age_seconds"].as<long long>();
|
||||
const std::string message_id = existing[0]["message_id"].as<std::string>();
|
||||
if (age < ttl) {
|
||||
if (existing_hash != hash) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::KeyConflict, std::nullopt};
|
||||
}
|
||||
const Message replayed = load_message(tx, message_id);
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::Replayed, replayed};
|
||||
}
|
||||
|
||||
tx.exec(
|
||||
"DELETE FROM idempotency_keys "
|
||||
"WHERE user_id = $1 AND chat_id = $2 AND key = $3",
|
||||
pqxx::params{sender_id, chat_id, idem_key});
|
||||
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
|
||||
}
|
||||
const Message created =
|
||||
insert_message(tx, chat_id, sender_id, text, attachment_ids);
|
||||
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
|
||||
created.id)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::Created, created};
|
||||
}
|
||||
}
|
||||
|
||||
throw std::runtime_error("idempotency retry exhausted");
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct CreateResult {
|
||||
enum class Status { Created, Replayed, NotFound, KeyConflict };
|
||||
Status status;
|
||||
std::optional<Message> message;
|
||||
};
|
||||
|
||||
class MessageRepo {
|
||||
public:
|
||||
explicit MessageRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
std::optional<Message> create(const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids);
|
||||
|
||||
std::optional<MessagePage> list_page(
|
||||
const std::string& chat_id,
|
||||
int limit,
|
||||
const std::optional<std::string>& cursor);
|
||||
|
||||
CreateResult create_idempotent(const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids,
|
||||
const std::string& idem_key);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,86 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct User {
|
||||
std::string id;
|
||||
std::string username;
|
||||
std::string display_name;
|
||||
std::string password_hash;
|
||||
std::string created_at;
|
||||
};
|
||||
|
||||
struct Chat {
|
||||
std::string id;
|
||||
std::string title;
|
||||
std::vector<std::string> member_ids;
|
||||
std::string created_at;
|
||||
};
|
||||
|
||||
struct Message {
|
||||
std::string id;
|
||||
std::string chat_id;
|
||||
std::string sender_id;
|
||||
std::string text;
|
||||
std::string created_at;
|
||||
std::vector<std::string> attachment_ids;
|
||||
};
|
||||
|
||||
struct MessagePage {
|
||||
std::vector<Message> items;
|
||||
std::optional<std::string> next_cursor;
|
||||
};
|
||||
|
||||
struct ChatPage {
|
||||
std::vector<Chat> items;
|
||||
std::optional<std::string> next_cursor;
|
||||
};
|
||||
|
||||
// Public metadata; storage keys are derived internally.
|
||||
struct Variant {
|
||||
std::string name;
|
||||
std::string content_type;
|
||||
int size_bytes = 0;
|
||||
std::string sha256;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
std::string storage_key; // internal; filtered from API JSON
|
||||
};
|
||||
|
||||
struct Attachment {
|
||||
std::string id;
|
||||
std::string chat_id;
|
||||
std::string owner_id;
|
||||
std::string filename;
|
||||
std::string content_type;
|
||||
int size_bytes = 0;
|
||||
std::string sha256;
|
||||
std::string status; // queued|processing|ready|failed
|
||||
std::vector<Variant> variants;
|
||||
std::optional<std::string> error_code;
|
||||
std::string s3_key; // internal; never exposed in API JSON
|
||||
std::string created_at;
|
||||
std::string updated_at;
|
||||
};
|
||||
|
||||
// Track 1: pending presigned PUT session (not a public attachment until complete).
|
||||
struct DirectUpload {
|
||||
std::string id;
|
||||
std::string chat_id;
|
||||
std::string owner_id;
|
||||
std::string filename;
|
||||
std::string content_type;
|
||||
int expected_size = 0;
|
||||
std::string expected_sha256;
|
||||
std::string s3_key; // final key: attachments/{chat}/{id}
|
||||
std::string staging_key; // presigned PUT target: staging/{chat}/{id}
|
||||
std::string status; // pending|completed|aborted|expired
|
||||
bool overdue = false; // pending && expires_at < now(); not a column
|
||||
std::optional<std::string> attachment_id;
|
||||
std::string expires_at; // RFC3339 for JSON
|
||||
long long expires_at_epoch = 0; // for INSERT
|
||||
std::string created_at;
|
||||
std::string updated_at;
|
||||
};
|
||||
@@ -0,0 +1,37 @@
|
||||
#include "observability_monitor.hpp"
|
||||
#include "db.hpp"
|
||||
#include "telemetry.hpp"
|
||||
#include <fstream>
|
||||
#include <sys/resource.h>
|
||||
#include <unistd.h>
|
||||
|
||||
ObservabilityMonitor::ObservabilityMonitor(std::string db_url, bool queue)
|
||||
: thread_([this, db_url, queue] {
|
||||
std::unique_ptr<pqxx::connection> conn;
|
||||
while (!stopping_) {
|
||||
try {
|
||||
rusage usage{};
|
||||
if (getrusage(RUSAGE_SELF, &usage) == 0)
|
||||
telemetry::gauge("process.cpu.time", usage.ru_utime.tv_sec + usage.ru_stime.tv_sec +
|
||||
(usage.ru_utime.tv_usec + usage.ru_stime.tv_usec)/1e6);
|
||||
long total, resident;
|
||||
std::ifstream statm("/proc/self/statm");
|
||||
if (statm >> total >> resident)
|
||||
telemetry::gauge("process.memory.usage", resident * sysconf(_SC_PAGESIZE));
|
||||
if (queue) {
|
||||
if (!conn) conn = open_pg_connection(db_url, true);
|
||||
pqxx::nontransaction tx(*conn);
|
||||
// Logical durable backlog includes outbox, Redis pending and delayed retries,
|
||||
// without counting duplicate stream entries or ACKed historical entries.
|
||||
const auto rows = tx.exec("SELECT count(*), coalesce(extract(epoch FROM now()-min(created_at)),0) "
|
||||
"FROM jobs WHERE status IN ('queued','processing')");
|
||||
telemetry::gauge("queue.depth", rows[0][0].as<double>());
|
||||
telemetry::gauge("queue.oldest.age", rows[0][1].as<double>());
|
||||
telemetry::gauge("queue.observation.timestamp", std::chrono::duration<double>(
|
||||
std::chrono::system_clock::now().time_since_epoch()).count());
|
||||
}
|
||||
} catch (...) { conn.reset(); telemetry::event("telemetry.queue.observation.failed", true); }
|
||||
for (int i=0; i<50 && !stopping_; ++i) std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
}
|
||||
}) {}
|
||||
ObservabilityMonitor::~ObservabilityMonitor() { stopping_ = true; thread_.join(); }
|
||||
@@ -0,0 +1,13 @@
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
// Dedicated connection: telemetry queries never consume the application's pool.
|
||||
class ObservabilityMonitor {
|
||||
public:
|
||||
ObservabilityMonitor(std::string db_url, bool queue);
|
||||
~ObservabilityMonitor();
|
||||
private:
|
||||
std::atomic<bool> stopping_{false};
|
||||
std::thread thread_;
|
||||
};
|
||||
@@ -0,0 +1,80 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "outbox.hpp"
|
||||
#include "attachment_queue.hpp"
|
||||
#include "fault_injection.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <iostream>
|
||||
#include <utility>
|
||||
|
||||
void enqueue_attachment_job(pqxx::work& tx, const std::string& job_id) {
|
||||
telemetry::Span producer("attachment.enqueue", opentelemetry::trace::SpanKind::kProducer);
|
||||
producer.span->SetAttribute("messaging.system", "redis");
|
||||
producer.span->SetAttribute("messaging.destination.name", "jobs:attachments");
|
||||
const auto rows = tx.exec(
|
||||
"INSERT INTO outbox(payload, available_at) "
|
||||
"SELECT jsonb_strip_nulls(jsonb_build_object('traceparent', nullif($2::text,''), 'tracestate', $3::text, 'schema_version', 1, 'type', 'attachment.process.v1', "
|
||||
"'job_id', id::text, 'attachment_id', attachment_id::text, 'pipeline_version', pipeline_version, "
|
||||
"'attempt', retries + 1, 'created_at', "
|
||||
"to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"'))), "
|
||||
"next_attempt_at FROM jobs WHERE id=$1::uuid RETURNING id", pqxx::params{job_id, telemetry::inject(), telemetry::inject("tracestate")});
|
||||
if (rows.size() != 1) throw std::runtime_error("outbox job missing");
|
||||
}
|
||||
|
||||
OutboxPublisher::OutboxPublisher(std::string db_url, std::string broker_url)
|
||||
: thread_([this, db_url = std::move(db_url), broker_url = std::move(broker_url)] {
|
||||
run(db_url, broker_url);
|
||||
}) {}
|
||||
|
||||
OutboxPublisher::~OutboxPublisher() {
|
||||
stopping_.store(true);
|
||||
cv_.notify_all();
|
||||
thread_.join();
|
||||
}
|
||||
|
||||
bool OutboxPublisher::wait_interval() {
|
||||
std::unique_lock<std::mutex> lock(mutex_);
|
||||
return cv_.wait_for(lock, std::chrono::seconds(1), [this] { return stopping_.load(); });
|
||||
}
|
||||
|
||||
void OutboxPublisher::run(const std::string& db_url, const std::string& broker_url) {
|
||||
while (!stopping_.load()) {
|
||||
try {
|
||||
// Dedicated pools keep broker failure/reconnect off the HTTP path.
|
||||
PgPool pg(db_url, 1);
|
||||
RedisPool redis(broker_url, 1);
|
||||
AttachmentQueue queue(redis);
|
||||
while (!stopping_.load()) {
|
||||
{
|
||||
auto lease = pg.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const auto rows = tx.exec(
|
||||
"SELECT id, payload FROM outbox WHERE NOT published AND available_at<=now() "
|
||||
"ORDER BY available_at, id LIMIT 32 FOR UPDATE SKIP LOCKED");
|
||||
for (const auto& row : rows) {
|
||||
auto payload = nlohmann::json::parse(row["payload"].as<std::string>());
|
||||
telemetry::Span publish("jobs:attachments publish", opentelemetry::trace::SpanKind::kProducer,
|
||||
payload.value("traceparent", ""), payload.value("tracestate", ""));
|
||||
publish.span->SetAttribute("messaging.system", "redis");
|
||||
publish.span->SetAttribute("messaging.destination.name", "jobs:attachments");
|
||||
const auto parent = telemetry::inject();
|
||||
if (!parent.empty()) payload["traceparent"] = parent;
|
||||
else payload.erase("traceparent");
|
||||
payload["tracestate"] = telemetry::inject("tracestate");
|
||||
queue.publish(payload.dump());
|
||||
telemetry::event("queue.published");
|
||||
lab_crash_at("outbox_after_publish");
|
||||
tx.exec("UPDATE outbox SET published=true WHERE id=$1",
|
||||
pqxx::params{row["id"].as<long long>()});
|
||||
}
|
||||
tx.commit();
|
||||
}
|
||||
if (wait_interval()) break;
|
||||
}
|
||||
} catch (const std::exception&) {
|
||||
telemetry::event("queue.publish.unavailable", true);
|
||||
if (wait_interval()) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
|
||||
// Must share the transaction that inserts/updates the job.
|
||||
void enqueue_attachment_job(pqxx::work& tx, const std::string& job_id);
|
||||
|
||||
class OutboxPublisher {
|
||||
public:
|
||||
OutboxPublisher(std::string db_url, std::string broker_url);
|
||||
~OutboxPublisher();
|
||||
OutboxPublisher(const OutboxPublisher&) = delete;
|
||||
OutboxPublisher& operator=(const OutboxPublisher&) = delete;
|
||||
private:
|
||||
void run(const std::string& db_url, const std::string& broker_url);
|
||||
bool wait_interval();
|
||||
std::atomic<bool> stopping_{false};
|
||||
std::mutex mutex_;
|
||||
std::condition_variable cv_;
|
||||
std::thread thread_;
|
||||
};
|
||||
@@ -0,0 +1,66 @@
|
||||
#include "pg_session_repo.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
#include <sodium.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string random_token_hex() {
|
||||
unsigned char buf[32];
|
||||
randombytes_buf(buf, sizeof(buf));
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.resize(sizeof(buf) * 2);
|
||||
for (std::size_t i = 0; i < sizeof(buf); ++i) {
|
||||
out[i * 2] = hex[buf[i] >> 4];
|
||||
out[i * 2 + 1] = hex[buf[i] & 0x0f];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::int64_t now_unix() {
|
||||
return static_cast<std::int64_t>(std::time(nullptr));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ISessionRepo::Created PgSessionRepo::create(const std::string& user_id) {
|
||||
const std::string token = random_token_hex();
|
||||
const std::int64_t exp = now_unix() + ttl_;
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
tx.exec(
|
||||
"INSERT INTO sessions (token, user_id, expires_at) "
|
||||
"VALUES ($1, $2, to_timestamp($3::bigint))",
|
||||
pqxx::params{token, user_id, exp});
|
||||
tx.commit();
|
||||
|
||||
return Created{token, exp};
|
||||
}
|
||||
|
||||
std::optional<std::string> PgSessionRepo::lookup(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT user_id FROM sessions "
|
||||
"WHERE token = $1 AND expires_at > now()",
|
||||
pqxx::params{token});
|
||||
tx.commit();
|
||||
if (r.empty()) return std::nullopt;
|
||||
return r[0]["user_id"].as<std::string>();
|
||||
}
|
||||
|
||||
bool PgSessionRepo::revoke(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"DELETE FROM sessions WHERE token = $1",
|
||||
pqxx::params{token});
|
||||
tx.commit();
|
||||
return r.affected_rows() > 0;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "session_repo_base.hpp"
|
||||
|
||||
class PgSessionRepo : public ISessionRepo {
|
||||
public:
|
||||
PgSessionRepo(PgPool& pool, int ttl_seconds)
|
||||
: pool_(pool), ttl_(ttl_seconds) {}
|
||||
|
||||
Created create(const std::string& user_id) override;
|
||||
std::optional<std::string> lookup(const std::string& token) override;
|
||||
bool revoke(const std::string& token) override;
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
int ttl_;
|
||||
};
|
||||
@@ -0,0 +1,216 @@
|
||||
#include "redis_pool.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr timeval kConnectTimeout{2, 0};
|
||||
constexpr timeval kCommandTimeout{2, 0};
|
||||
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
|
||||
|
||||
struct RedisUrl {
|
||||
std::string host = "127.0.0.1";
|
||||
int port = 6379;
|
||||
std::string password;
|
||||
int db = 0;
|
||||
};
|
||||
|
||||
RedisUrl parse_redis_url(const std::string& url) {
|
||||
RedisUrl out;
|
||||
const std::string prefix = "redis://";
|
||||
if (url.compare(0, prefix.size(), prefix) != 0) {
|
||||
throw std::runtime_error("redis url must start with redis://");
|
||||
}
|
||||
std::string rest = url.substr(prefix.size());
|
||||
|
||||
const auto slash = rest.find('/');
|
||||
if (slash != std::string::npos) {
|
||||
const std::string db_str = rest.substr(slash + 1);
|
||||
rest.resize(slash);
|
||||
if (!db_str.empty()) {
|
||||
out.db = std::atoi(db_str.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
const auto at = rest.find('@');
|
||||
std::string hostport = rest;
|
||||
if (at != std::string::npos) {
|
||||
const std::string creds = rest.substr(0, at);
|
||||
hostport = rest.substr(at + 1);
|
||||
const auto colon = creds.find(':');
|
||||
if (colon != std::string::npos) {
|
||||
out.password = creds.substr(colon + 1);
|
||||
} else {
|
||||
out.password = creds;
|
||||
}
|
||||
}
|
||||
|
||||
const auto colon = hostport.find(':');
|
||||
if (colon == std::string::npos) {
|
||||
out.host = hostport;
|
||||
} else {
|
||||
out.host = hostport.substr(0, colon);
|
||||
out.port = std::atoi(hostport.substr(colon + 1).c_str());
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
redisContext* make_connection(const RedisUrl& u) {
|
||||
redisContext* c = redisConnectWithTimeout(u.host.c_str(), u.port, kConnectTimeout);
|
||||
if (c == nullptr) {
|
||||
throw std::runtime_error("redisConnect: allocation failed");
|
||||
}
|
||||
if (c->err) {
|
||||
const std::string err = c->errstr;
|
||||
redisFree(c);
|
||||
throw DependencyUnavailable("redisConnect failed: " + err);
|
||||
}
|
||||
if (redisSetTimeout(c, kCommandTimeout) != REDIS_OK) {
|
||||
redisFree(c);
|
||||
throw DependencyUnavailable("redisSetTimeout failed");
|
||||
}
|
||||
if (!u.password.empty()) {
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "AUTH %s", u.password.c_str()));
|
||||
if (r == nullptr) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis AUTH: no reply");
|
||||
}
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
if (!ok) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis AUTH failed");
|
||||
}
|
||||
}
|
||||
if (u.db != 0) {
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "SELECT %d", u.db));
|
||||
if (r == nullptr) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis SELECT: no reply");
|
||||
}
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
if (!ok) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis SELECT failed");
|
||||
}
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
RedisPool::RedisPool(std::string url, std::size_t n)
|
||||
: url_(std::move(url)) {
|
||||
if (n == 0) {
|
||||
throw std::runtime_error("RedisPool: n must be > 0");
|
||||
}
|
||||
const RedisUrl parsed = parse_redis_url(url_);
|
||||
all_.reserve(n);
|
||||
idle_.reserve(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
ConnPtr c(make_connection(parsed));
|
||||
idle_.push_back(c.get());
|
||||
all_.push_back(std::move(c));
|
||||
}
|
||||
}
|
||||
|
||||
RedisPool::~RedisPool() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
shutting_down_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
RedisPool::Lease::Lease(RedisPool& pool)
|
||||
: pool_(&pool), ctx_(pool.take()) {}
|
||||
|
||||
RedisPool::Lease::~Lease() {
|
||||
if (pool_ != nullptr && ctx_ != nullptr) {
|
||||
pool_->give_back(ctx_);
|
||||
}
|
||||
}
|
||||
|
||||
RedisPool::Lease::Lease(Lease&& other) noexcept
|
||||
: pool_(other.pool_), ctx_(other.ctx_) {
|
||||
other.pool_ = nullptr;
|
||||
other.ctx_ = nullptr;
|
||||
}
|
||||
|
||||
RedisPool::Lease RedisPool::acquire() {
|
||||
return Lease(*this);
|
||||
}
|
||||
|
||||
redisContext* RedisPool::take() {
|
||||
std::unique_lock<std::mutex> lk(mu_);
|
||||
if (!cv_.wait_for(lk, kAcquireTimeout,
|
||||
[&]() { return shutting_down_ || !idle_.empty(); })) {
|
||||
throw DependencyUnavailable("redis pool: acquire timeout");
|
||||
}
|
||||
if (shutting_down_) {
|
||||
throw DependencyUnavailable("redis pool is shutting down");
|
||||
}
|
||||
redisContext* c = idle_.back();
|
||||
idle_.pop_back();
|
||||
lk.unlock();
|
||||
|
||||
bool alive = false;
|
||||
if (!c->err) {
|
||||
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
|
||||
if (r != nullptr && r->type == REDIS_REPLY_STATUS) {
|
||||
alive = true;
|
||||
}
|
||||
if (r != nullptr) freeReplyObject(r);
|
||||
}
|
||||
if (alive) {
|
||||
return c;
|
||||
}
|
||||
|
||||
ConnPtr fresh;
|
||||
try {
|
||||
fresh.reset(make_connection(parse_redis_url(url_)));
|
||||
} catch (const std::exception& e) {
|
||||
// Слот возвращается в пул, иначе после нескольких неудач пул опустеет навсегда.
|
||||
give_back(c);
|
||||
throw DependencyUnavailable(std::string("redis reconnect failed: ") + e.what());
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lk2(mu_);
|
||||
for (auto& up : all_) {
|
||||
if (up.get() == c) {
|
||||
up = std::move(fresh);
|
||||
return up.get();
|
||||
}
|
||||
}
|
||||
throw std::logic_error("RedisPool: connection not owned by pool");
|
||||
}
|
||||
|
||||
void RedisPool::give_back(redisContext* c) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
idle_.push_back(c);
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
bool RedisPool::ping() {
|
||||
try {
|
||||
auto lease = acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
|
||||
if (r == nullptr) return false;
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
return ok;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#pragma once
|
||||
|
||||
#include <hiredis/hiredis.h>
|
||||
|
||||
#include <condition_variable>
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class RedisPool {
|
||||
public:
|
||||
RedisPool(std::string url, std::size_t n);
|
||||
~RedisPool();
|
||||
|
||||
RedisPool(const RedisPool&) = delete;
|
||||
RedisPool& operator=(const RedisPool&) = delete;
|
||||
|
||||
class Lease {
|
||||
public:
|
||||
explicit Lease(RedisPool& pool);
|
||||
~Lease();
|
||||
Lease(Lease&& other) noexcept;
|
||||
Lease& operator=(Lease&&) = delete;
|
||||
Lease(const Lease&) = delete;
|
||||
Lease& operator=(const Lease&) = delete;
|
||||
|
||||
redisContext* ctx() const { return ctx_; }
|
||||
|
||||
private:
|
||||
RedisPool* pool_;
|
||||
redisContext* ctx_;
|
||||
};
|
||||
|
||||
Lease acquire();
|
||||
|
||||
bool ping();
|
||||
|
||||
private:
|
||||
friend class Lease;
|
||||
|
||||
struct RedisDeleter {
|
||||
void operator()(redisContext* c) const noexcept {
|
||||
if (c != nullptr) ::redisFree(c);
|
||||
}
|
||||
};
|
||||
using ConnPtr = std::unique_ptr<redisContext, RedisDeleter>;
|
||||
|
||||
redisContext* take();
|
||||
void give_back(redisContext* c);
|
||||
|
||||
std::string url_;
|
||||
std::vector<ConnPtr> all_;
|
||||
std::vector<redisContext*> idle_;
|
||||
std::mutex mu_;
|
||||
std::condition_variable cv_;
|
||||
bool shutting_down_ = false;
|
||||
};
|
||||
@@ -0,0 +1,290 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "s3_client.hpp"
|
||||
|
||||
#include <miniocpp/args.h>
|
||||
#include <miniocpp/client.h>
|
||||
#include <miniocpp/providers.h>
|
||||
#include <miniocpp/request.h>
|
||||
#include <miniocpp/response.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string read_file_trim(const std::string& path) {
|
||||
std::ifstream in(path);
|
||||
if (!in) {
|
||||
throw std::runtime_error("cannot open secret/file: " + path);
|
||||
}
|
||||
std::string s((std::istreambuf_iterator<char>(in)),
|
||||
std::istreambuf_iterator<char>());
|
||||
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) {
|
||||
s.pop_back();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string env_or(const char* key, const char* fallback = "") {
|
||||
const char* v = std::getenv(key);
|
||||
return (v && *v) ? std::string(v) : std::string(fallback);
|
||||
}
|
||||
|
||||
void parse_endpoint(const std::string& endpoint, std::string& host, unsigned& port,
|
||||
bool& https) {
|
||||
https = true;
|
||||
host = endpoint;
|
||||
port = 0;
|
||||
if (host.rfind("http://", 0) == 0) {
|
||||
https = false;
|
||||
host = host.substr(7);
|
||||
} else if (host.rfind("https://", 0) == 0) {
|
||||
https = true;
|
||||
host = host.substr(8);
|
||||
}
|
||||
// strip path
|
||||
const auto slash = host.find('/');
|
||||
if (slash != std::string::npos) {
|
||||
host = host.substr(0, slash);
|
||||
}
|
||||
const auto colon = host.rfind(':');
|
||||
if (colon != std::string::npos) {
|
||||
port = static_cast<unsigned>(std::stoul(host.substr(colon + 1)));
|
||||
host = host.substr(0, colon);
|
||||
}
|
||||
if (port == 0) {
|
||||
port = https ? 443u : 80u;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct S3Client::Impl {
|
||||
minio::s3::BaseUrl base_url;
|
||||
minio::s3::BaseUrl presign_url;
|
||||
std::unique_ptr<minio::creds::StaticProvider> provider;
|
||||
std::unique_ptr<minio::s3::Client> client;
|
||||
std::unique_ptr<minio::s3::Client> presign_client; // null → use client
|
||||
};
|
||||
|
||||
S3Client::Config S3Client::from_env() {
|
||||
Config c;
|
||||
// Ops endpoint (Stat/Get/Put/Delete). Presign may use S3_PRESIGN_ENDPOINT
|
||||
// (read in ctor) so signed Host matches the client-reachable address.
|
||||
c.endpoint = env_or("S3_ENDPOINT", "http://minio:9000");
|
||||
c.region = env_or("S3_REGION", "us-east-1");
|
||||
c.bucket = env_or("S3_BUCKET");
|
||||
if (c.bucket.empty()) {
|
||||
throw std::runtime_error("S3_BUCKET is required");
|
||||
}
|
||||
|
||||
const std::string key_file = env_or("S3_ACCESS_KEY_ID_FILE");
|
||||
const std::string secret_file = env_or("S3_SECRET_ACCESS_KEY_FILE");
|
||||
if (key_file.empty() || secret_file.empty()) {
|
||||
throw std::runtime_error(
|
||||
"S3_ACCESS_KEY_ID_FILE and S3_SECRET_ACCESS_KEY_FILE are required");
|
||||
}
|
||||
c.access_key = read_file_trim(key_file);
|
||||
c.secret_key = read_file_trim(secret_file);
|
||||
if (c.access_key.empty() || c.secret_key.empty()) {
|
||||
throw std::runtime_error("S3 credentials files are empty");
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
S3Client::S3Client(Config cfg) : cfg_(std::move(cfg)), impl_(std::make_unique<Impl>()) {
|
||||
if (cfg_.endpoint.empty() || cfg_.bucket.empty() ||
|
||||
cfg_.access_key.empty() || cfg_.secret_key.empty()) {
|
||||
throw std::runtime_error("S3Client: incomplete config");
|
||||
}
|
||||
|
||||
std::string host;
|
||||
unsigned port = 0;
|
||||
bool https = false;
|
||||
parse_endpoint(cfg_.endpoint, host, port, https);
|
||||
|
||||
impl_->base_url = minio::s3::BaseUrl(host, https, cfg_.region);
|
||||
impl_->base_url.port = port;
|
||||
// MinIO requires path-style addressing.
|
||||
impl_->base_url.virtual_style = false;
|
||||
|
||||
impl_->provider = std::make_unique<minio::creds::StaticProvider>(
|
||||
cfg_.access_key, cfg_.secret_key);
|
||||
impl_->client =
|
||||
std::make_unique<minio::s3::Client>(impl_->base_url, impl_->provider.get());
|
||||
|
||||
// Separate BaseUrl for presigning when S3_PRESIGN_ENDPOINT differs.
|
||||
const std::string presign_ep = env_or("S3_PRESIGN_ENDPOINT");
|
||||
if (!presign_ep.empty() && presign_ep != cfg_.endpoint) {
|
||||
std::string ph;
|
||||
unsigned pp = 0;
|
||||
bool phttps = false;
|
||||
parse_endpoint(presign_ep, ph, pp, phttps);
|
||||
impl_->presign_url = minio::s3::BaseUrl(ph, phttps, cfg_.region);
|
||||
impl_->presign_url.port = pp;
|
||||
impl_->presign_url.virtual_style = false;
|
||||
impl_->presign_client = std::make_unique<minio::s3::Client>(
|
||||
impl_->presign_url, impl_->provider.get());
|
||||
}
|
||||
}
|
||||
|
||||
S3Client::~S3Client() = default;
|
||||
|
||||
void S3Client::put_object(const std::string& key,
|
||||
std::istream& body,
|
||||
std::int64_t size_bytes,
|
||||
const std::string& content_type) {
|
||||
telemetry::Span operation("s3.put_object", opentelemetry::trace::SpanKind::kClient);
|
||||
if (size_bytes < 0) {
|
||||
throw std::invalid_argument("put_object: negative size");
|
||||
}
|
||||
|
||||
// part_size=0 → library chooses multipart sizing; streams from `body`.
|
||||
minio::s3::PutObjectArgs args(body, static_cast<long>(size_bytes), 0);
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
if (!content_type.empty()) {
|
||||
args.content_type = content_type;
|
||||
}
|
||||
|
||||
minio::s3::PutObjectResponse resp = impl_->client->PutObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 PutObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
void S3Client::get_object(const std::string& key, std::ostream& out) {
|
||||
telemetry::Span operation("s3.get_object", opentelemetry::trace::SpanKind::kClient);
|
||||
get_object(key, [&](const char* data, std::size_t n) {
|
||||
out.write(data, static_cast<std::streamsize>(n));
|
||||
if (!out) {
|
||||
throw std::runtime_error("get_object: write failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::int64_t S3Client::get_object(
|
||||
const std::string& key,
|
||||
const std::function<void(const char*, std::size_t)>& on_chunk) {
|
||||
telemetry::Span operation("s3.get_object", opentelemetry::trace::SpanKind::kClient);
|
||||
std::int64_t total = 0;
|
||||
minio::s3::GetObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
args.datafunc = [&](minio::http::DataFunctionArgs da) -> bool {
|
||||
on_chunk(da.datachunk.data(), da.datachunk.size());
|
||||
total += static_cast<std::int64_t>(da.datachunk.size());
|
||||
return true;
|
||||
};
|
||||
|
||||
minio::s3::GetObjectResponse resp = impl_->client->GetObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 GetObject failed: " + resp.Error().String());
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
void S3Client::copy_object(const std::string& src_key,
|
||||
const std::string& dst_key) {
|
||||
telemetry::Span operation("s3.copy_object", opentelemetry::trace::SpanKind::kClient);
|
||||
if (src_key.empty() || dst_key.empty() || src_key == dst_key) {
|
||||
throw std::invalid_argument("copy_object: bad keys");
|
||||
}
|
||||
minio::s3::CopyObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = dst_key;
|
||||
args.source.bucket = cfg_.bucket;
|
||||
args.source.object = src_key;
|
||||
minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
void S3Client::delete_object(const std::string& key) {
|
||||
telemetry::Span operation("s3.delete_object", opentelemetry::trace::SpanKind::kClient);
|
||||
minio::s3::RemoveObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
minio::s3::RemoveObjectResponse resp = impl_->client->RemoveObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 DeleteObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
bool S3Client::stat_object(const std::string& key, std::int64_t& size_bytes,
|
||||
std::string& etag) {
|
||||
telemetry::Span operation("s3.stat_object", opentelemetry::trace::SpanKind::kClient);
|
||||
minio::s3::StatObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
minio::s3::StatObjectResponse resp = impl_->client->StatObject(args);
|
||||
if (!resp) {
|
||||
// NoSuchKey / Not Found → absent
|
||||
const std::string err = resp.Error().String();
|
||||
if (resp.code == "NoSuchKey" || resp.status_code == 404) {
|
||||
return false;
|
||||
}
|
||||
throw std::runtime_error("S3 StatObject failed: " + err);
|
||||
}
|
||||
size_bytes = static_cast<std::int64_t>(resp.size);
|
||||
etag = resp.etag;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string S3Client::presign_put(const std::string& key,
|
||||
unsigned expiry_seconds,
|
||||
const std::string& content_type) {
|
||||
telemetry::Span operation("s3.presign_put", opentelemetry::trace::SpanKind::kClient);
|
||||
if (expiry_seconds < 1 || expiry_seconds > 7 * 24 * 3600u) {
|
||||
throw std::invalid_argument("presign_put: bad expiry");
|
||||
}
|
||||
minio::s3::Client* c =
|
||||
impl_->presign_client ? impl_->presign_client.get() : impl_->client.get();
|
||||
|
||||
// Note: minio-cpp query-presign signs method/host/path/query only — it does
|
||||
// not bind Content-Type into the signature. Clients should still send
|
||||
// Content-Type so the object metadata is correct; complete() re-validates
|
||||
// magic/sha256 against the session.
|
||||
(void)content_type;
|
||||
minio::s3::GetPresignedObjectUrlArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
args.method = minio::http::Method::kPut;
|
||||
args.expiry_seconds = expiry_seconds;
|
||||
|
||||
minio::s3::GetPresignedObjectUrlResponse resp = c->GetPresignedObjectUrl(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 PresignPut failed: " + resp.Error().String());
|
||||
}
|
||||
return resp.url;
|
||||
}
|
||||
|
||||
std::vector<std::string> S3Client::old_staging(const std::string& cursor,
|
||||
int retention_seconds,
|
||||
std::string& next_cursor) {
|
||||
telemetry::Span operation("s3.old_staging", opentelemetry::trace::SpanKind::kClient);
|
||||
minio::s3::ListObjectsArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.prefix = "staging/";
|
||||
args.recursive = true;
|
||||
args.max_keys = 100;
|
||||
args.start_after = cursor;
|
||||
auto cutoff = minio::utils::UtcTime::Now();
|
||||
cutoff.Add(-retention_seconds);
|
||||
auto result = impl_->client->ListObjects(args);
|
||||
std::vector<std::string> keys;
|
||||
next_cursor.clear();
|
||||
for (int count = 0; result && count < 100; ++count) {
|
||||
const auto& item = *result;
|
||||
if (!item) throw std::runtime_error("S3 staging listing failed");
|
||||
if (!item.is_prefix && item.last_modified <= cutoff) keys.push_back(item.name);
|
||||
if (count == 99) {
|
||||
next_cursor = item.name;
|
||||
break;
|
||||
}
|
||||
++result;
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <istream>
|
||||
#include <memory>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Thin wrapper over AWS SDK for C++ (S3 only). Credentials and SigV4 come from
|
||||
// the SDK — no hand-rolled signing.
|
||||
|
||||
class S3Client {
|
||||
public:
|
||||
struct Config {
|
||||
std::string endpoint; // e.g. http://minio:9000
|
||||
std::string region; // e.g. us-east-1
|
||||
std::string bucket;
|
||||
std::string access_key;
|
||||
std::string secret_key;
|
||||
};
|
||||
|
||||
explicit S3Client(Config cfg);
|
||||
~S3Client();
|
||||
|
||||
S3Client(const S3Client&) = delete;
|
||||
S3Client& operator=(const S3Client&) = delete;
|
||||
|
||||
static Config from_env();
|
||||
|
||||
const std::string& bucket() const { return cfg_.bucket; }
|
||||
|
||||
// Stream upload. `body` must yield exactly `size_bytes` bytes.
|
||||
void put_object(const std::string& key,
|
||||
std::istream& body,
|
||||
std::int64_t size_bytes,
|
||||
const std::string& content_type);
|
||||
|
||||
// Stream download into `out` (chunked reads from SDK body stream).
|
||||
void get_object(const std::string& key, std::ostream& out);
|
||||
|
||||
// Stream download with a consumer callback (chunk, size). Returns total bytes.
|
||||
std::int64_t get_object(const std::string& key,
|
||||
const std::function<void(const char*, std::size_t)>& on_chunk);
|
||||
|
||||
void delete_object(const std::string& key);
|
||||
|
||||
// At most 100 entries examined per call; cursor wraps to empty at end.
|
||||
std::vector<std::string> old_staging(const std::string& cursor,
|
||||
int retention_seconds, std::string& next_cursor);
|
||||
|
||||
// Server-side copy inside the bucket (complete: staging → final).
|
||||
void copy_object(const std::string& src_key, const std::string& dst_key);
|
||||
|
||||
// HeadObject-style metadata. Returns false if the object does not exist.
|
||||
bool stat_object(const std::string& key, std::int64_t& size_bytes,
|
||||
std::string& etag);
|
||||
|
||||
// Presigned PUT URL via minio-cpp (SigV4). Optional required Content-Type
|
||||
// is bound into the signature when non-empty.
|
||||
// expiry_seconds: typically 60–600 for Track 1.
|
||||
std::string presign_put(const std::string& key,
|
||||
unsigned expiry_seconds,
|
||||
const std::string& content_type);
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
Config cfg_;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
@@ -0,0 +1,87 @@
|
||||
#include "session_repo.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <sodium.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string random_token_hex() {
|
||||
unsigned char buf[32];
|
||||
randombytes_buf(buf, sizeof(buf));
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.resize(sizeof(buf) * 2);
|
||||
for (std::size_t i = 0; i < sizeof(buf); ++i) {
|
||||
out[i * 2] = hex[buf[i] >> 4];
|
||||
out[i * 2 + 1] = hex[buf[i] & 0x0f];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::int64_t now_unix() {
|
||||
return static_cast<std::int64_t>(std::time(nullptr));
|
||||
}
|
||||
|
||||
redisReply* check_reply(redisContext* c, redisReply* r) {
|
||||
if (r == nullptr) {
|
||||
const std::string err = c->err ? c->errstr : "no reply";
|
||||
throw DependencyUnavailable("redis command failed: " + err);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ISessionRepo::Created SessionRepo::create(const std::string& user_id) {
|
||||
const std::string token = random_token_hex();
|
||||
const std::int64_t exp = now_unix() + ttl_;
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "SET session:%s %s EX %d",
|
||||
token.c_str(), user_id.c_str(), ttl_));
|
||||
check_reply(c, r);
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
if (!ok) {
|
||||
throw std::runtime_error("redis SET returned non-status");
|
||||
}
|
||||
|
||||
return Created{token, exp};
|
||||
}
|
||||
|
||||
std::optional<std::string> SessionRepo::lookup(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "GET session:%s", token.c_str()));
|
||||
check_reply(c, r);
|
||||
|
||||
std::optional<std::string> out;
|
||||
if (r->type == REDIS_REPLY_STRING) {
|
||||
out = std::string(r->str, r->len);
|
||||
}
|
||||
freeReplyObject(r);
|
||||
return out;
|
||||
}
|
||||
|
||||
bool SessionRepo::revoke(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "DEL session:%s", token.c_str()));
|
||||
check_reply(c, r);
|
||||
|
||||
const bool existed = (r->type == REDIS_REPLY_INTEGER && r->integer == 1);
|
||||
freeReplyObject(r);
|
||||
return existed;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "redis_pool.hpp"
|
||||
#include "session_repo_base.hpp"
|
||||
|
||||
class SessionRepo : public ISessionRepo {
|
||||
public:
|
||||
SessionRepo(RedisPool& pool, int ttl_seconds)
|
||||
: pool_(pool), ttl_(ttl_seconds) {}
|
||||
|
||||
Created create(const std::string& user_id) override;
|
||||
std::optional<std::string> lookup(const std::string& token) override;
|
||||
bool revoke(const std::string& token) override;
|
||||
|
||||
private:
|
||||
RedisPool& pool_;
|
||||
int ttl_;
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
|
||||
class ISessionRepo {
|
||||
public:
|
||||
virtual ~ISessionRepo() = default;
|
||||
|
||||
struct Created {
|
||||
std::string token;
|
||||
std::int64_t expires_at; // unix-время в секундах UTC
|
||||
};
|
||||
|
||||
virtual Created create(const std::string& user_id) = 0;
|
||||
|
||||
virtual std::optional<std::string> lookup(const std::string& token) = 0;
|
||||
|
||||
virtual bool revoke(const std::string& token) = 0;
|
||||
};
|
||||
@@ -0,0 +1,77 @@
|
||||
#include "staging_cleaner.hpp"
|
||||
#include "attachment_storage.hpp"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <regex>
|
||||
|
||||
void StagingCleaner::clean(const std::string& key) {
|
||||
static const std::regex pattern(
|
||||
"^staging/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/"
|
||||
"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$");
|
||||
std::smatch match;
|
||||
if (!std::regex_match(key, match, pattern)) return;
|
||||
const std::string id = match[1];
|
||||
auto lease = pg_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
if (!tx.exec("SELECT pg_try_advisory_xact_lock(hashtextextended('staging:' || $1::uuid::text, 0))",
|
||||
pqxx::params{id})[0][0].as<bool>()) return;
|
||||
auto rows = tx.exec(
|
||||
"SELECT a.chat_id, a.staging_key, a.s3_key, a.size_bytes, a.variants, a.status, j.id AS job_id, "
|
||||
"a.updated_at < now()-($2::bigint * interval '1 second') AS expired "
|
||||
"FROM attachments a LEFT JOIN jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version "
|
||||
"WHERE a.id=$1::uuid FOR UPDATE OF a SKIP LOCKED", pqxx::params{id, retention_seconds_});
|
||||
if (rows.empty()) {
|
||||
// An existing locked row is not an orphan.
|
||||
if (!tx.exec("SELECT 1 FROM attachments WHERE id=$1::uuid", pqxx::params{id}).empty()) return;
|
||||
} else {
|
||||
const auto& row = rows[0];
|
||||
if (row["staging_key"].is_null() || row["staging_key"].as<std::string>() != key) return;
|
||||
const std::string status = row["status"].as<std::string>();
|
||||
if (status != "ready" && !(status == "failed" && row["expired"].as<bool>())) return;
|
||||
if (!row["job_id"].is_null() && !tx.exec(
|
||||
"SELECT pg_try_advisory_xact_lock(hashtextextended($1::uuid::text, 0))",
|
||||
pqxx::params{row["job_id"].as<std::string>()})[0][0].as<bool>()) return;
|
||||
if (!tx.exec("SELECT 1 FROM jobs WHERE attachment_id=$1::uuid AND source_key=$2 "
|
||||
"AND status IN ('queued','processing')", pqxx::params{id,key}).empty()) return;
|
||||
if (status == "ready") {
|
||||
std::int64_t size = 0;
|
||||
std::string etag;
|
||||
if (!s3_.stat_object(row["s3_key"].as<std::string>(), size, etag) ||
|
||||
size != row["size_bytes"].as<int>()) return;
|
||||
for (const auto& variant : nlohmann::json::parse(row["variants"].as<std::string>())) {
|
||||
const std::string object_key = variant.value("storage_key",
|
||||
attachment_variant_key(row["chat_id"].as<std::string>(), id, variant.at("name").get<std::string>()));
|
||||
if (!s3_.stat_object(object_key, size, etag) ||
|
||||
size != variant.at("size_bytes").get<std::int64_t>()) return;
|
||||
}
|
||||
}
|
||||
}
|
||||
// A still-valid presigned URL can recreate the key: wait for its expiry.
|
||||
if (!tx.exec("SELECT 1 FROM direct_uploads WHERE staging_key=$1 AND expires_at>now()",
|
||||
pqxx::params{key}).empty()) return;
|
||||
s3_.delete_object(key);
|
||||
if (!rows.empty()) {
|
||||
// Internal cleanup does not change public metadata/updated_at.
|
||||
tx.exec("UPDATE attachments SET staging_cleaned_at=now() WHERE id=$1::uuid", pqxx::params{id});
|
||||
}
|
||||
tx.commit();
|
||||
}
|
||||
|
||||
void StagingCleaner::sweep() {
|
||||
std::vector<std::string> known;
|
||||
{
|
||||
auto lease = pg_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
for (const auto& row : tx.exec(
|
||||
"SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL "
|
||||
"AND staging_cleaned_at IS NULL AND (status='ready' OR "
|
||||
"(status='failed' AND updated_at<now()-($1::bigint * interval '1 second'))) "
|
||||
"ORDER BY updated_at, id LIMIT 32", pqxx::params{retention_seconds_})) {
|
||||
known.push_back(row[0].as<std::string>());
|
||||
}
|
||||
}
|
||||
for (const auto& key : known) clean(key);
|
||||
std::string next;
|
||||
for (const auto& key : s3_.old_staging(cursor_, retention_seconds_, next)) clean(key);
|
||||
cursor_ = std::move(next);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "s3_client.hpp"
|
||||
#include <string>
|
||||
|
||||
class StagingCleaner {
|
||||
public:
|
||||
StagingCleaner(PgPool& pg, S3Client& s3, int retention_seconds)
|
||||
: pg_(pg), s3_(s3), retention_seconds_(retention_seconds) {}
|
||||
void clean(const std::string& key);
|
||||
void sweep();
|
||||
private:
|
||||
PgPool& pg_;
|
||||
S3Client& s3_;
|
||||
int retention_seconds_;
|
||||
std::string cursor_;
|
||||
};
|
||||
@@ -0,0 +1,270 @@
|
||||
#include "telemetry.hpp"
|
||||
#include <opentelemetry/context/propagation/text_map_propagator.h>
|
||||
#include <opentelemetry/trace/propagation/http_trace_context.h>
|
||||
#include <opentelemetry/exporters/otlp/otlp_http_exporter_factory.h>
|
||||
#include <opentelemetry/exporters/otlp/otlp_http_metric_exporter_factory.h>
|
||||
#include <opentelemetry/exporters/otlp/otlp_http_log_record_exporter_factory.h>
|
||||
#include <opentelemetry/sdk/trace/batch_span_processor_factory.h>
|
||||
#include <opentelemetry/sdk/trace/tracer_provider.h>
|
||||
#include <opentelemetry/sdk/trace/samplers/parent.h>
|
||||
#include <opentelemetry/sdk/metrics/meter_provider.h>
|
||||
#include <opentelemetry/sdk/metrics/export/periodic_exporting_metric_reader_factory.h>
|
||||
#include <opentelemetry/sdk/metrics/aggregation/aggregation_config.h>
|
||||
#include <opentelemetry/sdk/logs/batch_log_record_processor_factory.h>
|
||||
#include <opentelemetry/sdk/logs/logger_provider.h>
|
||||
#include <opentelemetry/logs/log_record.h>
|
||||
#include <opentelemetry/metrics/observer_result.h>
|
||||
#include <opentelemetry/sdk/common/global_log_handler.h>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cstdlib>
|
||||
#include <exception>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <regex>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace telemetry {
|
||||
namespace ot = opentelemetry;
|
||||
namespace ts = ot::sdk::trace;
|
||||
namespace ms = ot::sdk::metrics;
|
||||
namespace ls = ot::sdk::logs;
|
||||
namespace ex = ot::exporter::otlp;
|
||||
namespace {
|
||||
std::shared_ptr<ts::TracerProvider> traces;
|
||||
std::shared_ptr<ms::MeterProvider> metrics;
|
||||
std::shared_ptr<ls::LoggerProvider> logs;
|
||||
ot::nostd::shared_ptr<ot::logs::Logger> logger;
|
||||
ot::nostd::unique_ptr<ot::metrics::Counter<uint64_t>> requests, jobs;
|
||||
ot::nostd::unique_ptr<ot::metrics::Histogram<double>> http_duration, job_duration, ready_duration;
|
||||
struct Gauge {
|
||||
std::atomic<double> value{0};
|
||||
ot::nostd::shared_ptr<ot::metrics::ObservableInstrument> instrument;
|
||||
};
|
||||
std::map<std::string, Gauge> gauges;
|
||||
thread_local HttpRequest* current_http = nullptr;
|
||||
struct Carrier : ot::context::propagation::TextMapCarrier {
|
||||
std::map<std::string, std::string> fields;
|
||||
ot::nostd::string_view Get(ot::nostd::string_view key) const noexcept override {
|
||||
auto i = fields.find(std::string(key));
|
||||
return i == fields.end() ? ot::nostd::string_view{} : ot::nostd::string_view(i->second);
|
||||
}
|
||||
void Set(ot::nostd::string_view key, ot::nostd::string_view value) noexcept override {
|
||||
fields[std::string(key)] = std::string(value);
|
||||
}
|
||||
};
|
||||
ot::trace::propagation::HttpTraceContext propagation;
|
||||
class Diagnostics : public ot::sdk::common::internal_log::LogHandler {
|
||||
public:
|
||||
void Handle(ot::sdk::common::internal_log::LogLevel level, const char*, int, const char* message,
|
||||
const ot::sdk::common::AttributeMap&) noexcept override {
|
||||
// Count diagnostics without copying backend errors, URLs or payloads into logs.
|
||||
delta("telemetry.sdk.errors", 1);
|
||||
if (message && std::string_view(message).find("BatchSpanProcessor queue is full") != std::string_view::npos)
|
||||
delta("telemetry.sdk.dropped.spans", 1);
|
||||
}
|
||||
};
|
||||
void observe(ot::metrics::ObserverResult result, void* state) {
|
||||
ot::nostd::get<ot::nostd::shared_ptr<ot::metrics::ObserverResultT<double>>>(result)
|
||||
->Observe(static_cast<Gauge*>(state)->value.load());
|
||||
}
|
||||
}
|
||||
Runtime::Runtime(const char* service) noexcept {
|
||||
try {
|
||||
if (const auto* disabled = std::getenv("OTEL_SDK_DISABLED"); disabled && std::string(disabled) == "true") return;
|
||||
// Do not dump exporter payloads or third party error strings to stderr.
|
||||
ot::sdk::common::internal_log::GlobalLogHandler::SetLogLevel(
|
||||
ot::sdk::common::internal_log::LogLevel::None);
|
||||
char host[256]{};
|
||||
gethostname(host, sizeof(host)-1);
|
||||
auto resource = ot::sdk::resource::Resource::Create({
|
||||
{"service.name", service}, {"service.version", "lab6-1.0"},
|
||||
{"service.instance.id", host}, {"deployment.environment.name", "lab"}});
|
||||
ex::OtlpHttpExporterOptions to;
|
||||
to.timeout = std::chrono::milliseconds(500);
|
||||
to.console_debug = false;
|
||||
to.retry_policy_max_attempts = 1;
|
||||
to.content_type = ex::HttpRequestContentType::kBinary;
|
||||
ts::BatchSpanProcessorOptions batch;
|
||||
batch.max_queue_size = 1024;
|
||||
batch.max_export_batch_size = 128;
|
||||
batch.schedule_delay_millis = std::chrono::milliseconds(500);
|
||||
traces = std::make_shared<ts::TracerProvider>(
|
||||
ts::BatchSpanProcessorFactory::Create(ex::OtlpHttpExporterFactory::Create(to), batch), resource,
|
||||
std::make_unique<ts::ParentBasedSampler>(std::make_shared<ts::AlwaysOnSampler>()));
|
||||
std::shared_ptr<ot::trace::TracerProvider> api_traces = traces;
|
||||
ot::trace::Provider::SetTracerProvider(api_traces);
|
||||
ex::OtlpHttpMetricExporterOptions mo;
|
||||
mo.timeout = std::chrono::milliseconds(500);
|
||||
mo.console_debug = false;
|
||||
mo.retry_policy_max_attempts = 1;
|
||||
mo.content_type = ex::HttpRequestContentType::kBinary;
|
||||
metrics = std::make_shared<ms::MeterProvider>(std::make_unique<ms::ViewRegistry>(), resource);
|
||||
for (auto name : {"http.server.request.duration", "worker.job.duration", "attachment.ready.duration"}) {
|
||||
auto config = std::make_shared<ms::HistogramAggregationConfig>();
|
||||
config->boundaries_ = {0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60,120};
|
||||
metrics->AddView(std::make_unique<ms::InstrumentSelector>(ms::InstrumentType::kHistogram, name, "s"),
|
||||
std::make_unique<ms::MeterSelector>("messenger", "1.0", ""),
|
||||
std::make_unique<ms::View>(name, "", "s", ms::AggregationType::kHistogram, config));
|
||||
}
|
||||
ms::PeriodicExportingMetricReaderOptions ro;
|
||||
ro.export_interval_millis = std::chrono::milliseconds(5000);
|
||||
ro.export_timeout_millis = std::chrono::milliseconds(1000);
|
||||
metrics->AddMetricReader(ms::PeriodicExportingMetricReaderFactory::Create(
|
||||
ex::OtlpHttpMetricExporterFactory::Create(mo), ro));
|
||||
auto meter = metrics->GetMeter("messenger", "1.0");
|
||||
requests = meter->CreateUInt64Counter("http.server.requests", "Business HTTP requests", "{request}");
|
||||
jobs = meter->CreateUInt64Counter("worker.jobs", "Delivery outcomes", "{job}");
|
||||
http_duration = meter->CreateDoubleHistogram("http.server.request.duration", "Request latency", "s");
|
||||
job_duration = meter->CreateDoubleHistogram("worker.job.duration", "Delivery processing latency", "s");
|
||||
ready_duration = meter->CreateDoubleHistogram("attachment.ready.duration", "Durable acceptance to ready", "s");
|
||||
for (auto name : {"queue.depth", "queue.oldest.age", "queue.observation.timestamp",
|
||||
"db.pool.used", "db.pool.waiting", "db.pool.capacity", "worker.busy", "worker.capacity",
|
||||
"process.cpu.time", "process.memory.usage", "telemetry.sdk.errors", "telemetry.sdk.dropped.spans"}) {
|
||||
auto& g = gauges[name];
|
||||
g.instrument = meter->CreateDoubleObservableGauge(name);
|
||||
g.instrument->AddCallback(observe, &g);
|
||||
}
|
||||
ex::OtlpHttpLogRecordExporterOptions lo;
|
||||
lo.timeout = std::chrono::milliseconds(500);
|
||||
lo.console_debug = false;
|
||||
lo.retry_policy_max_attempts = 1;
|
||||
lo.content_type = ex::HttpRequestContentType::kBinary;
|
||||
ls::BatchLogRecordProcessorOptions lb;
|
||||
lb.max_queue_size = 1024;
|
||||
lb.max_export_batch_size = 128;
|
||||
lb.schedule_delay_millis = std::chrono::milliseconds(500);
|
||||
logs = std::make_shared<ls::LoggerProvider>(ls::BatchLogRecordProcessorFactory::Create(
|
||||
ex::OtlpHttpLogRecordExporterFactory::Create(lo), lb), resource);
|
||||
logger = logs->GetLogger("messenger", "messenger", "1.0", "", ot::common::NoopKeyValueIterable());
|
||||
ot::sdk::common::internal_log::GlobalLogHandler::SetLogHandler(
|
||||
ot::nostd::shared_ptr<ot::sdk::common::internal_log::LogHandler>(new Diagnostics));
|
||||
ot::sdk::common::internal_log::GlobalLogHandler::SetLogLevel(
|
||||
ot::sdk::common::internal_log::LogLevel::Warning);
|
||||
event("service.started");
|
||||
} catch (...) { /* Telemetry setup must not prevent application startup. */ }
|
||||
}
|
||||
Runtime::~Runtime() {
|
||||
if (metrics) metrics->Shutdown(std::chrono::seconds(2));
|
||||
if (logs) logs->Shutdown(std::chrono::seconds(2));
|
||||
if (traces) traces->Shutdown(std::chrono::seconds(2));
|
||||
}
|
||||
Span::Span(const std::string& name, ot::trace::SpanKind kind,
|
||||
const std::string& parent, const std::string& state) : exceptions_(std::uncaught_exceptions()) {
|
||||
ot::trace::StartSpanOptions options;
|
||||
options.kind = kind;
|
||||
if (!parent.empty()) {
|
||||
Carrier carrier;
|
||||
if (parent.size() <= 256) carrier.fields["traceparent"] = parent;
|
||||
if (state.size() <= 512) carrier.fields["tracestate"] = state;
|
||||
auto root = ot::context::Context{};
|
||||
options.parent = propagation.Extract(carrier, root);
|
||||
}
|
||||
span = ot::trace::Provider::GetTracerProvider()->GetTracer("messenger", "1.0")->StartSpan(name, options);
|
||||
scope_ = std::make_unique<ot::trace::Scope>(span);
|
||||
}
|
||||
Span::~Span() {
|
||||
if (std::uncaught_exceptions() > exceptions_) error();
|
||||
span->End();
|
||||
}
|
||||
void Span::error() noexcept { span->SetStatus(ot::trace::StatusCode::kError); }
|
||||
std::string inject(const char* field) {
|
||||
Carrier carrier;
|
||||
auto context = ot::context::RuntimeContext::GetCurrent();
|
||||
propagation.Inject(carrier, context);
|
||||
return carrier.fields[field];
|
||||
}
|
||||
void event(const char* name, bool failed, const std::string& request_id) noexcept {
|
||||
if (!logger) return;
|
||||
auto record = logger->CreateLogRecord();
|
||||
record->SetTimestamp(std::chrono::system_clock::now());
|
||||
record->SetSeverity(failed ? ot::logs::Severity::kError : ot::logs::Severity::kInfo);
|
||||
record->SetBody(name);
|
||||
record->SetAttribute("event", name);
|
||||
if (!request_id.empty()) record->SetAttribute("request_id", request_id);
|
||||
const auto context = ot::trace::GetSpan(ot::context::RuntimeContext::GetCurrent())->GetContext();
|
||||
if (context.IsValid()) {
|
||||
record->SetTraceId(context.trace_id());
|
||||
record->SetSpanId(context.span_id());
|
||||
record->SetTraceFlags(context.trace_flags());
|
||||
}
|
||||
logger->EmitLogRecord(std::move(record));
|
||||
}
|
||||
void gauge(const char* name, double value) noexcept {
|
||||
auto i = gauges.find(name);
|
||||
if (i != gauges.end()) i->second.value.store(value);
|
||||
}
|
||||
void delta(const char* name, long value) noexcept {
|
||||
auto i = gauges.find(name);
|
||||
if (i != gauges.end()) {
|
||||
auto old = i->second.value.load();
|
||||
while (!i->second.value.compare_exchange_weak(old, old + value)) {}
|
||||
}
|
||||
}
|
||||
void duration(const char*, double seconds) noexcept {
|
||||
if (ready_duration) ready_duration->Record(seconds, ot::context::Context{});
|
||||
}
|
||||
std::string route_template(const std::string& path) {
|
||||
// Whitelist full templates; unknown/hostile paths never enter telemetry.
|
||||
static const std::pair<std::regex, const char*> routes[] = {
|
||||
{std::regex("^/api/v1/chats/[^/]+/messages$"), "/api/v1/chats/{chat_id}/messages"},
|
||||
{std::regex("^/api/v1/chats/[^/]+/attachments$"), "/api/v1/chats/{chat_id}/attachments"},
|
||||
{std::regex("^/api/v1/chats/[^/]+/direct-uploads$"), "/api/v1/chats/{chat_id}/direct-uploads"},
|
||||
{std::regex("^/api/v1/chats/[^/]+$"), "/api/v1/chats/{chat_id}"},
|
||||
{std::regex("^/api/v1/attachments/[^/]+/content$"), "/api/v1/attachments/{attachment_id}/content"},
|
||||
{std::regex("^/api/v1/attachments/[^/]+$"), "/api/v1/attachments/{attachment_id}"},
|
||||
{std::regex("^/api/v1/direct-uploads/[^/]+/complete$"), "/api/v1/direct-uploads/{upload_id}/complete"},
|
||||
{std::regex("^/api/v1/direct-uploads/[^/]+$"), "/api/v1/direct-uploads/{upload_id}"}};
|
||||
for (auto exact : {"/api/v1/users", "/api/v1/users/me", "/api/v1/chats", "/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/health/live", "/health/ready"})
|
||||
if (path == exact) return exact;
|
||||
for (const auto& route : routes) if (std::regex_match(path, route.first)) return route.second;
|
||||
return "unmatched";
|
||||
}
|
||||
HttpRequest::HttpRequest() : start_(std::chrono::steady_clock::now()), previous_(current_http) { current_http = this; }
|
||||
void HttpRequest::configure(const std::string& method, const std::string& path,
|
||||
const std::string& parent, const std::string& state, const std::string& id) {
|
||||
for (auto known : {"GET","POST","PUT","DELETE","PATCH","HEAD","OPTIONS"}) if (method == known) method_ = known;
|
||||
status = 500;
|
||||
route_ = route_template(path);
|
||||
id_ = id;
|
||||
span_ = std::make_unique<Span>(method_ + " " + route_, ot::trace::SpanKind::kServer, parent, state);
|
||||
span_->span->SetAttribute("http.request.method", method_);
|
||||
span_->span->SetAttribute("http.route", route_);
|
||||
}
|
||||
HttpRequest::~HttpRequest() {
|
||||
current_http = previous_;
|
||||
if (status == 0 || route_.rfind("/health/", 0) == 0) return;
|
||||
const std::map<std::string, std::string> attrs = {{"http.request.method", method_},
|
||||
{"http.route", route_}, {"http.response.status_class", std::to_string(status/100)+"xx"}};
|
||||
if (requests) requests->Add(1, attrs);
|
||||
if (http_duration) http_duration->Record(std::chrono::duration<double>(std::chrono::steady_clock::now()-start_).count(), attrs, ot::context::Context{});
|
||||
if (span_) {
|
||||
span_->span->SetAttribute("http.response.status_code", status);
|
||||
if (status >= 500) span_->error();
|
||||
}
|
||||
event("http.request.completed", status >= 500, id_);
|
||||
}
|
||||
void response(const std::string& bytes) noexcept {
|
||||
if (current_http && bytes.size() >= 12 && bytes.compare(0,9,"HTTP/1.1 ") == 0 &&
|
||||
bytes[9] >= '1' && bytes[9] <= '5' && bytes[10] >= '0' && bytes[10] <= '9' && bytes[11] >= '0' && bytes[11] <= '9')
|
||||
current_http->status = (bytes[9]-'0')*100+(bytes[10]-'0')*10+bytes[11]-'0';
|
||||
}
|
||||
Job::Job(const std::string& parent, const std::string& state)
|
||||
: span_("attachment.process", ot::trace::SpanKind::kConsumer, parent, state), start_(std::chrono::steady_clock::now()) {
|
||||
span_.span->SetAttribute("messaging.system", "redis");
|
||||
span_.span->SetAttribute("messaging.destination.name", "jobs:attachments");
|
||||
delta("worker.busy", 1);
|
||||
}
|
||||
void Job::finish(const char* outcome) { outcome_ = outcome; }
|
||||
Job::~Job() {
|
||||
if (std::uncaught_exceptions() > exceptions_) outcome_ = "failed";
|
||||
delta("worker.busy", -1);
|
||||
const std::map<std::string,std::string> attrs{{"job.type","attachment.process.v1"},{"outcome",outcome_}};
|
||||
if (jobs) jobs->Add(1, attrs);
|
||||
if (job_duration) job_duration->Record(std::chrono::duration<double>(std::chrono::steady_clock::now()-start_).count(), attrs, ot::context::Context{});
|
||||
bool failed = std::string(outcome_) == "failed" || std::string(outcome_) == "retry";
|
||||
span_.span->SetAttribute("job.outcome", outcome_);
|
||||
if (failed) span_.error();
|
||||
event("worker.job.completed", failed);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
#pragma once
|
||||
#include <opentelemetry/trace/provider.h>
|
||||
#include <opentelemetry/trace/scope.h>
|
||||
#include <opentelemetry/context/context.h>
|
||||
#include <chrono>
|
||||
#include <exception>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace telemetry {
|
||||
class Runtime {
|
||||
public:
|
||||
explicit Runtime(const char* service) noexcept;
|
||||
~Runtime();
|
||||
};
|
||||
// Only trusted constants and bounded route templates may be passed as names/attributes.
|
||||
class Span {
|
||||
public:
|
||||
explicit Span(const std::string& name,
|
||||
opentelemetry::trace::SpanKind kind = opentelemetry::trace::SpanKind::kInternal,
|
||||
const std::string& traceparent = "", const std::string& tracestate = "");
|
||||
~Span();
|
||||
void error() noexcept;
|
||||
opentelemetry::nostd::shared_ptr<opentelemetry::trace::Span> span;
|
||||
private:
|
||||
int exceptions_;
|
||||
std::unique_ptr<opentelemetry::trace::Scope> scope_;
|
||||
};
|
||||
std::string inject(const char* field = "traceparent");
|
||||
std::string route_template(const std::string& path);
|
||||
void event(const char* name, bool error = false, const std::string& request_id = "") noexcept;
|
||||
void gauge(const char* name, double value) noexcept;
|
||||
void delta(const char* name, long value) noexcept;
|
||||
void duration(const char* name, double seconds) noexcept;
|
||||
class HttpRequest {
|
||||
public:
|
||||
HttpRequest();
|
||||
~HttpRequest();
|
||||
void configure(const std::string& method, const std::string& path,
|
||||
const std::string& parent, const std::string& state, const std::string& id);
|
||||
int status = 0;
|
||||
private:
|
||||
std::chrono::steady_clock::time_point start_;
|
||||
std::unique_ptr<Span> span_;
|
||||
std::string method_ = "_OTHER", route_ = "unmatched", id_;
|
||||
HttpRequest* previous_;
|
||||
};
|
||||
void response(const std::string& bytes) noexcept;
|
||||
class Job {
|
||||
public:
|
||||
Job(const std::string& parent, const std::string& state);
|
||||
~Job();
|
||||
void finish(const char* outcome);
|
||||
private:
|
||||
Span span_;
|
||||
std::chrono::steady_clock::time_point start_;
|
||||
int exceptions_ = std::uncaught_exceptions();
|
||||
const char* outcome_ = "failed";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
#include "user_repo.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string make_uuid() {
|
||||
static std::mt19937_64 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
|
||||
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
|
||||
std::ostringstream out;
|
||||
out << std::hex << std::setfill('0');
|
||||
out << std::setw(8) << d32(rng);
|
||||
out << '-' << std::setw(4) << d16(rng);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
|
||||
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch()).count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
User row_to_user(const pqxx::row& r) {
|
||||
User u;
|
||||
u.id = r["id"].as<std::string>();
|
||||
u.username = r["username"].as<std::string>();
|
||||
u.display_name = r["display_name"].as<std::string>();
|
||||
u.password_hash = r["password_hash"].as<std::string>();
|
||||
u.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
return u;
|
||||
}
|
||||
|
||||
constexpr const char* kUserColumns =
|
||||
"id, username, display_name, password_hash, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
|
||||
|
||||
}
|
||||
|
||||
std::optional<User> UserRepo::create(const std::string& username,
|
||||
const std::string& display_name,
|
||||
const std::string& password_hash) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
try {
|
||||
const std::string id = make_uuid();
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at) "
|
||||
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
|
||||
"RETURNING " + std::string(kUserColumns),
|
||||
pqxx::params{id, username, display_name, password_hash, now_epoch_str()});
|
||||
tx.commit();
|
||||
return row_to_user(r[0]);
|
||||
} catch (const pqxx::unique_violation&) {
|
||||
tx.abort();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<User> UserRepo::find_by_id(const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kUserColumns) + " FROM users WHERE id = $1",
|
||||
pqxx::params{id});
|
||||
tx.commit();
|
||||
if (r.empty()) return std::nullopt;
|
||||
return row_to_user(r[0]);
|
||||
}
|
||||
|
||||
std::optional<User> UserRepo::find_by_username(const std::string& username) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kUserColumns) + " FROM users WHERE username = $1",
|
||||
pqxx::params{username});
|
||||
tx.commit();
|
||||
if (r.empty()) return std::nullopt;
|
||||
return row_to_user(r[0]);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
class UserRepo {
|
||||
public:
|
||||
explicit UserRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
std::optional<User> create(const std::string& username,
|
||||
const std::string& display_name,
|
||||
const std::string& password_hash);
|
||||
|
||||
std::optional<User> find_by_id(const std::string& id);
|
||||
std::optional<User> find_by_username(const std::string& username);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,236 @@
|
||||
#include "telemetry.hpp"
|
||||
#include "observability_monitor.hpp"
|
||||
#include "attachment_queue.hpp"
|
||||
#include "attachment_repo.hpp"
|
||||
#include "attachment_storage.hpp"
|
||||
#include "staging_cleaner.hpp"
|
||||
#include "image_pipeline.hpp"
|
||||
#include "job_repo.hpp"
|
||||
#include "fault_injection.hpp"
|
||||
#include "s3_client.hpp"
|
||||
|
||||
#include <sodium.h>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <regex>
|
||||
#include <sstream>
|
||||
#include <thread>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace {
|
||||
volatile std::sig_atomic_t stop_requested = 0;
|
||||
void stop(int) { stop_requested = 1; }
|
||||
|
||||
class Watchdog {
|
||||
public:
|
||||
explicit Watchdog(int seconds) : seconds_(seconds), thread_([this] {
|
||||
while (!stopping_.load()) {
|
||||
const auto deadline = deadline_.load();
|
||||
if (deadline && now_ms() >= deadline) {
|
||||
std::cerr << "worker job timed out; exiting for durable recovery\n";
|
||||
std::_Exit(124);
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(100));
|
||||
}
|
||||
}) {}
|
||||
~Watchdog() { stopping_.store(true); thread_.join(); }
|
||||
void arm() { deadline_.store(now_ms() + seconds_ * 1000LL); }
|
||||
void clear() { deadline_.store(0); }
|
||||
private:
|
||||
static long long now_ms() {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::steady_clock::now().time_since_epoch()).count();
|
||||
}
|
||||
int seconds_;
|
||||
std::atomic<long long> deadline_{0};
|
||||
std::atomic<bool> stopping_{false};
|
||||
std::thread thread_;
|
||||
};
|
||||
|
||||
struct Deadline {
|
||||
Watchdog& timer;
|
||||
explicit Deadline(Watchdog& timer) : timer(timer) { timer.arm(); }
|
||||
~Deadline() { timer.clear(); }
|
||||
};
|
||||
|
||||
int setting(const char* name, int fallback, int minimum, int maximum) {
|
||||
const char* value = std::getenv(name);
|
||||
if (!value) return fallback;
|
||||
std::size_t end = 0;
|
||||
const int result = std::stoi(value, &end);
|
||||
if (value[end] || result < minimum || result > maximum) {
|
||||
throw std::runtime_error("invalid worker setting");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string sha256(const std::string& bytes) {
|
||||
unsigned char hash[crypto_hash_sha256_BYTES];
|
||||
crypto_hash_sha256(hash, reinterpret_cast<const unsigned char*>(bytes.data()), bytes.size());
|
||||
char hex[crypto_hash_sha256_BYTES * 2 + 1];
|
||||
sodium_bin2hex(hex, sizeof(hex), hash, sizeof(hash));
|
||||
return hex;
|
||||
}
|
||||
|
||||
bool valid_job(const nlohmann::json& j) {
|
||||
static const std::regex uuid("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
||||
return j.is_object() && j.value("schema_version", 0) == 1 &&
|
||||
j.value("type", "") == "attachment.process.v1" && j.value("pipeline_version", 0) >= 1 && j.value("pipeline_version", 0) <= 2 &&
|
||||
j.value("attempt", 0) >= 1 && j.contains("created_at") && j.at("created_at").is_string() &&
|
||||
std::regex_match(j.value("job_id", ""), uuid) &&
|
||||
std::regex_match(j.value("attachment_id", ""), uuid);
|
||||
}
|
||||
|
||||
void verify_ready(S3Client& s3, const Attachment& a) {
|
||||
std::int64_t size = 0;
|
||||
std::string etag;
|
||||
if (!s3.stat_object(a.s3_key, size, etag) || size != a.size_bytes) {
|
||||
throw std::runtime_error("ready original missing");
|
||||
}
|
||||
bool thumbnail = false;
|
||||
for (const auto& v : a.variants) {
|
||||
thumbnail = thumbnail || v.name == "thumbnail";
|
||||
if (!s3.stat_object((v.storage_key.empty() ? attachment_variant_key(a.chat_id, a.id, v.name) : v.storage_key), size, etag) ||
|
||||
size != v.size_bytes) throw std::runtime_error("ready variant missing");
|
||||
}
|
||||
if (!thumbnail) throw std::runtime_error("ready thumbnail missing");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main() {
|
||||
telemetry::Runtime telemetry_runtime("messenger-worker");
|
||||
telemetry::gauge("worker.capacity", 1);
|
||||
std::signal(SIGTERM, stop);
|
||||
std::signal(SIGINT, stop);
|
||||
try {
|
||||
if (sodium_init() < 0) return 1;
|
||||
const RetryPolicy retry{
|
||||
setting("JOB_MAX_ATTEMPTS", 6, 1, 100),
|
||||
setting("JOB_RETRY_BASE_MS", 1000, 1, 3600000),
|
||||
setting("JOB_RETRY_MAX_MS", 60000, 1, 3600000)};
|
||||
if (retry.base_delay_ms > retry.max_delay_ms) throw std::runtime_error("invalid retry policy");
|
||||
const int timeout = setting("JOB_TIMEOUT_SECONDS", 60, 1, 3600);
|
||||
const int cleanup_interval = setting("STAGING_CLEANUP_INTERVAL_SECONDS", 60, 1, 86400);
|
||||
const int retention = setting("STAGING_RETENTION_SECONDS", 86400, 1, 31536000);
|
||||
const int reclaim_idle = setting("JOB_RECLAIM_IDLE_MS", 5000, 1000, 3600000);
|
||||
const auto db_url = read_secret_file("DATABASE_URL_FILE");
|
||||
{
|
||||
auto conn = open_pg_connection(db_url, false);
|
||||
apply_migrations(*conn, "./migrations");
|
||||
}
|
||||
PgPool pg(db_url, 2);
|
||||
AttachmentRepo attachments(pg);
|
||||
JobRepo jobs(pg, retry);
|
||||
const char* broker_file = std::getenv("BROKER_URL_FILE");
|
||||
RedisPool redis(read_secret_file(
|
||||
broker_file && *broker_file ? "BROKER_URL_FILE" : "CACHE_URL_FILE"), 2);
|
||||
AttachmentQueue queue(redis);
|
||||
S3Client s3(S3Client::from_env());
|
||||
char random[33];
|
||||
unsigned char entropy[16];
|
||||
randombytes_buf(entropy, sizeof(entropy));
|
||||
sodium_bin2hex(random, sizeof(random), entropy, sizeof(entropy));
|
||||
const std::string consumer = "worker-" + std::string(random);
|
||||
std::clog << "worker started consumer=" << consumer << " max_attempts=" << retry.max_attempts << '\n';
|
||||
StagingCleaner cleaner(pg, s3, retention);
|
||||
Watchdog watchdog(timeout);
|
||||
ObservabilityMonitor monitor(db_url, false);
|
||||
auto next_sweep = std::chrono::steady_clock::now();
|
||||
while (!stop_requested) {
|
||||
try {
|
||||
if (std::chrono::steady_clock::now() >= next_sweep) {
|
||||
next_sweep = std::chrono::steady_clock::now() + std::chrono::seconds(cleanup_interval);
|
||||
Deadline deadline(watchdog);
|
||||
cleaner.sweep();
|
||||
}
|
||||
queue.ensure_group();
|
||||
const auto message = queue.read(consumer, reclaim_idle);
|
||||
if (!message) continue;
|
||||
nlohmann::json j;
|
||||
bool valid = false;
|
||||
try {
|
||||
j = nlohmann::json::parse(message->payload);
|
||||
valid = valid_job(j);
|
||||
} catch (const nlohmann::json::exception&) {}
|
||||
if (!valid) {
|
||||
telemetry::event("worker.job.invalid", true);
|
||||
queue.reject(message->stream_id);
|
||||
continue;
|
||||
}
|
||||
auto context_field = [&](const char* name) {
|
||||
auto i = j.find(name);
|
||||
return i != j.end() && i->is_string() && i->get_ref<const std::string&>().size() <= 512
|
||||
? i->get<std::string>() : std::string{};
|
||||
};
|
||||
telemetry::Job observation(context_field("traceparent"), context_field("tracestate"));
|
||||
const std::string id = j.at("attachment_id").get<std::string>();
|
||||
Deadline deadline(watchdog);
|
||||
const auto result = jobs.run(j.at("job_id").get<std::string>(), id,
|
||||
j.at("pipeline_version").get<int>(), j.at("attempt").get<int>(), [&](const JobInput& input) {
|
||||
const auto a = attachments.find_by_id(id);
|
||||
if (!a) throw std::runtime_error("attachment missing");
|
||||
std::string original;
|
||||
original.reserve(a->size_bytes);
|
||||
s3.get_object(input.source_key, [&](const char* data, std::size_t n) {
|
||||
if (original.size() > static_cast<std::size_t>(a->size_bytes) ||
|
||||
n > static_cast<std::size_t>(a->size_bytes) - original.size()) {
|
||||
throw InvalidImage("source_mismatch");
|
||||
}
|
||||
original.append(data, n);
|
||||
});
|
||||
if (original.size() != static_cast<std::size_t>(a->size_bytes) ||
|
||||
sha256(original) != a->sha256) throw InvalidImage("source_mismatch");
|
||||
const auto images = make_variants(std::vector<std::uint8_t>(original.begin(), original.end()), input.pipeline_version);
|
||||
const std::string prefix = pipeline_prefix(a->chat_id, a->id, input.pipeline_version);
|
||||
const std::string key = prefix + "original";
|
||||
// Publish verified bytes even if a presigned staging key changes.
|
||||
std::istringstream source(original);
|
||||
s3.put_object(key, source, original.size(), a->content_type);
|
||||
nlohmann::json variants = nlohmann::json::array();
|
||||
for (const auto& v : images) {
|
||||
std::istringstream body(v.image.bytes);
|
||||
const std::string variant_key = prefix + v.name + (v.content_type == "image/webp" ? ".webp" : ".jpg");
|
||||
s3.put_object(variant_key, body,
|
||||
v.image.bytes.size(), v.content_type);
|
||||
variants.push_back({{"name", v.name}, {"content_type", v.content_type},
|
||||
{"size_bytes", v.image.bytes.size()}, {"sha256", sha256(v.image.bytes)},
|
||||
{"width", v.image.width}, {"height", v.image.height}, {"storage_key", variant_key}});
|
||||
}
|
||||
lab_worker_fault("worker_after_effect", j.at("job_id").get<std::string>(), input.pipeline_version);
|
||||
return JobResult{key, variants.dump()};
|
||||
});
|
||||
observation.finish(result == JobRepo::Result::Scheduled ? "retry" :
|
||||
result == JobRepo::Result::Busy ? "busy" :
|
||||
result == JobRepo::Result::Missing ? "failed" : "superseded");
|
||||
if (result == JobRepo::Result::Missing) {
|
||||
queue.reject(message->stream_id);
|
||||
} else if (result == JobRepo::Result::Done) {
|
||||
const auto a = attachments.find_by_id(id);
|
||||
observation.finish(a && a->status == "ready" ? "success" : "failed");
|
||||
if (a && a->status == "ready") verify_ready(s3, *a);
|
||||
lab_worker_fault("worker_before_ack", j.at("job_id").get<std::string>(), j.at("pipeline_version").get<int>());
|
||||
queue.ack(message->stream_id);
|
||||
if (a && a->status == "ready") cleaner.clean("staging/" + a->chat_id + "/" + a->id);
|
||||
} else if (result == JobRepo::Result::Superseded) {
|
||||
queue.ack(message->stream_id);
|
||||
} else if (result == JobRepo::Result::Scheduled) {
|
||||
// ACK only after the next attempt was committed to outbox.
|
||||
queue.ack(message->stream_id);
|
||||
}
|
||||
// Busy remains pending; an active job is protected by its advisory lock.
|
||||
} catch (const std::exception&) {
|
||||
telemetry::event("worker.operation.failed", true);
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
} catch (const std::exception&) {
|
||||
std::cerr << "worker initialization failed\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Controlled lab6 failures, with unconditional service recovery and sanitized evidence."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
from observability import get, eventually
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||||
parser.add_argument('--only', choices=['all', 'retry', 'backend'], default='all')
|
||||
args = parser.parse_args()
|
||||
compose = shlex.split(os.environ.get('COMPOSE', 'docker compose'))
|
||||
def command(*cmd):
|
||||
subprocess.run(compose + list(cmd), cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
def query(expr):
|
||||
return get(args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': expr}))['data']['result']
|
||||
def value(expr):
|
||||
rows = query(expr)
|
||||
return float(rows[0]['value'][1]) if rows else 0
|
||||
def firing(name):
|
||||
alerts = get(args.prometheus_url + '/api/v1/alerts')['data']['alerts']
|
||||
return any(a['labels']['alertname'] == name and a['state'] == 'firing' for a in alerts)
|
||||
c = Client(args.base_url)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
events = json.loads((ROOT / "evidence/incidents.json").read_text()) if args.only != "all" else []
|
||||
def record(name, **fields):
|
||||
events.append({'event': name, 'timestamp': datetime.now(timezone.utc).isoformat(), **fields})
|
||||
(ROOT / 'evidence/incidents.json').write_text(json.dumps(events, indent=2) + '\n')
|
||||
print(name, flush=True)
|
||||
if args.only == "all":
|
||||
# 1. Durable queue ages while worker is stopped; resume drains the same job.
|
||||
command('stop', 'worker')
|
||||
try:
|
||||
att, _ = c.attachment(alice, chat)
|
||||
record('worker_stopped_upload_accepted')
|
||||
eventually(lambda: firing('QueueLagGrowing'), timeout=120)
|
||||
record('QueueLagGrowing_firing', depth=value('max(queue_depth)'), oldest_seconds=value('max(queue_oldest_age)'))
|
||||
finally:
|
||||
command('start', 'worker')
|
||||
c.wait_ready(bob, att)
|
||||
eventually(lambda: not firing('QueueLagGrowing') and value('max(queue_depth)') == 0, timeout=90)
|
||||
record('QueueLagGrowing_resolved_backlog_drained')
|
||||
# 2. PostgreSQL unavailable: valid authenticated reads must yield 5xx, never successful phantom writes.
|
||||
command('stop', 'db')
|
||||
statuses = {}
|
||||
try:
|
||||
record('database_stopped')
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
request = urllib.request.Request(args.base_url + '/api/v1/chats/' + chat['id'] + '/messages',
|
||||
headers={'Authorization': 'Bearer ' + alice['token']})
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=12)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
code = response.status
|
||||
statuses[str(code)] = statuses.get(str(code), 0) + 1
|
||||
if firing('ApiHighErrorRate'):
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(firing('ApiHighErrorRate'), 'API error alert did not fire')
|
||||
check(any(int(code) >= 500 for code in statuses), 'No server errors observed')
|
||||
record('ApiHighErrorRate_firing', response_counts=statuses)
|
||||
finally:
|
||||
command('start', 'db')
|
||||
def recovered():
|
||||
return c.call('GET', '/health/ready', schema='Health')['status'] == 200
|
||||
eventually(recovered, timeout=90)
|
||||
# New traffic moves the one-minute failure window out of the rate calculation.
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=alice, schema='MessagePage')
|
||||
if not firing('ApiHighErrorRate'):
|
||||
break
|
||||
time.sleep(1)
|
||||
check(not firing('ApiHighErrorRate'), 'API alert did not resolve')
|
||||
record('ApiHighErrorRate_resolved_database_recovered')
|
||||
if args.only != "backend":
|
||||
# A retriable S3 failure must preserve the upload trace across the next outbox row.
|
||||
import base64
|
||||
import uuid
|
||||
trace = uuid.uuid4().hex
|
||||
command('pause', 'worker')
|
||||
paused = True
|
||||
try:
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
|
||||
actor=alice, raw=content, headers={'Content-Type': 'image/png',
|
||||
'traceparent': '00-' + trace + '-0123456789abcdef-01'}, schema='Attachment')['json']
|
||||
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
||||
grafana = os.environ.get('GRAFANA_URL', 'http://localhost:3000')
|
||||
def retry_observed():
|
||||
data = get(grafana + '/api/datasources/proxy/uid/tempo/api/traces/' + trace, auth)
|
||||
def contains(obj):
|
||||
if isinstance(obj, dict):
|
||||
if obj.get('key') == 'job.outcome' and obj.get('value', {}).get('stringValue') == 'retry':
|
||||
return True
|
||||
return any(contains(v) for v in obj.values())
|
||||
return isinstance(obj, list) and any(contains(v) for v in obj)
|
||||
return contains(data)
|
||||
command('stop', 'minio')
|
||||
command('unpause', 'worker')
|
||||
paused = False
|
||||
eventually(retry_observed, timeout=45)
|
||||
record('s3_failure_scheduled_retry', trace_id=trace)
|
||||
finally:
|
||||
command('start', 'minio')
|
||||
if paused:
|
||||
command('unpause', 'worker')
|
||||
c.wait_ready(bob, att)
|
||||
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
||||
grafana = os.environ.get('GRAFANA_URL', 'http://localhost:3000')
|
||||
def retry_trace():
|
||||
data = get(grafana + '/api/datasources/proxy/uid/tempo/api/traces/' + trace, auth)
|
||||
spans = []
|
||||
def walk(obj):
|
||||
if isinstance(obj, dict):
|
||||
if 'spanId' in obj and 'name' in obj:
|
||||
spans.append(obj)
|
||||
for value_ in obj.values():
|
||||
walk(value_)
|
||||
elif isinstance(obj, list):
|
||||
for value_ in obj:
|
||||
walk(value_)
|
||||
walk(data)
|
||||
by_id = {s['spanId']: s for s in spans}
|
||||
# consumer -> enqueue -> publish -> next consumer (all inside the same trace)
|
||||
for child in spans:
|
||||
if child['name'] != 'attachment.process':
|
||||
continue
|
||||
producer = by_id.get(child.get('parentSpanId'), {})
|
||||
enqueue = by_id.get(producer.get('parentSpanId'), {})
|
||||
prior = by_id.get(enqueue.get('parentSpanId'), {})
|
||||
if producer.get('name') == 'jobs:attachments publish' and enqueue.get('name') == 'attachment.enqueue' and prior.get('name') == 'attachment.process':
|
||||
return True
|
||||
return False
|
||||
eventually(retry_trace)
|
||||
record('retry_same_trace_ready_verified', trace_id=trace)
|
||||
# 3. Broken telemetry storage must not break either application process.
|
||||
before = value('sum(otelcol_exporter_send_failed_spans)')
|
||||
command('stop', 'tempo', 'loki')
|
||||
try:
|
||||
record('tempo_loki_stopped')
|
||||
att, _ = c.attachment(alice, chat)
|
||||
c.wait_ready(bob, att)
|
||||
for _ in range(20):
|
||||
c.message(alice, chat, 'Synthetic telemetry outage probe')
|
||||
time.sleep(1)
|
||||
eventually(lambda: value('sum(otelcol_exporter_send_failed_spans)') > before, timeout=45)
|
||||
record('telemetry_outage_api_worker_healthy',
|
||||
failed_span_exports=value('sum(otelcol_exporter_send_failed_spans)')-before,
|
||||
collector_rss_bytes=value('max(otelcol_process_memory_rss)'))
|
||||
finally:
|
||||
command('start', 'tempo', 'loki')
|
||||
record('tempo_loki_restarted')
|
||||
print('PASS: worker lag and DB error alerts fired/resolved; API and worker survived telemetry backend outage.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live lab6 checks. No tokens, message contents or backend dumps are printed."""
|
||||
import argparse
|
||||
import base64
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def get(url, auth=None):
|
||||
headers = {'Accept': 'application/json'}
|
||||
if auth:
|
||||
headers['Authorization'] = 'Basic ' + base64.b64encode(auth.encode()).decode()
|
||||
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def eventually(fn, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
result = fn()
|
||||
if result:
|
||||
return result
|
||||
except (urllib.error.URLError, AssertionError, KeyError, ValueError):
|
||||
pass
|
||||
time.sleep(2)
|
||||
raise AssertionError('Telemetry check timed out')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--prometheus-url', default='http://localhost:9090')
|
||||
parser.add_argument('--grafana-url', default='http://localhost:3000')
|
||||
parser.add_argument('--output', default='evidence/observability.json')
|
||||
args = parser.parse_args()
|
||||
auth = os.environ.get('GRAFANA_ADMIN_USER', 'admin') + ':' + (ROOT / 'secrets/grafana_admin_password').read_text().strip()
|
||||
proxy = args.grafana_url + '/api/datasources/proxy/uid/'
|
||||
def prom(query):
|
||||
return get(args.prometheus_url + '/api/v1/query?' + urllib.parse.urlencode({'query': query}))['data']['result']
|
||||
c = Client(args.base_url)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
trace = uuid.uuid4().hex
|
||||
parent = '00-' + trace + '-0123456789abcdef-01'
|
||||
secret_marker = 'private-' + uuid.uuid4().hex
|
||||
c.message(alice, chat, secret_marker)
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
att = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', 202,
|
||||
actor=alice, raw=content, headers={'Content-Type': 'image/png', 'traceparent': parent}, schema='Attachment')['json']
|
||||
c.wait_ready(bob, att)
|
||||
def trace_complete():
|
||||
data = get(proxy + 'tempo/api/traces/' + trace, auth)
|
||||
spans = []
|
||||
def walk(obj):
|
||||
if isinstance(obj, dict):
|
||||
if 'spanId' in obj and 'name' in obj:
|
||||
spans.append(obj)
|
||||
for value in obj.values():
|
||||
walk(value)
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
walk(value)
|
||||
walk(data)
|
||||
names = {span['name'] for span in spans}
|
||||
return (data, spans) if {'attachment.enqueue', 'jobs:attachments publish', 'attachment.process', 's3.put_object', 'postgresql.publish_ready'} <= names else None
|
||||
trace_data, spans = eventually(trace_complete)
|
||||
by_id = {span['spanId']: span for span in spans}
|
||||
consumer = next(span for span in spans if span['name'] == 'attachment.process')
|
||||
publish = by_id.get(consumer['parentSpanId'])
|
||||
check(publish and publish['name'] == 'jobs:attachments publish', 'consumer parent must be producer')
|
||||
enqueue = by_id.get(publish['parentSpanId'])
|
||||
check(enqueue and enqueue['name'] == 'attachment.enqueue', 'publish must continue durable outbox context')
|
||||
server = by_id.get(enqueue['parentSpanId'])
|
||||
check(server and server['name'].startswith('POST '), 'enqueue parent must be HTTP server')
|
||||
query = '{service_name=~"messenger-api|messenger-worker"} | trace_id="' + trace + '"'
|
||||
def correlated_logs():
|
||||
result = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({'query': query, 'limit': 1000}), auth)['data']['result']
|
||||
services = {stream['stream'].get('service_name') for stream in result}
|
||||
return result if {'messenger-api', 'messenger-worker'} <= services else None
|
||||
logs = eventually(correlated_logs)
|
||||
required = ['http_server_requests_total', 'http_server_request_duration_seconds_bucket', 'worker_jobs_total',
|
||||
'worker_job_duration_seconds_bucket', 'attachment_ready_duration_seconds_bucket', 'queue_depth',
|
||||
'queue_oldest_age', 'db_pool_used', 'db_pool_waiting', 'worker_busy', 'worker_capacity',
|
||||
'process_memory_usage', 'process_cpu_time']
|
||||
for name in required:
|
||||
eventually(lambda: prom(name))
|
||||
panel_count = 0
|
||||
dashboard = get(args.grafana_url + '/api/dashboards/uid/messenger-observability', auth)['dashboard']
|
||||
check(len(dashboard['panels']) == len(json.loads((ROOT / 'infra/grafana/dashboards/messenger.json').read_text())['panels']), 'Provisioned dashboard differs')
|
||||
for panel in dashboard['panels']:
|
||||
if panel['datasource']['uid'] != 'prometheus':
|
||||
continue
|
||||
for target in panel['targets']:
|
||||
expr = target['expr'].replace('$__rate_interval', '1m')
|
||||
result = prom(expr)
|
||||
# Diagnostics are allowed to be absent when no failure has occurred.
|
||||
if 'otelcol_' not in expr:
|
||||
result = eventually(lambda: prom(expr), timeout=45)
|
||||
panel_count += 1
|
||||
for item in prom('http_server_requests_total'):
|
||||
labels = item['metric']
|
||||
check(not {'user_id','chat_id','message_id','attachment_id','request_id','filename'} & labels.keys(), 'High cardinality metric label')
|
||||
check(chat['id'] not in json.dumps(labels), 'Raw chat ID in labels')
|
||||
# Inspect full recent log/trace data in memory, never save private test values.
|
||||
all_logs = get(proxy + 'loki/loki/api/v1/query_range?' + urllib.parse.urlencode({
|
||||
'query': '{service_name=~"messenger-api|messenger-worker"}', 'limit': 5000}), auth)
|
||||
exported = json.dumps([all_logs, trace_data])
|
||||
for forbidden in [secret_marker, alice['token'], alice['registration']['password'], base64.b64encode(content).decode()]:
|
||||
check(forbidden not in exported, 'Private test data leaked to telemetry')
|
||||
report = {'timestamp': datetime.now(timezone.utc).isoformat(), 'result': 'PASS', 'trace_id': trace,
|
||||
'span_names': sorted({s['name'] for s in spans}), 'consumer_parent_is_producer': True,
|
||||
'log_services': sorted({s['stream']['service_name'] for s in logs}),
|
||||
'metric_names_checked': required, 'dashboard_queries_checked': panel_count,
|
||||
'privacy_canaries': 'absent in recent logs and upload trace'}
|
||||
(ROOT / args.output).write_text(json.dumps(report, indent=2) + '\n')
|
||||
print('PASS: real API -> outbox -> Redis -> worker -> S3/DB trace, correlated logs, metrics and dashboard queries; privacy canaries absent.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,33 @@
|
||||
#include "telemetry.hpp"
|
||||
#include <cassert>
|
||||
#include <regex>
|
||||
|
||||
int main() {
|
||||
telemetry::Runtime runtime("messenger-test");
|
||||
assert(telemetry::route_template("/api/v1/chats/private-id/messages") == "/api/v1/chats/{chat_id}/messages");
|
||||
assert(telemetry::route_template("/secret-token") == "unmatched");
|
||||
assert(telemetry::route_template("/api/v1/attachments/private/content") == "/api/v1/attachments/{attachment_id}/content");
|
||||
const std::string parent = "00-11223344556677889900aabbccddeeff-0123456789abcdef-01";
|
||||
std::string producer;
|
||||
{
|
||||
telemetry::Span request("test.server", opentelemetry::trace::SpanKind::kServer, parent);
|
||||
assert(telemetry::inject().substr(3,32) == parent.substr(3,32));
|
||||
{
|
||||
telemetry::Span publish("test.publish", opentelemetry::trace::SpanKind::kProducer);
|
||||
producer = telemetry::inject();
|
||||
assert(producer.substr(3,32) == parent.substr(3,32));
|
||||
assert(producer != parent);
|
||||
}
|
||||
assert(telemetry::inject() != producer);
|
||||
}
|
||||
{
|
||||
telemetry::Span consumer("test.consume", opentelemetry::trace::SpanKind::kConsumer, producer);
|
||||
assert(telemetry::inject().substr(3,32) == parent.substr(3,32));
|
||||
}
|
||||
for (const auto& invalid : {"garbage", "00-00000000000000000000000000000000-0000000000000000-01"}) {
|
||||
telemetry::Span span("test.invalid", opentelemetry::trace::SpanKind::kConsumer, invalid);
|
||||
const auto actual = telemetry::inject();
|
||||
assert(std::regex_match(actual, std::regex("00-[a-f0-9]{32}-[a-f0-9]{16}-01")));
|
||||
assert(actual.substr(3,32) != parent.substr(3,32));
|
||||
}
|
||||
}
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
Headers from https://github.com/nothings/stb at the commit in VERSION:
|
||||
- stb_image.h: PNG/JPEG validation and decoding
|
||||
- deprecated/stb_image_resize.h: aspect-preserving resize
|
||||
- stb_image_write.h: JPEG encoding (quality 85)
|
||||
|
||||
Each header includes its upstream license.
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
013ac3beddff3dbffafd5177e7972067cd2b5083
|
||||
Vendored
+7988
File diff suppressed because it is too large
Load Diff
Vendored
+2634
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user