Author SHA1 Message Date
AndrewNikkk 580b33efe8 lab5 2026-11-16 09:28:38 +03:00
99 changed files with 23335 additions and 27 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
# Local build/test outputs can contain disposable credentials.
build/
+21 -4
View File
@@ -11,15 +11,32 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
# Same Valkey instance serves sessions and Streams.
BROKER_URL_FILE=/run/secrets/cache_url
WORKER_CONCURRENCY=1
# One initial attempt plus five retries.
JOB_MAX_ATTEMPTS=6
JOB_RECLAIM_IDLE_MS=5000
JOB_RETRY_BASE_MS=1000
JOB_RETRY_MAX_MS=60000
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
STAGING_CLEANUP_INTERVAL_SECONDS=60
# Local Compose bootstrap; choose your own values and matching secret files.
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change_me_locally
POSTGRES_DB=messenger
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=change_me_minio_root_locally
MINIO_USER_ACCESS_KEY=messenger-s3
MINIO_USER_SECRET_KEY=change_me_s3_user_locally
MAX_CONCURRENT_UPLOADS=3
DIRECT_UPLOAD_TTL_SECONDS=300
+124
View File
@@ -0,0 +1,124 @@
cmake_minimum_required(VERSION 3.16)
project(mtusi_messenger CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# Official MinIO C++ SDK (S3-compatible). Pinned tag; not a hand-rolled SigV4.
set(MTUSI_MINIO_CPP_VERSION "v0.3.0" CACHE STRING "minio-cpp git tag")
find_package(Threads REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PQXX REQUIRED IMPORTED_TARGET libpqxx>=7.8)
pkg_check_modules(HIREDIS REQUIRED IMPORTED_TARGET hiredis)
pkg_check_modules(SODIUM REQUIRED IMPORTED_TARGET libsodium)
pkg_check_modules(EXIF REQUIRED IMPORTED_TARGET libexif)
pkg_check_modules(WEBP REQUIRED IMPORTED_TARGET libwebp)
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ImportMinioDeps.cmake")
include(FetchContent)
set(MINIO_CPP_TEST OFF CACHE BOOL "" FORCE)
set(MINIO_CPP_MAKE_DOC OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
miniocpp
GIT_REPOSITORY https://github.com/minio/minio-cpp.git
GIT_TAG ${MTUSI_MINIO_CPP_VERSION}
GIT_SHALLOW TRUE
)
# Inject find_package stubs before miniocpp configures.
function(_mtusi_provide_minio_find_packages)
# miniocpp calls find_package(unofficial-curlpp CONFIG) etc.
# We already created ALIAS targets; create tiny config files in a prefix.
set(_cfg_dir "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps")
file(MAKE_DIRECTORY "${_cfg_dir}")
file(WRITE "${_cfg_dir}/unofficial-curlpp-config.cmake" "")
file(WRITE "${_cfg_dir}/unofficial-inih-config.cmake" "")
list(PREPEND CMAKE_PREFIX_PATH "${_cfg_dir}")
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
endfunction()
_mtusi_provide_minio_find_packages()
FetchContent_GetProperties(miniocpp)
if(NOT miniocpp_POPULATED)
FetchContent_Populate(miniocpp)
# Neutralize CONFIG find_package calls by providing empty configs + targets.
set(CMAKE_FIND_PACKAGE_REDIRECTS_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-find-redirects")
file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
# CMake 3.24+ redirects; for older, patch via policy: set variables that
# make find_package(CONFIG) succeed when targets already exist.
set(unofficial-curlpp_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
set(unofficial-inih_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
# Provide package config bodies that are no-ops (targets already defined).
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-curlpp-config.cmake"
"# provided by ImportMinioDeps.cmake\n")
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-inih-config.cmake"
"# provided by ImportMinioDeps.cmake\n")
# Skip miniocpp install(EXPORT) — our INTERFACE system deps are not exportable.
set(CMAKE_SKIP_INSTALL_RULES ON)
add_subdirectory(${miniocpp_SOURCE_DIR} ${miniocpp_BINARY_DIR})
set(CMAKE_SKIP_INSTALL_RULES OFF)
endif()
add_library(messenger_core STATIC
src/db.cpp
src/auth.cpp
src/user_repo.cpp
src/chat_repo.cpp
src/message_repo.cpp
src/redis_pool.cpp
src/session_repo.cpp
src/pg_session_repo.cpp
src/s3_client.cpp
src/image_util.cpp
src/attachment_repo.cpp
src/attachment_queue.cpp
src/outbox.cpp
src/staging_cleaner.cpp
src/job_repo.cpp
src/image_pipeline.cpp
src/direct_upload_repo.cpp
)
target_include_directories(messenger_core PUBLIC
src
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
)
target_compile_definitions(messenger_core PUBLIC
MTUSI_MINIO_CPP_VERSION="${MTUSI_MINIO_CPP_VERSION}"
)
target_link_libraries(messenger_core PUBLIC
Threads::Threads
PkgConfig::PQXX
PkgConfig::HIREDIS
PkgConfig::SODIUM
PkgConfig::EXIF
PkgConfig::WEBP
miniocpp::miniocpp
)
add_executable(messenger src/main.cpp)
target_link_libraries(messenger PRIVATE messenger_core)
add_executable(worker src/worker_main.cpp)
target_link_libraries(worker PRIVATE messenger_core)
add_executable(s3_smoke tools/s3_smoke.cpp)
target_link_libraries(s3_smoke PRIVATE messenger_core)
add_executable(image_smoke tools/image_smoke.cpp)
target_link_libraries(image_smoke PRIVATE messenger_core)
target_compile_definitions(image_smoke PRIVATE
IMAGE_SMOKE_FIXTURES="${CMAKE_SOURCE_DIR}/tools/fixtures"
)
add_executable(thumbnail_smoke tools/thumbnail_smoke.cpp)
target_link_libraries(thumbnail_smoke PRIVATE messenger_core)
add_executable(variants_smoke tools/variants_smoke.cpp)
target_link_libraries(variants_smoke PRIVATE messenger_core)
add_executable(job_admin src/job_admin_main.cpp)
target_link_libraries(job_admin PRIVATE messenger_core)
+26
View File
@@ -0,0 +1,26 @@
{
"version": 6,
"cmakeMinimumRequired": {
"major": 3,
"minor": 16,
"patch": 0
},
"configurePresets": [
{
"name": "release",
"displayName": "Release with minio-cpp",
"generator": "Unix Makefiles",
"binaryDir": "${sourceDir}/build",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"MTUSI_MINIO_CPP_VERSION": "v0.3.0"
}
}
],
"buildPresets": [
{
"name": "release",
"configurePreset": "release"
}
]
}
+117
View File
@@ -0,0 +1,117 @@
# Clean-clone build: Ubuntu 24.04 + libpqxx from a pinned git tag.
# S3 client: official minio-cpp v0.3.0 via FetchContent (library SigV4).
# No dependency on mtusi2026lab3-api or other prior-lab images.
ARG PQXX_VERSION=7.10.0
ARG MTUSI_MINIO_CPP_VERSION=v0.3.0
# ---------------------------------------------------------------------------
# Stage: build libpqxx into /opt/libpqxx
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS pqxx
ARG PQXX_VERSION
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake git ca-certificates pkg-config \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
RUN git clone --depth 1 --branch "${PQXX_VERSION}" \
https://github.com/jtv/libpqxx.git libpqxx-src \
&& cmake -S libpqxx-src -B libpqxx-build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/opt/libpqxx \
-DBUILD_SHARED_LIBS=ON \
-DSKIP_BUILD_TEST=ON \
&& cmake --build libpqxx-build -j"$(nproc)" \
&& cmake --install libpqxx-build \
&& rm -rf /tmp/libpqxx-src /tmp/libpqxx-build
# ---------------------------------------------------------------------------
# Stage: build messenger + worker + smokes
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS builder
ARG MTUSI_MINIO_CPP_VERSION
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake pkg-config git ca-certificates \
libpq-dev \
libhiredis-dev \
libsodium-dev libexif-dev libwebp-dev \
nlohmann-json3-dev \
libcurl4-openssl-dev libssl-dev zlib1g-dev \
libpugixml-dev \
libcurlpp-dev \
libinih-dev \
&& rm -rf /var/lib/apt/lists/*
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
ENV PKG_CONFIG_PATH=/opt/libpqxx/lib/pkgconfig:/opt/libpqxx/lib/x86_64-linux-gnu/pkgconfig
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
ENV CMAKE_PREFIX_PATH=/opt/libpqxx
WORKDIR /src
COPY CMakeLists.txt CMakePresets.json ./
COPY cmake ./cmake
COPY src ./src
COPY tools ./tools
COPY migrations ./migrations
COPY third_party ./third_party
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \
-DMTUSI_MINIO_CPP_VERSION=${MTUSI_MINIO_CPP_VERSION} \
&& cmake --build build -j"$(nproc)"
# ---------------------------------------------------------------------------
# Runtime
# ---------------------------------------------------------------------------
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
libpq5 \
libhiredis1.1.0 \
libsodium23 libexif12 libwebp7 \
libssl3t64 \
libcurl4t64 \
zlib1g \
libpugixml1v5 \
libcurlpp0t64 \
libinih1 \
libinireader0 \
util-linux \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --gid 10001 app \
&& useradd --uid 10001 --gid app --shell /usr/sbin/nologin --no-create-home app
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
WORKDIR /app
COPY --from=builder /src/build/messenger /app/messenger
COPY --from=builder /src/build/worker /app/worker
COPY --from=builder /src/build/job_admin /app/job_admin
COPY --from=builder /src/build/s3_smoke /app/s3_smoke
COPY --from=builder /src/build/image_smoke /app/image_smoke
COPY --from=builder /src/build/thumbnail_smoke /app/thumbnail_smoke
COPY --from=builder /src/build/variants_smoke /app/variants_smoke
COPY --from=builder /src/migrations /app/migrations
COPY tools/fixtures /app/fixtures
COPY docker-entrypoint.sh /docker-entrypoint.sh
RUN chmod 0755 /docker-entrypoint.sh \
&& chown -R app:app /app
# Entrypoint starts as root to stage S3 secret files for uid 10001, then drops.
USER root
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=3 \
CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["/app/messenger"]
+22 -2
View File
@@ -1,13 +1,14 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
TEST_ARGS ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-async test-upload-async test-worker-down test-outbox-crash test-reliability test-track2
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; test-async: disposable Compose suite; load: synthetic writes"
check:
$(PYTHON) scripts/check_contract.py
@@ -28,3 +29,22 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# Creates/removes only a disposable Compose project with synthetic data.
test-async:
$(PYTHON) student-tests/async_suite.py all $(TEST_ARGS)
test-upload-async:
PYTHON="$(PYTHON)" student-tests/test_upload_async.sh $(TEST_ARGS)
test-worker-down:
PYTHON="$(PYTHON)" student-tests/test_worker_down.sh $(TEST_ARGS)
test-outbox-crash:
PYTHON="$(PYTHON)" student-tests/test_outbox_crash.sh $(TEST_ARGS)
test-reliability:
PYTHON="$(PYTHON)" student-tests/test_reliability.sh $(TEST_ARGS)
test-track2:
PYTHON="$(PYTHON)" student-tests/test_pipeline_idempotency.sh $(TEST_ARGS)
+7 -1
View File
@@ -2,7 +2,13 @@
Отделить приём файла от тяжёлой обработки, корректно переживать повторную доставку и остановку worker.
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
В репозитории реализованы C++17 API и отдельный worker, Redis Streams, transactional outbox,
thumbnail/crop/WebP, retry, replay и очистка staging. Запуск — [REPORT.md](REPORT.md),
воспроизводимые проверки — [student-tests/README.md](student-tests/README.md).
Трек 2 уровня 5: [версии pipeline и исследование идемпотентности](infra/track2.md).
Исходное задание поставлялось как **skeleton**. Ниже сохранены условия курса,
контракт и публичные проверки; UI не обязателен.
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
+249 -20
View File
@@ -1,37 +1,266 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Никитин Андрей Александрович, БВТ2301
- Целевой уровень: 5, трек 2 — доказанная идемпотентность versioned pipeline.
- Commit/PR реализации:
- Помощь ИИ: Codex, DeepSeek
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Linux amd64, Docker Engine с Compose v2, Python 3.10+, доступ к Ubuntu/GitHub
для первой сборки. API и worker собираются здесь из C++17; предыдущие lab-образы
не используются. При локальной CMake-сборке нужны зависимости из Dockerfile,
включая libpqxx >=7.8, hiredis, libsodium, libexif, libwebp и зависимости minio-cpp.
Быстрая проверка в полностью отдельном стенде, без рабочего `.env` и secrets:
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt -r student-tests/requirements.txt
make check validate PYTHON=.venv/bin/python
make test-async PYTHON=.venv/bin/python TEST_ARGS="--output evidence/async-suite.json"
```
Runner создаёт случайный Compose-проект, синтетические credentials и volumes,
проверяет сценарии и удаляет только собственный стенд. Подробнее:
[student-tests/README.md](student-tests/README.md).
Для обычного локального стенда следующие команды предназначены для чистого
clone без существующих `.env`/secrets. Они генерируют случайные учебные пароли
и согласованные file-secrets, не печатая значения:
```sh
python3 - <<'PY'
from pathlib import Path
import secrets
assert not Path('.env').exists() and not Path('secrets').exists(), 'Use existing local configuration'
text = Path('.env.example').read_text()
passwords = {k: secrets.token_hex(24) for k in
('POSTGRES_PASSWORD', 'MINIO_ROOT_PASSWORD', 'MINIO_USER_SECRET_KEY')}
lines = []
for line in text.splitlines():
key = line.partition('=')[0]
lines.append(key + '=' + passwords[key] if key in passwords else line)
Path('.env').write_text('\n'.join(lines) + '\n')
Path('.env').chmod(0o600)
values = dict(line.split('=', 1) for line in lines if '=' in line and not line.startswith('#'))
Path('secrets').mkdir(mode=0o700)
files = {
'database_url': 'postgresql://'+values['POSTGRES_USER']+':'+values['POSTGRES_PASSWORD']+'@db:5432/'+values['POSTGRES_DB'],
'cache_url': 'redis://cache:6379/0',
's3_access_key_id': values['MINIO_USER_ACCESS_KEY'],
's3_secret_access_key': values['MINIO_USER_SECRET_KEY'],
}
for name, value in files.items():
path = Path('secrets') / name
path.write_text(value + '\n')
path.chmod(0o644) # container uid 10001; the host parent directory is private
PY
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Дождаться HTTP 200:
curl --fail http://localhost:8080/health/ready
make test
```
Миграции 001–011 выполняются API/worker при старте под общим advisory lock;
версия фиксируется в `schema_migrations`. MinIO init создаёт приватный bucket,
а user-init — ограниченного S3-пользователя и policy. Consumer group создаёт
worker с позиции 0. Init/migrations допускают повторный запуск с прежними volumes.
Безопасная остановка: `docker compose down` — named volumes и локальные secrets
сохраняются. **Намеренный сброс только своего учебного стенда:**
`docker compose down --volumes` удаляет его PostgreSQL/Valkey/S3 данные.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Перенесены код и миграции lab4: пользователи, сессии, участники, сообщения,
keyset pagination, idempotency, S3 и проверка PNG/JPEG. Общие репозитории,
пулы, S3 и изображения собраны в статическую `messenger_core`. Отдельные
бинарники: `messenger`, `worker`, операторский `job_admin` и image smoke tools.
| Роль | Compose service / версия | Persistence и доступ |
| --- | --- | --- |
| API | api, образ собирается из Dockerfile, 2 реплики по умолчанию | Только LB публикует основной HTTP :8080; credentials через file-secrets |
| Worker | worker, тот же образ, `/app/worker` | COUNT 1, одна работа за раз, 512 MiB, watchdog/restart |
| БД | db, postgres:16.15-alpine | pgdata, synchronous_commit включён |
| Брокер/Redis-сессии | cache, valkey/valkey:8.1.10-alpine | valkeydata, AOF yes, appendfsync always, без host-порта |
| S3 | minio, RELEASE.2025-04-22T22-12-26Z | miniodata, приватный bucket, без host-портов 9000/9001 |
| S3 bootstrap | minio-init / minio-user-init, mc RELEASE.2025-04-16T18-13-26Z | Root используется только bootstrap; API/worker — ограниченный пользователь |
| LB | lb, nginx:1.28.0-alpine | Доступ к API-репликам; :8080 |
SDK MinIO закреплён v0.3.0, stb — commit
`013ac3beddff3dbffafd5177e7972067cd2b5083`. libwebp/libexif берутся из Ubuntu 24.04.
APT-пакеты не закреплены по отдельным версиям; ID протестированных образов есть
в evidence. В Docker context исключены `.env`, secrets, build и приватные файлы.
### Upload и durable handoff
Обязательный endpoint — `POST /api/v1/chats/{chat_id}/attachments?filename=…`,
сырые PNG/JPEG bytes, не multipart. Проверяются идентификация/доступ, filename,
MIME по содержимому, 10 MiB и 40 млн пикселей до полной распаковки. API проверяет
целостность, но resize/crop/WebP выполняет только worker.
API PUT сохраняет оригинал в `staging/{chat_id}/{attachment_id}`. Затем одной
транзакцией сохраняются attachment queued, job и outbox. До COMMIT 202 не выдаётся.
Весь upload и PUT остаются синхронными; асинхронна дальнейшая обработка/публикация.
Возвращается неизменяемый снимок queued, даже если worker успеет раньше первого GET.
Отдельный поток API выбирает до 32 due outbox через `FOR UPDATE SKIP LOCKED`,
делает XADD и фиксирует published=true. Его пулы независимы от HTTP.
COMMIT→XADD восстанавливается из outbox; XADD→COMMIT marker может дать дубль.
Доставка at least once; exactly-once не заявляется. `payload` соответствует
`contracts/jobs.schema.json`: IDs/служебные поля, без bytes, credentials или URL.
В `.env.example` сессии PostgreSQL. При таком выборе брокер не нужен для upload
или readiness: проверяется PostgreSQL. При Redis-сессиях Redis остаётся синхронной
зависимостью аутентификации/readiness. Runtime readiness API не выполняет S3 HEAD;
S3 имеет отдельный service healthcheck, а upload не подтверждается при ошибке PUT.
### Worker, изображения и повторы
Worker читает consumer group `attachment-workers` через XREADGROUP, подбирает
pending через XAUTOCLAIM. Job защищена session advisory lock PostgreSQL, который
освобождается при смерти процесса. Размер/SHA-256 staging проверяются до эффектов.
Оригинал публикуется из проверенных bytes побайтно, включая возможный EXIF.
Растр декодируется один раз, EXIF orientation применяется, alpha компонуется на
белый. Имена итоговых объектов детерминированы. Новые jobs создают три варианта:
| Вариант | Геометрия | Кодирование |
| --- | --- | --- |
| thumbnail | Вписан в 256×256, без увеличения, floor/min 1 px | JPEG, v1 quality 85 / v2 quality 90 |
| crop | Центральный квадрат → 128×128, увеличение разрешено | JPEG, v1 quality 85 / v2 quality 90 |
| optimized | Вписан в 1280×1280, без увеличения, floor/min 1 px | lossy WebP, v1 quality 80 / v2 quality 90 |
EXIF/ICC/XMP в варианты не переносятся. WebP не обязан быть меньше любого
маленького исходника. Ready/variants публикуются одной DB-транзакцией после всех
четырёх объектов (оригинал + варианты); затем ACK. Дубль ready проверяет объекты
и не изменяет результат. Метаданные/SHA оригинала всегда описывают исходные bytes.
Content идёт через авторизованный API; без ранее опубликованного результата
queued/processing/failed отвечают 409. При upgrade старый ready-снимок сохраняется
до успешного COMMIT новой версии.
Временные ошибки сохраняют retries, next_attempt_at и следующий outbox атомарно;
после этого ACK старой доставки безопасен. Backoff: ceiling=min(max,base*2^(n-1)),
случайная задержка в [ceil(ceiling/2),ceiling], n — номер неудачи текущего
бюджета (retries-retry_base); defaults 1000/60000 ms. До due
не публикует publisher и не начинает worker. Попытки 1..6 ограничены; постоянный
invalid_image/source_mismatch сразу failed, исчерпание — processing_failed.
Failed виден в jobs, некорректные события — в `jobs:attachments:invalid`.
Watchdog (default 60 s) завершает зависший процесс с 124, Compose перезапускает его.
Очистка staging и operator replay подробно описаны в [infra/worker.md](infra/worker.md).
Ready-staging удаляется после проверки результатов; live presigned URL откладывает
удаление до expiry. Failed/orphan retention — 86400 s, период GC — 60 s.
Queued/processing и PUT до DB COMMIT защищены, batch GC ограничен. Replay проверяет
source checksum, разрешён только failed с сохранённым staging, сохраняет job_id,
audit и новый outbox, не сбрасывая монотонный retries. После retention нужен новый upload.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Основная команда — `make test-async PYTHON=.venv/bin/python
TEST_ARGS="--no-build --output evidence/async-suite.json"` (после первой сборки).
Все воздействия относятся только к изолированному проекту. JSON содержит
контейнеры/StartedAt/volumes, observed checks и source SHA; secrets не сохраняются.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий | Воспроизводимый сценарий | Наблюдение / evidence |
| --- | --- | --- |
| POST 202, состояния, оригинал/thumbnail | test_upload_async.sh | queued наблюдается при остановленном worker; ready, bytes/SHA и реальный JPEG 256×160 проверены |
| Durable staging/queue, restart stores | test_worker_down.sh | сообщение с queued attachment; прежняя сессия и job переживают API/PostgreSQL/Valkey/MinIO restart с прежними volumes |
| Outbox COMMIT→publish | test_outbox_crash.sh | exit 86 в реальном API, committed outbox подхвачен после рестарта без нового POST |
| Publish→marker duplicate | test_outbox_crash.sh | два события одного job_id, один готовый набор вариантов |
| Crop/WebP, metadata/download/access | Все сценарии ready | Реальные JPEG 128×128 и WebP 320×200 fixture; размеры, MIME, SHA, отсутствие EXIF и доступ участников |
| DB atomicity и bounded retry | test_reliability.sh | ошибка INSERT outbox откатывает attachment/job; attempts 1..6, backoff/jitter/due и final failed |
| Replay/DLQ register | test_reliability.sh | повреждённый source не replay; audit, attempt 7, старое событие не повторяет эффект, ready replay отказан |
| Crash до ACK, stuck processing | test_reliability.sh | SIGKILL после S3 и watchdog: восстановление pending, тот же эффект |
| Ограниченная конкуренция/идемпотентность | test_reliability.sh | 2 worker на дублях, 2 API-публикатора на outbox, без второго набора вариантов |
| GC и retention | test_reliability.sh | ready/orphan/expired failed удаляются, queued/live presigned и PUT→COMMIT защищены |
| Память | test_reliability.sh | RGB PNG 8000×5000 обработан при 512 MiB без swap; это не измерение RSS всех возможных форматов |
| Публичный API | Завершение каждого shell-теста | Неизменённый tests/smoke.py проходит |
| Схемы | make check validate | Offline structure и полная OpenAPI 3.1 / JSON Schema validation проходят |
## Эксперименты и отказы
Сводка полного прогона: [evidence/async-suite.json](evidence/async-suite.json).
Финальный прогон 15.11.2026 (Europe/Moscow): Python 3.14.4, Pillow 12.1.1,
18 групп проверок, 121.048 s вместе с запуском/удалением стенда.
Cleanup контейнеров, volumes, сети и disposable credentials отдельно подтверждён.
Отдельный upload-прогон со сборкой: [evidence/upload-async.json](evidence/upload-async.json).
Геометрия/EXIF/alpha отдельно проверяются `test_thumbnail.py` и `test_variants.py`;
команды и зависимости — в student-tests/README.md.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Методика и ограничения
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Проверки функциональные, не нагрузочный эксперимент. Основной fixture —
320×200 PNG, 111817 bytes, SHA-256 из tests/fixtures/README.md. Memory fixture —
один синтетический 8-bit RGB PNG 40 млн пикселей. По умолчанию worker serial,
для дублей временно запущены 2 процесса; неограниченного prefetch нет.
В тестовом override API получает 512 MiB/1 CPU, timeout worker — 3 s,
retry base/max — 200/800 ms, GC — 1 s; рабочие defaults сохранены в Compose.
Фактические image IDs/лимиты и elapsed test run находятся в JSON evidence.
RPS, p50/p95/p99 и статистический memory benchmark не измерялись.
## Ограничения и следующие шаги
Не выдерживается потеря DB/S3/брокер volumes или самого единственного хоста;
это не HA-кластер. Сессии Redis при остановке общего Valkey недоступны, поэтому
outbox fault-tests используют PostgreSQL-сессии. При выключенных API-публикаторах
outbox/retry остаётся durable, но не публикуется до запуска API. Рабочий стенд
учебный HTTP; lab7 TLS и rate limiting сюда не входят.
Готовые записи предыдущих шагов сохраняют прежние варианты; обновление pipeline
не откатывает ready автоматически. Публикация кода/commit/PR не выполнялась.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
## Уровень 5: трек 2
Гипотеза, инварианты, модель отказов, алгоритм защиты и границы результата:
[infra/track2.md](infra/track2.md). Для v1/v2 используются раздельные S3 namespaces,
unique jobs/results на attachment/version и повторная проверка desired version
под row lock перед переключением published pointer. Source/recipe захвачены из job;
carrier не может подменить версию записи БД. Legacy storage keys продолжают читаться.
```sh
make test-track2 PYTHON=.venv/bin/python \
TEST_ARGS="--output evidence/track2.json"
# Повтор уже собранного неизменённого кода:
make test-track2 PYTHON=.venv/bin/python \
TEST_ARGS="--no-build --repetitions 3 --output evidence/track2.json"
```
Исследование использует три независимых холодных Compose-стенда и фиксированную
матрицу доставок/отказов. Каждый повтор включает 16 валидных uploads, 27 логических
jobs для версий 1/2 и проверки результатов после рестарта PostgreSQL/Valkey/MinIO/API.
Четыре crash points выполняются для обеих версий. Отдельно старый worker
останавливается до эффекта, после эффекта и после COMMIT; v2 завершается на втором
worker, затем тот же старый процесс возобновляется. Сравниваются actual S3 SHA,
метаданные, version pointer, PK counts и terminal states всех committed intents.
Сводка итоговых повторов: [evidence/track2.json](evidence/track2.json).
В ней хранятся source/method SHA, image IDs, фактические Docker die exit codes,
хеши versioned объектов, результаты инвариантов и время каждого повтора.
Измерения RPS/p95/RSS не заявляются: это выбранный трек идемпотентности.
| Повтор | Проверки | Attachments / jobs | Время со стендом, s | Итог |
| --- | --- | --- | --- | --- |
| 1 | 20 | 16 / 27 | 187.385 | PASS |
| 2 | 20 | 16 / 27 | 185.107 | PASS |
| 3 | 20 | 16 / 27 | 187.243 | PASS |
Итог 15.11.2026 (Europe/Moscow): 48 принятых вложений, 81 committed job,
24 подтверждённых crash с exit code 86 и 9 возобновлений старого процесса после
публикации v2. Потерянных вложений, незавершённых jobs и дублей result rows — 0;
новый снимок не изменился после поздней работы v1. Все три стенда и их credentials
удалены. Гипотеза подтверждена в описанной модели отказов; исчерпывающий перебор
распределённых расписаний не выполнялся.
Exactly-once доставка не заявляется: XADD/PUT могут повторяться после crash.
Защищён один логический результат `(attachment_id,pipeline_version,variant)` и
монотонное переключение опубликованного снимка. Изоляция ключей защищает bytes
v2 от поздних PUT v1; CAS защищает DB pointer от позднего COMMIT v1. Исторические
versioned outputs сохраняются, их retention пока не задан. Два HTTP-запроса
по versionless endpoint вокруг upgrade могут увидеть разные published snapshots;
обязательный API не расширен новым version-download контрактом.
## Источники и библиотеки
- [Контракт и состояния](contracts/README.md), [attachment-states](contracts/attachment-states.md), [jobs schema](contracts/jobs.schema.json).
- [minio-cpp v0.3.0](https://github.com/minio/minio-cpp/tree/v0.3.0), [hiredis](https://github.com/redis/hiredis), [libpqxx](https://github.com/jtv/libpqxx).
- [stb, закреплённый commit](https://github.com/nothings/stb/tree/013ac3beddff3dbffafd5177e7972067cd2b5083), [libwebp](https://github.com/webmproject/libwebp), [libexif](https://github.com/libexif/libexif).
- libsodium — KDF/сессии/контрольные суммы; Pillow — независимое декодирование в тестах. Код lab4 перенесён из соседнего репозитория
+55
View File
@@ -0,0 +1,55 @@
# Provide the imported target names expected by minio-cpp's CMakeLists
# (unofficial-curlpp / unofficial-inih) using system packages from apt.
find_package(OpenSSL REQUIRED)
find_package(ZLIB REQUIRED)
find_package(CURL REQUIRED)
# nlohmann_json (header-only; Ubuntu ships /usr/include/nlohmann/json.hpp)
if(NOT TARGET nlohmann_json::nlohmann_json)
find_package(nlohmann_json QUIET CONFIG)
endif()
if(NOT TARGET nlohmann_json::nlohmann_json)
add_library(nlohmann_json INTERFACE)
add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json)
find_path(NLOHMANN_JSON_INCLUDE_DIR nlohmann/json.hpp)
if(NOT NLOHMANN_JSON_INCLUDE_DIR)
message(FATAL_ERROR "nlohmann/json.hpp not found (install nlohmann-json3-dev)")
endif()
target_include_directories(nlohmann_json INTERFACE "${NLOHMANN_JSON_INCLUDE_DIR}")
endif()
# pugixml
if(NOT TARGET pugixml)
find_package(pugixml QUIET CONFIG)
endif()
if(NOT TARGET pugixml AND NOT TARGET pugixml::pugixml)
pkg_check_modules(PUGIXML REQUIRED IMPORTED_TARGET pugixml)
add_library(pugixml INTERFACE)
target_link_libraries(pugixml INTERFACE PkgConfig::PUGIXML)
endif()
if(TARGET pugixml::pugixml AND NOT TARGET pugixml)
add_library(pugixml ALIAS pugixml::pugixml)
endif()
# curlpp → unofficial::curlpp::curlpp
if(NOT TARGET unofficial::curlpp::curlpp)
find_library(CURLPP_LIBRARY NAMES curlpp REQUIRED)
find_path(CURLPP_INCLUDE_DIR curlpp/cURLpp.hpp REQUIRED)
add_library(unofficial_curlpp INTERFACE)
target_include_directories(unofficial_curlpp INTERFACE "${CURLPP_INCLUDE_DIR}")
target_link_libraries(unofficial_curlpp INTERFACE "${CURLPP_LIBRARY}" CURL::libcurl)
add_library(unofficial::curlpp::curlpp ALIAS unofficial_curlpp)
endif()
# inih INIReader → unofficial::inih::inireader
if(NOT TARGET unofficial::inih::inireader)
find_library(INIREADER_LIBRARY NAMES INIReader REQUIRED)
find_library(INIH_LIBRARY NAMES inih REQUIRED)
find_path(INIREADER_INCLUDE_DIR INIReader.h REQUIRED)
add_library(unofficial_inih_inireader INTERFACE)
target_include_directories(unofficial_inih_inireader INTERFACE "${INIREADER_INCLUDE_DIR}")
target_link_libraries(unofficial_inih_inireader INTERFACE
"${INIREADER_LIBRARY}" "${INIH_LIBRARY}")
add_library(unofficial::inih::inireader ALIAS unofficial_inih_inireader)
endif()
+273
View File
@@ -0,0 +1,273 @@
services:
db:
image: postgres:16.15-alpine
command: postgres -c max_connections=200 -c shared_buffers=256MB
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
cpus: ${DB_CPU_LIMIT:-1.0}
mem_limit: 512m
stop_grace_period: 20s
cache:
image: valkey/valkey:8.1.10-alpine
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "always"]
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
start_period: 5s
cpus: "0.50"
mem_limit: 256m
stop_grace_period: 15s
api: &api
build: .
image: mtusi2026lab5-api:local
depends_on:
db:
condition: service_healthy
cache:
condition: service_healthy
minio:
condition: service_healthy
minio-user-init:
condition: service_completed_successfully
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-redis}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
BROKER_URL_FILE: /run/secrets/cache_url
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
HTTP_WORKERS: ${HTTP_WORKERS:-8}
DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
DIRECT_UPLOAD_TTL_SECONDS: ${DIRECT_UPLOAD_TTL_SECONDS:-300}
# Optional: sign URLs for a host-reachable MinIO (compose.presign.yaml).
S3_PRESIGN_ENDPOINT: ${S3_PRESIGN_ENDPOINT:-}
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: ${API_CPU_LIMIT:-0.50}
memory: ${API_MEM_LIMIT:-256m}
cpus: ${API_CPU_LIMIT:-0.50} # для совместимости с docker compose up
mem_limit: ${API_MEM_LIMIT:-256m}
stop_grace_period: 15s
worker:
<<: *api
# Keep the shared entrypoint: it stages secrets and drops root privileges.
entrypoint: ["/docker-entrypoint.sh", "/app/worker"]
command: []
restart: unless-stopped
environment:
DATABASE_URL_FILE: /run/secrets/database_url
BROKER_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-6}
JOB_RECLAIM_IDLE_MS: ${JOB_RECLAIM_IDLE_MS:-5000}
JOB_RETRY_BASE_MS: ${JOB_RETRY_BASE_MS:-1000}
JOB_RETRY_MAX_MS: ${JOB_RETRY_MAX_MS:-60000}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
STAGING_CLEANUP_INTERVAL_SECONDS: ${STAGING_CLEANUP_INTERVAL_SECONDS:-60}
healthcheck:
disable: true
deploy:
replicas: 1
resources:
limits:
cpus: "0.50"
memory: 512m
cpus: "0.50"
mem_limit: 512m
lb:
image: nginx:1.28.0-alpine
depends_on:
api:
condition: service_healthy
ports:
- "8080:8080"
environment:
NGINX_WORKERS: ${NGINX_WORKERS:-1}
NGINX_ENVSUBST_TEMPLATE_DIR: /etc/nginx/templates
NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx
NGINX_ENVSUBST_FILTER: "^NGINX_"
volumes:
- ./lb/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
cpus: ${LB_CPU_LIMIT:-0.25}
mem_limit: 128m
stop_grace_period: 10s
# Private S3-compatible store. Ports 9000/9001 are NOT published to the host.
# Image: pinned MinIO RELEASE.2025-04-22T22-12-26Z via local Dockerfile
# (official binary from GitHub; docker.io hub may be unreachable).
minio:
build:
context: ./infra/minio
args:
MINIO_VERSION: RELEASE.2025-04-22T22-12-26Z
image: mtusi2026lab5-minio:RELEASE.2025-04-22T22-12-26Z
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
volumes:
- miniodata:/data
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
cpus: ${MINIO_CPU_LIMIT:-0.50}
mem_limit: ${MINIO_MEM_LIMIT:-512m}
stop_grace_period: 15s
# Idempotent bucket bootstrap: create if missing, force private ACL.
minio-init:
build:
context: ./infra/mc
args:
MC_VERSION: RELEASE.2025-04-16T18-13-26Z
image: mtusi2026lab5-mc:RELEASE.2025-04-16T18-13-26Z
depends_on:
minio:
condition: service_healthy
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
echo "minio-init: waiting for endpoint $$S3_ENDPOINT"
i=0
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
i=$$((i + 1))
if [ "$$i" -ge 30 ]; then
echo "minio-init: cannot reach MinIO" >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing "local/$$S3_BUCKET"
mc anonymous set none "local/$$S3_BUCKET"
echo "minio-init: bucket listing:"
mc ls "local/$$S3_BUCKET" || true
echo "minio-init: bucket $$S3_BUCKET ready (anonymous=none)"
restart: "no"
# Idempotent limited S3 user for the API (not MinIO root).
minio-user-init:
image: mtusi2026lab5-mc:RELEASE.2025-04-16T18-13-26Z
depends_on:
minio:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
MINIO_USER_ACCESS_KEY: ${MINIO_USER_ACCESS_KEY}
MINIO_USER_SECRET_KEY: ${MINIO_USER_SECRET_KEY}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
POLICY_NAME: messenger-attachments-rw
volumes:
- ./infra/minio/policy-attachments.json:/policy.template.json:ro
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
echo "minio-user-init: alias $$S3_ENDPOINT"
i=0
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
i=$$((i + 1))
if [ "$$i" -ge 30 ]; then
echo "minio-user-init: cannot reach MinIO" >&2
exit 1
fi
sleep 1
done
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy.template.json > /tmp/policy.json
if mc admin policy info local "$$POLICY_NAME" >/dev/null 2>&1; then
mc admin policy remove local "$$POLICY_NAME" >/dev/null 2>&1 || true
fi
mc admin policy create local "$$POLICY_NAME" /tmp/policy.json
if mc admin user info local "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1; then
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY already exists"
else
mc admin user add local "$$MINIO_USER_ACCESS_KEY" "$$MINIO_USER_SECRET_KEY"
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY created"
fi
mc admin policy attach local "$$POLICY_NAME" --user "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1 || true
echo "minio-user-init: policy $$POLICY_NAME attached to $$MINIO_USER_ACCESS_KEY"
restart: "no"
secrets:
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata: {}
valkeydata: {}
miniodata: {}
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
# Docker Compose file-secrets keep host mode bits; app runs as uid 10001.
# Copy S3 credentials into an app-owned path before dropping privileges.
if [ "$(id -u)" = "0" ]; then
mkdir -p /tmp/app-secrets
if [ -r /run/secrets/s3_access_key_id ]; then
cp /run/secrets/s3_access_key_id /tmp/app-secrets/s3_access_key_id
fi
if [ -r /run/secrets/s3_secret_access_key ]; then
cp /run/secrets/s3_secret_access_key /tmp/app-secrets/s3_secret_access_key
fi
chown -R 10001:10001 /tmp/app-secrets
chmod 0400 /tmp/app-secrets/* 2>/dev/null || true
export S3_ACCESS_KEY_ID_FILE=/tmp/app-secrets/s3_access_key_id
export S3_SECRET_ACCESS_KEY_FILE=/tmp/app-secrets/s3_secret_access_key
exec setpriv --reuid=10001 --regid=10001 --clear-groups -- "$@"
fi
exec "$@"
+30
View File
@@ -3,3 +3,33 @@
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
## Lab5: итоговый функциональный прогон
- [async-suite.json](async-suite.json): `make test-async PYTHON=build/.venv/bin/python TEST_ARGS="--no-build --output evidence/async-suite.json"`.
- Результат: PASS, 18 групп проверок, 121.048 s (подъём и удаление стенда включены), Python 3.14.4, Pillow 12.1.1.
- JSON фиксирует source SHA-256, image/container IDs, StartedAt/volumes до и после рестартов, лимиты и наблюдения fault-tests.
- [upload-async.json](upload-async.json): отдельный успешный запуск shell-теста с предварительной сборкой образов.
Методика, профиль и ограничения: [REPORT.md](../REPORT.md) и
[student-tests/README.md](../student-tests/README.md). Это функциональные проверки;
нагрузочные перцентили не измерялись. Трек 2 уровня 5 проверен отдельно ниже. Credentials и
ресурсы disposable-проектов удалены после проверок.
## Уровень 5: трек 2
[track2.json](track2.json): три независимых повтора, по 20 групп проверок — PASS.
Время со стартом/удалением стенда: 187.385 / 185.107 / 187.243 s.
Всего 48 принятых attachments, 81 committed job, 24 реальных crash с exit code 86,
9 возобновлений старого worker после публикации v2. Потерь, незавершённых jobs,
дублей логических результатов и перезаписей v2 старой версией не обнаружено.
Методика и ограничения: [infra/track2.md](../infra/track2.md). JSON содержит source
и method SHA-256, image IDs, object SHA и фактические коды падений.
```sh
make test-track2 PYTHON=build/.venv/bin/python \
TEST_ARGS="--no-build --repetitions 3 --output evidence/track2.json"
```
Три изолированных проекта и их credentials удалены. Гарантия касается
логического versioned результата: физические XADD/PUT могут повторяться.
+313
View File
@@ -0,0 +1,313 @@
{
"scenario": "all",
"project": "lab5teste6c77bd21515",
"source_sha256": "60094f93be259bc3b3f6d6ec922dbdfbcb126f7587b1ba8e77158ce52b9c4795",
"python": "3.14.4",
"pillow": "12.1.1",
"started_at": "2026-11-14T23:17:29.320266+00:00",
"passed": true,
"cleanup": "removed own project and volumes; disposable secrets deleted",
"events": [
{
"check": "isolated stack started",
"project": "lab5teste6c77bd21515",
"containers": {
"api": [
{
"id": "004a042f3938ea83bb36133ac530ce3f35494bda8f5a5747edfc173bb8bae7cd",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:17:37.538664566Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
}
],
"worker": [
{
"id": "efe87e1c4e29124cb99105a8e252d8e9882bfd0b3f3fa0617d35e549fd0e5ba9",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:17:37.325636872Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
}
],
"db": [
{
"id": "6be56781aef8cca2c9edbeaef847ea0a5220245cca7842b8c02652082fc3ee0d",
"image": "sha256:f4610795587492cd4fb887c2dc3de301b813278bb6f616992061141b5f15ac36",
"started_at": "2026-11-14T23:17:30.462343085Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": [
"lab5teste6c77bd21515_pgdata"
]
}
],
"cache": [
{
"id": "f562a69df006bdee697532eaed6a944d0193b4094ba0ff40f832226aefaa1c42",
"image": "sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507",
"started_at": "2026-11-14T23:17:30.261027338Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 268435456,
"volumes": [
"lab5teste6c77bd21515_valkeydata"
]
}
],
"minio": [
{
"id": "3411bb6744c61e30770404239ac3e62f935db50c38959fd6ba913c62ee8019cc",
"image": "sha256:1fc9d7f3aae16d7573c9784271f15a05d483a1c33d6af4e417a89c1825141cfe",
"started_at": "2026-11-14T23:17:30.054292532Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": [
"lab5teste6c77bd21515_miniodata"
]
}
]
}
},
{
"check": "upload 202 -> GET queued -> ready, original and 3 variants verified",
"attachment_id": "81cc30ca-4c28-4453-8eb5-c8959785a113"
},
{
"check": "worker down: accepted job/session survive API, PostgreSQL, Valkey and S3 restart",
"attachment_id": "d5ec1997-a35d-426e-8fde-607333c2609a",
"before": {
"api": [
{
"id": "004a042f3938ea83bb36133ac530ce3f35494bda8f5a5747edfc173bb8bae7cd",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:17:37.538664566Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
}
],
"db": [
{
"id": "6be56781aef8cca2c9edbeaef847ea0a5220245cca7842b8c02652082fc3ee0d",
"image": "sha256:f4610795587492cd4fb887c2dc3de301b813278bb6f616992061141b5f15ac36",
"started_at": "2026-11-14T23:17:30.462343085Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": [
"lab5teste6c77bd21515_pgdata"
]
}
],
"cache": [
{
"id": "f562a69df006bdee697532eaed6a944d0193b4094ba0ff40f832226aefaa1c42",
"image": "sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507",
"started_at": "2026-11-14T23:17:30.261027338Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 268435456,
"volumes": [
"lab5teste6c77bd21515_valkeydata"
]
}
],
"minio": [
{
"id": "3411bb6744c61e30770404239ac3e62f935db50c38959fd6ba913c62ee8019cc",
"image": "sha256:1fc9d7f3aae16d7573c9784271f15a05d483a1c33d6af4e417a89c1825141cfe",
"started_at": "2026-11-14T23:17:30.054292532Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": [
"lab5teste6c77bd21515_miniodata"
]
}
]
},
"after": {
"api": [
{
"id": "004a042f3938ea83bb36133ac530ce3f35494bda8f5a5747edfc173bb8bae7cd",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:17:53.593930727Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
}
],
"db": [
{
"id": "6be56781aef8cca2c9edbeaef847ea0a5220245cca7842b8c02652082fc3ee0d",
"image": "sha256:f4610795587492cd4fb887c2dc3de301b813278bb6f616992061141b5f15ac36",
"started_at": "2026-11-14T23:17:47.099866146Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": [
"lab5teste6c77bd21515_pgdata"
]
}
],
"cache": [
{
"id": "f562a69df006bdee697532eaed6a944d0193b4094ba0ff40f832226aefaa1c42",
"image": "sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507",
"started_at": "2026-11-14T23:17:47.044747087Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 268435456,
"volumes": [
"lab5teste6c77bd21515_valkeydata"
]
}
],
"minio": [
{
"id": "3411bb6744c61e30770404239ac3e62f935db50c38959fd6ba913c62ee8019cc",
"image": "sha256:1fc9d7f3aae16d7573c9784271f15a05d483a1c33d6af4e417a89c1825141cfe",
"started_at": "2026-11-14T23:17:47.081708096Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": [
"lab5teste6c77bd21515_miniodata"
]
}
]
}
},
{
"check": "broker down: upload 202 and offline API restart preserve outbox"
},
{
"check": "crash after COMMIT before XADD: same attachment/job recovers without new POST",
"attachment_id": "715b3027-4756-4425-a170-6329fe19bb2e",
"exit_code": 86
},
{
"check": "crash after XADD before published marker: duplicate delivery, one ready effect",
"attachment_id": "bb925d74-c11c-40f8-a4c8-ae26daadb23b",
"stream_deliveries": 2,
"exit_code": 86
},
{
"check": "attachment/job/outbox transaction rolls back on outbox INSERT failure"
},
{
"check": "backoff+jitter persisted, attempts 1..6, no early processing, finite failed",
"delays": [
{
"attempt": 2,
"delay_ms": 193.0
},
{
"attempt": 3,
"delay_ms": 296.0
},
{
"attempt": 4,
"delay_ms": 722.0
},
{
"attempt": 5,
"delay_ms": 772.0
},
{
"attempt": 6,
"delay_ms": 742.0
}
]
},
{
"check": "audited replay verifies SHA, keeps job_id, fences stale events, refuses queued/ready",
"job_id": "b2b5743c-bbd1-40f1-91bc-d663797335f9"
},
{
"check": "SIGKILL after S3 before COMMIT/ACK: pending recovers the same effect",
"attachment_id": "92851587-7299-4bcd-9be2-33be1ca4b7c8"
},
{
"check": "watchdog bounds hung processing and Compose restarts worker",
"restart_count_before": 0,
"restart_count_after": 1
},
{
"check": "two workers serialize duplicate processing; variants and ready metadata stay unique",
"workers": [
{
"id": "efe87e1c4e29124cb99105a8e252d8e9882bfd0b3f3fa0617d35e549fd0e5ba9",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:18:52.539090243Z",
"exit_code": 0,
"restart_count": 1,
"memory_limit_bytes": 536870912,
"volumes": []
},
{
"id": "879ccd8c40bf96073dc9ff006ecb39c3ee27bb78698b887667505e57de248afe",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:18:56.32044115Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
}
],
"stream_deliveries": 5
},
{
"check": "two API publishers drain outbox with SKIP LOCKED",
"apis": [
{
"id": "dfb6a357bace91557e4f593f1ad397e0f09fd3675456a174af5997e01f06070d",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:18:20.837454075Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
},
{
"id": "237addf1f14359024ddd076325a28650d544f2a0bda3de04d705ec16bafec983",
"image": "sha256:301ec25bc35502f8e47d845720acad46ef5d19a206b568457df08d6d42ca2a47",
"started_at": "2026-11-14T23:19:03.201241928Z",
"exit_code": 0,
"restart_count": 0,
"memory_limit_bytes": 536870912,
"volumes": []
}
],
"jobs": 8
},
{
"check": "retention deletes orphan/ready staging, protects queued and live presigned source"
},
{
"check": "GC/upload race: PUT-before-COMMIT source is protected"
},
{
"check": "expired failed staging is collected; replay refuses cleaned source"
},
{
"check": "40M pixel RGB PNG produces three variants with 512 MiB and no swap",
"pixels": 40000000,
"memory_limit_bytes": 536870912,
"swap_bytes": 0
},
{
"check": "unchanged lab5 public smoke passed"
}
],
"elapsed_seconds": 121.048
}
+1311
View File
File diff suppressed because it is too large Load Diff
+21
View File
@@ -0,0 +1,21 @@
{
"scenario": "upload",
"project": "lab5test37f0067cc042",
"started_at": "2026-11-14T22:14:32.221211+00:00",
"passed": true,
"cleanup": "removed own project and volumes; disposable secrets deleted",
"events": [
{
"check": "isolated stack started",
"project": "lab5test37f0067cc042"
},
{
"check": "upload 202 -> GET queued -> ready, original and 3 variants verified",
"attachment_id": "ba5b7e16-cead-4dc7-a208-40cf0de8518d"
},
{
"check": "unchanged lab5 public smoke passed"
}
],
"elapsed_seconds": 27.777
}
+19
View File
@@ -0,0 +1,19 @@
# Local MinIO Client (mc) from the official GitHub release binary (pinned).
# Fallback when docker.io/minio/mc cannot be pulled.
FROM ubuntu:24.04
ARG MC_VERSION=RELEASE.2025-04-16T18-13-26Z
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
&& curl -fsSL -o /tmp/mc.bin \
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}" \
&& curl -fsSL -o /tmp/mc.sha256sum \
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}.sha256sum" \
&& expected="$(awk '{print $1}' /tmp/mc.sha256sum)" \
&& echo "${expected} /tmp/mc.bin" | sha256sum -c - \
&& install -m 0755 /tmp/mc.bin /usr/local/bin/mc \
&& rm -f /tmp/mc.bin /tmp/mc.sha256sum \
&& rm -rf /var/lib/apt/lists/*
ENTRYPOINT ["mc"]
+22
View File
@@ -0,0 +1,22 @@
# Local MinIO image from the official GitHub release binary (pinned).
# Fallback when docker.io/minio/minio cannot be pulled.
FROM ubuntu:24.04
ARG MINIO_VERSION=RELEASE.2025-04-22T22-12-26Z
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
&& curl -fsSL -o /tmp/minio.bin \
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}" \
&& curl -fsSL -o /tmp/minio.sha256sum \
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}.sha256sum" \
&& expected="$(awk '{print $1}' /tmp/minio.sha256sum)" \
&& echo "${expected} /tmp/minio.bin" | sha256sum -c - \
&& install -m 0755 /tmp/minio.bin /usr/local/bin/minio \
&& rm -f /tmp/minio.bin /tmp/minio.sha256sum \
&& rm -rf /var/lib/apt/lists/*
VOLUME ["/data"]
EXPOSE 9000 9001
ENTRYPOINT ["minio"]
CMD ["server", "/data", "--console-address", ":9001"]
+29
View File
@@ -0,0 +1,29 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucketMultipartUploads",
"s3:GetBucketLocation",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__/*"
]
}
]
}
+172
View File
@@ -0,0 +1,172 @@
# Трек 2: versioned pipeline и идемпотентный эффект
## Гипотеза и модель отказов
При повторной/переставленной доставке и crash worker в выбранных границах,
устойчиво принятое валидное вложение получает ready актуальной запрошенной версии.
Дубль не создаёт второй логический результат, а устаревшая доставка или уже
работающий старый worker не изменяет опубликованные объекты/метаданные новой версии.
Модель: PostgreSQL — источник истины и атомарной публикации, Redis Streams —
at least once, S3 PUT одного объекта атомарен. Volumes сохраняются; API publisher
и worker после сбоя возвращаются. Оригинал и алгоритм каждой версии неизменны.
Проверяется фиксированный набор отказов, а не все возможные распределённые истории.
## Версии и границы публикации
Миграция 011 добавляет:
- `attachments.pipeline_version` — последняя запрошенная версия;
- `attachments.published_version` — версия текущего complete ready-снимка;
- `jobs.pipeline_version/source_key`, unique `(attachment_id,pipeline_version)`;
- `attachment_results` с PK `(attachment_id,pipeline_version)` и одним полным
набором метаданных; INSERT не заменяет существующий результат;
- `pipeline_requests` — actor/reason/job_id запроса новой версии.
Job ID стабилен при redelivery/retry/replay одной версии. Новая версия — новая
логическая job. Wire schema остаётся `schema_version=1`, type
`attachment.process.v1`; поле pipeline_version различает рецепты обработки.
Обязательные OpenAPI и jobs schema не менялись. Внутренние storage_key из JSONB
не включаются в публичный JSON Attachment/Variant.
| Pipeline | JPEG thumbnail/crop | WebP optimized | Геометрия/alpha/EXIF |
| --- | --- | --- | --- |
| v1, default новых uploads | quality 85 | quality 80 | Как в основном контракте |
| v2, операторский upgrade | quality 90 | quality 90 | Та же геометрия; EXIF orientation, white alpha, metadata stripped |
Разница версий проверяется реальными различающимися SHA вариантов одного fixture.
Изменение рецепта/зависимостей, влияющее на bytes, требует новой версии; эксперимент
использует один и тот же image ID для обеих версий.
Результаты новых jobs:
```text
attachments/{chat_id}/{attachment_id}/v{pipeline_version}/original
attachments/{chat_id}/{attachment_id}/v{pipeline_version}/thumbnail.jpg
attachments/{chat_id}/{attachment_id}/v{pipeline_version}/crop.jpg
attachments/{chat_id}/{attachment_id}/v{pipeline_version}/optimized.webp
```
Ключ эффекта — `(attachment_id,pipeline_version,variant)`, включая original.
Legacy ready-записи мигрируются без изменения прежних ключей/метаданных;
download поддерживает старые варианты без внутреннего storage_key.
Worker сверяет version carrier с версией job в БД. Source key и рецепт захватываются
из job до обработки. Поэтому поздний worker не выбирает пути из новой версии
attachment. Jobs старее последней запрошенной версии отбрасываются до S3; queued/
processing становятся superseded, история ready/failed сохраняется.
После S3 worker повторно блокирует job/attachment. В одной транзакции сохраняется
versioned result и переключается публичный указатель **только если** desired
version по-прежнему равна версии job. Иначе старый завершившийся worker сохраняет
лишь свой исторический результат и получает superseded. Старые S3 PUT адресуют
только его namespace. Проверка до работы экономит ресурсы; проверка после работы
защищает от смены версии во время обработки.
Если уже есть ready v1, запрос/обработка/неудача v2 не скрывает v1. Публичный
снимок и download переключаются на v2 после полного успешного COMMIT. Начальные
вложения без ready-снимка по-прежнему проходят queued/processing/ready/failed.
Upgrade использует сохранённый опубликованный original, поэтому удалённый staging
не препятствует новой версии. При overlapping initial jobs GC удерживает source,
пока на него ссылается queued/processing job любой версии.
## Инварианты и аргумент защиты
1. **Устойчивое намерение.** До 202 оригинал сохранён, attachment/job/outbox
зафиксированы. Upgrade также фиксирует version/job/audit/outbox атомарно.
В эксперименте каждая принятая attachment заканчивается актуальным ready;
каждая зафиксированная job — ready или явно superseded.
2. **Один логический результат версии.** Unique job и PK result допускают одну
запись версии. Worker создаёт ровно три различных variant metadata; четыре
детерминированных S3-ключа не размножаются от доставки/перезапуска.
3. **Изоляция bytes разных версий.** Recipe/source захвачены из job, carrier не
может подменить её версию. PUT v1 не адресует v2. Проверяется SHA каждого
объекта v2 до и после поздней работы v1.
4. **Монотонная публикация.** Запросы версий сериализуются по attachment и не
уменьшают desired version. Row lock + условная публикация проверяют актуальность
в той же транзакции, что меняет pointer. После v2 версия v1 не может совпасть
с desired и опубликоваться поверх неё. DB CHECK дополнительно требует
published_version <= pipeline_version.
5. **ACK после durable handoff.** Успех подтверждается после COMMIT, retry — после
COMMIT следующего outbox. Смерть до ACK оставляет pending. После COMMIT результата
redelivery не вычисляет его заново и не увеличивает retries.
Повторные физические PUT/XADD возможны. Например, crash после PUT до DB COMMIT
оставляет объекты, которые retry запишет по тем же ключам. Гарантия относится к
логическому опубликованному эффекту, поэтому это **не exactly-once доставка**.
## Операторский интерфейс
```sh
docker compose exec -T api /docker-entrypoint.sh /app/job_admin upgrade ATTACHMENT_UUID \
--version 2 --actor operator-name --reason "quality pipeline upgrade"
```
Ответ — JSON с job_id, pipeline_version и created. Повтор версии 2 возвращает тот
же job_id и created=false без новой audit/outbox. Понижение версии отвергается.
Failed текущей версии можно replay прежней командой; устаревшая failed job не
replay. При неудачном upgrade прежний ready сохраняется до успешного replay v2.
Административные действия доступны только оператору DB/S3 secrets, не через HTTP.
## Воспроизводимый эксперимент
```sh
make test-track2 PYTHON=.venv/bin/python \
TEST_ARGS="--output evidence/track2.json"
# Для уже собранного неизменённого образа: добавить --no-build.
# Default — три независимых холодных стенда; --repetitions 1 для отладки.
```
Требования — student-tests/README.md. Каждый повтор создаёт свои DB/S3/Redis,
синтетический fixture sample.png (320×200, 111817 bytes), пользователей и IDs.
Worker serial; для состязания версий запускаются два процесса. COUNT 1, memory
512 MiB. Для fault-барьеров timeout 60 s, в обычном тестовом override — 3 s,
retry 200/800 ms, GC 1 s. Эти ускорения не меняют транзакционные guards.
Сводка хранит source/method SHA-256, реальные image IDs, object hashes, exit codes,
состояния и рестарты. Пароли, токены и signed URLs не сохраняются.
| Проверка каждого повтора | Воздействие и проверяемый результат |
| --- | --- |
| Legacy migration | Схема уровня 4 → 011, ready/key не меняются; unique/CHECK ограничения работают |
| 8 crash случаев | 4 точки ниже для v1 и v2, наблюдается Docker die exitCode=86; результат один |
| Дубли текущей версии | Пять доставок, одна result row и четыре S3-объекта, retries=0 |
| Два порядка версий | [1,2,1,2] и [2,1,2,1] при desired=2; v1 superseded без S3, v2 ready |
| Поздняя готовая v1 | v1 ready → v2 ready → старые/новые дубли; оба архивных результата неизменны |
| Несовпадение carrier/job | Подмена версии старой job направляется в invalid register без эффекта |
| Три overlapping барьера | v1 приостановлена, v2 ready, тот же старый процесс возобновлён; v2 SHA/pointer прежние |
| Failed upgrade/replay | v2 исчерпывает retry, v1 остаётся доступна; replay публикует v2 |
| Рестарт хранилищ | Все принятые IDs/снимки сохраняются после PostgreSQL/Valkey/MinIO/API restart |
| Публичный smoke | Неизменённый API scenario проходит |
Fault points, включаемые только через LAB_ENABLE_FAULT_INJECTION=1:
| LAB_FAULT_POINT | Граница | Ожидаемые retries после восстановления |
| --- | --- | --- |
| worker_before_effect | Processing уже durable, до GET/PUT/encode | 1 |
| worker_after_effect | Все четыре PUT завершены, DB result ещё не опубликован | 1 |
| worker_after_commit | Result/pointer durable, до ACK | 0 |
| worker_before_ack | Объекты дополнительно проверены, перед XACK | 0 |
LAB_FAULT_JOB_ID и LAB_FAULT_PIPELINE_VERSION выбирают ровно одну работу.
Default action — crash 86 один раз на container/job/point; marker сохраняется при
рестарте процесса. Action=pause создаёт barrier до появления release-файла;
watchdog остаётся включён. В normal Compose эти flags отсутствуют.
В overlapping case проверяются before_effect, after_effect и after_commit.
Первый даёт реальные **PUT старой версии после публикации новой**; второй — поздний
старый DB COMMIT; третий подтверждает ready в БД и pending в Redis до позднего ACK.
Container ID/restart count доказывают, что старый процесс возобновился, а не был
заменён новым. Проверка действительного содержимого S3 дополняет состояние БД.
## Границы результата
Эксперимент не является модельчекером всех сетевых расписаний и не измеряет RPS/
p95/RSS под нагрузкой. Не покрываются потеря volumes, произвольные изменения БД
оператором, недетерминированная замена алгоритма под тем же version number и
удаление опубликованных S3-объектов внешним процессом. Исторические versioned
outputs сохраняются; очистка staging не задаёт retention архивных версий.
API показывает последний опубликованный снимок. Два отдельных HTTP-запроса,
разделённые upgrade COMMIT, могут увидеть разные версии; обязательный контракт
не расширялся version-specific download URL. Каждый опубликованный снимок в БД
и его versioned S3-объекты остаются отдельно проверяемыми оператором.
+151
View File
@@ -0,0 +1,151 @@
# Pipeline: уровни 3–5
API сохраняет bytes в приватном `staging/{chat_id}/{id}`. В одной транзакции
PostgreSQL создаются attachment, job и outbox. После COMMIT возвращается
снимок `202 queued`; HTTP-путь не обращается к брокеру. Presigned complete
использует ту же схему и при повторе не создаёт новую job/outbox-запись.
В API работает отдельный поток-публикатор с собственными пулами соединений.
Раз в секунду он выбирает до 32 готовых строк через `FOR UPDATE SKIP LOCKED`,
делает XADD в `jobs:attachments` и ставит `published=true` в транзакции.
Несколько API-реплик могут публиковать одновременно. После XADD, но до
COMMIT отметки, возможен дубль; это доставка at least once, не exactly once.
Непубликованные строки переживают рестарт. Миграция 009 также создаёт
outbox-намерения для незавершённых jobs предыдущего шага.
Поле `payload` Stream содержит JSON по `contracts/jobs.schema.json`, без bytes,
URL и credentials. `job_id` стабилен в пределах версии; поддерживаются pipeline_version 1 и 2. Для новой управляемой
попытки формируется новое событие с `attempt=jobs.retries+1`. Старое событие
не запускает более новую попытку преждевременно и не откатывает ready.
Сессии могут храниться в PostgreSQL или Redis. При `SESSION_STORE=postgres`
API стартует и принимает upload даже при выключенном брокере; /health/ready
проверяет PostgreSQL. При Redis-сессиях Valkey остаётся синхронной зависимостью
аутентификации и readiness. S3 необходим для устойчивого приёма bytes до 202.
## Worker и повторные попытки
Worker — `/app/worker`, одна работа за раз. Consumer group `attachment-workers`
создаётся с позиции `0`, включая загрузки до первого старта worker.
XAUTOCLAIM подбирает pending после `JOB_RECLAIM_IDLE_MS` (по умолчанию 5000 ms).
Session advisory lock PostgreSQL на UUID job защищает эффект от одновременного
выполнения; смерть процесса освобождает lock. Незавершённый processing после
смерти worker считается неудачей и получает новый durable retry через outbox.
Временная ошибка атомарно сохраняет `queued`, увеличенный retries,
`next_attempt_at` и следующую outbox-запись с `available_at=next_attempt_at`.
Только после этого старое событие получает ACK. Публикатор не публикует
событие до available_at, worker дополнительно проверяет due time в БД.
Backoff: ceiling=min(JOB_RETRY_MAX_MS, JOB_RETRY_BASE_MS * 2^(retries-retry_base-1)),
задержка равномерно выбирается от ceil(ceiling/2) до ceiling включительно.
Период публикатора может добавить до секунды к задержке при свободном стенде.
`JOB_MAX_ATTEMPTS=6` означает начальную попытку и пять повторов. После шестой
неудачи текущего бюджета — failed/processing_failed и ACK. Невалидный исходник получает failed
сразу с безопасным кодом. Невалидные события регистрируются в
`jobs:attachments:invalid` с ID исходного события, без копирования bytes.
Watchdog ограничивает всю обработку job (включая S3/БД/ACK) значением
`JOB_TIMEOUT_SECONDS` (по умолчанию 60). При превышении процесс завершается
с кодом 124. Compose `restart: unless-stopped` запускает его снова; pending и
advisory lock обеспечивают восстановление. При запуске без Compose оператор
должен обеспечить рестарт процесса. SIGTERM позволяет закончить текущую job
в пределах того же таймаута. Worker читает COUNT 1, память ограничена 512 MiB.
## Изображения
Worker сверяет размер и SHA-256 staging, проверяет лимит 10 MiB и 40 миллионов
пикселей до декодирования. libexif применяет EXIF orientation (включая PNG eXIf).
RGBA компонуется на белый фон. Thumbnail вписан в 256×256 без увеличения,
размеры округляются вниз, минимум 1 px. JPEG: stb, quality=85 для v1 / 90 для v2, EXIF не переносится.
Из того же декодированного RGB-растра формируются crop и optimized:
- crop — центральный квадрат, затем JPEG 128×128 (маленький квадрат увеличивается);
при нечётной разнице отступ округляется вниз;
- optimized — lossy WebP quality=80 для v1 / 90 для v2, вписан в 1280×1280 без увеличения,
размеры округляются вниз, минимум 1 px. Используется libwebp WebPEncodeRGB,
EXIF/ICC/XMP не переносятся. Для WebP alpha также компонуется на белый.
Уменьшение размера каждого файла не гарантируется.
Заголовки stb закреплены commit из `third_party/stb/VERSION`.
Новые outputs публикуются в `attachments/{chat_id}/{id}/v{version}/`: original,
thumbnail.jpg, crop.jpg, optimized.webp. Прежние `{id}_256.jpg` и другие legacy
keys остаются доступны через fallback для старых metadata. Статусы job/attachment и все три variants
публикуются атомарно только после сохранения всех объектов S3. Ready-дубль
проверяет наличие и размеры опубликованных объектов, делает ACK и сохраняет
прежние метаданные. Готовые записи предыдущих шагов сохраняют прежние варианты;
ready не откатывается в processing ради обновления pipeline.
## Проверки и fault injection
Отдельно thumbnail: `python3 student-tests/test_thumbnail.py --binary build/thumbnail_smoke`.
Для независимой проверки JPEG и EXIF нужен Pillow; `--workdir PATH` выбирает
каталог временных синтетических файлов. Новые варианты: `python3 student-tests/test_variants.py --binary build/variants_smoke`.
Публичный smoke: `make test`.
Для локального теста crash предусмотрены два явных opt-in env:
`LAB_ENABLE_FAULT_INJECTION=1` и `LAB_FAULT_POINT=upload_after_commit` либо
`outbox_after_publish`. Первый завершает API с кодом 86 после COMMIT upload,
второй — после XADD до отметки published. Эти переменные не включены в Compose
по умолчанию и не управляются HTTP-запросом. Для детерминированного теста первого
окна остановите брокер, используйте PostgreSQL-сессии, вызовите upload; затем
перезапустите API без fault flags и верните брокер. Новый POST не нужен: та же
запись outbox будет опубликована и та же job завершится. Ответ при crash может
не дойти до клиента; это не отменяет уже зафиксированную транзакцию.
## Очистка staging
После ready и проверки итоговых объектов worker удаляет staging. Если действует
presigned URL, удаление откладывается до `direct_uploads.expires_at`, чтобы URL
не мог сразу восстановить удалённый объект. Внутренний marker
`staging_cleaned_at` не меняет публичный `updated_at`.
Раз в `STAGING_CLEANUP_INTERVAL_SECONDS` (по умолчанию 60) worker проверяет до
32 известных ready/истёкших failed исходников и до 100 объектов S3-prefix staging
с курсором. Для failed применяется retention от `attachments.updated_at`, для
orphan — от S3 LastModified; по умолчанию `STAGING_RETENTION_SECONDS=86400`.
Queued/processing защищены независимо от возраста. Действующие presigned-сессии
также защищены. Истёкшие failed и orphan удаляются; метаданные failed/audit остаются.
Ключи вне серверного формата `staging/{chat UUID}/{attachment UUID}` не удаляются.
При недоступности S3/БД очистка повторяется в следующем цикле; ready не откатывается.
API удерживает PostgreSQL advisory xact lock на `staging:<attachment UUID>` от
PUT до COMMIT метаданных. GC берёт тот же lock неблокирующим образом и повторно
проверяет ссылки в БД. Поэтому пауза API после PUT до INSERT не превращает
текущую загрузку в удаляемый orphan. GC дополнительно сериализован с worker/replay
через job lock и row lock. При GC/COMMIT crash повторное удаление идемпотентно.
Рабочие исходники и последний staging при неполном ready-результате не удаляются.
## Операторский replay
Команда доступна только оператору с доступом к DB/S3 secrets контейнера;
публичного административного HTTP endpoint нет:
```sh
docker compose exec -T api /docker-entrypoint.sh /app/job_admin replay JOB_UUID \
--actor operator-name --reason "temporary storage failure fixed"
```
Replay принимает только failed job актуальной версии с сохранённым source,
сверяет его размер и SHA-256. Source — staging или immutable original предыдущей
версии. Отказ для queued/processing/ready, устаревшей версии и удалённого source. Под job
и row lock одной транзакцией записываются actor/reason/предыдущая ошибка в
`job_replays`, состояния queued и новый outbox. Job ID и ключи эффектов прежние для replay одной версии.
`retries` монотонен, `retry_base` задаёт начало нового бюджета попыток, поэтому
старые stream-события не выполняют новый replay. Повтор команды для уже queued
или ready отказывается без дополнительного audit/outbox. Actor — до 100 UTF-8
байт, reason — до 500; не записывайте туда credentials. Реестр failed доступен
оператору в `jobs WHERE status='failed'`; журнал replay хранится в `job_replays`.
После удаления единственного staging source replay невозможен: нужен новый upload.
Upgrade с уже опубликованного original не зависит от staging retention.
## Versioned pipeline (трек 2)
Миграция 011 и расширение worker/operator CLI описаны в [track2.md](track2.md).
Новые outputs используют `attachments/{chat}/{id}/v{version}/…`; legacy keys
продолжают читаться. Public JSON не раскрывает storage_key. Jobs уникальны по
attachment/version; version 2 меняет quality JPEG/WebP на 90. Старый ready остаётся
доступным до успешной атомарной публикации новой версии; ошибки upgrade наблюдаются
в jobs. GC не удаляет source, пока его использует active job любой версии.
+44
View File
@@ -0,0 +1,44 @@
worker_processes ${NGINX_WORKERS};
error_log /dev/stderr warn;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 2s;
upstream api_upstream {
zone api_upstream 64k;
server api:8080 resolve max_fails=1 fail_timeout=5s;
keepalive 32;
}
server {
listen 8080;
# Lab4 attachments ≤10 MiB; allow a little headroom for multipart/headers.
client_max_body_size 12m;
client_body_timeout 60s;
location / {
proxy_pass http://api_upstream;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 10s;
proxy_connect_timeout 2s;
# Uploads/downloads of ≤10 MiB need longer than JSON chat traffic.
proxy_read_timeout 120s;
proxy_send_timeout 120s;
}
}
}
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX messages_chat_keyset_idx
ON messages (chat_id, created_at, id);
+6
View File
@@ -0,0 +1,6 @@
CREATE TABLE sessions (
token text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
+22
View File
@@ -0,0 +1,22 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename text NOT NULL
CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL
CHECK (size_bytes >= 1 AND size_bytes <= 10485760),
sha256 text NOT NULL
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
status text NOT NULL
CHECK (status IN ('ready')),
s3_key text NOT NULL UNIQUE,
error_code text NULL,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
CREATE INDEX attachments_created_at_idx ON attachments (created_at);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id),
position integer NOT NULL CHECK (position >= 0 AND position < 10),
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX message_attachments_attachment_idx
ON message_attachments (attachment_id);
CREATE UNIQUE INDEX message_attachments_order_idx
ON message_attachments (message_id, position);
+26
View File
@@ -0,0 +1,26 @@
-- Track 1 (grade 5): pending direct/presigned upload sessions.
-- Attachment rows stay status=ready only; incomplete uploads live here.
CREATE TABLE direct_uploads (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename text NOT NULL
CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CHECK (content_type IN ('image/png', 'image/jpeg')),
expected_size integer NOT NULL
CHECK (expected_size >= 1 AND expected_size <= 10485760),
expected_sha256 text NOT NULL
CHECK (expected_sha256 ~ '^[a-f0-9]{64}$'),
s3_key text NOT NULL UNIQUE,
status text NOT NULL
CHECK (status IN ('pending', 'completed', 'aborted', 'expired')),
attachment_id uuid NULL REFERENCES attachments(id),
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX direct_uploads_status_expires_idx
ON direct_uploads (status, expires_at);
CREATE INDEX direct_uploads_owner_idx ON direct_uploads (owner_id);
+12
View File
@@ -0,0 +1,12 @@
-- Track 1: presigned PUT lands in staging/; complete copies to attachments/.
-- Existing rows keep the old key as staging so in-flight sessions still resolve.
ALTER TABLE direct_uploads ADD COLUMN staging_key text;
UPDATE direct_uploads
SET staging_key = s3_key
WHERE staging_key IS NULL;
ALTER TABLE direct_uploads ALTER COLUMN staging_key SET NOT NULL;
ALTER TABLE direct_uploads
ADD CONSTRAINT direct_uploads_staging_key_key UNIQUE (staging_key);
+21
View File
@@ -0,0 +1,21 @@
-- Keep existing lab4 originals ready; new jobs set queued explicitly.
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
ALTER TABLE attachments
ADD CONSTRAINT attachments_status_check
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]'::jsonb
CHECK (jsonb_typeof(variants) = 'array');
CREATE TABLE jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL UNIQUE
REFERENCES attachments(id) ON DELETE CASCADE,
status text NOT NULL DEFAULT 'queued'
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
retries integer NOT NULL DEFAULT 0 CHECK (retries >= 0),
error_code text,
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX jobs_status_created_at_idx ON jobs (status, created_at);
+20
View File
@@ -0,0 +1,20 @@
ALTER TABLE jobs ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now();
CREATE TABLE outbox (
id bigserial PRIMARY KEY,
payload jsonb NOT NULL CHECK (jsonb_typeof(payload) = 'object'),
published boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT now(),
available_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE NOT published;
-- Upgrade jobs accepted before outbox existed; duplicates are safe.
INSERT INTO outbox (payload, available_at)
SELECT jsonb_build_object(
'schema_version', 1, 'type', 'attachment.process.v1',
'job_id', id::text, 'attachment_id', attachment_id::text,
'pipeline_version', 1, 'attempt', retries + 1,
'created_at', to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')
), next_attempt_at FROM jobs WHERE status IN ('queued', 'processing');
+22
View File
@@ -0,0 +1,22 @@
ALTER TABLE attachments
ADD COLUMN staging_key text,
ADD COLUMN staging_cleaned_at timestamptz;
UPDATE attachments a SET staging_key='staging/' || a.chat_id::text || '/' || a.id::text
WHERE EXISTS (SELECT 1 FROM jobs j WHERE j.attachment_id=a.id);
CREATE INDEX attachments_staging_cleanup_idx ON attachments(status, updated_at)
WHERE staging_key IS NOT NULL AND staging_cleaned_at IS NULL;
ALTER TABLE jobs ADD COLUMN retry_base integer NOT NULL DEFAULT 0
CHECK (retry_base >= 0 AND retry_base <= retries);
CREATE TABLE job_replays (
id bigserial PRIMARY KEY,
job_id uuid NOT NULL REFERENCES jobs(id),
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100),
reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500),
previous_error_code text,
retries_before integer NOT NULL CHECK (retries_before >= 0),
created_at timestamptz NOT NULL DEFAULT now()
);
+41
View File
@@ -0,0 +1,41 @@
ALTER TABLE attachments
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
ADD COLUMN published_version integer CHECK (published_version > 0),
ADD CONSTRAINT attachments_version_order CHECK (published_version <= pipeline_version);
UPDATE attachments SET published_version=1 WHERE status='ready';
ALTER TABLE jobs
ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version > 0),
ADD COLUMN source_key text;
UPDATE jobs j SET source_key=a.s3_key FROM attachments a WHERE a.id=j.attachment_id;
ALTER TABLE jobs ALTER COLUMN source_key SET NOT NULL;
ALTER TABLE jobs DROP CONSTRAINT jobs_attachment_id_key;
ALTER TABLE jobs ADD CONSTRAINT jobs_attachment_version_key UNIQUE (attachment_id, pipeline_version);
ALTER TABLE jobs DROP CONSTRAINT jobs_status_check;
ALTER TABLE jobs ADD CONSTRAINT jobs_status_check
CHECK (status IN ('queued','processing','ready','failed','superseded'));
-- One immutable logical result per (attachment, pipeline version).
CREATE TABLE attachment_results (
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
job_id uuid UNIQUE REFERENCES jobs(id),
original_key text NOT NULL,
variants jsonb NOT NULL CHECK (jsonb_typeof(variants)='array'),
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (attachment_id, pipeline_version)
);
INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants)
SELECT a.id,1,j.id,a.s3_key,a.variants FROM attachments a
LEFT JOIN jobs j ON j.attachment_id=a.id AND j.pipeline_version=1 WHERE a.status='ready';
CREATE TABLE pipeline_requests (
id bigserial PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK (pipeline_version > 0),
job_id uuid NOT NULL UNIQUE REFERENCES jobs(id),
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 100),
reason text NOT NULL CHECK (char_length(reason) BETWEEN 1 AND 500),
created_at timestamptz NOT NULL DEFAULT now(),
UNIQUE(attachment_id,pipeline_version)
);
+93
View File
@@ -0,0 +1,93 @@
#include "attachment_queue.hpp"
#include "errors.hpp"
#include <memory>
void AttachmentQueue::publish(const std::string& payload) {
auto lease = pool_.acquire();
std::unique_ptr<redisReply, decltype(&freeReplyObject)> reply(
static_cast<redisReply*>(redisCommand(
lease.ctx(), "XADD jobs:attachments * payload %b",
payload.data(), payload.size())), &freeReplyObject);
if (!reply || reply->type != REDIS_REPLY_STRING || reply->len == 0) {
throw DependencyUnavailable("attachment queue publish failed");
}
}
namespace {
using Reply = std::unique_ptr<redisReply, decltype(&freeReplyObject)>;
Reply checked(redisReply* reply) {
Reply owned(reply, &freeReplyObject);
if (!owned || owned->type == REDIS_REPLY_ERROR) {
throw DependencyUnavailable("attachment stream command failed");
}
return owned;
}
JobDelivery delivery(const redisReply* entry) {
if (entry->type != REDIS_REPLY_ARRAY || entry->elements != 2 ||
entry->element[0]->type != REDIS_REPLY_STRING ||
entry->element[1]->type != REDIS_REPLY_ARRAY) {
throw DependencyUnavailable("invalid stream response");
}
JobDelivery result{std::string(entry->element[0]->str, entry->element[0]->len), {}};
const auto* fields = entry->element[1];
for (std::size_t i = 0; i + 1 < fields->elements; i += 2) {
if (fields->element[i]->type == REDIS_REPLY_STRING &&
std::string(fields->element[i]->str, fields->element[i]->len) == "payload" &&
fields->element[i + 1]->type == REDIS_REPLY_STRING && fields->element[i + 1]->len <= 4096) {
result.payload.assign(fields->element[i + 1]->str, fields->element[i + 1]->len);
}
}
return result;
}
} // namespace
void AttachmentQueue::ensure_group() {
auto lease = pool_.acquire();
Reply reply(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XGROUP CREATE jobs:attachments attachment-workers 0 MKSTREAM")), &freeReplyObject);
if (!reply || (reply->type == REDIS_REPLY_ERROR &&
std::string(reply->str, reply->len).rfind("BUSYGROUP", 0) != 0)) {
throw DependencyUnavailable("attachment consumer group unavailable");
}
}
std::optional<JobDelivery> AttachmentQueue::read(const std::string& consumer, int reclaim_idle_ms) {
auto lease = pool_.acquire();
auto pending = checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XAUTOCLAIM jobs:attachments attachment-workers %s %d %s COUNT 1",
consumer.c_str(), reclaim_idle_ms, claim_cursor_.c_str())));
if (pending->type != REDIS_REPLY_ARRAY || pending->elements < 2 ||
pending->element[0]->type != REDIS_REPLY_STRING ||
pending->element[1]->type != REDIS_REPLY_ARRAY) {
throw DependencyUnavailable("invalid pending response");
}
claim_cursor_.assign(pending->element[0]->str, pending->element[0]->len);
if (pending->element[1]->elements) return delivery(pending->element[1]->element[0]);
auto fresh = checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XREADGROUP GROUP attachment-workers %s COUNT 1 BLOCK 1000 STREAMS jobs:attachments >",
consumer.c_str())));
if (fresh->type == REDIS_REPLY_NIL) return std::nullopt;
if (fresh->type != REDIS_REPLY_ARRAY || fresh->elements != 1 ||
fresh->element[0]->type != REDIS_REPLY_ARRAY || fresh->element[0]->elements != 2 ||
fresh->element[0]->element[1]->type != REDIS_REPLY_ARRAY ||
fresh->element[0]->element[1]->elements != 1) {
throw DependencyUnavailable("invalid stream read response");
}
return delivery(fresh->element[0]->element[1]->element[0]);
}
void AttachmentQueue::ack(const std::string& stream_id) {
auto lease = pool_.acquire();
checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XACK jobs:attachments attachment-workers %s", stream_id.c_str())));
}
void AttachmentQueue::reject(const std::string& stream_id) {
{
auto lease = pool_.acquire();
checked(static_cast<redisReply*>(redisCommand(lease.ctx(),
"XADD jobs:attachments:invalid * stream_id %s error_code invalid_job", stream_id.c_str())));
}
ack(stream_id);
}
+27
View File
@@ -0,0 +1,27 @@
#pragma once
#include "redis_pool.hpp"
#include <string>
#include <optional>
struct JobDelivery {
std::string stream_id;
std::string payload;
};
class AttachmentQueue {
public:
explicit AttachmentQueue(RedisPool& pool) : pool_(pool) {}
void publish(const std::string& payload);
void ensure_group();
std::optional<JobDelivery> read(const std::string& consumer, int reclaim_idle_ms);
void ack(const std::string& stream_id);
void reject(const std::string& stream_id);
bool ping() { return pool_.ping(); }
private:
RedisPool& pool_;
std::string claim_cursor_ = "0-0";
};
+156
View File
@@ -0,0 +1,156 @@
#include "attachment_repo.hpp"
#include "outbox.hpp"
#include <pqxx/pqxx>
#include <nlohmann/json.hpp>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <sstream>
namespace {
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch())
.count();
return std::to_string(secs);
}
Attachment row_to_attachment(const pqxx::row& r) {
Attachment a;
a.id = r["id"].as<std::string>();
a.chat_id = r["chat_id"].as<std::string>();
a.owner_id = r["owner_id"].as<std::string>();
a.filename = r["filename"].as<std::string>();
a.content_type = r["content_type"].as<std::string>();
a.size_bytes = r["size_bytes"].as<int>();
a.sha256 = r["sha256"].as<std::string>();
a.status = r["status"].as<std::string>();
a.s3_key = r["s3_key"].as<std::string>();
if (r["error_code"].is_null()) {
a.error_code = std::nullopt;
} else {
a.error_code = r["error_code"].as<std::string>();
}
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
for (const auto& v : nlohmann::json::parse(r["variants"].as<std::string>())) {
a.variants.push_back(Variant{v.at("name").get<std::string>(),
v.at("content_type").get<std::string>(), v.at("size_bytes").get<int>(),
v.at("sha256").get<std::string>(), v.at("width").get<int>(),
v.at("height").get<int>(), v.value("storage_key", "")});
}
return a;
}
constexpr const char* kAttachmentColumns =
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, error_code, variants, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
} // namespace
Attachment AttachmentRepo::insert(const Attachment& in) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
pqxx::result r;
if (in.error_code) {
r = tx.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, "
" to_timestamp($11::bigint), to_timestamp($12::bigint)"
") RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, *in.error_code, ts, ts});
} else {
r = tx.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
") RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, ts, ts});
}
tx.commit();
return row_to_attachment(r[0]);
}
std::optional<Attachment> AttachmentRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kAttachmentColumns) +
" FROM attachments WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to_attachment(r[0]);
}
Attachment AttachmentRepo::insert_queued(const Attachment& in,
const std::string& job_id,
const std::function<void()>& save_original) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
tx.exec("SELECT pg_advisory_xact_lock(hashtextextended('staging:' || $1::uuid::text, 0))",
pqxx::params{in.id});
save_original();
auto r = tx.exec(
"INSERT INTO attachments ("
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, staging_key, created_at, updated_at) VALUES ("
"$1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, 'queued', $8, $8, now(), now()) "
"RETURNING " + std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.s3_key});
tx.exec("INSERT INTO jobs (id, attachment_id, source_key) VALUES ($1::uuid, $2::uuid, $3)",
pqxx::params{job_id, in.id, in.s3_key});
enqueue_attachment_job(tx, job_id);
Attachment saved = row_to_attachment(r[0]);
tx.commit();
return saved;
}
bool AttachmentRepo::all_belong_to_chat(const std::string& chat_id,
const std::vector<std::string>& ids) {
if (ids.empty()) {
return true;
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
for (const auto& id : ids) {
auto r = tx.exec(
"SELECT 1 FROM attachments "
"WHERE id = $1::uuid AND chat_id = $2::uuid AND status <> 'failed'",
pqxx::params{id, chat_id});
if (r.empty()) {
return false;
}
}
return true;
}
+28
View File
@@ -0,0 +1,28 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <functional>
#include <string>
#include <vector>
class AttachmentRepo {
public:
explicit AttachmentRepo(PgPool& pool) : pool_(pool) {}
Attachment insert(const Attachment& a);
Attachment insert_queued(const Attachment& a, const std::string& job_id,
const std::function<void()>& save_original);
std::optional<Attachment> find_by_id(const std::string& id);
// All IDs must exist, belong to chat_id, and not have status=failed.
// Returns false if any id is missing/foreign/failed.
bool all_belong_to_chat(const std::string& chat_id,
const std::vector<std::string>& ids);
private:
PgPool& pool_;
};
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <stdexcept>
#include <string>
inline std::string attachment_variant_key(const std::string& chat_id,
const std::string& id,
const std::string& variant) {
const char* suffix = variant == "thumbnail" ? "_256.jpg" :
variant == "crop" ? "_128.jpg" :
variant == "optimized" ? "_optimized.webp" : nullptr;
if (!suffix) throw std::invalid_argument("unknown variant");
return "attachments/" + chat_id + "/" + id + suffix;
}
inline std::string pipeline_prefix(const std::string& chat_id, const std::string& id,
int version) {
if (version < 1) throw std::invalid_argument("invalid pipeline version");
return "attachments/" + chat_id + "/" + id + "/v" + std::to_string(version) + "/";
}
+39
View File
@@ -0,0 +1,39 @@
#include "auth.hpp"
#include <sodium.h>
#include <stdexcept>
namespace {
struct SodiumInit {
SodiumInit() {
if (sodium_init() < 0) {
throw std::runtime_error("sodium_init failed");
}
}
};
const SodiumInit g_sodium_init;
}
std::string hash_password(const std::string& plain) {
char hashed[crypto_pwhash_STRBYTES];
if (crypto_pwhash_str(
hashed,
plain.data(), plain.size(),
crypto_pwhash_OPSLIMIT_INTERACTIVE,
crypto_pwhash_MEMLIMIT_INTERACTIVE) != 0) {
throw std::runtime_error("crypto_pwhash_str failed (out of memory?)");
}
return std::string(hashed);
}
bool verify_password(const std::string& plain, const std::string& hash) {
if (hash.empty() || hash.size() >= crypto_pwhash_STRBYTES) {
return false;
}
return crypto_pwhash_str_verify(
hash.c_str(),
plain.data(), plain.size()) == 0;
}
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#include <string>
std::string hash_password(const std::string& plain);
bool verify_password(const std::string& plain, const std::string& hash);
+186
View File
@@ -0,0 +1,186 @@
#include "chat_repo.hpp"
#include "cursor.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <random>
#include <sstream>
#include <string>
namespace {
std::string make_uuid() {
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
Chat row_to_chat(const pqxx::row& r) {
Chat c;
c.id = r["id"].as<std::string>();
c.title = r["title"].as<std::string>();
c.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return c;
}
std::vector<std::string> load_member_ids(pqxx::work& tx,
const std::string& chat_id) {
std::vector<std::string> ids;
auto r = tx.exec(
"SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id",
pqxx::params{chat_id});
ids.reserve(r.size());
for (const auto& row : r) {
ids.push_back(row["user_id"].as<std::string>());
}
return ids;
}
constexpr const char* kChatSelect =
"id, title, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
}
std::optional<Chat> ChatRepo::create(const std::string& title,
const std::vector<std::string>& member_ids) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string chat_id = make_uuid();
const std::string created = now_epoch_str();
try {
tx.exec(
"INSERT INTO chats (id, title, created_at) "
"VALUES ($1, $2, to_timestamp($3::bigint))",
pqxx::params{chat_id, title, created});
for (const auto& uid : member_ids) {
tx.exec(
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
pqxx::params{chat_id, uid});
}
} catch (const pqxx::foreign_key_violation&) {
tx.abort();
return std::nullopt;
} catch (const pqxx::unique_violation&) {
tx.abort();
return std::nullopt;
}
auto r = tx.exec(
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
pqxx::params{chat_id});
Chat c = row_to_chat(r[0]);
c.member_ids = load_member_ids(tx, chat_id);
tx.commit();
return c;
}
std::optional<Chat> ChatRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
pqxx::params{id});
if (r.empty()) {
tx.commit();
return std::nullopt;
}
Chat c = row_to_chat(r[0]);
c.member_ids = load_member_ids(tx, id);
tx.commit();
return c;
}
std::optional<ChatPage> ChatRepo::list_for_user(
const std::string& user_id,
int limit,
const std::optional<std::string>& cursor) {
std::optional<Cursor> cur;
if (cursor) {
cur = parse_cursor(*cursor);
if (!cur) {
return std::nullopt;
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
if (cur) {
auto owner_check = tx.exec(
"SELECT 1 FROM chats WHERE id = $1 AND id IN "
"(SELECT chat_id FROM chat_members WHERE user_id = $2)",
pqxx::params{cur->id, user_id});
if (owner_check.empty()) {
tx.commit();
return std::nullopt;
}
}
std::string sql =
"SELECT " + std::string(kChatSelect) + ", " + kCreatedUsSql + " "
"FROM chats "
"WHERE id IN (SELECT chat_id FROM chat_members WHERE user_id = $1) ";
if (cur) {
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
}
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
pqxx::result r;
if (cur) {
r = tx.exec(sql, pqxx::params{user_id, limit + 1, cur->created_us, cur->id});
} else {
r = tx.exec(sql, pqxx::params{user_id, limit + 1});
}
ChatPage page;
const bool has_more = (static_cast<int>(r.size()) > limit);
const int take = has_more ? limit : static_cast<int>(r.size());
page.items.reserve(take);
for (int i = 0; i < take; ++i) {
Chat c = row_to_chat(r[static_cast<std::size_t>(i)]);
c.member_ids = load_member_ids(tx, c.id);
page.items.push_back(std::move(c));
}
if (has_more && take > 0) {
const auto& last = r[static_cast<std::size_t>(take - 1)];
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
last["id"].as<std::string>());
}
tx.commit();
return page;
}
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
class ChatRepo {
public:
explicit ChatRepo(PgPool& pool) : pool_(pool) {}
std::optional<Chat> create(const std::string& title,
const std::vector<std::string>& member_ids);
std::optional<Chat> find_by_id(const std::string& id);
std::optional<ChatPage> list_for_user(const std::string& user_id, int limit, const std::optional<std::string>& cursor);
private:
PgPool& pool_;
};
+47
View File
@@ -0,0 +1,47 @@
#pragma once
#include <algorithm>
#include <cctype>
#include <optional>
#include <regex>
#include <string>
// Непрозрачный для клиента курсор keyset-пагинации: "<created_at в мкс от epoch>|<uuid>".
// Микросекунды сохраняют точный created_at строки, даже если в JSON время округлено до секунд.
struct Cursor {
std::string created_us;
std::string id;
};
// Выражение SQL, восстанавливающее timestamptz из created_us без потери точности.
constexpr const char* kCursorTimestampSql =
"(timestamptz 'epoch' + $3::bigint * interval '1 microsecond')";
constexpr const char* kCreatedUsSql =
"(EXTRACT(EPOCH FROM created_at) * 1000000)::bigint AS created_us";
inline bool is_uuid(const std::string& s) {
static const std::regex re(
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
return std::regex_match(s, re);
}
inline std::optional<Cursor> parse_cursor(const std::string& raw) {
const auto bar = raw.find('|');
if (bar == std::string::npos) {
return std::nullopt;
}
Cursor c{raw.substr(0, bar), raw.substr(bar + 1)};
const bool digits = !c.created_us.empty() && c.created_us.size() <= 18 &&
std::all_of(c.created_us.begin(), c.created_us.end(),
[](unsigned char ch) { return std::isdigit(ch); });
if (!digits || !is_uuid(c.id)) {
return std::nullopt;
}
return c;
}
inline std::string make_cursor(const std::string& created_us, const std::string& id) {
return created_us + "|" + id;
}
+360
View File
@@ -0,0 +1,360 @@
#include "db.hpp"
#include "errors.hpp"
#include <chrono>
#include <fstream>
#include <future>
#include <thread>
#include <sys/socket.h>
#include <sys/time.h>
#include <stdexcept>
#include <algorithm>
#include <filesystem>
#include <vector>
#include <set>
#include <iostream>
#include <nlohmann/json.hpp>
namespace fs = std::filesystem;
namespace {
constexpr int kConnectTimeoutSeconds = 2;
constexpr int kTcpUserTimeoutMs = 5000;
constexpr int kStatementTimeoutMs = 5000;
constexpr int kClientIoTimeoutSeconds = 5;
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
std::string add_conn_param(std::string conninfo, const std::string& key,
const std::string& value) {
if (conninfo.find(key + "=") != std::string::npos) {
return conninfo;
}
const bool is_url = conninfo.rfind("postgres://", 0) == 0 ||
conninfo.rfind("postgresql://", 0) == 0;
if (is_url) {
conninfo += (conninfo.find('?') == std::string::npos ? "?" : "&");
} else {
conninfo += " ";
}
return conninfo + key + "=" + value;
}
bool is_alive(pqxx::connection* conn) {
std::promise<void> done;
auto fut = done.get_future();
std::thread th([conn, &done]() {
try {
pqxx::nontransaction tx(*conn);
tx.exec("SELECT 1");
done.set_value();
} catch (...) {
done.set_exception(std::current_exception());
}
});
if (fut.wait_for(std::chrono::seconds(kClientIoTimeoutSeconds)) !=
std::future_status::ready) {
const int fd = conn->sock();
if (fd >= 0) {
::shutdown(fd, SHUT_RDWR);
}
th.join();
return false;
}
th.join();
try {
fut.get();
return true;
} catch (...) {
return false;
}
}
struct Migration {
int version;
std::string path;
};
constexpr int kMigrationsAdvisoryLockKey = 0x6d747573;
Migration parse_migration_name(const fs::path& p) {
const std::string name = p.filename().string();
const auto dot = name.find('.');
if (dot == std::string::npos || name.substr(dot) != ".sql") {
throw std::runtime_error("migration file must end with .sql: " + name);
}
const auto us = name.find('_');
if (us == std::string::npos || us == 0) {
throw std::runtime_error(
"migration file must be named NNN_name.sql: " + name);
}
const std::string num = name.substr(0, us);
if (!std::all_of(num.begin(), num.end(),
[](unsigned char c){ return std::isdigit(c); })) {
throw std::runtime_error(
"migration file prefix must be digits: " + name);
}
Migration m;
m.version = std::stoi(num);
m.path = p.string();
return m;
}}
void apply_migrations(pqxx::connection& conn, const std::string& dir) {
{
pqxx::work tx(conn);
tx.exec("SELECT pg_advisory_lock($1)",
pqxx::params{kMigrationsAdvisoryLockKey});
tx.commit();
}
std::clog << "migration advisory lock acquired\n";
struct Unlock {
pqxx::connection& c;
~Unlock() {
try {
pqxx::work tx(c);
tx.exec("SELECT pg_advisory_unlock($1)",
pqxx::params{kMigrationsAdvisoryLockKey});
tx.commit();
} catch (...) {}
}
} unlock_guard{conn};
{
pqxx::work tx(conn);
tx.exec(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version int PRIMARY KEY,"
" applied_at timestamptz NOT NULL DEFAULT now()"
")");
tx.commit();
}
if (!fs::exists(dir) || !fs::is_directory(dir)) {
throw std::runtime_error("migrations dir not found: " + dir);
}
std::vector<Migration> files;
for (const auto& entry : fs::directory_iterator(dir)) {
if (entry.is_regular_file()) {
files.push_back(parse_migration_name(entry.path()));
}
}
std::sort(files.begin(), files.end(),
[](const Migration& a, const Migration& b){
return a.version < b.version;
});
for (std::size_t i = 1; i < files.size(); ++i) {
if (files[i].version == files[i-1].version) {
throw std::runtime_error(
"duplicate migration version: " + std::to_string(files[i].version));
}
}
std::set<int> applied;
{
pqxx::work tx(conn);
for (const auto& row : tx.exec("SELECT version FROM schema_migrations")) {
applied.insert(row[0].as<int>());
}
tx.commit();
}
for (const auto& m : files) {
if (applied.count(m.version)) {
continue;
}
std::ifstream f(m.path, std::ios::binary);
if (!f) {
throw std::runtime_error("cannot open migration: " + m.path);
}
std::stringstream buf;
buf << f.rdbuf();
const std::string sql = buf.str();
pqxx::work tx(conn);
tx.exec(sql);
tx.exec("INSERT INTO schema_migrations(version) VALUES ($1)",
m.version);
tx.commit();
std::clog << "applied migration " << m.version
<< " (" << m.path << ")\n";
}
}
std::string read_secret_file(const char* env_name) {
const char* path = std::getenv(env_name);
if (path == nullptr || *path == '\0') {
throw std::runtime_error(
std::string("environment variable not set: ") + env_name);
}
std::ifstream f(path, std::ios::binary);
if (!f) {
throw std::runtime_error(
std::string("cannot open secret file: ") + path);
}
std::string s;
std::getline(f, s);
if (!s.empty() && s.back() == '\r') {
s.pop_back();
}
if (s.empty()) {
throw std::runtime_error(
std::string("secret file is empty: ") + path);
}
return s;
}
bool is_dependency_failure(const std::exception& e) {
if (dynamic_cast<const DependencyUnavailable*>(&e) != nullptr ||
dynamic_cast<const pqxx::broken_connection*>(&e) != nullptr ||
dynamic_cast<const pqxx::in_doubt_error*>(&e) != nullptr) {
return true;
}
if (const auto* sql = dynamic_cast<const pqxx::sql_error*>(&e)) {
// 08xxx — ошибки соединения, 57xxx — отмена (statement_timeout) и остановка сервера.
const std::string& state = sql->sqlstate();
return state.rfind("08", 0) == 0 || state.rfind("57", 0) == 0;
}
return false;
}
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
bool statement_timeout) {
std::string full = add_conn_param(conninfo, "connect_timeout",
std::to_string(kConnectTimeoutSeconds));
full = add_conn_param(full, "tcp_user_timeout", std::to_string(kTcpUserTimeoutMs));
auto conn = std::make_unique<pqxx::connection>(full);
const int fd = conn->sock();
if (fd >= 0) {
timeval tv{};
tv.tv_sec = kClientIoTimeoutSeconds;
tv.tv_usec = 0;
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
}
if (statement_timeout) {
pqxx::nontransaction tx(*conn);
tx.exec("SET statement_timeout = " + std::to_string(kStatementTimeoutMs));
}
return conn;
}
PgPool::PgPool(std::string conninfo, std::size_t n)
: conninfo_(std::move(conninfo)) {
if (n == 0) {
throw std::runtime_error("PgPool: n must be > 0");
}
all_.reserve(n);
idle_.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
auto conn = open_pg_connection(conninfo_, true);
idle_.push_back(conn.get());
all_.push_back(std::move(conn));
}
}
PgPool::~PgPool() {
{
std::lock_guard<std::mutex> lk(mu_);
shutting_down_ = true;
}
cv_.notify_all();
}
pqxx::connection* PgPool::take() {
std::unique_lock<std::mutex> lk(mu_);
if (!cv_.wait_for(lk, kAcquireTimeout,
[&]() { return shutting_down_ || !idle_.empty(); })) {
throw DependencyUnavailable("postgres pool: acquire timeout");
}
if (shutting_down_) {
throw DependencyUnavailable("postgres pool is shutting down");
}
pqxx::connection* conn = idle_.back();
idle_.pop_back();
lk.unlock();
if (is_alive(conn)) {
return conn;
}
std::unique_ptr<pqxx::connection> fresh;
try {
fresh = open_pg_connection(conninfo_, true);
} catch (const std::exception& e) {
// Мёртвое соединение возвращается в пул: следующий take() снова
// попробует переподключиться, и пул не теряет слот.
give_back(conn);
throw DependencyUnavailable(std::string("postgres reconnect failed: ") + e.what());
}
std::lock_guard<std::mutex> lk2(mu_);
for (auto& up : all_) {
if (up.get() == conn) {
up = std::move(fresh);
return up.get();
}
}
throw std::logic_error("PgPool: connection not owned by pool");
}
void PgPool::give_back(pqxx::connection* conn) {
{
std::lock_guard<std::mutex> lk(mu_);
idle_.push_back(conn);
}
cv_.notify_one();
}
PgPool::Lease::Lease(PgPool& pool) : pool_(&pool), conn_(pool.take()) {}
PgPool::Lease::~Lease() {
if (pool_ != nullptr && conn_ != nullptr) {
pool_->give_back(conn_);
}
}
PgPool::Lease::Lease(Lease&& other) noexcept
: pool_(other.pool_), conn_(other.conn_) {
other.pool_ = nullptr;
other.conn_ = nullptr;
}
PgPool::Lease PgPool::acquire() {
return Lease(*this);
}
bool PgPool::ping() {
try {
auto lease = acquire();
pqxx::work tx(*lease);
tx.exec("SELECT 1");
tx.commit();
return true;
} catch (...) {
return false;
}
}
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include <condition_variable>
#include <pqxx/pqxx>
#include <cstddef>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
std::string read_secret_file(const char* env_name);
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
bool statement_timeout);
void apply_migrations(pqxx::connection& conn, const std::string& dir);
class PgPool {
public:
bool ping();
PgPool(std::string conninfo, std::size_t n);
~PgPool();
PgPool(const PgPool&) = delete;
PgPool& operator=(const PgPool&) = delete;
class Lease {
public:
explicit Lease(PgPool& pool);
~Lease();
Lease(Lease&& other) noexcept;
Lease& operator=(Lease&&) = delete;
Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
pqxx::connection& operator*() const { return *conn_; }
pqxx::connection* operator->() const { return conn_; }
private:
PgPool* pool_;
pqxx::connection* conn_;
};
Lease acquire();
private:
friend class Lease;
pqxx::connection* take();
void give_back(pqxx::connection* conn);
std::string conninfo_;
std::vector<std::unique_ptr<pqxx::connection>> all_;
std::vector<pqxx::connection*> idle_;
std::mutex mu_;
std::condition_variable cv_;
bool shutting_down_ = false;
};
+292
View File
@@ -0,0 +1,292 @@
#include "direct_upload_repo.hpp"
#include "outbox.hpp"
#include <pqxx/pqxx>
#include <nlohmann/json.hpp>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <sstream>
namespace {
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch())
.count();
return std::to_string(secs);
}
DirectUpload row_to(const pqxx::row& r) {
DirectUpload d;
d.id = r["id"].as<std::string>();
d.chat_id = r["chat_id"].as<std::string>();
d.owner_id = r["owner_id"].as<std::string>();
d.filename = r["filename"].as<std::string>();
d.content_type = r["content_type"].as<std::string>();
d.expected_size = r["expected_size"].as<int>();
d.expected_sha256 = r["expected_sha256"].as<std::string>();
d.s3_key = r["s3_key"].as<std::string>();
d.staging_key = r["staging_key"].as<std::string>();
d.status = r["status"].as<std::string>();
d.overdue = r["overdue"].as<bool>();
if (r["attachment_id"].is_null()) {
d.attachment_id = std::nullopt;
} else {
d.attachment_id = r["attachment_id"].as<std::string>();
}
d.expires_at = epoch_to_rfc3339(r["expires_at_epoch"].as<long long>());
d.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
d.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
return d;
}
constexpr const char* kCols =
"id, chat_id, owner_id, filename, content_type, expected_size, "
"expected_sha256, s3_key, staging_key, status, attachment_id, "
"(status = 'pending' AND expires_at < now()) AS overdue, "
"EXTRACT(EPOCH FROM expires_at)::bigint AS expires_at_epoch, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
constexpr const char* kAttachmentColumns =
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, error_code, variants, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
Attachment attachment_from_row(const pqxx::row& r) {
Attachment a;
a.id = r["id"].as<std::string>();
a.chat_id = r["chat_id"].as<std::string>();
a.owner_id = r["owner_id"].as<std::string>();
a.filename = r["filename"].as<std::string>();
a.content_type = r["content_type"].as<std::string>();
a.size_bytes = r["size_bytes"].as<int>();
a.sha256 = r["sha256"].as<std::string>();
a.status = r["status"].as<std::string>();
a.s3_key = r["s3_key"].as<std::string>();
if (r["error_code"].is_null()) {
a.error_code = std::nullopt;
} else {
a.error_code = r["error_code"].as<std::string>();
}
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
for (const auto& v : nlohmann::json::parse(r["variants"].as<std::string>())) {
a.variants.push_back(Variant{v.at("name").get<std::string>(),
v.at("content_type").get<std::string>(), v.at("size_bytes").get<int>(),
v.at("sha256").get<std::string>(), v.at("width").get<int>(),
v.at("height").get<int>(), v.value("storage_key", "")});
}
return a;
}
} // namespace
DirectUpload DirectUploadRepo::insert(const DirectUpload& in) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
// expires_at is already an absolute epoch string from the caller.
auto r = tx.exec(
"INSERT INTO direct_uploads ("
" id, chat_id, owner_id, filename, content_type, expected_size, "
" expected_sha256, s3_key, staging_key, status, attachment_id, "
" expires_at, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, NULL, "
" to_timestamp($11::bigint), to_timestamp($12::bigint), "
" to_timestamp($13::bigint)"
") RETURNING " +
std::string(kCols),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.expected_size, in.expected_sha256,
in.s3_key, in.staging_key, in.status, in.expires_at_epoch,
ts, ts});
tx.commit();
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::find_by_id(
const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kCols) +
" FROM direct_uploads WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::mark_completed(
const std::string& id, const std::string& attachment_id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'completed', "
" attachment_id = $2::uuid, "
" updated_at = to_timestamp($3::bigint) "
"WHERE id = $1::uuid AND status = 'pending' "
"RETURNING " +
std::string(kCols),
pqxx::params{id, attachment_id, ts});
if (r.empty()) {
return std::nullopt;
}
tx.commit();
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::mark_aborted(
const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'aborted', "
" updated_at = to_timestamp($2::bigint) "
"WHERE id = $1::uuid AND status = 'pending' "
"RETURNING " +
std::string(kCols),
pqxx::params{id, ts});
if (r.empty()) {
return std::nullopt;
}
tx.commit();
return row_to(r[0]);
}
std::vector<DirectUpload> DirectUploadRepo::expire_stale() {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'expired', "
" updated_at = to_timestamp($1::bigint) "
"WHERE status = 'pending' AND expires_at < now() "
"RETURNING " +
std::string(kCols),
pqxx::params{ts});
tx.commit();
std::vector<DirectUpload> out;
out.reserve(r.size());
for (const auto& row : r) {
out.push_back(row_to(row));
}
return out;
}
std::vector<std::string> DirectUploadRepo::pending_s3_keys() {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT staging_key FROM direct_uploads WHERE status = 'pending'");
std::vector<std::string> out;
out.reserve(r.size());
for (const auto& row : r) {
out.push_back(row["staging_key"].as<std::string>());
}
return out;
}
void DirectUploadRepo::with_lock(const std::function<void(UploadLock&)>& fn) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
UploadLock lock(tx);
fn(lock);
}
std::optional<DirectUpload> UploadLock::lock(const std::string& id) {
auto r = tx_.exec(
"SELECT " + std::string(kCols) +
" FROM direct_uploads WHERE id = $1::uuid FOR UPDATE",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to(r[0]);
}
std::optional<Attachment> UploadLock::find_attachment(const std::string& id) {
auto r = tx_.exec(
"SELECT " + std::string(kAttachmentColumns) +
" FROM attachments WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return attachment_from_row(r[0]);
}
std::optional<Attachment> UploadLock::insert_queued(const Attachment& in,
bool& inserted) {
inserted = false;
const std::string ts = now_epoch_str();
auto r = tx_.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, staging_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $9, NULL, "
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
") ON CONFLICT (id) DO NOTHING RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, ts, ts});
if (r.empty()) {
return std::nullopt;
}
tx_.exec("INSERT INTO jobs(id, attachment_id, source_key) VALUES($1::uuid, $1::uuid, $2)",
pqxx::params{in.id, in.s3_key});
enqueue_attachment_job(tx_, in.id);
inserted = true;
return attachment_from_row(r[0]);
}
void UploadLock::mark_completed(const std::string& id,
const std::string& attachment_id) {
const std::string ts = now_epoch_str();
auto r = tx_.exec(
"UPDATE direct_uploads SET status = 'completed', "
" attachment_id = $2::uuid, "
" updated_at = to_timestamp($3::bigint) "
"WHERE id = $1::uuid AND status = 'pending'",
pqxx::params{id, attachment_id, ts});
if (r.affected_rows() != 1) {
throw std::runtime_error("direct upload complete lost the pending row");
}
}
void UploadLock::mark_expired(const std::string& id) {
const std::string ts = now_epoch_str();
tx_.exec(
"UPDATE direct_uploads SET status = 'expired', "
" updated_at = to_timestamp($2::bigint) "
"WHERE id = $1::uuid AND status = 'pending'",
pqxx::params{id, ts});
}
void UploadLock::commit() {
if (committed_) {
return;
}
tx_.commit();
committed_ = true;
}
+63
View File
@@ -0,0 +1,63 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <functional>
#include <optional>
#include <string>
#include <vector>
class UploadLock;
class DirectUploadRepo {
public:
explicit DirectUploadRepo(PgPool& pool) : pool_(pool) {}
DirectUpload insert(const DirectUpload& in);
std::optional<DirectUpload> find_by_id(const std::string& id);
// Mark completed and link attachment_id. Returns nullopt if not pending
// (or already finished / wrong owner handled by caller).
std::optional<DirectUpload> mark_completed(const std::string& id,
const std::string& attachment_id);
std::optional<DirectUpload> mark_aborted(const std::string& id);
// pending → expired where expires_at < now; returns expired rows.
std::vector<DirectUpload> expire_stale();
// Staging keys of still-pending sessions (orphan GC must keep these).
std::vector<std::string> pending_s3_keys();
// One PG transaction. Callback may call S3 while the row lock is held.
// Commit explicitly via UploadLock::commit(); otherwise the work rolls back.
void with_lock(const std::function<void(UploadLock&)>& fn);
private:
PgPool& pool_;
};
// Row lock for a single complete(). Not safe to use after with_lock returns.
class UploadLock {
public:
std::optional<DirectUpload> lock(const std::string& id);
std::optional<Attachment> find_attachment(const std::string& id);
// Inserts attachment and its job in this transaction; job_id equals attachment id.
// ON CONFLICT (id) DO NOTHING. inserted=false and nullopt → row already exists.
std::optional<Attachment> insert_queued(const Attachment& in, bool& inserted);
void mark_completed(const std::string& id, const std::string& attachment_id);
void mark_expired(const std::string& id);
void commit();
private:
friend class DirectUploadRepo;
explicit UploadLock(pqxx::work& tx) : tx_(tx) {}
pqxx::work& tx_;
bool committed_ = false;
};
+15
View File
@@ -0,0 +1,15 @@
#pragma once
#include <exception>
#include <stdexcept>
// PostgreSQL или Valkey недоступны либо не ответили вовремя: клиент получает 503.
struct DependencyUnavailable : std::runtime_error {
using std::runtime_error::runtime_error;
};
struct AttachmentFailed : std::runtime_error {
AttachmentFailed() : std::runtime_error("attachment processing failed") {}
};
bool is_dependency_failure(const std::exception& e);
+42
View File
@@ -0,0 +1,42 @@
#pragma once
#include <cstdlib>
#include <cstring>
// Explicit local-test opt-in; no HTTP parameter can enable fault injection.
inline void lab_crash_at(const char* point) {
const char* enabled = std::getenv("LAB_ENABLE_FAULT_INJECTION");
const char* selected = std::getenv("LAB_FAULT_POINT");
if (enabled && std::strcmp(enabled, "1") == 0 && selected &&
std::strcmp(selected, point) == 0) std::_Exit(86);
}
#include <chrono>
#include <filesystem>
#include <fstream>
#include <string>
#include <thread>
// One injection per container/job/point; marker survives a process restart.
inline void lab_worker_fault(const char* point, const std::string& job_id, int version) {
const char* enabled = std::getenv("LAB_ENABLE_FAULT_INJECTION");
const char* selected = std::getenv("LAB_FAULT_POINT");
if (!enabled || std::strcmp(enabled, "1") || !selected || std::strcmp(selected, point)) return;
const char* job = std::getenv("LAB_FAULT_JOB_ID");
const char* target_version = std::getenv("LAB_FAULT_PIPELINE_VERSION");
if ((job && job_id != job) || (target_version && std::to_string(version) != target_version)) return;
const std::string base = "/tmp/lab-fault/" + std::string(point) + "." + job_id;
std::filesystem::create_directories("/tmp/lab-fault");
if (std::filesystem::exists(base + ".reached")) return;
std::ofstream marker(base + ".reached");
marker << version << '\n';
marker.close();
const char* action = std::getenv("LAB_FAULT_ACTION");
if (action && std::strcmp(action, "pause") == 0) {
while (!std::filesystem::exists(base + ".release")) {
std::this_thread::sleep_for(std::chrono::milliseconds(25));
}
} else {
std::_Exit(86);
}
}
+151
View File
@@ -0,0 +1,151 @@
#include "image_pipeline.hpp"
#include "image_util.hpp"
#include <stb_image.h>
#define STB_IMAGE_RESIZE_IMPLEMENTATION
#include <stb_image_resize.h>
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include <stb_image_write.h>
#include <libexif/exif-data.h>
#include <webp/encode.h>
#include <algorithm>
#include <cstring>
#include <memory>
namespace {
int orientation(const std::vector<std::uint8_t>& bytes) {
const unsigned char* data = bytes.data();
std::size_t size = bytes.size();
std::vector<unsigned char> exif;
// PNG eXIf stores a TIFF header without the JPEG Exif prefix.
if (size >= 8 && std::memcmp(data, "\x89PNG\r\n\x1a\n", 8) == 0) {
for (std::size_t pos = 8; pos + 12 <= size;) {
const std::uint32_t len = (std::uint32_t(data[pos]) << 24) |
(std::uint32_t(data[pos + 1]) << 16) |
(std::uint32_t(data[pos + 2]) << 8) | data[pos + 3];
if (len > size - pos - 12) break;
if (std::memcmp(data + pos + 4, "eXIf", 4) == 0) {
exif.assign({'E', 'x', 'i', 'f', 0, 0});
exif.insert(exif.end(), data + pos + 8, data + pos + 8 + len);
data = exif.data();
size = exif.size();
break;
}
pos += std::size_t(len) + 12;
}
}
std::unique_ptr<ExifData, decltype(&exif_data_unref)> metadata(
exif_data_new_from_data(data, static_cast<unsigned int>(size)), exif_data_unref);
if (!metadata) return 1;
const auto* entry = exif_content_get_entry(metadata->ifd[EXIF_IFD_0], EXIF_TAG_ORIENTATION);
if (!entry || entry->format != EXIF_FORMAT_SHORT || entry->size < 2) return 1;
const int result = exif_get_short(entry->data, exif_data_get_byte_order(metadata.get()));
return result >= 1 && result <= 8 ? result : 1;
}
struct Raster {
std::vector<unsigned char> pixels;
int width;
int height;
};
Raster decode(const std::vector<std::uint8_t>& bytes) {
const ImageLimits limits;
if (bytes.empty() || bytes.size() > static_cast<std::size_t>(limits.max_bytes) ||
!sniff_content_type(bytes.data(), bytes.size())) {
throw InvalidImage("invalid_image");
}
int w = 0, h = 0, channels = 0;
if (!stbi_info_from_memory(bytes.data(), static_cast<int>(bytes.size()), &w, &h, &channels) ||
w <= 0 || h <= 0) throw InvalidImage("invalid_image");
if (std::int64_t(w) * h > limits.max_pixels) throw InvalidImage("image_too_large");
std::unique_ptr<stbi_uc, decltype(&stbi_image_free)> rgba(
stbi_load_from_memory(bytes.data(), static_cast<int>(bytes.size()), &w, &h, &channels, 4),
stbi_image_free);
if (!rgba) throw InvalidImage("invalid_image");
const int o = orientation(bytes);
const int width = o >= 5 ? h : w;
const int height = o >= 5 ? w : h;
std::vector<unsigned char> rgb(std::size_t(width) * height * 3);
for (int y = 0; y < h; ++y) {
for (int x = 0; x < w; ++x) {
int dx = x, dy = y;
switch (o) {
case 2: dx = w - 1 - x; break;
case 3: dx = w - 1 - x; dy = h - 1 - y; break;
case 4: dy = h - 1 - y; break;
case 5: dx = y; dy = x; break;
case 6: dx = h - 1 - y; dy = x; break;
case 7: dx = h - 1 - y; dy = w - 1 - x; break;
case 8: dx = y; dy = w - 1 - x; break;
}
const auto* pixel = rgba.get() + (std::size_t(y) * w + x) * 4;
auto* target = rgb.data() + (std::size_t(dy) * width + dx) * 3;
for (int c = 0; c < 3; ++c) {
target[c] = static_cast<unsigned char>(
(pixel[c] * pixel[3] + 255 * (255 - pixel[3]) + 127) / 255);
}
}
}
rgba.reset();
return Raster{std::move(rgb), width, height};
}
Raster resize(const unsigned char* pixels, int width, int height, int stride,
int out_w, int out_h) {
Raster result{std::vector<unsigned char>(std::size_t(out_w) * out_h * 3), out_w, out_h};
if (!stbir_resize_uint8(pixels, width, height, stride,
result.pixels.data(), out_w, out_h, 0, 3)) {
throw std::runtime_error("resize failed");
}
return result;
}
Raster fit(const Raster& input, int limit) {
const int extent = std::max(input.width, input.height);
const int w = extent > limit ? std::max<std::int64_t>(1, std::int64_t(input.width) * limit / extent) : input.width;
const int h = extent > limit ? std::max<std::int64_t>(1, std::int64_t(input.height) * limit / extent) : input.height;
return resize(input.pixels.data(), input.width, input.height, 0, w, h);
}
EncodedImage jpeg(const Raster& raster, int quality = 85) {
EncodedImage image{{}, raster.width, raster.height};
const auto write = [](void* context, void* data, int size) {
static_cast<std::string*>(context)->append(static_cast<const char*>(data), size);
};
if (!stbi_write_jpg_to_func(write, &image.bytes, raster.width, raster.height,
3, raster.pixels.data(), quality)) {
throw std::runtime_error("JPEG encoding failed");
}
return image;
}
EncodedImage webp(const Raster& raster, float quality) {
unsigned char* output = nullptr;
const auto size = WebPEncodeRGB(raster.pixels.data(), raster.width, raster.height,
raster.width * 3, quality, &output);
std::unique_ptr<unsigned char, decltype(&WebPFree)> owned(output, WebPFree);
if (!size) throw std::runtime_error("WebP encoding failed");
return EncodedImage{std::string(reinterpret_cast<const char*>(output), size),
raster.width, raster.height};
}
} // namespace
EncodedImage make_thumbnail(const std::vector<std::uint8_t>& bytes) {
return jpeg(fit(decode(bytes), 256));
}
std::vector<EncodedVariant> make_variants(const std::vector<std::uint8_t>& bytes, int version) {
if (version != 1 && version != 2) throw std::invalid_argument("unsupported pipeline version");
const int quality = version == 1 ? 85 : 90;
const auto raster = decode(bytes);
std::vector<EncodedVariant> result;
result.push_back({"thumbnail", "image/jpeg", jpeg(fit(raster, 256), quality)});
const int side = std::min(raster.width, raster.height);
const int left = (raster.width - side) / 2;
const int top = (raster.height - side) / 2;
const auto* center = raster.pixels.data() + (std::size_t(top) * raster.width + left) * 3;
result.push_back({"crop", "image/jpeg", jpeg(resize(center, side, side, raster.width * 3, 128, 128), quality)});
result.push_back({"optimized", "image/webp", webp(fit(raster, 1280), version == 1 ? 80.0f : 90.0f)});
return result;
}
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <cstdint>
#include <stdexcept>
#include <string>
#include <vector>
struct InvalidImage : std::runtime_error {
explicit InvalidImage(const char* code) : std::runtime_error(code) {}
};
struct EncodedImage {
std::string bytes;
int width;
int height;
};
struct EncodedVariant {
std::string name;
std::string content_type;
EncodedImage image;
};
std::vector<EncodedVariant> make_variants(const std::vector<std::uint8_t>& original, int version = 1);
EncodedImage make_thumbnail(const std::vector<std::uint8_t>& original);
+90
View File
@@ -0,0 +1,90 @@
#include "image_util.hpp"
#define STB_IMAGE_IMPLEMENTATION
#define STBI_NO_STDIO
#define STBI_ONLY_JPEG
#define STBI_ONLY_PNG
#include "stb_image.h"
#include <cstring>
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
std::size_t len) {
if (!data || len < 3) {
return std::nullopt;
}
// JPEG: FF D8 FF
if (data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) {
return std::string("image/jpeg");
}
// PNG: 89 50 4E 47
if (len >= 4 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E &&
data[3] == 0x47) {
return std::string("image/png");
}
return std::nullopt;
}
std::string_view image_reject_code(ImageReject r) {
switch (r) {
case ImageReject::unsupported_media_type:
return "unsupported_media_type";
case ImageReject::payload_too_large:
return "payload_too_large";
case ImageReject::invalid_image:
return "invalid_image";
}
return "invalid_image";
}
std::optional<ImageReject> validate_image(const std::uint8_t* data,
std::size_t len,
const ImageLimits& limits,
ImageInfo& out) {
if (!data || len == 0) {
return ImageReject::unsupported_media_type;
}
if (static_cast<std::int64_t>(len) > limits.max_bytes) {
return ImageReject::payload_too_large;
}
auto sniffed = sniff_content_type(data, len);
if (!sniffed) {
return ImageReject::unsupported_media_type;
}
int w = 0;
int h = 0;
int comp = 0;
if (!stbi_info_from_memory(reinterpret_cast<const stbi_uc*>(data),
static_cast<int>(len), &w, &h, &comp)) {
return ImageReject::invalid_image;
}
if (w <= 0 || h <= 0) {
return ImageReject::invalid_image;
}
const std::int64_t pixels =
static_cast<std::int64_t>(w) * static_cast<std::int64_t>(h);
if (pixels > limits.max_pixels) {
return ImageReject::payload_too_large;
}
int load_w = 0;
int load_h = 0;
int load_comp = 0;
stbi_uc* decoded = stbi_load_from_memory(
reinterpret_cast<const stbi_uc*>(data), static_cast<int>(len), &load_w,
&load_h, &load_comp, 0);
if (!decoded) {
return ImageReject::invalid_image;
}
// Integrity only — do not retain decoded raster.
stbi_image_free(decoded);
out.content_type = *sniffed;
out.width = w;
out.height = h;
out.pixel_count = pixels;
return std::nullopt;
}
+41
View File
@@ -0,0 +1,41 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
struct ImageLimits {
std::int64_t max_bytes = 10'485'760; // 10 MiB
std::int64_t max_pixels = 40'000'000; // decoded width*height
};
enum class ImageReject {
unsupported_media_type, // → 415
payload_too_large, // → 413 (bytes or pixels)
invalid_image, // → 415 (corrupt / undecodable)
};
struct ImageInfo {
std::string content_type; // image/png | image/jpeg
int width = 0;
int height = 0;
std::int64_t pixel_count = 0;
};
// Magic sniff only (does not decode). Returns nullopt if neither PNG nor JPEG.
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
std::size_t len);
// Full validation pipeline from contracts:
// sniff → stbi_info (dims) → pixel limit → stbi_load (integrity) → free.
// On success fills `out`. On failure returns ImageReject; decoded pixels are
// never retained after return.
std::optional<ImageReject> validate_image(const std::uint8_t* data,
std::size_t len,
const ImageLimits& limits,
ImageInfo& out);
// Human-readable error_code for Attachment failed path / API body.
std::string_view image_reject_code(ImageReject r);
+110
View File
@@ -0,0 +1,110 @@
#include "db.hpp"
#include "outbox.hpp"
#include "s3_client.hpp"
#include <sodium.h>
#include <iostream>
#include <regex>
#include <nlohmann/json.hpp>
namespace {
void verify_source(S3Client& s3, const std::string& key, std::size_t expected,
const std::string& checksum) {
crypto_hash_sha256_state state;
crypto_hash_sha256_init(&state);
std::size_t received = 0;
s3.get_object(key, [&](const char* data, std::size_t n) {
if (received > expected || n > expected - received) throw std::runtime_error("source mismatch");
crypto_hash_sha256_update(&state, reinterpret_cast<const unsigned char*>(data), n);
received += n;
});
unsigned char digest[crypto_hash_sha256_BYTES];
crypto_hash_sha256_final(&state, digest);
char hex[crypto_hash_sha256_BYTES * 2 + 1];
sodium_bin2hex(hex, sizeof(hex), digest, sizeof(digest));
if (received != expected || checksum != hex) throw std::runtime_error("source checksum mismatch");
}
}
int main(int argc, char** argv) {
const bool replay = argc == 7 && std::string(argv[1]) == "replay" &&
std::string(argv[3]) == "--actor" && std::string(argv[5]) == "--reason";
const bool upgrade = argc == 9 && std::string(argv[1]) == "upgrade" &&
std::string(argv[3]) == "--version" && std::string(argv[5]) == "--actor" &&
std::string(argv[7]) == "--reason";
if (!replay && !upgrade) {
std::cerr << "usage: job_admin replay JOB_UUID --actor NAME --reason REASON\n"
" job_admin upgrade ATTACHMENT_UUID --version 2 --actor NAME --reason REASON\n";
return 2;
}
const std::string id = argv[2], actor = argv[replay ? 4 : 6], reason = argv[replay ? 6 : 8];
const std::regex uuid("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
if (!std::regex_match(id, uuid) || actor.empty() || actor.size() > 100 || reason.empty() ||
reason.size() > 500 || actor.find_first_not_of(" \t\r\n") == std::string::npos ||
reason.find_first_not_of(" \t\r\n") == std::string::npos) return 2;
if (upgrade && std::string(argv[4]) != "1" && std::string(argv[4]) != "2") return 2;
try {
if (sodium_init() < 0) return 1;
auto conn = open_pg_connection(read_secret_file("DATABASE_URL_FILE"), true);
S3Client s3(S3Client::from_env());
pqxx::work tx(*conn);
if (upgrade) {
const int version = std::stoi(argv[4]);
tx.exec("SELECT pg_advisory_xact_lock(hashtextextended('pipeline:' || $1::uuid::text,0))", pqxx::params{id});
auto rows = tx.exec("SELECT pipeline_version,s3_key,size_bytes,sha256 FROM attachments "
"WHERE id=$1::uuid FOR UPDATE", pqxx::params{id});
if (rows.empty() || version < rows[0]["pipeline_version"].as<int>()) return 1;
if (version == rows[0]["pipeline_version"].as<int>()) {
const auto jobs = tx.exec("SELECT id FROM jobs WHERE attachment_id=$1::uuid AND pipeline_version=$2",
pqxx::params{id, version});
if (jobs.empty()) return 1;
std::cout << nlohmann::json{{"job_id", jobs[0][0].as<std::string>()},
{"pipeline_version", version}, {"created", false}}.dump() << '\n';
return 0;
}
const auto& row = rows[0];
verify_source(s3, row["s3_key"].as<std::string>(), row["size_bytes"].as<std::size_t>(), row["sha256"].as<std::string>());
const auto job = tx.exec("INSERT INTO jobs(id,attachment_id,pipeline_version,source_key) "
"VALUES(gen_random_uuid(),$1::uuid,$2,$3) RETURNING id",
pqxx::params{id, version, row["s3_key"].as<std::string>()})[0][0].as<std::string>();
tx.exec("UPDATE attachments SET pipeline_version=$2, "
"status=CASE WHEN published_version IS NULL THEN 'queued' ELSE status END, "
"error_code=CASE WHEN published_version IS NULL THEN NULL ELSE error_code END, "
"updated_at=CASE WHEN published_version IS NULL THEN now() ELSE updated_at END WHERE id=$1::uuid",
pqxx::params{id, version});
tx.exec("INSERT INTO pipeline_requests(attachment_id,pipeline_version,job_id,actor,reason) "
"VALUES($1::uuid,$2,$3::uuid,$4,$5)", pqxx::params{id, version, job, actor, reason});
enqueue_attachment_job(tx, job);
tx.commit();
std::cout << nlohmann::json{{"job_id", job}, {"pipeline_version", version}, {"created", true}}.dump() << '\n';
return 0;
}
if (!tx.exec("SELECT pg_try_advisory_xact_lock(hashtextextended($1::uuid::text,0))",
pqxx::params{id})[0][0].as<bool>()) return 1;
const auto rows = tx.exec(
"SELECT j.attachment_id,j.status,j.retries,j.source_key,j.pipeline_version, "
"a.pipeline_version AS desired_version,a.staging_cleaned_at,a.size_bytes,a.sha256 "
"FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{id});
if (rows.empty() || rows[0]["status"].as<std::string>() != "failed" ||
rows[0]["pipeline_version"].as<int>() != rows[0]["desired_version"].as<int>()) return 1;
const auto& row = rows[0];
const std::string source = row["source_key"].as<std::string>();
if (source.rfind("staging/",0) == 0 && !row["staging_cleaned_at"].is_null()) return 1;
verify_source(s3, source, row["size_bytes"].as<std::size_t>(), row["sha256"].as<std::string>());
tx.exec("INSERT INTO job_replays(job_id,actor,reason,previous_error_code,retries_before) "
"SELECT id,$2,$3,error_code,retries FROM jobs WHERE id=$1::uuid", pqxx::params{id, actor, reason});
tx.exec("UPDATE jobs SET status='queued',retry_base=retries,error_code=NULL, "
"next_attempt_at=now(),updated_at=now() WHERE id=$1::uuid", pqxx::params{id});
tx.exec("UPDATE attachments SET status='queued',s3_key=$2,error_code=NULL,variants='[]'::jsonb,updated_at=now() "
"WHERE id=$1::uuid AND published_version IS NULL",
pqxx::params{row["attachment_id"].as<std::string>(), source});
enqueue_attachment_job(tx, id);
tx.commit();
std::cout << "replayed job=" << id << '\n';
return 0;
} catch (const std::exception&) {
std::cerr << "operator action failed; transaction was not confirmed\n";
return 1;
}
}
+137
View File
@@ -0,0 +1,137 @@
#include "job_repo.hpp"
#include "image_pipeline.hpp"
#include "outbox.hpp"
#include "fault_injection.hpp"
#include <iostream>
#include <optional>
#include <algorithm>
#include <random>
JobRepo::Result JobRepo::run(const std::string& job_id, const std::string& attachment_id,
int version, int attempt, const std::function<JobResult(const JobInput&)>& process) {
auto lease = pool_.acquire();
{
pqxx::nontransaction tx(*lease);
if (!tx.exec("SELECT pg_try_advisory_lock(hashtextextended($1::uuid::text, 0))",
pqxx::params{job_id})[0][0].as<bool>()) return Result::Busy;
}
struct Unlock {
pqxx::connection& conn;
const std::string& id;
~Unlock() {
try {
pqxx::nontransaction tx(conn);
tx.exec("SELECT pg_advisory_unlock(hashtextextended($1::uuid::text, 0))", pqxx::params{id});
} catch (...) {}
}
} unlock{*lease, job_id};
JobInput input;
{
pqxx::work tx(*lease);
auto rows = tx.exec(
"SELECT j.status, j.retries, j.retry_base, j.source_key, j.pipeline_version AS job_version, "
"a.pipeline_version AS desired_version, j.next_attempt_at<=now() AS due, "
"a.status AS attachment_status FROM jobs j "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid AND j.attachment_id=$2::uuid FOR UPDATE OF j, a",
pqxx::params{job_id, attachment_id});
if (rows.empty()) return Result::Missing;
const auto& row = rows[0];
if (row["job_version"].as<int>() != version) return Result::Missing;
const std::string status = row["status"].as<std::string>();
if (version < row["desired_version"].as<int>() || status == "superseded") {
tx.exec("UPDATE jobs SET status='superseded', updated_at=now() "
"WHERE id=$1::uuid AND status IN ('queued','processing')", pqxx::params{job_id});
tx.commit();
return Result::Superseded;
}
if (version > row["desired_version"].as<int>()) return Result::Missing;
if (status == "ready" || status == "failed") return Result::Done;
input = JobInput{version, row["source_key"].as<std::string>()};
const int retries = row["retries"].as<int>();
// A previous processing state with no advisory lock means its worker died.
if (status == "processing") {
return fail(tx, job_id, attachment_id, version, retries + 1, row["retry_base"].as<int>(), "processing_failed", false);
}
// A newer attempt is already durable in outbox/the stream; discard an old delivery.
if (attempt < retries + 1) return Result::Scheduled;
if (attempt > retries + 1) return Result::Missing;
if (!row["due"].as<bool>()) return Result::Busy;
tx.exec("UPDATE jobs SET status='processing', error_code=NULL, updated_at=now() "
"WHERE id=$1::uuid", pqxx::params{job_id});
tx.exec("UPDATE attachments SET status='processing', error_code=NULL, updated_at=now() "
"WHERE id=$1::uuid AND pipeline_version=$2 AND published_version IS NULL",
pqxx::params{attachment_id, version});
tx.commit();
}
try {
lab_worker_fault("worker_before_effect", job_id, version);
const JobResult result = process(input);
pqxx::work tx(*lease);
const auto current = tx.exec(
"SELECT a.pipeline_version FROM jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.id=$1::uuid FOR UPDATE OF j,a", pqxx::params{job_id});
tx.exec("INSERT INTO attachment_results(attachment_id,pipeline_version,job_id,original_key,variants) "
"VALUES($1::uuid,$2,$3::uuid,$4,$5::jsonb) ON CONFLICT DO NOTHING",
pqxx::params{attachment_id, version, job_id, result.original_key, result.variants_json});
const bool publish = current[0][0].as<int>() == version;
if (publish) {
// The pointer and complete result become visible in one transaction.
tx.exec("UPDATE attachments a SET status='ready', published_version=$2, "
"s3_key=r.original_key, variants=r.variants, error_code=NULL, updated_at=now() "
"FROM attachment_results r WHERE a.id=$1::uuid AND a.pipeline_version=$2 "
"AND r.attachment_id=a.id AND r.pipeline_version=$2",
pqxx::params{attachment_id, version});
}
tx.exec("UPDATE jobs SET status=$2, error_code=NULL, updated_at=now() WHERE id=$1::uuid",
pqxx::params{job_id, publish ? "ready" : "superseded"});
tx.commit();
lab_worker_fault("worker_after_commit", job_id, version);
return publish ? Result::Done : Result::Superseded;
} catch (const std::exception& e) {
const auto* invalid = dynamic_cast<const InvalidImage*>(&e);
const std::string code = invalid ? invalid->what() : "processing_failed";
pqxx::work tx(*lease);
auto rows = tx.exec("SELECT j.status,j.retries,j.retry_base,a.pipeline_version FROM jobs j "
"JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1::uuid FOR UPDATE OF j,a",
pqxx::params{job_id});
if (rows.empty()) return Result::Missing;
if (rows[0]["pipeline_version"].as<int>() != version) {
tx.exec("UPDATE jobs SET status='superseded',updated_at=now() WHERE id=$1::uuid "
"AND status IN ('queued','processing')", pqxx::params{job_id});
tx.commit();
return Result::Superseded;
}
const std::string status = rows[0]["status"].as<std::string>();
// A lost COMMIT reply must not roll back an already published ready result.
if (status == "ready" || status == "failed") return Result::Done;
return fail(tx, job_id, attachment_id, version, rows[0]["retries"].as<int>() + 1,
rows[0]["retry_base"].as<int>(), code, invalid != nullptr);
}
}
JobRepo::Result JobRepo::fail(pqxx::work& tx, const std::string& job_id,
const std::string& attachment_id, int version, int retries, int retry_base,
const std::string& code, bool permanent) {
const bool failed = permanent || retries - retry_base >= policy_.max_attempts;
long long ceiling = policy_.base_delay_ms;
for (int i = 1; i < retries - retry_base; ++i) ceiling = std::min<long long>(policy_.max_delay_ms, ceiling * 2);
ceiling = std::min<long long>(policy_.max_delay_ms, ceiling);
thread_local std::mt19937 rng(std::random_device{}());
const int delay = failed ? 0 : std::uniform_int_distribution<int>(
static_cast<int>((ceiling + 1) / 2), static_cast<int>(ceiling))(rng);
tx.exec("UPDATE jobs SET status=$2, retries=$3, error_code=$4, "
"next_attempt_at=now()+($5::bigint * interval '1 millisecond'), updated_at=now() "
"WHERE id=$1::uuid",
pqxx::params{job_id, failed ? "failed" : "queued", retries, code, delay});
tx.exec("UPDATE attachments SET status=$2, error_code=$3, updated_at=now() "
"WHERE id=$1::uuid AND status='processing' AND pipeline_version=$4 AND published_version IS NULL",
pqxx::params{attachment_id, failed ? "failed" : "queued",
failed ? std::optional<std::string>(code) : std::nullopt, version});
if (!failed) enqueue_attachment_job(tx, job_id);
tx.commit();
std::clog << "job=" << job_id << " retries=" << retries << " error_code=" << code
<< " retry_delay_ms=" << delay << '\n';
return failed ? Result::Done : Result::Scheduled;
}
+35
View File
@@ -0,0 +1,35 @@
#pragma once
#include "db.hpp"
#include <functional>
#include <string>
struct JobInput {
int pipeline_version;
std::string source_key;
};
struct JobResult {
std::string original_key;
std::string variants_json;
};
struct RetryPolicy {
int max_attempts = 6;
int base_delay_ms = 1000;
int max_delay_ms = 60000;
};
class JobRepo {
public:
enum class Result { Done, Scheduled, Busy, Missing, Superseded };
JobRepo(PgPool& pool, RetryPolicy policy) : pool_(pool), policy_(policy) {}
Result run(const std::string& job_id, const std::string& attachment_id,
int version, int attempt, const std::function<JobResult(const JobInput&)>& process);
private:
Result fail(pqxx::work& tx, const std::string& job_id,
const std::string& attachment_id, int version, int retries, int retry_base,
const std::string& code, bool permanent);
PgPool& pool_;
RetryPolicy policy_;
};
+2352
View File
File diff suppressed because it is too large Load Diff
+377
View File
@@ -0,0 +1,377 @@
#include "message_repo.hpp"
#include "cursor.hpp"
#include "errors.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <cstdlib>
#include <ctime>
#include <iomanip>
#include <mutex>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <sodium.h>
#include <vector>
namespace {
std::string make_uuid() {
static std::mutex rng_mu;
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::lock_guard<std::mutex> lock(rng_mu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string sha256_hex(const std::string& s) {
unsigned char out[crypto_hash_sha256_BYTES];
crypto_hash_sha256(out, reinterpret_cast<const unsigned char*>(s.data()), s.size());
static const char hex[] = "0123456789abcdef";
std::string r;
r.resize(sizeof(out) * 2);
for (std::size_t i = 0; i < sizeof(out); ++i) {
r[i * 2] = hex[out[i] >> 4];
r[i * 2 + 1] = hex[out[i] & 0x0f];
}
return r;
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
Message row_to_message(const pqxx::row& r) {
Message m;
m.id = r["id"].as<std::string>();
m.chat_id = r["chat_id"].as<std::string>();
m.sender_id = r["sender_id"].as<std::string>();
m.text = r["text"].as<std::string>();
m.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return m;
}
constexpr const char* kMessageSelect =
"id, chat_id, sender_id, text, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
std::string canonical_message_body(const std::string& text,
const std::vector<std::string>& attachment_ids) {
std::ostringstream out;
out << "text=" << text << "\nattachment_ids=";
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
if (i > 0) {
out << ',';
}
out << attachment_ids[i];
}
return out.str();
}
int idempotency_ttl_seconds() {
const char* env = std::getenv("IDEMPOTENCY_TTL_SECONDS");
if (env == nullptr || *env == '\0') {
return 86400;
}
try {
const int v = std::stoi(env);
if (v <= 0) {
return 86400;
}
return v;
} catch (...) {
return 86400;
}
}
bool attachments_belong_to_chat(pqxx::work& tx,
const std::string& chat_id,
const std::vector<std::string>& attachment_ids) {
bool failed = false;
for (const auto& id : attachment_ids) {
auto r = tx.exec(
"SELECT status FROM attachments "
"WHERE id = $1 AND chat_id = $2 FOR SHARE",
pqxx::params{id, chat_id});
if (r.empty()) {
return false;
}
failed = failed || r[0]["status"].as<std::string>() == "failed";
}
if (failed) {
throw AttachmentFailed();
}
return true;
}
void link_attachments(pqxx::work& tx,
const std::string& message_id,
const std::vector<std::string>& attachment_ids) {
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
tx.exec(
"INSERT INTO message_attachments (message_id, attachment_id, position) "
"VALUES ($1, $2, $3)",
pqxx::params{message_id, attachment_ids[i], static_cast<int>(i)});
}
}
std::vector<std::string> load_attachment_ids(pqxx::work& tx,
const std::string& message_id) {
auto r = tx.exec(
"SELECT attachment_id FROM message_attachments "
"WHERE message_id = $1 ORDER BY position ASC",
pqxx::params{message_id});
std::vector<std::string> ids;
ids.reserve(r.size());
for (const auto& row : r) {
ids.push_back(row["attachment_id"].as<std::string>());
}
return ids;
}
void fill_attachment_ids(pqxx::work& tx, Message& m) {
m.attachment_ids = load_attachment_ids(tx, m.id);
}
void fill_attachment_ids(pqxx::work& tx, std::vector<Message>& items) {
for (auto& m : items) {
fill_attachment_ids(tx, m);
}
}
Message insert_message(pqxx::work& tx,
const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids) {
const std::string id = make_uuid();
const std::string created = now_epoch_str();
auto r = tx.exec(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
"RETURNING " + std::string(kMessageSelect),
pqxx::params{id, chat_id, sender_id, text, created});
Message m = row_to_message(r[0]);
link_attachments(tx, m.id, attachment_ids);
m.attachment_ids = attachment_ids;
return m;
}
bool insert_idempotency_key(pqxx::work& tx,
const std::string& user_id,
const std::string& chat_id,
const std::string& idem_key,
const std::string& request_hash,
const std::string& message_id) {
auto inserted = tx.exec(
"INSERT INTO idempotency_keys "
"(user_id, chat_id, key, request_hash, message_id) "
"VALUES ($1, $2, $3, $4, $5) "
"ON CONFLICT (user_id, chat_id, key) DO NOTHING "
"RETURNING message_id",
pqxx::params{user_id, chat_id, idem_key, request_hash, message_id});
return !inserted.empty();
}
Message load_message(pqxx::work& tx, const std::string& message_id) {
auto r = tx.exec(
"SELECT " + std::string(kMessageSelect) + " FROM messages WHERE id = $1",
pqxx::params{message_id});
if (r.empty()) {
throw std::runtime_error("idempotent message row is missing");
}
Message m = row_to_message(r[0]);
fill_attachment_ids(tx, m);
return m;
}
} // namespace
std::optional<Message> MessageRepo::create(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto member_check = tx.exec(
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
pqxx::params{chat_id, sender_id});
if (member_check.empty()) {
tx.commit();
return std::nullopt;
}
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return std::nullopt;
}
Message created = insert_message(tx, chat_id, sender_id, text, attachment_ids);
tx.commit();
return created;
}
std::optional<MessagePage> MessageRepo::list_page(
const std::string& chat_id,
int limit,
const std::optional<std::string>& cursor) {
std::optional<Cursor> cur;
if (cursor) {
cur = parse_cursor(*cursor);
if (!cur) {
return std::nullopt;
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
if (cur) {
auto owner_check = tx.exec(
"SELECT 1 FROM messages WHERE id = $1 AND chat_id = $2",
pqxx::params{cur->id, chat_id});
if (owner_check.empty()) {
tx.commit();
return std::nullopt;
}
}
std::string sql =
"SELECT " + std::string(kMessageSelect) + ", " + kCreatedUsSql + " "
"FROM messages "
"WHERE chat_id = $1 ";
if (cur) {
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
}
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
pqxx::result r;
if (cur) {
r = tx.exec(sql, pqxx::params{chat_id, limit + 1, cur->created_us, cur->id});
} else {
r = tx.exec(sql, pqxx::params{chat_id, limit + 1});
}
MessagePage page;
const bool has_more = (static_cast<int>(r.size()) > limit);
const int take = has_more ? limit : static_cast<int>(r.size());
page.items.reserve(take);
for (int i = 0; i < take; ++i) {
page.items.push_back(row_to_message(r[static_cast<std::size_t>(i)]));
}
fill_attachment_ids(tx, page.items);
if (has_more && take > 0) {
const auto& last = r[static_cast<std::size_t>(take - 1)];
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
last["id"].as<std::string>());
}
tx.commit();
return page;
}
CreateResult MessageRepo::create_idempotent(
const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids,
const std::string& idem_key) {
const int ttl = idempotency_ttl_seconds();
const std::string hash = sha256_hex(canonical_message_body(text, attachment_ids));
for (int attempt = 0; attempt < 3; ++attempt) {
{
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto member_check = tx.exec(
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
pqxx::params{chat_id, sender_id});
if (member_check.empty()) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
const Message created =
insert_message(tx, chat_id, sender_id, text, attachment_ids);
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
created.id)) {
tx.commit();
return CreateResult{CreateResult::Status::Created, created};
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto existing = tx.exec(
"SELECT request_hash, message_id, "
"EXTRACT(EPOCH FROM (now() - created_at))::bigint AS age_seconds "
"FROM idempotency_keys "
"WHERE user_id = $1 AND chat_id = $2 AND key = $3 "
"FOR UPDATE",
pqxx::params{sender_id, chat_id, idem_key});
if (existing.empty()) {
tx.commit();
continue;
}
const std::string existing_hash = existing[0]["request_hash"].as<std::string>();
const long long age = existing[0]["age_seconds"].as<long long>();
const std::string message_id = existing[0]["message_id"].as<std::string>();
if (age < ttl) {
if (existing_hash != hash) {
tx.commit();
return CreateResult{CreateResult::Status::KeyConflict, std::nullopt};
}
const Message replayed = load_message(tx, message_id);
tx.commit();
return CreateResult{CreateResult::Status::Replayed, replayed};
}
tx.exec(
"DELETE FROM idempotency_keys "
"WHERE user_id = $1 AND chat_id = $2 AND key = $3",
pqxx::params{sender_id, chat_id, idem_key});
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
const Message created =
insert_message(tx, chat_id, sender_id, text, attachment_ids);
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
created.id)) {
tx.commit();
return CreateResult{CreateResult::Status::Created, created};
}
}
throw std::runtime_error("idempotency retry exhausted");
}
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
struct CreateResult {
enum class Status { Created, Replayed, NotFound, KeyConflict };
Status status;
std::optional<Message> message;
};
class MessageRepo {
public:
explicit MessageRepo(PgPool& pool) : pool_(pool) {}
std::optional<Message> create(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids);
std::optional<MessagePage> list_page(
const std::string& chat_id,
int limit,
const std::optional<std::string>& cursor);
CreateResult create_idempotent(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids,
const std::string& idem_key);
private:
PgPool& pool_;
};
+86
View File
@@ -0,0 +1,86 @@
#pragma once
#include <optional>
#include <string>
#include <vector>
struct User {
std::string id;
std::string username;
std::string display_name;
std::string password_hash;
std::string created_at;
};
struct Chat {
std::string id;
std::string title;
std::vector<std::string> member_ids;
std::string created_at;
};
struct Message {
std::string id;
std::string chat_id;
std::string sender_id;
std::string text;
std::string created_at;
std::vector<std::string> attachment_ids;
};
struct MessagePage {
std::vector<Message> items;
std::optional<std::string> next_cursor;
};
struct ChatPage {
std::vector<Chat> items;
std::optional<std::string> next_cursor;
};
// Public metadata; storage keys are derived internally.
struct Variant {
std::string name;
std::string content_type;
int size_bytes = 0;
std::string sha256;
int width = 0;
int height = 0;
std::string storage_key; // internal; filtered from API JSON
};
struct Attachment {
std::string id;
std::string chat_id;
std::string owner_id;
std::string filename;
std::string content_type;
int size_bytes = 0;
std::string sha256;
std::string status; // queued|processing|ready|failed
std::vector<Variant> variants;
std::optional<std::string> error_code;
std::string s3_key; // internal; never exposed in API JSON
std::string created_at;
std::string updated_at;
};
// Track 1: pending presigned PUT session (not a public attachment until complete).
struct DirectUpload {
std::string id;
std::string chat_id;
std::string owner_id;
std::string filename;
std::string content_type;
int expected_size = 0;
std::string expected_sha256;
std::string s3_key; // final key: attachments/{chat}/{id}
std::string staging_key; // presigned PUT target: staging/{chat}/{id}
std::string status; // pending|completed|aborted|expired
bool overdue = false; // pending && expires_at < now(); not a column
std::optional<std::string> attachment_id;
std::string expires_at; // RFC3339 for JSON
long long expires_at_epoch = 0; // for INSERT
std::string created_at;
std::string updated_at;
};
+65
View File
@@ -0,0 +1,65 @@
#include "outbox.hpp"
#include "attachment_queue.hpp"
#include "fault_injection.hpp"
#include <chrono>
#include <iostream>
#include <utility>
void enqueue_attachment_job(pqxx::work& tx, const std::string& job_id) {
const auto rows = tx.exec(
"INSERT INTO outbox(payload, available_at) "
"SELECT jsonb_build_object('schema_version', 1, 'type', 'attachment.process.v1', "
"'job_id', id::text, 'attachment_id', attachment_id::text, 'pipeline_version', pipeline_version, "
"'attempt', retries + 1, 'created_at', "
"to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"')), "
"next_attempt_at FROM jobs WHERE id=$1::uuid RETURNING id", pqxx::params{job_id});
if (rows.size() != 1) throw std::runtime_error("outbox job missing");
}
OutboxPublisher::OutboxPublisher(std::string db_url, std::string broker_url)
: thread_([this, db_url = std::move(db_url), broker_url = std::move(broker_url)] {
run(db_url, broker_url);
}) {}
OutboxPublisher::~OutboxPublisher() {
stopping_.store(true);
cv_.notify_all();
thread_.join();
}
bool OutboxPublisher::wait_interval() {
std::unique_lock<std::mutex> lock(mutex_);
return cv_.wait_for(lock, std::chrono::seconds(1), [this] { return stopping_.load(); });
}
void OutboxPublisher::run(const std::string& db_url, const std::string& broker_url) {
while (!stopping_.load()) {
try {
// Dedicated pools keep broker failure/reconnect off the HTTP path.
PgPool pg(db_url, 1);
RedisPool redis(broker_url, 1);
AttachmentQueue queue(redis);
while (!stopping_.load()) {
{
auto lease = pg.acquire();
pqxx::work tx(*lease);
const auto rows = tx.exec(
"SELECT id, payload FROM outbox WHERE NOT published AND available_at<=now() "
"ORDER BY available_at, id LIMIT 32 FOR UPDATE SKIP LOCKED");
for (const auto& row : rows) {
queue.publish(row["payload"].as<std::string>());
lab_crash_at("outbox_after_publish");
tx.exec("UPDATE outbox SET published=true WHERE id=$1",
pqxx::params{row["id"].as<long long>()});
}
tx.commit();
}
if (wait_interval()) break;
}
} catch (const std::exception&) {
std::clog << "outbox publish unavailable; durable rows will be retried\n";
if (wait_interval()) break;
}
}
}
+26
View File
@@ -0,0 +1,26 @@
#pragma once
#include "db.hpp"
#include <atomic>
#include <condition_variable>
#include <mutex>
#include <string>
#include <thread>
// Must share the transaction that inserts/updates the job.
void enqueue_attachment_job(pqxx::work& tx, const std::string& job_id);
class OutboxPublisher {
public:
OutboxPublisher(std::string db_url, std::string broker_url);
~OutboxPublisher();
OutboxPublisher(const OutboxPublisher&) = delete;
OutboxPublisher& operator=(const OutboxPublisher&) = delete;
private:
void run(const std::string& db_url, const std::string& broker_url);
bool wait_interval();
std::atomic<bool> stopping_{false};
std::mutex mutex_;
std::condition_variable cv_;
std::thread thread_;
};
+66
View File
@@ -0,0 +1,66 @@
#include "pg_session_repo.hpp"
#include <pqxx/pqxx>
#include <sodium.h>
#include <ctime>
#include <stdexcept>
#include <string>
namespace {
std::string random_token_hex() {
unsigned char buf[32];
randombytes_buf(buf, sizeof(buf));
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(sizeof(buf) * 2);
for (std::size_t i = 0; i < sizeof(buf); ++i) {
out[i * 2] = hex[buf[i] >> 4];
out[i * 2 + 1] = hex[buf[i] & 0x0f];
}
return out;
}
std::int64_t now_unix() {
return static_cast<std::int64_t>(std::time(nullptr));
}
}
ISessionRepo::Created PgSessionRepo::create(const std::string& user_id) {
const std::string token = random_token_hex();
const std::int64_t exp = now_unix() + ttl_;
auto lease = pool_.acquire();
pqxx::work tx(*lease);
tx.exec(
"INSERT INTO sessions (token, user_id, expires_at) "
"VALUES ($1, $2, to_timestamp($3::bigint))",
pqxx::params{token, user_id, exp});
tx.commit();
return Created{token, exp};
}
std::optional<std::string> PgSessionRepo::lookup(const std::string& token) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT user_id FROM sessions "
"WHERE token = $1 AND expires_at > now()",
pqxx::params{token});
tx.commit();
if (r.empty()) return std::nullopt;
return r[0]["user_id"].as<std::string>();
}
bool PgSessionRepo::revoke(const std::string& token) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"DELETE FROM sessions WHERE token = $1",
pqxx::params{token});
tx.commit();
return r.affected_rows() > 0;
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "db.hpp"
#include "session_repo_base.hpp"
class PgSessionRepo : public ISessionRepo {
public:
PgSessionRepo(PgPool& pool, int ttl_seconds)
: pool_(pool), ttl_(ttl_seconds) {}
Created create(const std::string& user_id) override;
std::optional<std::string> lookup(const std::string& token) override;
bool revoke(const std::string& token) override;
private:
PgPool& pool_;
int ttl_;
};
+216
View File
@@ -0,0 +1,216 @@
#include "redis_pool.hpp"
#include "errors.hpp"
#include <chrono>
#include <cstdlib>
#include <stdexcept>
#include <string>
namespace {
constexpr timeval kConnectTimeout{2, 0};
constexpr timeval kCommandTimeout{2, 0};
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
struct RedisUrl {
std::string host = "127.0.0.1";
int port = 6379;
std::string password;
int db = 0;
};
RedisUrl parse_redis_url(const std::string& url) {
RedisUrl out;
const std::string prefix = "redis://";
if (url.compare(0, prefix.size(), prefix) != 0) {
throw std::runtime_error("redis url must start with redis://");
}
std::string rest = url.substr(prefix.size());
const auto slash = rest.find('/');
if (slash != std::string::npos) {
const std::string db_str = rest.substr(slash + 1);
rest.resize(slash);
if (!db_str.empty()) {
out.db = std::atoi(db_str.c_str());
}
}
const auto at = rest.find('@');
std::string hostport = rest;
if (at != std::string::npos) {
const std::string creds = rest.substr(0, at);
hostport = rest.substr(at + 1);
const auto colon = creds.find(':');
if (colon != std::string::npos) {
out.password = creds.substr(colon + 1);
} else {
out.password = creds;
}
}
const auto colon = hostport.find(':');
if (colon == std::string::npos) {
out.host = hostport;
} else {
out.host = hostport.substr(0, colon);
out.port = std::atoi(hostport.substr(colon + 1).c_str());
}
return out;
}
redisContext* make_connection(const RedisUrl& u) {
redisContext* c = redisConnectWithTimeout(u.host.c_str(), u.port, kConnectTimeout);
if (c == nullptr) {
throw std::runtime_error("redisConnect: allocation failed");
}
if (c->err) {
const std::string err = c->errstr;
redisFree(c);
throw DependencyUnavailable("redisConnect failed: " + err);
}
if (redisSetTimeout(c, kCommandTimeout) != REDIS_OK) {
redisFree(c);
throw DependencyUnavailable("redisSetTimeout failed");
}
if (!u.password.empty()) {
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "AUTH %s", u.password.c_str()));
if (r == nullptr) {
redisFree(c);
throw std::runtime_error("redis AUTH: no reply");
}
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
redisFree(c);
throw std::runtime_error("redis AUTH failed");
}
}
if (u.db != 0) {
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "SELECT %d", u.db));
if (r == nullptr) {
redisFree(c);
throw std::runtime_error("redis SELECT: no reply");
}
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
redisFree(c);
throw std::runtime_error("redis SELECT failed");
}
}
return c;
}
}
RedisPool::RedisPool(std::string url, std::size_t n)
: url_(std::move(url)) {
if (n == 0) {
throw std::runtime_error("RedisPool: n must be > 0");
}
const RedisUrl parsed = parse_redis_url(url_);
all_.reserve(n);
idle_.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
ConnPtr c(make_connection(parsed));
idle_.push_back(c.get());
all_.push_back(std::move(c));
}
}
RedisPool::~RedisPool() {
{
std::lock_guard<std::mutex> lk(mu_);
shutting_down_ = true;
}
cv_.notify_all();
}
RedisPool::Lease::Lease(RedisPool& pool)
: pool_(&pool), ctx_(pool.take()) {}
RedisPool::Lease::~Lease() {
if (pool_ != nullptr && ctx_ != nullptr) {
pool_->give_back(ctx_);
}
}
RedisPool::Lease::Lease(Lease&& other) noexcept
: pool_(other.pool_), ctx_(other.ctx_) {
other.pool_ = nullptr;
other.ctx_ = nullptr;
}
RedisPool::Lease RedisPool::acquire() {
return Lease(*this);
}
redisContext* RedisPool::take() {
std::unique_lock<std::mutex> lk(mu_);
if (!cv_.wait_for(lk, kAcquireTimeout,
[&]() { return shutting_down_ || !idle_.empty(); })) {
throw DependencyUnavailable("redis pool: acquire timeout");
}
if (shutting_down_) {
throw DependencyUnavailable("redis pool is shutting down");
}
redisContext* c = idle_.back();
idle_.pop_back();
lk.unlock();
bool alive = false;
if (!c->err) {
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
if (r != nullptr && r->type == REDIS_REPLY_STATUS) {
alive = true;
}
if (r != nullptr) freeReplyObject(r);
}
if (alive) {
return c;
}
ConnPtr fresh;
try {
fresh.reset(make_connection(parse_redis_url(url_)));
} catch (const std::exception& e) {
// Слот возвращается в пул, иначе после нескольких неудач пул опустеет навсегда.
give_back(c);
throw DependencyUnavailable(std::string("redis reconnect failed: ") + e.what());
}
std::lock_guard<std::mutex> lk2(mu_);
for (auto& up : all_) {
if (up.get() == c) {
up = std::move(fresh);
return up.get();
}
}
throw std::logic_error("RedisPool: connection not owned by pool");
}
void RedisPool::give_back(redisContext* c) {
{
std::lock_guard<std::mutex> lk(mu_);
idle_.push_back(c);
}
cv_.notify_one();
}
bool RedisPool::ping() {
try {
auto lease = acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
if (r == nullptr) return false;
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
return ok;
} catch (...) {
return false;
}
}
+59
View File
@@ -0,0 +1,59 @@
#pragma once
#include <hiredis/hiredis.h>
#include <condition_variable>
#include <cstddef>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
class RedisPool {
public:
RedisPool(std::string url, std::size_t n);
~RedisPool();
RedisPool(const RedisPool&) = delete;
RedisPool& operator=(const RedisPool&) = delete;
class Lease {
public:
explicit Lease(RedisPool& pool);
~Lease();
Lease(Lease&& other) noexcept;
Lease& operator=(Lease&&) = delete;
Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
redisContext* ctx() const { return ctx_; }
private:
RedisPool* pool_;
redisContext* ctx_;
};
Lease acquire();
bool ping();
private:
friend class Lease;
struct RedisDeleter {
void operator()(redisContext* c) const noexcept {
if (c != nullptr) ::redisFree(c);
}
};
using ConnPtr = std::unique_ptr<redisContext, RedisDeleter>;
redisContext* take();
void give_back(redisContext* c);
std::string url_;
std::vector<ConnPtr> all_;
std::vector<redisContext*> idle_;
std::mutex mu_;
std::condition_variable cv_;
bool shutting_down_ = false;
};
+281
View File
@@ -0,0 +1,281 @@
#include "s3_client.hpp"
#include <miniocpp/args.h>
#include <miniocpp/client.h>
#include <miniocpp/providers.h>
#include <miniocpp/request.h>
#include <miniocpp/response.h>
#include <cstdlib>
#include <fstream>
#include <stdexcept>
namespace {
std::string read_file_trim(const std::string& path) {
std::ifstream in(path);
if (!in) {
throw std::runtime_error("cannot open secret/file: " + path);
}
std::string s((std::istreambuf_iterator<char>(in)),
std::istreambuf_iterator<char>());
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) {
s.pop_back();
}
return s;
}
std::string env_or(const char* key, const char* fallback = "") {
const char* v = std::getenv(key);
return (v && *v) ? std::string(v) : std::string(fallback);
}
void parse_endpoint(const std::string& endpoint, std::string& host, unsigned& port,
bool& https) {
https = true;
host = endpoint;
port = 0;
if (host.rfind("http://", 0) == 0) {
https = false;
host = host.substr(7);
} else if (host.rfind("https://", 0) == 0) {
https = true;
host = host.substr(8);
}
// strip path
const auto slash = host.find('/');
if (slash != std::string::npos) {
host = host.substr(0, slash);
}
const auto colon = host.rfind(':');
if (colon != std::string::npos) {
port = static_cast<unsigned>(std::stoul(host.substr(colon + 1)));
host = host.substr(0, colon);
}
if (port == 0) {
port = https ? 443u : 80u;
}
}
} // namespace
struct S3Client::Impl {
minio::s3::BaseUrl base_url;
minio::s3::BaseUrl presign_url;
std::unique_ptr<minio::creds::StaticProvider> provider;
std::unique_ptr<minio::s3::Client> client;
std::unique_ptr<minio::s3::Client> presign_client; // null → use client
};
S3Client::Config S3Client::from_env() {
Config c;
// Ops endpoint (Stat/Get/Put/Delete). Presign may use S3_PRESIGN_ENDPOINT
// (read in ctor) so signed Host matches the client-reachable address.
c.endpoint = env_or("S3_ENDPOINT", "http://minio:9000");
c.region = env_or("S3_REGION", "us-east-1");
c.bucket = env_or("S3_BUCKET");
if (c.bucket.empty()) {
throw std::runtime_error("S3_BUCKET is required");
}
const std::string key_file = env_or("S3_ACCESS_KEY_ID_FILE");
const std::string secret_file = env_or("S3_SECRET_ACCESS_KEY_FILE");
if (key_file.empty() || secret_file.empty()) {
throw std::runtime_error(
"S3_ACCESS_KEY_ID_FILE and S3_SECRET_ACCESS_KEY_FILE are required");
}
c.access_key = read_file_trim(key_file);
c.secret_key = read_file_trim(secret_file);
if (c.access_key.empty() || c.secret_key.empty()) {
throw std::runtime_error("S3 credentials files are empty");
}
return c;
}
S3Client::S3Client(Config cfg) : cfg_(std::move(cfg)), impl_(std::make_unique<Impl>()) {
if (cfg_.endpoint.empty() || cfg_.bucket.empty() ||
cfg_.access_key.empty() || cfg_.secret_key.empty()) {
throw std::runtime_error("S3Client: incomplete config");
}
std::string host;
unsigned port = 0;
bool https = false;
parse_endpoint(cfg_.endpoint, host, port, https);
impl_->base_url = minio::s3::BaseUrl(host, https, cfg_.region);
impl_->base_url.port = port;
// MinIO requires path-style addressing.
impl_->base_url.virtual_style = false;
impl_->provider = std::make_unique<minio::creds::StaticProvider>(
cfg_.access_key, cfg_.secret_key);
impl_->client =
std::make_unique<minio::s3::Client>(impl_->base_url, impl_->provider.get());
// Separate BaseUrl for presigning when S3_PRESIGN_ENDPOINT differs.
const std::string presign_ep = env_or("S3_PRESIGN_ENDPOINT");
if (!presign_ep.empty() && presign_ep != cfg_.endpoint) {
std::string ph;
unsigned pp = 0;
bool phttps = false;
parse_endpoint(presign_ep, ph, pp, phttps);
impl_->presign_url = minio::s3::BaseUrl(ph, phttps, cfg_.region);
impl_->presign_url.port = pp;
impl_->presign_url.virtual_style = false;
impl_->presign_client = std::make_unique<minio::s3::Client>(
impl_->presign_url, impl_->provider.get());
}
}
S3Client::~S3Client() = default;
void S3Client::put_object(const std::string& key,
std::istream& body,
std::int64_t size_bytes,
const std::string& content_type) {
if (size_bytes < 0) {
throw std::invalid_argument("put_object: negative size");
}
// part_size=0 → library chooses multipart sizing; streams from `body`.
minio::s3::PutObjectArgs args(body, static_cast<long>(size_bytes), 0);
args.bucket = cfg_.bucket;
args.object = key;
if (!content_type.empty()) {
args.content_type = content_type;
}
minio::s3::PutObjectResponse resp = impl_->client->PutObject(args);
if (!resp) {
throw std::runtime_error("S3 PutObject failed: " + resp.Error().String());
}
}
void S3Client::get_object(const std::string& key, std::ostream& out) {
get_object(key, [&](const char* data, std::size_t n) {
out.write(data, static_cast<std::streamsize>(n));
if (!out) {
throw std::runtime_error("get_object: write failed");
}
});
}
std::int64_t S3Client::get_object(
const std::string& key,
const std::function<void(const char*, std::size_t)>& on_chunk) {
std::int64_t total = 0;
minio::s3::GetObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
args.datafunc = [&](minio::http::DataFunctionArgs da) -> bool {
on_chunk(da.datachunk.data(), da.datachunk.size());
total += static_cast<std::int64_t>(da.datachunk.size());
return true;
};
minio::s3::GetObjectResponse resp = impl_->client->GetObject(args);
if (!resp) {
throw std::runtime_error("S3 GetObject failed: " + resp.Error().String());
}
return total;
}
void S3Client::copy_object(const std::string& src_key,
const std::string& dst_key) {
if (src_key.empty() || dst_key.empty() || src_key == dst_key) {
throw std::invalid_argument("copy_object: bad keys");
}
minio::s3::CopyObjectArgs args;
args.bucket = cfg_.bucket;
args.object = dst_key;
args.source.bucket = cfg_.bucket;
args.source.object = src_key;
minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args);
if (!resp) {
throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String());
}
}
void S3Client::delete_object(const std::string& key) {
minio::s3::RemoveObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
minio::s3::RemoveObjectResponse resp = impl_->client->RemoveObject(args);
if (!resp) {
throw std::runtime_error("S3 DeleteObject failed: " + resp.Error().String());
}
}
bool S3Client::stat_object(const std::string& key, std::int64_t& size_bytes,
std::string& etag) {
minio::s3::StatObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
minio::s3::StatObjectResponse resp = impl_->client->StatObject(args);
if (!resp) {
// NoSuchKey / Not Found → absent
const std::string err = resp.Error().String();
if (resp.code == "NoSuchKey" || resp.status_code == 404) {
return false;
}
throw std::runtime_error("S3 StatObject failed: " + err);
}
size_bytes = static_cast<std::int64_t>(resp.size);
etag = resp.etag;
return true;
}
std::string S3Client::presign_put(const std::string& key,
unsigned expiry_seconds,
const std::string& content_type) {
if (expiry_seconds < 1 || expiry_seconds > 7 * 24 * 3600u) {
throw std::invalid_argument("presign_put: bad expiry");
}
minio::s3::Client* c =
impl_->presign_client ? impl_->presign_client.get() : impl_->client.get();
// Note: minio-cpp query-presign signs method/host/path/query only — it does
// not bind Content-Type into the signature. Clients should still send
// Content-Type so the object metadata is correct; complete() re-validates
// magic/sha256 against the session.
(void)content_type;
minio::s3::GetPresignedObjectUrlArgs args;
args.bucket = cfg_.bucket;
args.object = key;
args.method = minio::http::Method::kPut;
args.expiry_seconds = expiry_seconds;
minio::s3::GetPresignedObjectUrlResponse resp = c->GetPresignedObjectUrl(args);
if (!resp) {
throw std::runtime_error("S3 PresignPut failed: " + resp.Error().String());
}
return resp.url;
}
std::vector<std::string> S3Client::old_staging(const std::string& cursor,
int retention_seconds,
std::string& next_cursor) {
minio::s3::ListObjectsArgs args;
args.bucket = cfg_.bucket;
args.prefix = "staging/";
args.recursive = true;
args.max_keys = 100;
args.start_after = cursor;
auto cutoff = minio::utils::UtcTime::Now();
cutoff.Add(-retention_seconds);
auto result = impl_->client->ListObjects(args);
std::vector<std::string> keys;
next_cursor.clear();
for (int count = 0; result && count < 100; ++count) {
const auto& item = *result;
if (!item) throw std::runtime_error("S3 staging listing failed");
if (!item.is_prefix && item.last_modified <= cutoff) keys.push_back(item.name);
if (count == 99) {
next_cursor = item.name;
break;
}
++result;
}
return keys;
}
+71
View File
@@ -0,0 +1,71 @@
#pragma once
#include <cstdint>
#include <functional>
#include <istream>
#include <memory>
#include <ostream>
#include <string>
#include <vector>
// Thin wrapper over AWS SDK for C++ (S3 only). Credentials and SigV4 come from
// the SDK — no hand-rolled signing.
class S3Client {
public:
struct Config {
std::string endpoint; // e.g. http://minio:9000
std::string region; // e.g. us-east-1
std::string bucket;
std::string access_key;
std::string secret_key;
};
explicit S3Client(Config cfg);
~S3Client();
S3Client(const S3Client&) = delete;
S3Client& operator=(const S3Client&) = delete;
static Config from_env();
const std::string& bucket() const { return cfg_.bucket; }
// Stream upload. `body` must yield exactly `size_bytes` bytes.
void put_object(const std::string& key,
std::istream& body,
std::int64_t size_bytes,
const std::string& content_type);
// Stream download into `out` (chunked reads from SDK body stream).
void get_object(const std::string& key, std::ostream& out);
// Stream download with a consumer callback (chunk, size). Returns total bytes.
std::int64_t get_object(const std::string& key,
const std::function<void(const char*, std::size_t)>& on_chunk);
void delete_object(const std::string& key);
// At most 100 entries examined per call; cursor wraps to empty at end.
std::vector<std::string> old_staging(const std::string& cursor,
int retention_seconds, std::string& next_cursor);
// Server-side copy inside the bucket (complete: staging → final).
void copy_object(const std::string& src_key, const std::string& dst_key);
// HeadObject-style metadata. Returns false if the object does not exist.
bool stat_object(const std::string& key, std::int64_t& size_bytes,
std::string& etag);
// Presigned PUT URL via minio-cpp (SigV4). Optional required Content-Type
// is bound into the signature when non-empty.
// expiry_seconds: typically 60–600 for Track 1.
std::string presign_put(const std::string& key,
unsigned expiry_seconds,
const std::string& content_type);
private:
struct Impl;
Config cfg_;
std::unique_ptr<Impl> impl_;
};
+87
View File
@@ -0,0 +1,87 @@
#include "session_repo.hpp"
#include "errors.hpp"
#include <sodium.h>
#include <ctime>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
std::string random_token_hex() {
unsigned char buf[32];
randombytes_buf(buf, sizeof(buf));
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(sizeof(buf) * 2);
for (std::size_t i = 0; i < sizeof(buf); ++i) {
out[i * 2] = hex[buf[i] >> 4];
out[i * 2 + 1] = hex[buf[i] & 0x0f];
}
return out;
}
std::int64_t now_unix() {
return static_cast<std::int64_t>(std::time(nullptr));
}
redisReply* check_reply(redisContext* c, redisReply* r) {
if (r == nullptr) {
const std::string err = c->err ? c->errstr : "no reply";
throw DependencyUnavailable("redis command failed: " + err);
}
return r;
}
}
ISessionRepo::Created SessionRepo::create(const std::string& user_id) {
const std::string token = random_token_hex();
const std::int64_t exp = now_unix() + ttl_;
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "SET session:%s %s EX %d",
token.c_str(), user_id.c_str(), ttl_));
check_reply(c, r);
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
throw std::runtime_error("redis SET returned non-status");
}
return Created{token, exp};
}
std::optional<std::string> SessionRepo::lookup(const std::string& token) {
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "GET session:%s", token.c_str()));
check_reply(c, r);
std::optional<std::string> out;
if (r->type == REDIS_REPLY_STRING) {
out = std::string(r->str, r->len);
}
freeReplyObject(r);
return out;
}
bool SessionRepo::revoke(const std::string& token) {
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "DEL session:%s", token.c_str()));
check_reply(c, r);
const bool existed = (r->type == REDIS_REPLY_INTEGER && r->integer == 1);
freeReplyObject(r);
return existed;
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "redis_pool.hpp"
#include "session_repo_base.hpp"
class SessionRepo : public ISessionRepo {
public:
SessionRepo(RedisPool& pool, int ttl_seconds)
: pool_(pool), ttl_(ttl_seconds) {}
Created create(const std::string& user_id) override;
std::optional<std::string> lookup(const std::string& token) override;
bool revoke(const std::string& token) override;
private:
RedisPool& pool_;
int ttl_;
};
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string>
class ISessionRepo {
public:
virtual ~ISessionRepo() = default;
struct Created {
std::string token;
std::int64_t expires_at; // unix-время в секундах UTC
};
virtual Created create(const std::string& user_id) = 0;
virtual std::optional<std::string> lookup(const std::string& token) = 0;
virtual bool revoke(const std::string& token) = 0;
};
+77
View File
@@ -0,0 +1,77 @@
#include "staging_cleaner.hpp"
#include "attachment_storage.hpp"
#include <nlohmann/json.hpp>
#include <regex>
void StagingCleaner::clean(const std::string& key) {
static const std::regex pattern(
"^staging/[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/"
"([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$");
std::smatch match;
if (!std::regex_match(key, match, pattern)) return;
const std::string id = match[1];
auto lease = pg_.acquire();
pqxx::work tx(*lease);
if (!tx.exec("SELECT pg_try_advisory_xact_lock(hashtextextended('staging:' || $1::uuid::text, 0))",
pqxx::params{id})[0][0].as<bool>()) return;
auto rows = tx.exec(
"SELECT a.chat_id, a.staging_key, a.s3_key, a.size_bytes, a.variants, a.status, j.id AS job_id, "
"a.updated_at < now()-($2::bigint * interval '1 second') AS expired "
"FROM attachments a LEFT JOIN jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version "
"WHERE a.id=$1::uuid FOR UPDATE OF a SKIP LOCKED", pqxx::params{id, retention_seconds_});
if (rows.empty()) {
// An existing locked row is not an orphan.
if (!tx.exec("SELECT 1 FROM attachments WHERE id=$1::uuid", pqxx::params{id}).empty()) return;
} else {
const auto& row = rows[0];
if (row["staging_key"].is_null() || row["staging_key"].as<std::string>() != key) return;
const std::string status = row["status"].as<std::string>();
if (status != "ready" && !(status == "failed" && row["expired"].as<bool>())) return;
if (!row["job_id"].is_null() && !tx.exec(
"SELECT pg_try_advisory_xact_lock(hashtextextended($1::uuid::text, 0))",
pqxx::params{row["job_id"].as<std::string>()})[0][0].as<bool>()) return;
if (!tx.exec("SELECT 1 FROM jobs WHERE attachment_id=$1::uuid AND source_key=$2 "
"AND status IN ('queued','processing')", pqxx::params{id,key}).empty()) return;
if (status == "ready") {
std::int64_t size = 0;
std::string etag;
if (!s3_.stat_object(row["s3_key"].as<std::string>(), size, etag) ||
size != row["size_bytes"].as<int>()) return;
for (const auto& variant : nlohmann::json::parse(row["variants"].as<std::string>())) {
const std::string object_key = variant.value("storage_key",
attachment_variant_key(row["chat_id"].as<std::string>(), id, variant.at("name").get<std::string>()));
if (!s3_.stat_object(object_key, size, etag) ||
size != variant.at("size_bytes").get<std::int64_t>()) return;
}
}
}
// A still-valid presigned URL can recreate the key: wait for its expiry.
if (!tx.exec("SELECT 1 FROM direct_uploads WHERE staging_key=$1 AND expires_at>now()",
pqxx::params{key}).empty()) return;
s3_.delete_object(key);
if (!rows.empty()) {
// Internal cleanup does not change public metadata/updated_at.
tx.exec("UPDATE attachments SET staging_cleaned_at=now() WHERE id=$1::uuid", pqxx::params{id});
}
tx.commit();
}
void StagingCleaner::sweep() {
std::vector<std::string> known;
{
auto lease = pg_.acquire();
pqxx::work tx(*lease);
for (const auto& row : tx.exec(
"SELECT staging_key FROM attachments WHERE staging_key IS NOT NULL "
"AND staging_cleaned_at IS NULL AND (status='ready' OR "
"(status='failed' AND updated_at<now()-($1::bigint * interval '1 second'))) "
"ORDER BY updated_at, id LIMIT 32", pqxx::params{retention_seconds_})) {
known.push_back(row[0].as<std::string>());
}
}
for (const auto& key : known) clean(key);
std::string next;
for (const auto& key : s3_.old_staging(cursor_, retention_seconds_, next)) clean(key);
cursor_ = std::move(next);
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "db.hpp"
#include "s3_client.hpp"
#include <string>
class StagingCleaner {
public:
StagingCleaner(PgPool& pg, S3Client& s3, int retention_seconds)
: pg_(pg), s3_(s3), retention_seconds_(retention_seconds) {}
void clean(const std::string& key);
void sweep();
private:
PgPool& pg_;
S3Client& s3_;
int retention_seconds_;
std::string cursor_;
};
+100
View File
@@ -0,0 +1,100 @@
#include "user_repo.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <random>
#include <sstream>
#include <string>
namespace {
std::string make_uuid() {
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
User row_to_user(const pqxx::row& r) {
User u;
u.id = r["id"].as<std::string>();
u.username = r["username"].as<std::string>();
u.display_name = r["display_name"].as<std::string>();
u.password_hash = r["password_hash"].as<std::string>();
u.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return u;
}
constexpr const char* kUserColumns =
"id, username, display_name, password_hash, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
}
std::optional<User> UserRepo::create(const std::string& username,
const std::string& display_name,
const std::string& password_hash) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
try {
const std::string id = make_uuid();
auto r = tx.exec(
"INSERT INTO users (id, username, display_name, password_hash, created_at) "
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
"RETURNING " + std::string(kUserColumns),
pqxx::params{id, username, display_name, password_hash, now_epoch_str()});
tx.commit();
return row_to_user(r[0]);
} catch (const pqxx::unique_violation&) {
tx.abort();
return std::nullopt;
}
}
std::optional<User> UserRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kUserColumns) + " FROM users WHERE id = $1",
pqxx::params{id});
tx.commit();
if (r.empty()) return std::nullopt;
return row_to_user(r[0]);
}
std::optional<User> UserRepo::find_by_username(const std::string& username) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kUserColumns) + " FROM users WHERE username = $1",
pqxx::params{username});
tx.commit();
if (r.empty()) return std::nullopt;
return row_to_user(r[0]);
}
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
class UserRepo {
public:
explicit UserRepo(PgPool& pool) : pool_(pool) {}
std::optional<User> create(const std::string& username,
const std::string& display_name,
const std::string& password_hash);
std::optional<User> find_by_id(const std::string& id);
std::optional<User> find_by_username(const std::string& username);
private:
PgPool& pool_;
};
+220
View File
@@ -0,0 +1,220 @@
#include "attachment_queue.hpp"
#include "attachment_repo.hpp"
#include "attachment_storage.hpp"
#include "staging_cleaner.hpp"
#include "image_pipeline.hpp"
#include "job_repo.hpp"
#include "fault_injection.hpp"
#include "s3_client.hpp"
#include <sodium.h>
#include <nlohmann/json.hpp>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <csignal>
#include <cstdlib>
#include <iostream>
#include <regex>
#include <sstream>
#include <thread>
#include <unistd.h>
namespace {
volatile std::sig_atomic_t stop_requested = 0;
void stop(int) { stop_requested = 1; }
class Watchdog {
public:
explicit Watchdog(int seconds) : seconds_(seconds), thread_([this] {
while (!stopping_.load()) {
const auto deadline = deadline_.load();
if (deadline && now_ms() >= deadline) {
std::cerr << "worker job timed out; exiting for durable recovery\n";
std::_Exit(124);
}
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
}) {}
~Watchdog() { stopping_.store(true); thread_.join(); }
void arm() { deadline_.store(now_ms() + seconds_ * 1000LL); }
void clear() { deadline_.store(0); }
private:
static long long now_ms() {
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now().time_since_epoch()).count();
}
int seconds_;
std::atomic<long long> deadline_{0};
std::atomic<bool> stopping_{false};
std::thread thread_;
};
struct Deadline {
Watchdog& timer;
explicit Deadline(Watchdog& timer) : timer(timer) { timer.arm(); }
~Deadline() { timer.clear(); }
};
int setting(const char* name, int fallback, int minimum, int maximum) {
const char* value = std::getenv(name);
if (!value) return fallback;
std::size_t end = 0;
const int result = std::stoi(value, &end);
if (value[end] || result < minimum || result > maximum) {
throw std::runtime_error("invalid worker setting");
}
return result;
}
std::string sha256(const std::string& bytes) {
unsigned char hash[crypto_hash_sha256_BYTES];
crypto_hash_sha256(hash, reinterpret_cast<const unsigned char*>(bytes.data()), bytes.size());
char hex[crypto_hash_sha256_BYTES * 2 + 1];
sodium_bin2hex(hex, sizeof(hex), hash, sizeof(hash));
return hex;
}
bool valid_job(const nlohmann::json& j) {
static const std::regex uuid("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
return j.is_object() && j.value("schema_version", 0) == 1 &&
j.value("type", "") == "attachment.process.v1" && j.value("pipeline_version", 0) >= 1 && j.value("pipeline_version", 0) <= 2 &&
j.value("attempt", 0) >= 1 && j.contains("created_at") && j.at("created_at").is_string() &&
std::regex_match(j.value("job_id", ""), uuid) &&
std::regex_match(j.value("attachment_id", ""), uuid);
}
void verify_ready(S3Client& s3, const Attachment& a) {
std::int64_t size = 0;
std::string etag;
if (!s3.stat_object(a.s3_key, size, etag) || size != a.size_bytes) {
throw std::runtime_error("ready original missing");
}
bool thumbnail = false;
for (const auto& v : a.variants) {
thumbnail = thumbnail || v.name == "thumbnail";
if (!s3.stat_object((v.storage_key.empty() ? attachment_variant_key(a.chat_id, a.id, v.name) : v.storage_key), size, etag) ||
size != v.size_bytes) throw std::runtime_error("ready variant missing");
}
if (!thumbnail) throw std::runtime_error("ready thumbnail missing");
}
} // namespace
int main() {
std::signal(SIGTERM, stop);
std::signal(SIGINT, stop);
try {
if (sodium_init() < 0) return 1;
const RetryPolicy retry{
setting("JOB_MAX_ATTEMPTS", 6, 1, 100),
setting("JOB_RETRY_BASE_MS", 1000, 1, 3600000),
setting("JOB_RETRY_MAX_MS", 60000, 1, 3600000)};
if (retry.base_delay_ms > retry.max_delay_ms) throw std::runtime_error("invalid retry policy");
const int timeout = setting("JOB_TIMEOUT_SECONDS", 60, 1, 3600);
const int cleanup_interval = setting("STAGING_CLEANUP_INTERVAL_SECONDS", 60, 1, 86400);
const int retention = setting("STAGING_RETENTION_SECONDS", 86400, 1, 31536000);
const int reclaim_idle = setting("JOB_RECLAIM_IDLE_MS", 5000, 1000, 3600000);
const auto db_url = read_secret_file("DATABASE_URL_FILE");
{
auto conn = open_pg_connection(db_url, false);
apply_migrations(*conn, "./migrations");
}
PgPool pg(db_url, 2);
AttachmentRepo attachments(pg);
JobRepo jobs(pg, retry);
const char* broker_file = std::getenv("BROKER_URL_FILE");
RedisPool redis(read_secret_file(
broker_file && *broker_file ? "BROKER_URL_FILE" : "CACHE_URL_FILE"), 2);
AttachmentQueue queue(redis);
S3Client s3(S3Client::from_env());
char random[33];
unsigned char entropy[16];
randombytes_buf(entropy, sizeof(entropy));
sodium_bin2hex(random, sizeof(random), entropy, sizeof(entropy));
const std::string consumer = "worker-" + std::string(random);
std::clog << "worker started consumer=" << consumer << " max_attempts=" << retry.max_attempts << '\n';
StagingCleaner cleaner(pg, s3, retention);
Watchdog watchdog(timeout);
auto next_sweep = std::chrono::steady_clock::now();
while (!stop_requested) {
try {
if (std::chrono::steady_clock::now() >= next_sweep) {
next_sweep = std::chrono::steady_clock::now() + std::chrono::seconds(cleanup_interval);
Deadline deadline(watchdog);
cleaner.sweep();
}
queue.ensure_group();
const auto message = queue.read(consumer, reclaim_idle);
if (!message) continue;
nlohmann::json j;
bool valid = false;
try {
j = nlohmann::json::parse(message->payload);
valid = valid_job(j);
} catch (const nlohmann::json::exception&) {}
if (!valid) {
queue.reject(message->stream_id);
continue;
}
const std::string id = j.at("attachment_id").get<std::string>();
Deadline deadline(watchdog);
const auto result = jobs.run(j.at("job_id").get<std::string>(), id,
j.at("pipeline_version").get<int>(), j.at("attempt").get<int>(), [&](const JobInput& input) {
const auto a = attachments.find_by_id(id);
if (!a) throw std::runtime_error("attachment missing");
std::string original;
original.reserve(a->size_bytes);
s3.get_object(input.source_key, [&](const char* data, std::size_t n) {
if (original.size() > static_cast<std::size_t>(a->size_bytes) ||
n > static_cast<std::size_t>(a->size_bytes) - original.size()) {
throw InvalidImage("source_mismatch");
}
original.append(data, n);
});
if (original.size() != static_cast<std::size_t>(a->size_bytes) ||
sha256(original) != a->sha256) throw InvalidImage("source_mismatch");
const auto images = make_variants(std::vector<std::uint8_t>(original.begin(), original.end()), input.pipeline_version);
const std::string prefix = pipeline_prefix(a->chat_id, a->id, input.pipeline_version);
const std::string key = prefix + "original";
// Publish verified bytes even if a presigned staging key changes.
std::istringstream source(original);
s3.put_object(key, source, original.size(), a->content_type);
nlohmann::json variants = nlohmann::json::array();
for (const auto& v : images) {
std::istringstream body(v.image.bytes);
const std::string variant_key = prefix + v.name + (v.content_type == "image/webp" ? ".webp" : ".jpg");
s3.put_object(variant_key, body,
v.image.bytes.size(), v.content_type);
variants.push_back({{"name", v.name}, {"content_type", v.content_type},
{"size_bytes", v.image.bytes.size()}, {"sha256", sha256(v.image.bytes)},
{"width", v.image.width}, {"height", v.image.height}, {"storage_key", variant_key}});
}
lab_worker_fault("worker_after_effect", j.at("job_id").get<std::string>(), input.pipeline_version);
return JobResult{key, variants.dump()};
});
if (result == JobRepo::Result::Missing) {
queue.reject(message->stream_id);
} else if (result == JobRepo::Result::Done) {
const auto a = attachments.find_by_id(id);
if (a && a->status == "ready") verify_ready(s3, *a);
lab_worker_fault("worker_before_ack", j.at("job_id").get<std::string>(), j.at("pipeline_version").get<int>());
queue.ack(message->stream_id);
if (a && a->status == "ready") cleaner.clean("staging/" + a->chat_id + "/" + a->id);
} else if (result == JobRepo::Result::Superseded) {
queue.ack(message->stream_id);
} else if (result == JobRepo::Result::Scheduled) {
// ACK only after the next attempt was committed to outbox.
queue.ack(message->stream_id);
}
// Busy remains pending; an active job is protected by its advisory lock.
} catch (const std::exception&) {
std::clog << "worker operation failed; delivery remains pending\n";
std::this_thread::sleep_for(std::chrono::seconds(1));
}
}
return 0;
} catch (const std::exception&) {
std::cerr << "worker initialization failed\n";
return 1;
}
}
+102
View File
@@ -0,0 +1,102 @@
# Проверки реализации lab5
Нужны Linux amd64, Docker Engine с Compose v2, Python 3.10+ и Pillow. Полный прогон
создаёт только синтетические данные. Для сборки нужен доступ к пакетным
репозиториям Ubuntu и закреплённым исходникам GitHub.
```sh
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt -r student-tests/requirements.txt
make check validate PYTHON=.venv/bin/python
make test-async PYTHON=.venv/bin/python TEST_ARGS="--output evidence/async-suite.json"
```
Первый запуск собирает API/worker, MinIO и mc из этого репозитория. При повторе
можно добавить `--no-build`, если исходники и Dockerfiles не изменились.
Полный тест занимает несколько минут; отдельное ожидание ready ограничено 60 s.
Тесты запускайте последовательно: они кратковременно меняют состояние своих
API/брокеров и используют ресурсы Docker, хотя имена проектов изолированы.
## Отдельные сценарии
После активации venv или с `PYTHON=.venv/bin/python`:
```sh
student-tests/test_upload_async.sh --no-build --output evidence/upload-async.json
student-tests/test_worker_down.sh --no-build --output evidence/worker-down.json
student-tests/test_outbox_crash.sh --no-build --output evidence/outbox-crash.json
student-tests/test_reliability.sh --no-build --output evidence/reliability.json
```
Эквивалентные Make targets: `test-upload-async`, `test-worker-down`,
`test-outbox-crash`, `test-reliability`; параметры передаются через `TEST_ARGS`.
Каждый сценарий заканчивается неизменённым публичным `tests/smoke.py`.
- `test_upload_async.sh`: worker остановлен → POST 202 со снимком queued →
GET queued и content 409 → worker запущен → ready. Проверяются исходные bytes,
SHA-256, реальные JPEG/WebP, размеры и метаданные всех трёх вариантов, доступ
другого участника и отказ постороннему. Остановка делает GET queued
детерминированным: при работающем worker контракт разрешает первый GET ready.
- `test_worker_down.sh`: дополнительно создаёт сообщение с queued-вложением,
перезапускает PostgreSQL, Valkey, MinIO и API с сохранением volumes,
проверяет прежнюю сессию/вложение и запускает worker. В JSON evidence есть
реальные container IDs, StartedAt и имена volumes до/после рестарта.
- `test_outbox_crash.sh`: API принимает upload с выключенным брокером;
API падает с exit 86 после COMMIT до XADD; затем отдельно падает после XADD
до published=true. После рестарта та же job завершается без нового POST;
повтор события не создаёт второго эффекта. Используются PostgreSQL-сессии,
поэтому остановка Valkey не отключает синхронную аутентификацию.
- `test_reliability.sh`: атомарный rollback outbox; шесть попыток с backoff/jitter
и проверкой due time; replay с checksum и audit; устаревшее событие;
SIGKILL после S3 до COMMIT/ACK; watchdog; конкурентные дубли у двух worker;
два API-публикатора; retention, presigned complete и гонка GC/upload;
RGB PNG на 40 млн пикселей при 512 MiB без swap. Для GC/upload проверяется,
что DB INSERT всё ещё находится в паузе до и после S3-пробы; наблюдение после
COMMIT не принимается как доказательство гонки.
## Изоляция и результаты
Общий runner `async_suite.py` создаёт случайный `lab5test…` Compose-проект,
частный каталог `build/student-tests/<project>`, свои env/credentials/volumes и
случайный loopback-порт API. Основной LB не запускается. Рабочие `.env`, secrets,
volumes и сервисы других проектов не используются. Экспортированные S3/Compose
переменные не могут перенаправить тестовый стенд во внешнее хранилище.
В `finally` удаляются только собственные контейнеры/volumes и файлы credentials.
Каталог с диагностикой остаётся в игнорируемом build. `build/` исключён и из
Docker build context. При ошибке cleanup runner сообщает точное имя своего
проекта; не применяйте глобальный prune. Обрыв питания/SIGKILL самого runner
может потребовать удаления его проекта оператором.
`--output` сохраняет только итог, проверки, ID синтетических объектов,
идентификаторы контейнеров/образов, лимиты, время и SHA-256 исходников. Пароли,
Bearer-токены, подписанные URL и HTTP bodies в evidence не записываются.
Проверки header/JSON используют неизменённые публичные helpers; Pillow независимо
декодирует изображения. Измерение RSS/нагрузки не выполняется: memory case
подтверждает завершение одного RGB fixture при заданном cgroup limit.
## Геометрия изображений отдельно
Если binaries собраны локально с зависимостями из Dockerfile:
```sh
.venv/bin/python student-tests/test_thumbnail.py --binary build/thumbnail_smoke
.venv/bin/python student-tests/test_variants.py --binary build/variants_smoke
```
Второй тест также проверяет центральный crop, WebP без увеличения, округление,
EXIF orientation 1–8 для JPEG/PNG, большой PNG eXIf, alpha и удаление metadata.
## Уровень 5, трек 2
```sh
make test-track2 PYTHON=.venv/bin/python TEST_ARGS="--output evidence/track2.json"
# Если образы уже собраны из текущих исходников:
PYTHON=.venv/bin/python student-tests/test_pipeline_idempotency.sh --no-build
```
Default — 3 независимых повтора; `--repetitions 1` запускает отладочный прогон.
Методика, гипотеза, fault-точки и инварианты: [infra/track2.md](../infra/track2.md).
Используется тот же isolated runner и независимая проверка images; сохраняется
отдельный evidence с source/method hashes и SHA объектов каждой версии.
+599
View File
@@ -0,0 +1,599 @@
#!/usr/bin/env python3
"""Disposable Compose integration tests. Never operates on an existing project."""
import argparse
import concurrent.futures
import contextlib
import datetime as dt
import hashlib
import http.client
import io
import json
import os
import re
from pathlib import Path
import secrets
import shlex
import subprocess
import sys
import time
import uuid
from PIL import Image, __version__ as pillow_version
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
def wait_for(predicate, label, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(0.2)
raise Failure('timeout: ' + label)
class Stack:
def __init__(self):
self.project = 'lab5test' + uuid.uuid4().hex[:12]
self.directory = ROOT / 'build' / 'student-tests' / self.project
self.directory.mkdir(parents=True, mode=0o700)
self.events = []
self.client = None
self.probe_name = None
self.store_ids = {}
self.base_files = ['-f', str(ROOT / 'compose.yaml'), '-f', str(self.directory / 'compose.json')]
values = {
'POSTGRES_USER': 'messenger', 'POSTGRES_DB': 'messenger',
'POSTGRES_PASSWORD': secrets.token_hex(24),
'MINIO_ROOT_USER': 'lab5-test-root', 'MINIO_ROOT_PASSWORD': secrets.token_hex(24),
'MINIO_USER_ACCESS_KEY': 'lab5-test-access', 'MINIO_USER_SECRET_KEY': secrets.token_hex(24),
'S3_BUCKET': 'lab5-tests', 'SESSION_STORE': 'postgres', 'API_REPLICAS': '1',
'DB_POOL_SIZE': '2', 'HTTP_WORKERS': '2', 'DRAIN_SECONDS': '0',
'API_CPU_LIMIT': '1.0', 'API_MEM_LIMIT': '512m', 'MINIO_CPU_LIMIT': '1.0',
}
self.command_env = os.environ.copy()
# An exported S3_ENDPOINT must never redirect a disposable test to another store.
for name in re.findall(r'\$\{([A-Za-z_]\w*)', (ROOT / 'compose.yaml').read_text()):
self.command_env.pop(name, None)
self.env_file = self.directory / 'test.env'
self.env_file.write_text(''.join(k + '=' + v + '\n' for k, v in values.items()))
self.env_file.chmod(0o600)
credentials = {
'database_url': 'postgresql://messenger:' + values['POSTGRES_PASSWORD'] + '@db:5432/messenger',
'cache_url': 'redis://cache:6379/0', 's3_access_key_id': values['MINIO_USER_ACCESS_KEY'],
's3_secret_access_key': values['MINIO_USER_SECRET_KEY'],
}
self.secret_files = []
for name, value in credentials.items():
path = self.directory / name
path.write_text(value + '\n')
path.chmod(0o644) # Bind-mounted files must be readable by container uid 10001.
self.secret_files.append(path)
override = {
'services': {
'api': {'ports': ['127.0.0.1::8080'], 'deploy': {'replicas': 1},
'healthcheck': {'interval': '1s', 'start_period': '1s', 'retries': 60}},
'worker': {'environment': {
'JOB_RECLAIM_IDLE_MS': '1000', 'JOB_RETRY_BASE_MS': '200',
'JOB_RETRY_MAX_MS': '800', 'JOB_TIMEOUT_SECONDS': '3',
'STAGING_RETENTION_SECONDS': '30', 'STAGING_CLEANUP_INTERVAL_SECONDS': '1',
}},
},
'secrets': {name: {'file': str(self.directory / name)} for name in credentials},
}
(self.directory / 'compose.json').write_text(json.dumps(override))
self.prefix = ['docker', 'compose', '--project-directory', str(ROOT), '-p', self.project,
'--env-file', str(self.env_file)]
def command(self, args, label, data=None, allow_failure=False, timeout=180):
result = subprocess.run(args, input=data, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=ROOT, timeout=timeout, env=self.command_env)
if result.returncode and not allow_failure:
with (self.directory / 'control.log').open('ab') as log:
log.write(result.stdout + result.stderr)
raise Failure(label + ' failed; diagnostic: ' + str(self.directory / 'control.log'))
return result
def dc(self, *args, fault=None, data=None, allow_failure=False, timeout=180):
files = self.base_files.copy()
if fault:
path = self.directory / ('fault-' + fault + '.json')
path.write_text(json.dumps({'services': {'api': {'environment': {
'LAB_ENABLE_FAULT_INJECTION': '1', 'LAB_FAULT_POINT': fault}}}}))
files += ['-f', str(path)]
result = self.command(self.prefix + files + list(args), 'Compose ' + args[0],
data=data, allow_failure=allow_failure, timeout=timeout)
return result.stdout.decode().strip()
def record(self, name, **fields):
self.events.append({'check': name, **fields})
print('PASS:', name, flush=True)
def start(self, build):
self.dc('config', '--quiet')
if build:
self.dc('build', 'api', 'minio', 'minio-init', timeout=1800)
self.dc('up', '-d', '--wait', '--wait-timeout', '120', 'api', 'worker', timeout=180)
self.refresh()
self.store_ids = {name: self.dc('ps', '-q', name) for name in ('db', 'cache')}
self.record('isolated stack started', project=self.project,
containers={name: self.inspect(name) for name in ('api', 'worker', 'db', 'cache', 'minio')})
def refresh(self):
address = self.dc('port', '--index', '1', 'api', '8080')
if self.client is None:
self.client = Client('http://' + address, timeout=10)
else:
self.client.base = 'http://' + address
def healthy():
try:
self.client.call('GET', '/health/ready', schema='Health')
return True
except (Failure, http.client.HTTPException):
return False
wait_for(healthy, 'API readiness')
def sql(self, query):
# These containers are restarted, never recreated, during a run.
args = ['docker', 'exec', '-i', self.store_ids['db'], 'psql', '-X', '-v',
'ON_ERROR_STOP=1', '-U', 'messenger', '-d', 'messenger', '-Atqc', query]
return self.command(args, 'test SQL').stdout.decode().strip()
def redis(self, *args):
command = ['docker', 'exec', '-i', self.store_ids['cache'], 'valkey-cli', '--json', *args]
return json.loads(self.command(command, 'test Redis').stdout)
def inspect(self, service):
ids = self.dc('ps', '-a', '-q', service).splitlines()
if not ids:
return []
raw = self.command(['docker', 'inspect', *ids], 'inspect containers').stdout
return [{'id': x['Id'], 'image': x['Image'], 'started_at': x['State']['StartedAt'],
'exit_code': x['State']['ExitCode'], 'restart_count': x['RestartCount'],
'memory_limit_bytes': x['HostConfig']['Memory'],
'volumes': sorted(m['Name'] for m in x['Mounts'] if m['Type'] == 'volume')}
for x in json.loads(raw)]
def wait_store(self, service):
def healthy():
cid = self.dc('ps', '-q', service)
result = self.command(['docker', 'inspect', '--format',
'{{.State.Health.Status}}', cid], 'store health')
return result.stdout.decode().strip() == 'healthy'
wait_for(healthy, service + ' health', timeout=90)
def recreate_api(self, fault=None):
self.dc('up', '-d', '--no-deps', '--force-recreate', 'api', fault=fault)
self.refresh()
def mc(self, command, data=None, allow_failure=False):
if self.probe_name is None:
self.probe_name = self.project + '-mc-probe'
script = ('mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" '
'"$MINIO_ROOT_PASSWORD" >/dev/null; touch /tmp/probe-ready; exec sleep infinity')
self.dc('run', '--rm', '-d', '--no-deps', '--name', self.probe_name,
'--entrypoint', '/bin/sh', 'minio-user-init', '-ec', script)
wait_for(lambda: self.command(['docker', 'exec', self.probe_name, 'test', '-f',
'/tmp/probe-ready'], 'MinIO probe readiness',
allow_failure=True).returncode == 0,
'MinIO probe readiness', timeout=30)
return self.command(['docker', 'exec', '-i', self.probe_name, '/bin/sh', '-ec', command],
'MinIO test action', data=data, allow_failure=allow_failure)
def exists(self, key):
result = self.mc('mc stat "local/$S3_BUCKET/"' + shlex.quote(key), allow_failure=True)
return result.returncode == 0
def replay(self, job_id, success):
args = self.prefix + self.base_files + ['exec', '-T', 'api', '/docker-entrypoint.sh',
'/app/job_admin', 'replay', job_id, '--actor', 'integration-test',
'--reason', 'temporary failure fixed']
result = self.command(args, 'operator replay', allow_failure=True)
check((result.returncode == 0) == success, 'unexpected replay outcome')
def close(self):
try:
if self.probe_name:
self.command(['docker', 'rm', '-f', self.probe_name], 'remove own MinIO probe',
allow_failure=True)
self.dc('down', '--volumes', '--remove-orphans', timeout=120)
finally:
for path in [self.env_file, *self.secret_files]:
path.unlink(missing_ok=True)
class Suite:
def __init__(self, stack):
self.stack = stack
self.c = stack.client
self.alice, self.bob, self.eve, self.chat = self.c.bootstrap()
self.content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
def upload(self, expected=202):
result = self.c.call('POST', '/api/v1/chats/' + self.chat['id'] + '/attachments?filename=sample.png',
expected, actor=self.alice, raw=self.content,
headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None)
if expected != 202:
return result
att = result['json']
check(att['status'] == 'queued' and att['variants'] == [] and att['error_code'] is None,
'202 snapshot must be queued without variants')
check(att['sha256'] == hashlib.sha256(self.content).hexdigest() and
att['size_bytes'] == len(self.content), 'upload original metadata mismatch')
return att
def metadata(self, att):
return self.c.call('GET', '/api/v1/attachments/' + att['id'], actor=self.alice,
schema='Attachment')['json']
def job(self, att):
return self.stack.sql("SELECT id FROM jobs WHERE attachment_id='" + att['id'] + "'")
def key(self, att):
return 'staging/' + self.chat['id'] + '/' + att['id']
def ready(self, att):
ready = self.c.wait_ready(self.alice, att, wait_seconds=60)
check(ready['error_code'] is None, 'ready has error_code')
variants = {v['name']: v for v in ready['variants']}
check(len(ready['variants']) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'},
'ready must publish exactly three variants')
path = '/api/v1/attachments/' + att['id'] + '/content'
original = self.c.call('GET', path, actor=self.bob)
check(original['bytes'] == self.content and original['headers'].get_content_type() == 'image/png',
'original differs')
for name, size, mime, fmt in [('thumbnail', (256, 160), 'image/jpeg', 'JPEG'),
('crop', (128, 128), 'image/jpeg', 'JPEG'),
('optimized', (320, 200), 'image/webp', 'WEBP')]:
result = self.c.call('GET', path + '?variant=' + name, actor=self.bob)
v = variants[name]
with Image.open(io.BytesIO(result['bytes'])) as image:
image.load()
check(image.format == fmt and image.size == size and not image.getexif(),
'actual variant image mismatch: ' + name)
check(result['headers'].get_content_type() == mime and
len(result['bytes']) == v['size_bytes'] and
hashlib.sha256(result['bytes']).hexdigest() == v['sha256'] and
(v['width'], v['height']) == size, 'variant metadata mismatch: ' + name)
for suffix in ('', '/content?variant=optimized'):
self.c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=self.eve)
return ready
def events(self, att):
events = []
for _, fields in self.stack.redis('XRANGE', 'jobs:attachments', '-', '+'):
values = dict(zip(fields[::2], fields[1::2]))
with contextlib.suppress(ValueError, KeyError, AttributeError):
payload = json.loads(values['payload'])
if payload.get('attachment_id') == att['id']:
events.append(payload)
return events
def pending_empty(self):
return self.stack.redis('XPENDING', 'jobs:attachments', 'attachment-workers')[0] == 0
def upload_async(self):
s = self.stack
s.dc('stop', 'worker')
att = self.upload()
check(self.metadata(att)['status'] == 'queued', 'GET must be queued with stopped worker')
self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=self.alice)
s.dc('start', 'worker')
self.ready(att)
s.record('upload 202 -> GET queued -> ready, original and 3 variants verified', attachment_id=att['id'])
def worker_down(self):
s = self.stack
s.dc('stop', 'worker')
att = self.upload()
check(self.metadata(att)['status'] == 'queued', 'worker-down upload was processed')
self.c.message(self.alice, self.chat, '', attachments=[att['id']])
before = {name: s.inspect(name) for name in ('api', 'db', 'cache', 'minio')}
s.dc('restart', 'db', 'cache', 'minio')
for name in ('db', 'cache', 'minio'):
s.wait_store(name)
s.dc('restart', 'api')
s.refresh()
check(self.metadata(att)['status'] == 'queued', 'accepted job lost after store/API restart')
after = {name: s.inspect(name) for name in before}
for name in before:
check(before[name][0]['started_at'] != after[name][0]['started_at'], name + ' did not restart')
check(before[name][0]['volumes'] == after[name][0]['volumes'], name + ' changed volumes')
s.dc('start', 'worker')
self.ready(att)
s.record('worker down: accepted job/session survive API, PostgreSQL, Valkey and S3 restart',
attachment_id=att['id'], before=before, after=after)
def outbox_crash(self):
s = self.stack
s.dc('stop', 'worker', 'cache')
att = self.upload()
check(self.metadata(att)['status'] == 'queued', 'broker-down upload not queued')
s.dc('restart', 'api'); s.refresh()
check(self.metadata(att)['status'] == 'queued', 'outbox lost on offline restart')
s.dc('start', 'cache', 'worker'); self.ready(att)
s.record('broker down: upload 202 and offline API restart preserve outbox')
s.dc('stop', 'worker', 'cache')
s.recreate_api('upload_after_commit')
before = int(s.sql('SELECT count(*) FROM attachments'))
try:
self.upload()
except (Failure, http.client.HTTPException):
pass
else:
raise Failure('upload fault did not terminate API')
wait_for(lambda: s.inspect('api')[0]['exit_code'] == 86, 'upload crash exit 86')
check(int(s.sql('SELECT count(*) FROM attachments')) == before + 1, 'crash transaction absent')
att = {'id': s.sql('SELECT id FROM attachments ORDER BY created_at DESC LIMIT 1')}
check(s.sql("SELECT count(*) FROM outbox WHERE NOT published AND payload->>'attachment_id'='" +
att['id'] + "'") == '1', 'committed outbox intent absent')
s.recreate_api()
check(self.metadata(att)['status'] == 'queued', 'crash attachment missing')
s.dc('start', 'cache', 'worker'); self.ready(att)
s.record('crash after COMMIT before XADD: same attachment/job recovers without new POST',
attachment_id=att['id'], exit_code=86)
s.dc('stop', 'worker', 'cache')
att = self.upload()
s.recreate_api('outbox_after_publish'); s.dc('start', 'cache')
wait_for(lambda: s.inspect('api')[0]['exit_code'] == 86, 'publisher crash exit 86')
check(len(self.events(att)) == 1, 'XADD did not precede injected crash')
check(s.sql("SELECT count(*) FROM outbox WHERE NOT published AND payload->>'attachment_id'='" +
att['id'] + "'") == '1', 'publisher committed marker before crash')
s.recreate_api()
wait_for(lambda: len(self.events(att)) == 2, 'outbox redelivery')
s.dc('start', 'worker'); self.ready(att)
check(s.sql("SELECT retries FROM jobs WHERE id='" + self.job(att) + "'") == '0',
'publisher duplicate caused an extra processing attempt')
s.record('crash after XADD before published marker: duplicate delivery, one ready effect',
attachment_id=att['id'], stream_deliveries=2, exit_code=86)
def retry_replay(self):
s = self.stack
counts = s.sql('SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM jobs), (SELECT count(*) FROM outbox)')
s.sql("CREATE FUNCTION test_reject_outbox() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'test failure'; END $$; CREATE TRIGGER test_reject_outbox BEFORE INSERT ON outbox FOR EACH ROW EXECUTE FUNCTION test_reject_outbox()")
self.upload(expected=500)
check(s.sql('SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM jobs), (SELECT count(*) FROM outbox)') == counts,
'outbox failure did not roll back all metadata')
s.sql('DROP TRIGGER test_reject_outbox ON outbox; DROP FUNCTION test_reject_outbox()')
s.record('attachment/job/outbox transaction rolls back on outbox INSERT failure')
s.sql("CREATE TABLE test_attempts(job_id uuid, attempt int, observed_at timestamptz, due_at timestamptz); CREATE FUNCTION test_audit_attempt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.status='processing' AND OLD.status<>'processing' THEN INSERT INTO test_attempts VALUES(NEW.id,NEW.retries+1,now(),NEW.next_attempt_at); END IF; RETURN NEW; END $$; CREATE TRIGGER test_audit_attempt AFTER UPDATE ON jobs FOR EACH ROW EXECUTE FUNCTION test_audit_attempt()")
s.sql("CREATE FUNCTION test_reject_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.status='ready' THEN RAISE EXCEPTION 'test failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER test_reject_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION test_reject_ready()")
att = self.upload()
wait_for(lambda: self.metadata(att)['status'] == 'failed', 'retry exhaustion')
s.dc('stop', 'worker')
s.sql('DROP TRIGGER test_reject_ready ON attachments; DROP FUNCTION test_reject_ready()')
job = self.job(att)
check(s.sql("SELECT retries FROM jobs WHERE id='" + job + "'") == '6', 'retry limit not enforced')
check(self.metadata(att)['error_code'] == 'processing_failed', 'failed error_code absent')
check([p['attempt'] for p in self.events(att)] == list(range(1, 7)), 'managed attempts differ')
check(s.sql("SELECT count(*) FROM test_attempts WHERE job_id='" + job + "' AND observed_at<due_at") == '0',
'worker started before durable due time')
delays = []
rows = s.sql("SELECT payload->>'attempt', EXTRACT(EPOCH FROM (available_at-created_at))*1000 "
"FROM outbox WHERE payload->>'job_id'='" + job + "' ORDER BY id")
for row in rows.splitlines()[1:]:
attempt, delay = row.split('|'); attempt, delay = int(attempt), float(delay)
ceiling = min(800, 200 * 2 ** (attempt - 2))
check(ceiling / 2 <= delay <= ceiling, 'backoff/jitter outside configured range')
delays.append({'attempt': attempt, 'delay_ms': delay})
self.c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=self.alice)
s.record('backoff+jitter persisted, attempts 1..6, no early processing, finite failed', delays=delays)
# Corrupt bytes of the same size: replay must check SHA, not only HEAD/length.
key = shlex.quote(self.key(att))
s.mc('mc pipe "local/$S3_BUCKET/"' + key, data=bytes(len(self.content)))
s.replay(job, False)
check(s.sql("SELECT count(*) FROM job_replays WHERE job_id='" + job + "'") == '0',
'rejected replay changed audit')
s.mc('mc pipe "local/$S3_BUCKET/"' + key, data=self.content)
s.replay(job, True)
s.replay(job, False)
s.redis('XADD', 'jobs:attachments', '*', 'payload', json.dumps(self.events(att)[0]))
s.dc('start', 'worker'); self.ready(att)
s.replay(job, False)
check(s.sql("SELECT retry_base,retries FROM jobs WHERE id='" + job + "'") == '6|6',
'replay reset monotonic retry count')
check(max(p['attempt'] for p in self.events(att)) == 7, 'replay did not enqueue next managed attempt')
check(s.sql("SELECT actor,retries_before FROM job_replays WHERE job_id='" + job + "'") == 'integration-test|6',
'replay audit mismatch')
s.record('audited replay verifies SHA, keeps job_id, fences stale events, refuses queued/ready', job_id=job)
def worker_crashes(self):
s = self.stack
s.sql("CREATE FUNCTION test_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.status='ready' THEN PERFORM pg_sleep(10); END IF; RETURN NEW; END $$; CREATE TRIGGER test_pause_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION test_pause_ready()")
att = self.upload()
wait_for(lambda: s.sql("SELECT count(*) FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'UPDATE attachments a SET status=%'") == '1',
'worker after S3 before result COMMIT')
s.dc('kill', '-s', 'SIGKILL', 'worker')
s.sql('DROP TRIGGER test_pause_ready ON attachments; DROP FUNCTION test_pause_ready()')
check(s.exists('attachments/' + self.chat['id'] + '/' + att['id'] + '/v1/optimized.webp'),
'worker had not saved the final variant before crash')
s.dc('start', 'worker'); self.ready(att)
check(int(s.sql("SELECT retries FROM jobs WHERE id='" + self.job(att) + "'")) >= 1,
'crashed processing not counted')
s.record('SIGKILL after S3 before COMMIT/ACK: pending recovers the same effect', attachment_id=att['id'])
before = s.inspect('worker')[0]['restart_count']
s.sql("CREATE FUNCTION test_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.status='ready' THEN PERFORM pg_sleep(10); END IF; RETURN NEW; END $$; CREATE TRIGGER test_pause_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION test_pause_ready()")
att = self.upload()
wait_for(lambda: s.inspect('worker')[0]['restart_count'] > before, 'watchdog restart')
s.sql('DROP TRIGGER test_pause_ready ON attachments; DROP FUNCTION test_pause_ready()')
self.ready(att)
check('timed out' in s.dc('logs', '--no-color', 'worker'), 'watchdog timeout not observed')
s.record('watchdog bounds hung processing and Compose restarts worker',
restart_count_before=before, restart_count_after=s.inspect('worker')[0]['restart_count'])
def duplicates_publishers(self):
s = self.stack
s.dc('up', '-d', '--no-deps', '--scale', 'worker=2', 'worker')
s.sql("CREATE FUNCTION test_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.status='ready' THEN PERFORM pg_sleep(1); END IF; RETURN NEW; END $$; CREATE TRIGGER test_pause_ready BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION test_pause_ready()")
att = self.upload()
wait_for(lambda: self.events(att), 'initial stream event')
for _ in range(4):
s.redis('XADD', 'jobs:attachments', '*', 'payload', json.dumps(self.events(att)[0]))
ready = self.ready(att)
s.sql('DROP TRIGGER test_pause_ready ON attachments; DROP FUNCTION test_pause_ready()')
wait_for(self.pending_empty, 'duplicate ACKs')
check(self.metadata(att) == ready, 'duplicate changed ready metadata')
check(s.sql("SELECT retries FROM jobs WHERE id='" + self.job(att) + "'") == '0', 'duplicate became a retry')
s.record('two workers serialize duplicate processing; variants and ready metadata stay unique',
workers=s.inspect('worker'), stream_deliveries=len(self.events(att)))
s.dc('up', '-d', '--no-deps', '--scale', 'worker=1', 'worker')
s.dc('stop', 'worker', 'cache')
items = [self.upload() for _ in range(8)]
s.dc('up', '-d', '--no-deps', '--scale', 'api=2', 'api'); s.refresh()
s.dc('start', 'cache')
wait_for(lambda: s.sql('SELECT count(*) FROM outbox WHERE NOT published') == '0', 'concurrent publishers')
check(all(len(self.events(att)) == 1 for att in items), 'healthy publishers duplicated outbox rows')
s.dc('start', 'worker')
for att in items:
self.ready(att)
s.record('two API publishers drain outbox with SKIP LOCKED', apis=s.inspect('api'), jobs=len(items))
s.dc('up', '-d', '--no-deps', '--scale', 'api=1', 'api'); s.refresh()
def cleanup_races(self):
s = self.stack
# Short retention only inside this disposable project.
path = s.directory / 'cleanup.json'
path.write_text(json.dumps({'services': {'worker': {'environment': {'STAGING_RETENTION_SECONDS': '1'}}}}))
s.command(s.prefix + s.base_files + ['-f', str(path), 'up', '-d', '--no-deps', '--force-recreate', 'worker'],
'enable test retention')
s.dc('stop', 'cache')
att = self.upload()
direct = self.c.call('POST', '/api/v1/chats/' + self.chat['id'] + '/direct-uploads', 201,
actor=self.alice, body={'filename': 'sample.png', 'content_type': 'image/png',
'size_bytes': len(self.content), 'sha256': hashlib.sha256(self.content).hexdigest()})
direct = json.loads(direct['bytes'])
s.command(s.prefix + s.base_files + ['exec', '-T', 'api', 'curl', '-fsS', '-X', 'PUT',
'-H', 'Content-Type: image/png', '--data-binary', '@-', direct['upload_url']],
'presigned upload', data=self.content)
direct_key = 'staging/' + self.chat['id'] + '/' + direct['id']
orphan = 'staging/' + self.chat['id'] + '/' + str(uuid.uuid4())
s.mc('mc pipe "local/$S3_BUCKET/"' + shlex.quote(orphan), data=self.content)
wait_for(lambda: not s.exists(orphan), 'orphan retention', timeout=30)
check(s.exists(self.key(att)) and s.exists(direct_key), 'GC deleted an active upload')
s.dc('start', 'cache'); ready = self.ready(att)
wait_for(lambda: s.sql("SELECT staging_cleaned_at IS NOT NULL FROM attachments WHERE id='" + att['id'] + "'") == 't',
'ready staging cleanup')
check(not s.exists(self.key(att)) and self.metadata(att) == ready, 'ready cleanup changed content/metadata')
complete_path = '/api/v1/direct-uploads/' + direct['id'] + '/complete'
direct_att = self.c.call('POST', complete_path, 202, actor=self.alice, schema='Attachment')['json']
self.ready(direct_att)
check(s.exists(direct_key), 'GC deleted a key while its presigned URL is valid')
self.c.call('POST', complete_path, 200, actor=self.alice, schema='Attachment')
check(s.sql("SELECT count(*) FROM outbox WHERE payload->>'attachment_id'='" + direct['id'] + "'") == '1',
'presigned completion replay duplicated outbox')
s.record('retention deletes orphan/ready staging, protects queued and live presigned source')
# Hold an API upload in INSERT after its PUT. GC must respect its advisory lock.
listing = 'mc ls --json "local/$S3_BUCKET/staging/"' + shlex.quote(self.chat['id'] + '/')
old_keys = {json.loads(line)['key'] for line in s.mc(listing).stdout.splitlines()}
s.sql("CREATE FUNCTION test_pause_upload() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_sleep(4); RETURN NEW; END $$; CREATE TRIGGER test_pause_upload BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION test_pause_upload()")
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
future = executor.submit(self.upload)
wait_for(lambda: s.sql("SELECT count(*) FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'INSERT INTO attachments%'") == '1',
'API paused after PUT')
current = {json.loads(line)['key'] for line in s.mc(listing).stdout.splitlines()}
new_key = next(iter(current - old_keys))
time.sleep(2.2)
phase_query = "SELECT count(*) FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'INSERT INTO attachments%'"
check(s.sql(phase_query) == '1', 'GC race probe missed the pre-COMMIT window')
present = s.exists('staging/' + self.chat['id'] + '/' + new_key)
check(s.sql(phase_query) == '1', 'GC race probe crossed the COMMIT boundary')
check(present, 'GC deleted uncommitted upload')
raced = future.result()
s.sql('DROP TRIGGER test_pause_upload ON attachments; DROP FUNCTION test_pause_upload()')
self.ready(raced)
s.record('GC/upload race: PUT-before-COMMIT source is protected')
s.dc('stop', 'cache'); expired = self.upload()
s.sql("UPDATE jobs SET status='failed', retries=1, error_code='processing_failed' WHERE attachment_id='" + expired['id'] + "'; "
"UPDATE attachments SET status='failed', error_code='processing_failed', updated_at=now()-interval '1 day' WHERE id='" + expired['id'] + "'")
wait_for(lambda: s.sql("SELECT staging_cleaned_at IS NOT NULL FROM attachments WHERE id='" + expired['id'] + "'") == 't',
'failed retention')
check(not s.exists(self.key(expired)), 'expired failed source retained')
s.replay(self.job(expired), False)
s.dc('start', 'cache')
s.record('expired failed staging is collected; replay refuses cleaned source')
def memory(self):
s = self.stack
image = s.directory / '40m.png'
with Image.new('RGB', (8000, 5000), (30, 140, 200)) as fixture:
fixture.save(image)
image_id = s.inspect('api')[0]['image']
args = ['docker', 'run', '--rm', '--network', 'none', '--memory', '512m', '--memory-swap', '512m',
'--user', str(os.getuid()) + ':' + str(os.getgid()),
'-v', str(s.directory) + ':/work', '--entrypoint', '/app/variants_smoke', image_id,
'/work/40m.png', '/work/40m-output']
s.command(args, '40M pixel memory boundary', timeout=120)
for name, size in [('thumbnail.jpg', (256, 160)), ('crop.jpg', (128, 128)), ('optimized.webp', (1280, 800))]:
with Image.open(s.directory / '40m-output' / name) as result:
result.load()
check(result.size == size, 'large fixture output mismatch')
s.record('40M pixel RGB PNG produces three variants with 512 MiB and no swap', pixels=40000000,
memory_limit_bytes=512 * 1024 * 1024, swap_bytes=0)
def reliability(self):
self.retry_replay()
self.worker_crashes()
self.duplicates_publishers()
self.cleanup_races()
self.memory()
def public_smoke(self):
self.stack.command([sys.executable, str(ROOT / 'tests/smoke.py'), '--base-url', self.c.base],
'public smoke', timeout=180)
self.stack.record('unchanged lab5 public smoke passed')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('scenario', choices=['upload', 'worker_down', 'outbox', 'reliability', 'all'])
parser.add_argument('--no-build', action='store_true', help='reuse images built from this repository')
parser.add_argument('--output', type=Path, help='sanitized JSON evidence (no credentials/tokens)')
args = parser.parse_args()
stack = Stack()
started = time.monotonic()
digest = hashlib.sha256()
source_files = [ROOT / name for name in ('CMakeLists.txt', 'Dockerfile', 'compose.yaml', '.dockerignore')]
for folder in ('src', 'migrations', 'third_party'):
source_files += [p for p in (ROOT / folder).rglob('*') if p.is_file()]
for path in sorted(source_files):
digest.update(str(path.relative_to(ROOT)).encode() + b'\0' + path.read_bytes())
result = {'scenario': args.scenario, 'project': stack.project,
'source_sha256': digest.hexdigest(), 'python': sys.version.split()[0], 'pillow': pillow_version,
'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'passed': False}
failed = False
try:
stack.start(build=not args.no_build)
suite = Suite(stack)
scenarios = ['upload_async', 'worker_down', 'outbox_crash', 'reliability'] if args.scenario == 'all' else [
{'upload': 'upload_async', 'worker_down': 'worker_down', 'outbox': 'outbox_crash', 'reliability': 'reliability'}[args.scenario]]
for scenario in scenarios:
getattr(suite, scenario)()
suite.public_smoke()
result['passed'] = True
except (Failure, OSError, ValueError, http.client.HTTPException, subprocess.TimeoutExpired) as error:
failed = True
# Do not print command argv, HTTP bodies or environment on failure.
result['error'] = str(error) if isinstance(error, Failure) else type(error).__name__
print('FAIL:', result['error'], file=sys.stderr)
finally:
try:
stack.close()
result['cleanup'] = 'removed own project and volumes; disposable secrets deleted'
except (Failure, OSError, subprocess.TimeoutExpired):
failed = True
result['passed'] = False
result['cleanup'] = 'FAILED; remove only project ' + stack.project
print('FAIL: cleanup of own project ' + stack.project, file=sys.stderr)
result['events'] = stack.events
result['elapsed_seconds'] = round(time.monotonic() - started, 3)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
return 1 if failed else 0
if __name__ == '__main__':
raise SystemExit(main())
+1
View File
@@ -0,0 +1 @@
Pillow==12.1.1
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec "${PYTHON:-python3}" "$script_dir/async_suite.py" outbox "$@"
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec "${PYTHON:-python3}" "$script_dir/track2_suite.py" "$@"
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec "${PYTHON:-python3}" "$script_dir/async_suite.py" reliability "$@"
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Run against thumbnail_smoke; requires Pillow for independent JPEG inspection."""
import argparse
import subprocess
import struct
import zlib
import tempfile
from pathlib import Path
from PIL import Image, ImageOps
def run(binary, directory):
original = Image.new('RGB', (80, 40))
for color, box in [((250, 0, 0), (0, 0, 40, 20)),
((0, 250, 0), (40, 0, 80, 20)),
((0, 0, 250), (0, 20, 40, 40)),
((250, 250, 0), (40, 20, 80, 40))]:
original.paste(color, box)
def convert(source):
target = directory / (source.stem + '-thumbnail.jpg')
subprocess.run([binary, str(source), str(target)], check=True, capture_output=True)
result = Image.open(target)
assert result.format == 'JPEG'
assert not result.getexif()
return result
for suffix in ('jpg', 'png'):
for orientation in range(1, 9):
source = directory / f'orientation{orientation}.{suffix}'
exif = Image.Exif()
exif[274] = orientation
original.save(source, exif=exif, **({'quality': 100, 'subsampling': 0} if suffix == 'jpg' else {}))
expected = ImageOps.exif_transpose(Image.open(source)).convert('RGB')
actual = convert(source)
assert actual.size == expected.size, (source, actual.size, expected.size)
for x, y in [(actual.width // 4, actual.height // 4),
(3 * actual.width // 4, 3 * actual.height // 4)]:
assert max(abs(a - b) for a, b in zip(actual.getpixel((x, y)), expected.getpixel((x, y)))) < 15
for name, size, expected in [('standard', (320, 200), (256, 160)),
('round', (501, 333), (256, 170)),
('thin', (1, 300), (1, 256)),
('small', (17, 11), (17, 11))]:
source = directory / (name + '.png')
Image.new('RGB', size, (30, 110, 200)).save(source)
assert convert(source).size == expected
for alpha, expected in [(0, (255, 255, 255)), (128, (255, 127, 127))]:
source = directory / f'alpha{alpha}.png'
Image.new('RGBA', (10, 10), (255, 0, 0, alpha)).save(source)
actual = convert(source).getpixel((5, 5))
assert max(abs(a - b) for a, b in zip(actual, expected)) <= 3
source = directory / 'pixel-limit.png'
Image.new('RGB', (1, 1)).save(source)
data = bytearray(source.read_bytes())
data[16:24] = struct.pack('>II', 8000, 5001)
data[29:33] = struct.pack('>I', zlib.crc32(data[12:29]))
source.write_bytes(data)
rejected = subprocess.run([binary, str(source), str(directory / 'large.jpg')], capture_output=True)
assert rejected.returncode != 0 and b'image_too_large' in rejected.stderr
source = directory / 'invalid.png'
source.write_bytes(b'not an image')
assert subprocess.run([binary, str(source), str(directory / 'invalid.jpg')], capture_output=True).returncode != 0
print('PASS: JPEG dimensions, rounding, no upscale, all 8 EXIF orientations (JPEG/PNG), white alpha, pre-decode pixel limit, corrupt input')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--binary', default='build/thumbnail_smoke')
parser.add_argument('--workdir')
args = parser.parse_args()
binary = str(Path(args.binary).resolve())
with tempfile.TemporaryDirectory(dir=str(Path(args.workdir).resolve()) if args.workdir else None) as directory:
run(binary, Path(directory))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec "${PYTHON:-python3}" "$script_dir/async_suite.py" upload "$@"
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Independent inspection of crop/WebP through variants_smoke (requires Pillow)."""
import argparse
import json
import subprocess
import tempfile
from pathlib import Path
from PIL import Image, ImageOps
def run(binary, directory):
def convert(source):
output = directory / (source.stem + '-' + source.suffix[1:])
command = subprocess.run([binary, str(source), str(output)], check=True, capture_output=True)
manifest = json.loads(command.stdout)
result = {}
for name, extension, fmt in [('thumbnail', 'jpg', 'JPEG'), ('crop', 'jpg', 'JPEG'), ('optimized', 'webp', 'WEBP')]:
path = output / (name + '.' + extension)
img = Image.open(path)
assert img.format == fmt and not img.getexif()
assert img.size == (manifest[name]['width'], manifest[name]['height'])
assert path.stat().st_size == manifest[name]['size_bytes']
result[name] = img
return result
for size, optimized in [((320, 200), (320, 200)), ((2560, 1600), (1280, 800)),
((17, 11), (17, 11)), ((1, 300), (1, 300)),
((2001, 1333), (1280, 852))]:
source = directory / f'fit{size[0]}x{size[1]}.png'
Image.new('RGB', size, (40, 180, 80)).save(source)
images = convert(source)
assert images['crop'].size == (128, 128)
assert images['optimized'].size == optimized
# Cropping must discard both outer strips and keep the centered square.
source = directory / 'center.png'
image = Image.new('RGB', (200, 100), (255, 0, 0))
image.paste((0, 220, 0), (50, 0, 150, 100))
image.paste((0, 0, 255), (150, 0, 200, 100))
image.save(source)
cropped = convert(source)['crop']
for point in [(5, 5), (64, 64), (122, 122)]:
assert max(abs(a-b) for a, b in zip(cropped.getpixel(point), (0, 220, 0))) < 10
image = Image.new('RGB', (80, 40))
for color, box in [((250, 0, 0), (0, 0, 40, 20)), ((0, 250, 0), (40, 0, 80, 20)),
((0, 0, 250), (0, 20, 40, 40)), ((250, 250, 0), (40, 20, 80, 40))]:
image.paste(color, box)
for suffix in ('jpg', 'png'):
for orientation in range(1, 9):
source = directory / f'orientation{orientation}.{suffix}'
exif = Image.Exif(); exif[274] = orientation
image.save(source, exif=exif, **({'quality': 100, 'subsampling': 0} if suffix == 'jpg' else {}))
expected = ImageOps.exif_transpose(Image.open(source)).convert('RGB')
result = convert(source)
actual = result['optimized']
assert actual.size == expected.size
for x, y in [(actual.width // 4, actual.height // 4), (3 * actual.width // 4, 3 * actual.height // 4)]:
assert max(abs(a-b) for a,b in zip(actual.getpixel((x,y)),expected.getpixel((x,y)))) < 20
source = directory / 'large-exif.png'
exif = Image.Exif(); exif[274] = 6; exif[37510] = b'A' * 70000
image.save(source, exif=exif)
assert convert(source)['optimized'].size == (40, 80)
for alpha, color in [(0, (255, 255, 255)), (128, (255, 127, 127))]:
source = directory / f'alpha{alpha}.png'
Image.new('RGBA', (10, 10), (255, 0, 0, alpha)).save(source)
for image in convert(source).values():
assert max(abs(a-b) for a,b in zip(image.getpixel((5,5)),color)) <= 6
print('PASS: 3 variants, centered/upscaled crop, WebP fit/rounding/no upscale, EXIF stripped, orientations, alpha')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--binary', default='build/variants_smoke')
parser.add_argument('--workdir')
args = parser.parse_args()
with tempfile.TemporaryDirectory(dir=str(Path(args.workdir).resolve()) if args.workdir else None) as directory:
run(str(Path(args.binary).resolve()), Path(directory))
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec "${PYTHON:-python3}" "$script_dir/async_suite.py" worker_down "$@"
+385
View File
@@ -0,0 +1,385 @@
#!/usr/bin/env python3
"""Track 2: reproducible version/delivery/crash experiment on disposable stacks."""
import argparse
import contextlib
import datetime as dt
import hashlib
import http.client
import json
from pathlib import Path
import shlex
import subprocess
import sys
import time
import uuid
from async_suite import Stack, Suite, ROOT, Failure, check, wait_for, pillow_version
class Experiment(Suite):
def __init__(self, stack):
super().__init__(stack)
self.accepted = []
self.intents = set()
def upload(self, expected=202):
att = super().upload(expected)
if expected == 202:
self.accepted.append(att)
self.intents.add(self.job_for(att, 1))
return att
def job_for(self, att, version):
return self.stack.sql("SELECT id FROM jobs WHERE attachment_id='" + att['id'] + "' AND pipeline_version=" + str(version))
def payload(self, job):
return json.loads(self.stack.sql("SELECT payload FROM outbox WHERE payload->>'job_id'='" + job + "' ORDER BY id LIMIT 1"))
def inject(self, payload):
self.stack.redis('XADD', 'jobs:attachments', '*', 'payload', json.dumps(payload))
def upgrade(self, att, created=True):
s = self.stack
result = s.command(s.prefix + s.base_files + ['exec', '-T', 'api', '/docker-entrypoint.sh',
'/app/job_admin', 'upgrade', att['id'], '--version', '2', '--actor', 'track2',
'--reason', 'quality pipeline version 2'], 'upgrade request')
data = json.loads(result.stdout)
uuid.UUID(data['job_id'])
check(data['created'] == created and data['pipeline_version'] == 2, 'upgrade idempotence mismatch')
self.intents.add(data['job_id'])
return data['job_id']
def ready_version(self, att, version):
s = self.stack
wait_for(lambda: s.sql("SELECT published_version FROM attachments WHERE id='" + att['id'] + "'") == str(version),
'published pipeline ' + str(version))
return self.ready(att)
def result(self, att, version):
raw = self.stack.sql("SELECT row_to_json(r) FROM attachment_results r WHERE attachment_id='" +
att['id'] + "' AND pipeline_version=" + str(version))
return json.loads(raw) if raw else None
def keys(self, att, version):
prefix = 'attachments/' + self.chat['id'] + '/' + att['id'] + '/v' + str(version) + '/'
result = self.stack.mc('mc ls --json "local/$S3_BUCKET/"' + shlex.quote(prefix))
return sorted(json.loads(line)['key'] for line in result.stdout.splitlines())
def snapshot(self, att, version):
result = self.result(att, version)
check(result is not None, 'published logical result missing')
keys = {result['original_key']: hashlib.sha256(self.content).hexdigest()}
names = [v['name'] for v in result['variants']]
check(len(names) == len(set(names)) == 3, 'duplicate logical variants')
for v in result['variants']:
keys[v['storage_key']] = v['sha256']
check(len(keys) == 4, 'object keys collide')
for key, expected in keys.items():
data = self.stack.mc('mc cat "local/$S3_BUCKET/"' + shlex.quote(key)).stdout
check(hashlib.sha256(data).hexdigest() == expected, 'S3 bytes differ from immutable result')
check(self.keys(att, version) == ['crop.jpg', 'optimized.webp', 'original', 'thumbnail.jpg'],
'version has extra/missing S3 objects')
return {'metadata': result, 'objects_sha256': keys}
def worker_ids(self):
return self.stack.dc('ps', '-q', 'worker').splitlines()
def marker(self, cid, point, job):
return self.stack.command(['docker', 'exec', cid, 'test', '-f',
'/tmp/lab-fault/' + point + '.' + job + '.reached'], 'fault marker', allow_failure=True).returncode == 0
def release(self, cid, point, job):
self.stack.command(['docker', 'exec', cid, 'touch',
'/tmp/lab-fault/' + point + '.' + job + '.release'], 'release paused worker')
@contextlib.contextmanager
def fault(self, point, job, version, action='crash'):
s = self.stack
s.dc('stop', 'worker')
path = s.directory / 'worker-fault.json'
path.write_text(json.dumps({'services': {'worker': {'environment': {
'JOB_TIMEOUT_SECONDS': '60', 'LAB_ENABLE_FAULT_INJECTION': '1',
'LAB_FAULT_POINT': point, 'LAB_FAULT_ACTION': action,
'LAB_FAULT_JOB_ID': job, 'LAB_FAULT_PIPELINE_VERSION': str(version),
}}}}))
previous = s.base_files.copy()
s.base_files += ['-f', str(path)]
try:
s.dc('up', '-d', '--no-deps', '--force-recreate', '--scale', 'worker=1', 'worker')
cid = self.worker_ids()[0]
yield cid
finally:
s.base_files = previous
s.dc('up', '-d', '--no-deps', '--force-recreate', '--scale', 'worker=1', 'worker')
def crash_case(self, version, point):
s = self.stack
att = self.upload()
if version == 2:
self.ready_version(att, 1)
s.dc('stop', 'worker')
job = self.upgrade(att)
else:
# The worker must not consume the new job before fault configuration.
job = self.job_for(att, 1)
since = dt.datetime.now(dt.timezone.utc).isoformat()
with self.fault(point, job, version) as cid:
wait_for(lambda: self.marker(cid, point, job), point + ' reached')
wait_for(lambda: s.inspect('worker')[0]['restart_count'] >= 1, 'injected crash restart')
until = dt.datetime.now(dt.timezone.utc).isoformat()
deaths = s.command(['docker', 'events', '--since', since, '--until', until,
'--filter', 'container=' + cid, '--filter', 'event=die', '--format', '{{json .}}'],
'observed worker death')
exit_codes = [json.loads(line)['Actor']['Attributes'].get('exitCode')
for line in deaths.stdout.splitlines()]
check('86' in exit_codes, 'injected exit 86 not observed in Docker events')
ready = self.ready_version(att, version)
retries = int(s.sql("SELECT retries FROM jobs WHERE id='" + job + "'"))
check(retries == (1 if point in ('worker_before_effect', 'worker_after_effect') else 0),
'crash boundary counted the wrong managed attempt')
wait_for(self.pending_empty, 'ACK after crash recovery')
snapshot = self.snapshot(att, version)
check(s.sql("SELECT count(*) FROM attachment_results WHERE job_id='" + job + "'") == '1',
'crash duplicated logical result')
for _ in range(2):
self.inject(self.payload(job))
wait_for(self.pending_empty, 'duplicate ACKs')
check(self.metadata(att) == ready and self.snapshot(att, version) == snapshot,
'completed duplicate changed result')
s.record('track2 crash recovery', point=point, pipeline_version=version, job_id=job,
attachment_id=att['id'], logical_results=1, retries=retries, observed_exit_codes=exit_codes, object_hashes=snapshot['objects_sha256'])
def crash_matrix(self):
for version in (1, 2):
for point in ('worker_before_effect', 'worker_after_effect', 'worker_after_commit', 'worker_before_ack'):
self.stack.dc('stop', 'worker')
if version == 2:
self.stack.dc('start', 'worker')
self.crash_case(version, point)
def duplicate_and_reorder(self):
s = self.stack
s.dc('stop', 'worker')
att = self.upload(); job = self.job_for(att, 1)
wait_for(lambda: self.events(att), 'initial duplicate payload')
for _ in range(4):
self.inject(self.payload(job))
s.dc('start', 'worker')
ready = self.ready_version(att, 1); wait_for(self.pending_empty, 'duplicate drain')
snapshot = self.snapshot(att, 1)
check(s.sql("SELECT retries FROM jobs WHERE id='" + job + "'") == '0', 'duplicate created a new managed attempt')
check(self.metadata(att) == ready, 'duplicate changed ready timestamp')
s.record('track2 duplicate delivery', deliveries=5, logical_results=1, objects=4,
attachment_id=att['id'], object_hashes=snapshot['objects_sha256'])
for order in ([1, 2, 1, 2], [2, 1, 2, 1]):
s.dc('stop', 'worker', 'cache')
att = self.upload(); old_job = self.job_for(att, 1); new_job = self.upgrade(att)
# Delay normal publishing; native XADD below models the selected delivery order.
s.sql("UPDATE outbox SET available_at=now()+interval '1 hour' WHERE payload->>'attachment_id'='" + att['id'] + "'")
s.dc('start', 'cache')
payloads = {1: self.payload(old_job), 2: self.payload(new_job)}
for version in order:
self.inject(payloads[version])
s.sql("UPDATE outbox SET published=true WHERE payload->>'attachment_id'='" + att['id'] + "'")
s.dc('start', 'worker'); self.ready_version(att, 2)
wait_for(lambda: s.sql("SELECT status FROM jobs WHERE id='" + old_job + "'") == 'superseded', 'obsolete job discard')
wait_for(self.pending_empty, 'reordered events drain')
snapshot = self.snapshot(att, 2)
check(self.result(att, 1) is None and self.keys(att, 1) == [], 'obsolete queued job performed an effect')
s.record('track2 reordered versions', delivery_order=order, attachment_id=att['id'],
old_job_status='superseded', published_version=2, object_hashes=snapshot['objects_sha256'])
def stale_ready(self):
s = self.stack
att = self.upload(); first = self.ready_version(att, 1); old_job = self.job_for(att, 1)
wait_for(lambda: s.sql("SELECT staging_cleaned_at IS NOT NULL FROM attachments WHERE id='" + att['id'] + "'") == 't', 'initial staging cleanup')
old = self.snapshot(att, 1)
job = self.upgrade(att)
check(self.upgrade(att, created=False) == job, 'same version created a second job')
current = self.ready_version(att, 2); new = self.snapshot(att, 2)
check(first['variants'][2]['sha256'] != current['variants'][2]['sha256'], 'v2 does not change the image algorithm')
check(s.sql("SELECT source_key FROM jobs WHERE id='" + job + "'") == old['metadata']['original_key'],
'upgrade depends on deleted staging')
for version in (1, 2, 1, 1, 2):
self.inject(self.payload(old_job if version == 1 else job))
invalid_before = s.redis('XLEN', 'jobs:attachments:invalid')
forged = self.payload(old_job); forged['pipeline_version'] = 2
self.inject(forged)
wait_for(lambda: s.redis('XLEN', 'jobs:attachments:invalid') == invalid_before + 1,
'carrier/database version mismatch rejected')
wait_for(self.pending_empty, 'late old events')
check(self.metadata(att) == current and self.snapshot(att, 2) == new and self.snapshot(att, 1) == old,
'late version rewrote immutable data')
check(s.sql("SELECT count(*) FROM pipeline_requests WHERE attachment_id='" + att['id'] + "'") == '1', 'upgrade audit duplicated')
s.record('track2 old ready job after new version', attachment_id=att['id'], published_version=2,
old_result_unchanged=True, new_result_unchanged=True, upgrade_requests=1,
object_hashes=new['objects_sha256'])
def late_worker(self, point):
s = self.stack
s.dc('stop', 'worker'); att = self.upload(); old_job = self.job_for(att, 1)
with self.fault(point, old_job, 1, action='pause') as cid:
wait_for(lambda: self.marker(cid, point, old_job), 'old worker barrier')
if point == 'worker_after_commit':
check(s.sql("SELECT published_version FROM attachments WHERE id='" + att['id'] + "'") == '1', 'commit barrier is before COMMIT')
check(s.redis('XPENDING', 'jobs:attachments', 'attachment-workers')[0] >= 1, 'commit barrier is after ACK')
if point == 'worker_before_effect':
check(self.keys(att, 1) == [], 'before-effect barrier has written S3')
else:
check(len(self.keys(att, 1)) == 4, 'after-effect barrier lacks S3 data')
new_job = self.upgrade(att)
# Same fault config/filter; the second worker handles only the independent v2 job.
s.dc('up', '-d', '--no-deps', '--scale', 'worker=2', 'worker')
check(cid in self.worker_ids(), 'scaling replaced the paused old worker')
current = self.ready_version(att, 2); snapshot = self.snapshot(att, 2)
if point == 'worker_before_effect':
check(s.exists(self.key(att)), 'GC removed an in-flight version source')
before = next(x for x in s.inspect('worker') if x['id'].startswith(cid))['restart_count']
self.release(cid, point, old_job)
expected = 'ready' if point == 'worker_after_commit' else 'superseded'
wait_for(lambda: s.sql("SELECT status FROM jobs WHERE id='" + old_job + "'") == expected, 'late old worker finishes')
wait_for(self.pending_empty, 'late old ACK')
after = next(x for x in s.inspect('worker') if x['id'].startswith(cid))['restart_count']
check(before == after == 0, 'old worker restarted instead of resuming')
check(self.metadata(att) == current and self.snapshot(att, 2) == snapshot,
'late old worker replaced new published result')
check(len(self.keys(att, 1)) == 4, 'old worker did not execute its version effect')
s.record('track2 paused old worker resumes after v2 ready', point=point,
attachment_id=att['id'], old_job=old_job, new_job=new_job,
old_worker_resumed=True, published_version=2, old_status=expected,
object_hashes=snapshot['objects_sha256'])
def failed_upgrade(self):
s = self.stack
att = self.upload(); first = self.ready_version(att, 1)
s.sql("CREATE FUNCTION test_reject_v2() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.published_version=2 THEN RAISE EXCEPTION 'test failure'; END IF; RETURN NEW; END $$; CREATE TRIGGER test_reject_v2 BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION test_reject_v2()")
job = self.upgrade(att)
wait_for(lambda: s.sql("SELECT status FROM jobs WHERE id='" + job + "'") == 'failed', 'failed v2 upgrade')
check(self.metadata(att) == first, 'failed upgrade hides previous ready result')
s.sql('DROP TRIGGER test_reject_v2 ON attachments; DROP FUNCTION test_reject_v2()')
s.replay(job, True)
current = self.ready_version(att, 2)
check(current['variants'] != first['variants'], 'replayed v2 did not replace v1')
self.snapshot(att, 2)
s.record('track2 failed upgrade preserves v1; explicit replay publishes v2', attachment_id=att['id'], job_id=job)
def invariants_and_restart(self):
s = self.stack
before = {a['id']: self.metadata(a) for a in self.accepted}
stores_before = {name: s.inspect(name) for name in ('db', 'cache', 'minio')}
s.dc('stop', 'worker'); s.dc('restart', 'db', 'cache', 'minio')
for name in stores_before:
s.wait_store(name)
s.dc('restart', 'api'); s.refresh(); s.dc('start', 'worker')
for att in self.accepted:
check(self.metadata(att) == before[att['id']], 'confirmed attachment/result lost after restart')
self.ready(att)
check(s.sql("SELECT count(*) FROM attachment_results WHERE attachment_id='" + att['id'] +
"' AND pipeline_version=(SELECT published_version FROM attachments WHERE id='" + att['id'] + "')") == '1',
'current result not unique')
for job in self.intents:
check(s.sql("SELECT status FROM jobs WHERE id='" + job + "'") in ('ready', 'superseded'), 'committed intent lost or unfinished')
check(s.sql('SELECT count(*) FROM attachments WHERE published_version>pipeline_version') == '0', 'version order violated')
check(s.sql('SELECT count(*) FROM (SELECT attachment_id,pipeline_version,count(*) FROM attachment_results GROUP BY 1,2 HAVING count(*)>1) x') == '0',
'logical result duplicated')
after = {name: s.inspect(name) for name in stores_before}
for name in stores_before:
check(stores_before[name][0]['started_at'] != after[name][0]['started_at'] and
stores_before[name][0]['volumes'] == after[name][0]['volumes'], 'store restart did not preserve volumes')
s.record('track2 invariants survive persistent-store restart', accepted_attachments=len(self.accepted),
committed_jobs=len(self.intents), lost_attachments=0, unfinished_jobs=0,
duplicate_results=0, snapshots_unchanged=True, before=stores_before, after=after)
def migration(self):
s = self.stack
s.sql('CREATE DATABASE track2_legacy')
sql = ''.join(p.read_text() + '\n' for p in sorted((ROOT / 'migrations').glob('*.sql'))
if int(p.name.split('_')[0]) <= 10)
sql += """
INSERT INTO users VALUES ('00000000-0000-4000-8000-000000000001','legacy_user','Legacy','test',now());
INSERT INTO chats VALUES ('00000000-0000-4000-8000-000000000002','Legacy',now());
INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status,s3_key,created_at,updated_at)
VALUES('00000000-0000-4000-8000-000000000003','00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001','legacy.png','image/png',1,repeat('a',64),'ready','attachments/legacy',now(),now());
INSERT INTO jobs(id,attachment_id,status) VALUES('00000000-0000-4000-8000-000000000004','00000000-0000-4000-8000-000000000003','ready');
"""
sql += (ROOT / 'migrations/011_versioned_pipeline.sql').read_text()
sql += """
DO $$ BEGIN
IF NOT EXISTS(SELECT 1 FROM attachments WHERE status='ready' AND published_version=1 AND s3_key='attachments/legacy') THEN RAISE EXCEPTION 'legacy result changed'; END IF;
IF NOT EXISTS(SELECT 1 FROM attachment_results WHERE pipeline_version=1 AND original_key='attachments/legacy') THEN RAISE EXCEPTION 'legacy result missing'; END IF;
END $$;
UPDATE attachments SET pipeline_version=2;
INSERT INTO jobs(id,attachment_id,pipeline_version,source_key) VALUES('00000000-0000-4000-8000-000000000005','00000000-0000-4000-8000-000000000003',2,'attachments/legacy');
DO $$ BEGIN
BEGIN UPDATE attachments SET published_version=3; RAISE EXCEPTION 'invalid pointer accepted'; EXCEPTION WHEN check_violation THEN NULL; END;
BEGIN INSERT INTO jobs(id,attachment_id,pipeline_version,source_key) VALUES('00000000-0000-4000-8000-000000000006','00000000-0000-4000-8000-000000000003',2,'attachments/legacy'); RAISE EXCEPTION 'duplicate logical job accepted'; EXCEPTION WHEN unique_violation THEN NULL; END;
BEGIN INSERT INTO attachment_results(attachment_id,pipeline_version,original_key,variants) SELECT attachment_id,pipeline_version,original_key,variants FROM attachment_results; RAISE EXCEPTION 'duplicate result accepted'; EXCEPTION WHEN unique_violation THEN NULL; END;
END $$;
"""
s.command(['docker', 'exec', '-i', s.store_ids['db'], 'psql', '-X', '-v', 'ON_ERROR_STOP=1',
'-U', 'messenger', '-d', 'track2_legacy'], 'legacy pipeline migration', data=sql.encode())
s.sql('DROP DATABASE track2_legacy')
s.record('track2 legacy migration preserves ready and enforces version/job/result constraints')
def run(self):
self.migration()
self.crash_matrix()
self.duplicate_and_reorder()
self.stale_ready()
for point in ('worker_before_effect', 'worker_after_effect', 'worker_after_commit'):
self.late_worker(point)
self.failed_upgrade()
self.invariants_and_restart()
self.public_smoke()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--repetitions', type=int, default=3)
parser.add_argument('--no-build', action='store_true')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/track2.json')
args = parser.parse_args()
if not 1 <= args.repetitions <= 10:
parser.error('repetitions must be 1..10')
# Reuse the same fingerprint definition as the functional suite.
digest = hashlib.sha256()
files = [ROOT / n for n in ('CMakeLists.txt', 'Dockerfile', 'compose.yaml', '.dockerignore')]
for folder in ('src', 'migrations', 'third_party'):
files += [p for p in (ROOT / folder).rglob('*') if p.is_file()]
for path in sorted(files):
digest.update(str(path.relative_to(ROOT)).encode() + b'\0' + path.read_bytes())
report = {'track': 2, 'hypothesis': 'at-least-once delivery plus versioned keys and fenced publication preserve one logical result and reject stale publication',
'source_sha256': digest.hexdigest(),
'method_sha256': hashlib.sha256(Path(__file__).read_bytes() + (ROOT / 'student-tests/async_suite.py').read_bytes()).hexdigest(),
'python': sys.version.split()[0], 'pillow': pillow_version,
'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'passed': False, 'trials': []}
failure = False
for number in range(1, args.repetitions + 1):
stack = Stack(); trial = {'number': number, 'project': stack.project, 'passed': False}
started = time.monotonic()
try:
stack.start(build=not args.no_build and number == 1)
experiment = Experiment(stack); experiment.run()
trial['passed'] = True
except (Failure, OSError, ValueError, http.client.HTTPException, subprocess.TimeoutExpired) as error:
failure = True
trial['error'] = str(error) if isinstance(error, Failure) else type(error).__name__
print('FAIL:', trial['error'], file=sys.stderr)
finally:
try:
stack.close(); trial['cleanup'] = 'own project and credentials removed'
except (Failure, OSError, subprocess.TimeoutExpired):
failure = True; trial['passed'] = False; trial['cleanup'] = 'FAILED: ' + stack.project
trial['events'] = stack.events; trial['elapsed_seconds'] = round(time.monotonic()-started, 3)
report['trials'].append(trial)
report['passed'] = not failure and len(report['trials']) == args.repetitions
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
if failure:
break
print('PASS: track2 trial', number, flush=True)
return 1 if failure else 0
if __name__ == '__main__':
raise SystemExit(main())
+6
View File
@@ -0,0 +1,6 @@
Headers from https://github.com/nothings/stb at the commit in VERSION:
- stb_image.h: PNG/JPEG validation and decoding
- deprecated/stb_image_resize.h: aspect-preserving resize
- stb_image_write.h: JPEG encoding (quality 85)
Each header includes its upstream license.
+1
View File
@@ -0,0 +1 @@
013ac3beddff3dbffafd5177e7972067cd2b5083
+7988
View File
File diff suppressed because it is too large Load Diff
+2634
View File
File diff suppressed because it is too large Load Diff
+1724
View File
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 502 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 644 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 633 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 B

+111
View File
@@ -0,0 +1,111 @@
// Step 5 smoke: PNG / JPEG / truncated PNG / non-image / size limit.
#include "image_util.hpp"
#include <cstdint>
#include <fstream>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>
#ifndef IMAGE_SMOKE_FIXTURES
#define IMAGE_SMOKE_FIXTURES "tools/fixtures"
#endif
namespace {
std::vector<std::uint8_t> read_all(const std::string& path) {
std::ifstream in(path, std::ios::binary);
if (!in) {
throw std::runtime_error("cannot open " + path);
}
return std::vector<std::uint8_t>((std::istreambuf_iterator<char>(in)),
std::istreambuf_iterator<char>());
}
bool expect_ok(const char* name, const std::vector<std::uint8_t>& bytes,
const char* want_ct) {
ImageLimits lim;
ImageInfo info;
auto rej = validate_image(bytes.data(), bytes.size(), lim, info);
if (rej) {
std::cerr << name << ": expected OK, got " << image_reject_code(*rej)
<< "\n";
return false;
}
if (info.content_type != want_ct) {
std::cerr << name << ": content_type " << info.content_type << "\n";
return false;
}
std::cout << name << ": OK " << info.content_type << " " << info.width
<< "x" << info.height << "\n";
return true;
}
bool expect_reject(const char* name, const std::vector<std::uint8_t>& bytes,
ImageReject want) {
ImageLimits lim;
ImageInfo info;
auto rej = validate_image(bytes.data(), bytes.size(), lim, info);
if (!rej || *rej != want) {
std::cerr << name << ": expected " << image_reject_code(want) << ", got "
<< (rej ? image_reject_code(*rej) : "OK") << "\n";
return false;
}
std::cout << name << ": OK reject=" << image_reject_code(want) << "\n";
return true;
}
} // namespace
int main(int argc, char** argv) {
try {
const std::string dir =
(argc > 1) ? argv[1] : std::string(IMAGE_SMOKE_FIXTURES);
const auto png = read_all(dir + "/tiny.png");
const auto jpg = read_all(dir + "/tiny.jpg");
bool ok = true;
ok &= expect_ok("png", png, "image/png");
ok &= expect_ok("jpeg", jpg, "image/jpeg");
std::vector<std::uint8_t> trunc(png.begin(),
png.begin() + std::min<std::size_t>(16, png.size()));
ok &= expect_reject("truncated_png", trunc, ImageReject::invalid_image);
std::vector<std::uint8_t> spoof = jpg;
spoof[0] = 'G';
spoof[1] = 'I';
spoof[2] = 'F';
ok &= expect_reject("not_image", spoof, ImageReject::unsupported_media_type);
ImageLimits tiny_lim;
tiny_lim.max_bytes = 10;
ImageInfo info;
auto rej = validate_image(png.data(), png.size(), tiny_lim, info);
if (!rej || *rej != ImageReject::payload_too_large) {
std::cerr << "max_bytes: FAIL\n";
ok = false;
} else {
std::cout << "max_bytes: OK\n";
}
// JPEG bytes sniffed as jpeg regardless of any HTTP Content-Type.
auto sniffed = sniff_content_type(jpg.data(), jpg.size());
if (!sniffed || *sniffed != "image/jpeg") {
std::cerr << "sniff_jpeg: FAIL\n";
ok = false;
} else {
std::cout << "sniff_jpeg: OK (MIME spoof would be ignored)\n";
}
if (!ok) {
return 1;
}
std::cout << "image_smoke: OK\n";
return 0;
} catch (const std::exception& e) {
std::cerr << "image_smoke: FAIL: " << e.what() << "\n";
return 1;
}
}
+35
View File
@@ -0,0 +1,35 @@
// Small smoke tool for step 4: put → get → delete against MinIO.
#include "s3_client.hpp"
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <sstream>
#include <string>
int main() {
try {
auto cfg = S3Client::from_env();
S3Client s3(cfg);
const std::string key = "smoke/step4.txt";
const std::string payload = "lab4-s3-smoke-ok";
std::istringstream in(payload);
s3.put_object(key, in, static_cast<std::int64_t>(payload.size()),
"text/plain");
std::ostringstream out;
s3.get_object(key, out);
if (out.str() != payload) {
std::cerr << "mismatch: got [" << out.str() << "]\n";
return 2;
}
s3.delete_object(key);
std::cout << "s3_smoke: OK bucket=" << cfg.bucket << " key=" << key << "\n";
return 0;
} catch (const std::exception& e) {
std::cerr << "s3_smoke: FAIL: " << e.what() << "\n";
return 1;
}
}
+24
View File
@@ -0,0 +1,24 @@
#include "image_pipeline.hpp"
#include <fstream>
#include <iostream>
#include <iterator>
int main(int argc, char** argv) {
if (argc != 3) {
std::cerr << "usage: thumbnail_smoke INPUT OUTPUT.jpg\n";
return 2;
}
try {
std::ifstream input(argv[1], std::ios::binary);
if (!input) return 1;
const std::vector<std::uint8_t> bytes((std::istreambuf_iterator<char>(input)), {});
const auto image = make_thumbnail(bytes);
std::ofstream output(argv[2], std::ios::binary);
output.write(image.bytes.data(), image.bytes.size());
if (!output) return 1;
std::cout << image.width << 'x' << image.height << '\n';
} catch (const std::exception& e) {
std::cerr << e.what() << '\n';
return 1;
}
}
+32
View File
@@ -0,0 +1,32 @@
#include "image_pipeline.hpp"
#include <filesystem>
#include <fstream>
#include <iostream>
#include <iterator>
#include <nlohmann/json.hpp>
int main(int argc, char** argv) {
if (argc != 3) return 2;
try {
std::ifstream input(argv[1], std::ios::binary);
if (!input) return 1;
const std::vector<std::uint8_t> bytes((std::istreambuf_iterator<char>(input)), {});
const auto variants = make_variants(bytes);
std::filesystem::create_directories(argv[2]);
nlohmann::json manifest;
for (const auto& v : variants) {
const auto path = std::filesystem::path(argv[2]) /
(v.name + (v.content_type == "image/webp" ? ".webp" : ".jpg"));
std::ofstream out(path, std::ios::binary);
out.write(v.image.bytes.data(), v.image.bytes.size());
if (!out) return 1;
manifest[v.name] = {{"width", v.image.width}, {"height", v.image.height},
{"size_bytes", v.image.bytes.size()}};
}
std::cout << manifest.dump() << '\n';
return 0;
} catch (const std::exception& e) {
std::cerr << e.what() << '\n';
return 1;
}
}