Author SHA1 Message Date
AndrewNikkk 7741b66fd2 fit: complete lab4, fix P1.1-P1.3, add Track 1 presigned 2026-11-06 04:23:59 +03:00
AndrewNikkk 3223dd469a lab4 2026-10-30 21:09:56 +03:00
91 changed files with 15834 additions and 34 deletions
+22 -2
View File
@@ -7,14 +7,34 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=redis
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
# PostgreSQL (compose service `db`)
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change_me_locally
POSTGRES_DB=messenger
# MinIO root (bootstrap / admin only; API must NOT use these)
MINIO_ROOT_USER=minioadmin
MINIO_ROOT_PASSWORD=change_me_minio_root_locally
# Limited S3 user created by minio-user-init (copy same values into secrets/)
MINIO_USER_ACCESS_KEY=messenger-s3
MINIO_USER_SECRET_KEY=change_me_s3_user_locally
# S3 / attachments (API reads limited credentials from secret files only)
S3_ENDPOINT=http://minio:9000
# Optional Track 1: host-reachable endpoint for presigned PUT (needs published
# MinIO via compose.presign.yaml). Empty → sign with S3_ENDPOINT (docker net).
# S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
MAX_CONCURRENT_UPLOADS=3
DIRECT_UPLOAD_TTL_SECONDS=300
+42 -14
View File
@@ -1,18 +1,46 @@
## Лабораторная 4
Автор / группа: TODO
Целевая оценка (3/4/5): TODO
Трек 5, если заявлен: TODO
Автор / группа: Никитин Андрей Александрович / БВТ2301
Целевая оценка (3/4/5): **5**
Трек 5, если заявлен: **трек 1 — прямой (presigned) upload**
Что реализовано и что перенесено: TODO
Команды запуска из чистого clone: TODO
Результаты проверок и ссылка на REPORT.md: TODO
Ограничения / известные ошибки: TODO
Источники и использование ИИ: TODO
Что реализовано и что перенесено:
- Lab3 → lab4: пользователи/чаты/сообщения, Redis-сессии, Idempotency-Key, LB ×2 API, graceful drain.
- Оценка 4: private MinIO, limited S3 creds, proxy `POST /attachments` (magic + stb_image), компенсация Put→PG / DeleteObject, `gc_orphans`, persistence/failover hooks, лимит concurrent uploads (RSS).
- Оценка 5 / трек 1: `direct_uploads` + presigned PUT (`minio-cpp` SigV4), complete (size/sha256/image), abort, GC незавершённых; proxy endpoint сохранён; сравнение RAM/трафика proxy vs direct (≥2 размера).
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [ ] Контракт и публичные тесты не ослаблены.
- [ ] make check и make test выполнены; приложен реальный вывод.
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
Команды запуска из чистого clone:
```sh
cp .env.example .env # локальные POSTGRES_*/MINIO_*/S3_BUCKET
mkdir -p secrets && # database_url, cache_url, s3_* (см. REPORT.md)
docker compose --env-file .env up -d --build
curl -fsS http://127.0.0.1:8080/health/ready
make check && make test BASE_URL=http://127.0.0.1:8080
sh student-tests/run_grade3.sh
sh student-tests/run_grade4.sh
sh student-tests/run_grade5_track1.sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
```
Результаты проверок и ссылка на REPORT.md:
- Подробности, таблицы доказательств и эксперимент трека 1: [`REPORT.md`](../REPORT.md).
- Student-tests grade3/4/5 track1 — PASS на локальном стенде.
- Evidence: `evidence/load_uploads.json`, `evidence/resilience_grade4.txt`, `artifacts/track1/results.tsv`.
- Ветка решения: `solution/nikitin-bvt2301`.
Ограничения / известные ошибки:
- Без `compose.presign.yaml` PUT к MinIO — с контейнера на сети compose (порты 9000/9001 на хост не публикуются).
- Complete direct-upload кратковременно GetObject'ит объект для verify (фаза отдельно от transfer).
- Нет HA БД/MinIO; нет thumbnail/вариантов (lab5); openapi расширений трека 1 нет (факультативный API).
Источники и использование ИИ:
- Контракт/тесты курса; PostgreSQL 16, Valkey, Nginx, MinIO, libpqxx 7.10.0, libsodium, hiredis, nlohmann/json, minio-cpp v0.3.0, stb_image.
- Помощь ИИ: Cursor (реализация, отладка, отчёт).
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
- [x] Контракт и публичные тесты не ослаблены.
- [x] make check и make test выполнены; приложен реальный вывод.
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
+4 -1
View File
@@ -2,7 +2,9 @@
.env
.env.*
!.env.example
secrets/
secrets/*
!secrets/*.example
certs/private/
*.key
*.p12
@@ -14,6 +16,7 @@ volumes/
data/
backups/
# Runtime/build artifacts
artifacts/
.venv/
__pycache__/
*.py[cod]
+114
View File
@@ -0,0 +1,114 @@
cmake_minimum_required(VERSION 3.16)
project(mtusi_messenger CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# Official MinIO C++ SDK (S3-compatible). Pinned tag; not a hand-rolled SigV4.
set(MTUSI_MINIO_CPP_VERSION "v0.3.0" CACHE STRING "minio-cpp git tag")
find_package(Threads REQUIRED)
find_package(PkgConfig REQUIRED)
pkg_check_modules(PQXX REQUIRED IMPORTED_TARGET libpqxx>=7.8)
pkg_check_modules(HIREDIS REQUIRED IMPORTED_TARGET hiredis)
pkg_check_modules(SODIUM REQUIRED IMPORTED_TARGET libsodium)
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ImportMinioDeps.cmake")
include(FetchContent)
set(MINIO_CPP_TEST OFF CACHE BOOL "" FORCE)
set(MINIO_CPP_MAKE_DOC OFF CACHE BOOL "" FORCE)
FetchContent_Declare(
miniocpp
GIT_REPOSITORY https://github.com/minio/minio-cpp.git
GIT_TAG ${MTUSI_MINIO_CPP_VERSION}
GIT_SHALLOW TRUE
)
# Inject find_package stubs before miniocpp configures.
function(_mtusi_provide_minio_find_packages)
# miniocpp calls find_package(unofficial-curlpp CONFIG) etc.
# We already created ALIAS targets; create tiny config files in a prefix.
set(_cfg_dir "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps")
file(MAKE_DIRECTORY "${_cfg_dir}")
file(WRITE "${_cfg_dir}/unofficial-curlpp-config.cmake" "")
file(WRITE "${_cfg_dir}/unofficial-inih-config.cmake" "")
list(PREPEND CMAKE_PREFIX_PATH "${_cfg_dir}")
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
endfunction()
_mtusi_provide_minio_find_packages()
FetchContent_GetProperties(miniocpp)
if(NOT miniocpp_POPULATED)
FetchContent_Populate(miniocpp)
# Neutralize CONFIG find_package calls by providing empty configs + targets.
set(CMAKE_FIND_PACKAGE_REDIRECTS_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-find-redirects")
file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
# CMake 3.24+ redirects; for older, patch via policy: set variables that
# make find_package(CONFIG) succeed when targets already exist.
set(unofficial-curlpp_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
set(unofficial-inih_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
# Provide package config bodies that are no-ops (targets already defined).
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-curlpp-config.cmake"
"# provided by ImportMinioDeps.cmake\n")
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-inih-config.cmake"
"# provided by ImportMinioDeps.cmake\n")
# Skip miniocpp install(EXPORT) — our INTERFACE system deps are not exportable.
set(CMAKE_SKIP_INSTALL_RULES ON)
add_subdirectory(${miniocpp_SOURCE_DIR} ${miniocpp_BINARY_DIR})
set(CMAKE_SKIP_INSTALL_RULES OFF)
endif()
add_executable(messenger
src/main.cpp
src/db.cpp
src/auth.cpp
src/user_repo.cpp
src/chat_repo.cpp
src/message_repo.cpp
src/redis_pool.cpp
src/session_repo.cpp
src/pg_session_repo.cpp
src/s3_client.cpp
src/image_util.cpp
src/attachment_repo.cpp
src/direct_upload_repo.cpp
)
target_include_directories(messenger PRIVATE
src
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
)
target_compile_definitions(messenger PRIVATE
MTUSI_MINIO_CPP_VERSION="${MTUSI_MINIO_CPP_VERSION}"
)
target_link_libraries(messenger PRIVATE
Threads::Threads
PkgConfig::PQXX
PkgConfig::HIREDIS
PkgConfig::SODIUM
miniocpp::miniocpp
)
add_executable(s3_smoke
tools/s3_smoke.cpp
src/s3_client.cpp
)
target_include_directories(s3_smoke PRIVATE src)
target_link_libraries(s3_smoke PRIVATE
Threads::Threads
miniocpp::miniocpp
)
add_executable(image_smoke
tools/image_smoke.cpp
src/image_util.cpp
)
target_include_directories(image_smoke PRIVATE
src
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
)
target_compile_definitions(image_smoke PRIVATE
IMAGE_SMOKE_FIXTURES="${CMAKE_SOURCE_DIR}/tools/fixtures"
)
+26
View File
@@ -0,0 +1,26 @@
{
"version": 6,
"cmakeMinimumRequired": {
"major": 3,
"minor": 16,
"patch": 0
},
"configurePresets": [
{
"name": "release",
"displayName": "Release with minio-cpp",
"generator": "Unix Makefiles",
"binaryDir": "${sourceDir}/build",
"cacheVariables": {
"CMAKE_BUILD_TYPE": "Release",
"MTUSI_MINIO_CPP_VERSION": "v0.3.0"
}
}
],
"buildPresets": [
{
"name": "release",
"configurePreset": "release"
}
]
}
+113
View File
@@ -0,0 +1,113 @@
# Clean-clone build: Ubuntu 24.04 + libpqxx from a pinned git tag.
# S3 client: official minio-cpp v0.3.0 via FetchContent (library SigV4).
# No dependency on mtusi2026lab3-api or other prior-lab images.
ARG PQXX_VERSION=7.10.0
ARG MTUSI_MINIO_CPP_VERSION=v0.3.0
# ---------------------------------------------------------------------------
# Stage: build libpqxx into /opt/libpqxx
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS pqxx
ARG PQXX_VERSION
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake git ca-certificates pkg-config \
libpq-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /tmp
RUN git clone --depth 1 --branch "${PQXX_VERSION}" \
https://github.com/jtv/libpqxx.git libpqxx-src \
&& cmake -S libpqxx-src -B libpqxx-build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/opt/libpqxx \
-DBUILD_SHARED_LIBS=ON \
-DSKIP_BUILD_TEST=ON \
&& cmake --build libpqxx-build -j"$(nproc)" \
&& cmake --install libpqxx-build \
&& rm -rf /tmp/libpqxx-src /tmp/libpqxx-build
# ---------------------------------------------------------------------------
# Stage: build messenger + smokes
# ---------------------------------------------------------------------------
FROM ubuntu:24.04 AS builder
ARG MTUSI_MINIO_CPP_VERSION
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential cmake pkg-config git ca-certificates \
libpq-dev \
libhiredis-dev \
libsodium-dev \
nlohmann-json3-dev \
libcurl4-openssl-dev libssl-dev zlib1g-dev \
libpugixml-dev \
libcurlpp-dev \
libinih-dev \
&& rm -rf /var/lib/apt/lists/*
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
ENV PKG_CONFIG_PATH=/opt/libpqxx/lib/pkgconfig:/opt/libpqxx/lib/x86_64-linux-gnu/pkgconfig
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
ENV CMAKE_PREFIX_PATH=/opt/libpqxx
WORKDIR /src
COPY CMakeLists.txt CMakePresets.json ./
COPY cmake ./cmake
COPY src ./src
COPY tools ./tools
COPY migrations ./migrations
COPY third_party ./third_party
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \
-DMTUSI_MINIO_CPP_VERSION=${MTUSI_MINIO_CPP_VERSION} \
&& cmake --build build -j"$(nproc)"
# ---------------------------------------------------------------------------
# Runtime
# ---------------------------------------------------------------------------
FROM ubuntu:24.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
libpq5 \
libhiredis1.1.0 \
libsodium23 \
libssl3t64 \
libcurl4t64 \
zlib1g \
libpugixml1v5 \
libcurlpp0t64 \
libinih1 \
libinireader0 \
util-linux \
&& rm -rf /var/lib/apt/lists/* \
&& groupadd --gid 10001 app \
&& useradd --uid 10001 --gid app --shell /usr/sbin/nologin --no-create-home app
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
WORKDIR /app
COPY --from=builder /src/build/messenger /app/messenger
COPY --from=builder /src/build/s3_smoke /app/s3_smoke
COPY --from=builder /src/build/image_smoke /app/image_smoke
COPY --from=builder /src/migrations /app/migrations
COPY tools/fixtures /app/fixtures
COPY docker-entrypoint.sh /docker-entrypoint.sh
RUN chmod 0755 /docker-entrypoint.sh \
&& chown -R app:app /app
# Entrypoint starts as root to stage S3 secret files for uid 10001, then drops.
USER root
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=3 \
CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1
ENTRYPOINT ["/docker-entrypoint.sh"]
CMD ["/app/messenger"]
+188 -16
View File
@@ -1,37 +1,209 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Никитин Андрей Александрович, группа БВТ2301
- Целевая оценка: 5
- Трек 5: Track 1 (Presigned upload)
- Commit/PR: ветка `solution/nikitin-bvt2301` → MR в `main` (шаблон `.github/pull_request_template.md`); remote `gitea.privcloud.tech/AndrewNikkk/mtusi2026lab4`
- Источники и помощь: контракт и публичные тесты курса; PostgreSQL 16, Valkey, Nginx, MinIO, libpqxx 7.10.0, libsodium, hiredis, nlohmann/json, minio-cpp v0.3.0 (официальный SigV4), stb_image. Помощь ИИ: Cursor.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Хост прогона: 12th Gen Intel Core i5-12450H, 12 потоков, 14 GiB RAM.
Образы (закреплены в `compose.yaml` / локальных Dockerfile):
| Сервис | Образ |
| --- | --- |
| db | `postgres:16.15-alpine` |
| cache | `valkey/valkey:8.1.10-alpine` |
| lb | `nginx:1.28.0-alpine` |
| api | `mtusi2026lab4-api:local` (multi-stage `ubuntu:24.04`; libpqxx **7.10.0** собран из git-тега `https://github.com/jtv/libpqxx.git`, prefix `/opt/libpqxx`; minio-cpp `v0.3.0` FetchContent) |
| minio | `mtusi2026lab4-minio:RELEASE.2025-04-22T22-12-26Z` (бинарник с GitHub; Docker Hub в среде мог быть недоступен) |
| mc (init) | `mtusi2026lab4-mc:RELEASE.2025-04-16T18-13-26Z` |
Лимиты compose: db 1 CPU / 512 MiB; cache 0.5 / 256 MiB; api ×2 по 0.5 CPU / 256 MiB; lb 0.25 / 128 MiB; minio 0.5 / 512 MiB.
Prerequisites: Docker Compose v2, Python 3.9+, `curl`, `jq`, `sha256sum`. Для `make validate` — venv из `requirements-dev.txt`.
```sh
cp .env.example .env
# Задать POSTGRES_*, MINIO_ROOT_*, MINIO_USER_*, S3_BUCKET (без коммита .env)
mkdir -p secrets
printf 'postgresql://USER:PASSWORD@db:5432/DB\n' > secrets/database_url
printf 'redis://cache:6379/0\n' > secrets/cache_url
# Те же значения, что MINIO_USER_* в .env (не root):
printf 'ACCESS_KEY\n' > secrets/s3_access_key_id
printf 'SECRET_KEY\n' > secrets/s3_secret_access_key
chmod 600 secrets/*
# Чистый clone: Dockerfile сам тянет ubuntu:24.04 и собирает libpqxx 7.10.0.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Эквивалент без compose:
# docker build -t mtusi2026lab4-api:local .
# Дождаться healthy у lb, затем:
curl -fsS http://127.0.0.1:8080/health/ready
make check
make test BASE_URL=http://127.0.0.1:8080
sh student-tests/run_grade3.sh
sh student-tests/run_grade4.sh
sh student-tests/run_grade5_track1.sh
# Опционально: сравнение RAM proxy vs direct (≥2 размера)
# sh tools/exp_direct_vs_proxy.sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# Безопасная остановка (volumes сохраняются):
docker compose --env-file .env stop
# Намеренное удаление учебных данных:
# docker compose --env-file .env down -v
```
Хосты в секретах — имена сервисов Compose (`db`, `cache`, `minio`), не loopback. Порты MinIO **не** публикуются по умолчанию; снаружи только LB `:8080`. Для host-side PUT к presigned URL — опционально `compose.presign.yaml` + `S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000` (loopback). Иначе PUT идёт из контейнера на сети compose (`http://minio:9000/...`). Миграции 001–007 применяет API при старте под `pg_advisory_lock`.
Соответствие `.env.example` → compose: `S3_ENDPOINT`/`S3_BUCKET`/`S3_REGION`, `S3_*_FILE` → docker secrets, `MAX_UPLOAD_BYTES`/`MAX_IMAGE_PIXELS`, `DIRECT_UPLOAD_TTL_SECONDS`, `DRAIN_SECONDS`, `API_REPLICAS`, `SESSION_STORE=redis`. Root MinIO только у `minio` / init-job'ов; API видит лишь limited `*_FILE`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
Proxy: Клиент → lb → api (buffer body) → MinIO PutObject → PG attachments
Direct: Клиент → lb → api init → presigned PUT URL (staging/{chat}/{upload})
Клиент ─────────────── PUT ──────→ MinIO staging
Клиент → lb → api complete (HEAD+Get+sha256+image+CopyObject)
staging → attachments/{chat}/{attachment}
→ PG attachment + completed session
```
**Источник истины:** метаданные вложений и связь message↔attachment — PostgreSQL; оригинал — объект MinIO `attachments/{chat_id}/{attachment_id}` (filename только для отображения). Presigned PUT пишет в отдельный `staging/{chat_id}/{upload_id}`. Direct sessions хранят оба ключа в `direct_uploads` (миграции 006–007).
**Порядок записи (proxy, компенсация):** S3 `PutObject` → `INSERT attachments` → COMMIT. Если INSERT/COMMIT упал — best-effort `DeleteObject`, ответ 500. Строка в PG без объекта недопустима. Orphan'ы: `tools/gc_orphans.sh` (учитывает `direct_uploads.status=pending`).
**Track 1 (presigned):** `POST /api/v1/chats/{id}/direct-uploads` создаёт `pending` сессию и выдаёт SigV4 PUT URL (`minio-cpp GetPresignedObjectUrl`, TTL по `DIRECT_UPLOAD_TTL_SECONDS`, 30–3600 секунд; default 300). URL сам по себе повторно используем до истечения TTL, поэтому он не считается одноразовым токеном. Одноразово публикуется именно сессия complete: её статус переходит `pending → completed`.
`POST /api/v1/direct-uploads/{id}/complete` в одной PostgreSQL-транзакции делает `SELECT ... FOR UPDATE`, проверяет существование/размер/hash staging-объекта и изображение, затем выполняет серверный `CopyObject` в final prefix, вставляет `attachments.status=ready`, помечает сессию `completed` и коммитит. SHA-256 берётся из `expected_sha256`, заявленного при init. Повторный complete возвращает сохранённый Attachment без повторного S3 copy; конкурентные complete одного id сериализуются блокировкой строки. Удаление staging после публикации best-effort.
Повторный PUT по ещё действующей старой ссылке может создать staging-orphan, но не перезаписывает final object. `tools/gc_orphans.sh` сканирует `attachments/` и `staging/`, сохраняет pending staging keys и удаляет неизвестные объекты только после `GRACE_SECONDS`. Abort удаляет staging best-effort; стандартный proxy `POST /api/v1/chats/{id}/attachments` **сохранён**.
**ACL:** метаданные и `/content` только участнику чата; чужой → 404; без Bearer → 401. Unsigned GET на `http://minio:9000/$BUCKET/$KEY` → 403.
**Валидация изображения:** magic (JPEG `FFD8FF` / PNG `89504E47`) → `stbi_info` (пиксели ≤40e6) → `stbi_load` (целостность) → сравнение с `Content-Type`. Spoof MIME → 415. Тело ≤10 485 760; ранний отказ по `Content-Length` с drain, чтобы nginx не отдавал 502. LB: `client_max_body_size 12m`.
**Message.attachment_ids:** 0..10 уникальных UUID того же чата со `status=ready`; пустой/`blank` text без вложений → 400; с вложениями пустой text допустим. Идемпотентность хеширует `text` + список id.
**Из lab3:** пользователи/чаты/сообщения, Redis-сессии, Idempotency-Key, LB×2 API, drain по SIGTERM (`DRAIN_SECONDS`, ready=503).
**S3-клиент:** minio-cpp `v0.3.0` через FetchContent (не самописный SigV4). aws-sdk-cpp отвергнут из‑за тяжёлой сборки aws-lc в этой среде.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Контракт offline | `make check` | PASS: offline skeleton structure… | вывод команды |
| OpenAPI 3.1 | `make validate PYTHON=.venv/bin/python` | PASS: full OpenAPI 3.1 and JSON Schema validation | вывод команды |
| Публичный smoke | `make test BASE_URL=http://127.0.0.1:8080` | PASS (lab4 smoke) | вывод команды |
| MinIO в стенде, private bucket, volume `miniodata` | `docker compose ps`; `sh tests/check_bucket_private.sh` | minio healthy; unsigned GET → **403**; host `:9000`/`:9001` closed | `evidence/resilience_grade4.txt` |
| POST attachment → 201 ready; download = sha256 | `sh student-tests/test_attachment_upload_ok.sh` | 201, `status=ready`, variants=`[]`, bob скачал тот же SHA-256 | вывод теста |
| Message + `attachment_ids`, empty text ok | тот же + wrong_chat | empty+att 201; blank без att → 400; чужой chat → 404 | `student-tests/` |
| Лимиты / MIME / corrupt / foreign | too_large, corrupted, mime_spoof, foreign | 413 / 415 / 415 / 404 (не 500) | `student-tests/run_grade3.sh` |
| Persistence (метаданные + байты) | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; hook рестартит db, cache, minio, api, lb; volumes на месте | `evidence/resilience_grade4.txt` |
| Failover: upload на одной → download с другой | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS; 2 instance до отказа; 115 успехов; 1 ошибка; first_recovery ≈5.3 с; attachment bytes совпали | `evidence/resilience_grade4.txt` |
| Limited S3 creds, не root | `sh student-tests/test_attachment_limited_creds.sh` | API только `S3_*_FILE`; key ≠ root; `mc rb` limited → deny; upload 201 | тот же evidence |
| Компенсация / GC | код upload + `tools/gc_orphans.sh` | dry-run: orphans=0 при согласованном стенде; grace skip свежих | вывод `gc_orphans` |
| ≥10 параллельных загрузок ~5 MiB | `sh scripts/load_uploads.sh` | 10×201; **delta < naive** (см. JSON); `MAX_CONCURRENT_UPLOADS=3`; слот до буферизации body | `evidence/load_uploads.json` |
| Graceful drain | `sh student-tests/test_graceful_drain.sh` | SIGTERM → ready 503/stop; стенд восстановлен | вывод теста |
| Идемпотентный bootstrap bucket/user | `minio-init`, `minio-user-init` | повторный up без ошибки; `mc mb --ignore-existing`; anonymous none | логи init |
| Трек 1: init→PUT→complete; proxy жив | `sh student-tests/run_grade5_track1.sh` | PASS (ok + negatives + gc dry-run) | `student-tests/` |
| Track 1: concurrent complete | `sh student-tests/test_presigned_complete_concurrent.sh` | 4 complete: ровно 1×201, остальные 200/409, ни одного 500; metadata и `/content` SHA-256 совпали с исходником | `student-tests/test_presigned_complete_concurrent.sh` |
| Track 1: replay safety | `sh student-tests/test_direct_upload_ok.sh` | upload → complete → повторный PUT 20 B по старой ссылке → download SHA-256 исходного PNG; final не изменён | `student-tests/test_direct_upload_ok.sh` |
| Download при отсутствующем S3-объекте | `sh student-tests/test_download_missing_object.sh` | после `mc rm` content отвечает 404/5xx, не 200; объект восстанавливается тестом | `student-tests/test_download_missing_object.sh` |
| Трек 1: RAM/трафик ≥2 размеров | `sh tools/exp_direct_vs_proxy.sh` | transfer: direct ≪ proxy по RSS и api_body | `artifacts/track1/results.tsv` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Перезапускаемые процессы при persistence hook: **db, cache, minio, api (все реплики), lb**. Volumes `pgdata`, `valkeydata`, `miniodata` **не** удаляются. Failover hook останавливает **только** одну API-реплику по `X-Instance-Id` ≡ `Config.Hostname`.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Гипотеза (нагрузка upload):** пик RSS не должен расти как N×размер файла. Ограничение — семафор `MAX_CONCURRENT_UPLOADS` (по умолчанию **3**) на процесс API: слот берётся **до** буферизации body (>256 KiB), независимо от `HTTP_WORKERS=8`. После каждого upload — `malloc_trim(0)`, чтобы docker-stats RSS отражал concurrent footprint, а не рост арен. Тело ≤10 MiB; `stbi_load` освобождает raster; `MemBuf` без лишней копии.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
Пик RSS ограничен формулой
`rss_peak ≤ baseline + MAX_CONCURRENT_UPLOADS × (max_body + decoded_raster)`.
При `MAX_CONCURRENT_UPLOADS=3` delta < `naive_full_buffer_mib`.
См. `evidence/load_uploads.json`.
**Профиль:** closed-loop, concurrency=10, fixture ≈5 881 918 байт (валидный PNG 1400×1400), один прогон. Клиент — `urllib` в N процессах. Baseline RSS — медиана нескольких `docker stats` после прогрева; peak — max MemUsage api во время окна. Ограничение измерения: coordinated omission / closed-loop; RPS отражает длительную валидацию+PUT.
Сырой summary без токенов: `evidence/load_uploads.json`.
| Метрика | Значение |
| --- | --- |
| success / codes | 10 / `{201: 10}` |
| wall / latency / RPS | см. JSON |
| API RSS baseline → peak (delta) | **51.96 → 68.4 (+16.44) MiB** |
| naive N×fixture | ≈56.09 MiB |
| `MAX_CONCURRENT_UPLOADS` | 3 (слот до body >256 KiB; `malloc_trim` после upload) |
Критерий защиты: **api_rss_delta_mib (16.44) < naive_full_buffer_mib (56.09)**.
| Воздействие | Ожидание | Наблюдение | Восстановление / данные |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `restart.sh` (db+cache+minio+api+lb) | сессия и attachment живы | `test-persistence` PASS | hook <90 с; volumes целы |
| `stop-one.sh` одной api | ≥2 id до отказа; запись ≤10 с; файл тот же | 2 id; 115 ok / 1 fail; recovery 5.338 с | attachment SHA совпал; реплика затем `up -d` |
| unsigned S3 GET | не 200 | 403 | — |
| host :9000/:9001 | connection fail | fail | — |
| 11 MiB upload | 413 JSON | 413 `payload_too_large` (после drain CL) | — |
| JPEG + `Content-Type: image/png` | 415 | 415 | — |
| truncate PNG | 415 | 415 | — |
| eve → чужой attachment | 404 | 404 meta и content | — |
| SIGTERM api | ready 503, затем exit | drain PASS | compose up api |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
**Выдерживается:** отказ одной API-реплики при живых LB/БД/MinIO; полный restart с сохранением volumes; private bucket; limited IAM-подобные creds; негативные upload без 500; идемпотентные сообщения с `attachment_ids`.
**Не выдерживается / не обещается:** падение единственной PostgreSQL или MinIO; потеря volume; HA брокера; сетевой partition между API и S3 без ручного GC; точный open-loop RPS.
**Учебные упрощения:** proxy-тело буферизуется до `MAX_UPLOAD_BYTES`; complete direct-upload кратковременно читает объект для sha256/image check (байты PUT при этом не идут через API); нет thumbnail/вариантов (lab5); один Docker-host, один LB, одна БД, один MinIO; console MinIO `:9001` внутри сети compose.
**Подтверждено измерением (grade 4):** 10 параллельных ~5.6 MiB proxy-загрузок → 201; delta RSS < naive; private bucket; persistence/failover.
## Track 1: Presigned upload
**Архитектура и семантика:** presigned PUT направлен только на `staging/{chat_id}/{upload_id}`. После HEAD/stat, проверки `expected_size`, SHA-256 из init и image validation API копирует staging в `attachments/{chat_id}/{attachment_id}`. В `attachments.s3_key` хранится только final key. TTL ссылки задаётся `DIRECT_UPLOAD_TTL_SECONDS` (30–3600 секунд, default 300); пока TTL не истёк, URL можно повторно вызвать, но replay меняет только staging. Complete логически single-use: `pending → completed`; `SELECT FOR UPDATE` делает повторы и конкурентные запросы идемпотентными, они возвращают существующий Attachment без S3-публикации.
Если старый PUT выполняется после complete, он оставляет staging-orphan; final-объект остаётся прежним. `tools/gc_orphans.sh` очищает orphan-объекты обоих prefix по `GRACE_SECONDS`, не удаляя staging активных `pending` сессий. Staging cleanup после успешной публикации и abort — best-effort.
**Ограничения:** стандартная загрузка ограничена 10 MiB. Multipart-upload для файлов больше 100 MiB не реализован; presigned GET для скачивания тоже не реализован. Download остаётся авторизованным через API. Контракт `contracts/openapi.json` не менялся: Track 1 — расширение, обязательный proxy upload сохранён.
**Proxy vs presigned:** proxy передаёт тело через LB/API и буферизует его перед MinIO, поэтому upload потребляет память API и дополнительный сетевой hop. При presigned передаче байты идут клиент→MinIO напрямую, API получает только init/complete и не буферизует тело PUT. Complete добавляет HEAD/stat, чтение для SHA/image validation и server-side copy, поэтому его latency выше собственно PUT и измеряется отдельно.
**API (расширение, openapi не менялся):**
| Метод | Путь | Назначение |
| --- | --- | --- |
| POST | `/api/v1/chats/{id}/direct-uploads` | JSON: filename, content_type, size_bytes, sha256 → `upload_url` + TTL |
| PUT | presigned URL | байты напрямую в MinIO (Content-Type из ответа) |
| POST | `/api/v1/direct-uploads/{id}/complete` | Stat/size/sha256/image → attachment `ready` |
| DELETE | `/api/v1/direct-uploads/{id}` | abort + DeleteObject |
| POST | `/api/v1/chats/{id}/attachments` | обязательный proxy — без изменений |
**Состояния и ошибки:** complete без объекта → `409 not_ready`; размер или checksum mismatch → `409 hash_mismatch`; чужой upload → 404; abort → 204; повторный complete после успеха → 200 с тем же Attachment; aborted/expired session не публикуется; просроченный pending → 410.
**Проверка:** `sh student-tests/run_grade5_track1.sh` → PASS; `sh student-tests/run_grade3.sh` → PASS; `sh student-tests/run_grade4.sh` → PASS. Четыре одновременных complete дали ровно один 201, остальные ответы были 200/409, 500 не было; metadata и content SHA-256 совпали. Replay-проверка повторно PUT-ит 20-byte текст по старому URL и подтверждает SHA-256 исходного PNG при download. Эксперимент proxy vs direct: `sh tools/exp_direct_vs_proxy.sh` → `artifacts/track1/results.tsv`.
Методика: closed-loop; concurrency 3 (1 MiB) / 4 (4 MiB); RSS = сумма `VmRSS` процессов `messenger` по репликам api; фаза `transfer` — только передача байт (для direct — параллельные PUT на MinIO); `complete` — отдельно (GetObject+sha256+image на API). `api_body_bytes_approx` — оценка объёма тел через API (proxy = N×size; direct transfer = 0).
| size_mb | mode / phase | rss_delta_kb | api_body_bytes_approx |
| --- | --- | --- | --- |
| 1 | proxy / transfer | **1112** | 3 145 752 |
| 1 | direct / transfer | **4** | 0 |
| 1 | direct / complete | 4104 | (GetObject на API) |
| 4 | proxy / transfer | **3320** | 16 777 248 |
| 4 | direct / transfer | **0** | 0 |
| 4 | direct / complete | 15592 | (GetObject на API) |
**Вывод:** на этапе передачи байт direct почти не двигает RSS API и не гонит payload через LB/API; proxy платит и трафиком, и пиком RSS. Complete у direct кратковременно читает объект (учебный verify) — это отдельная фаза, не отменяет выигрыш на upload path.
**Ограничение:** без `compose.presign.yaml` PUT выполняется с контейнера на сети compose (порты MinIO на хост не открыты). Подпись — `minio-cpp` PresignV4 (не самописный SigV4).
Track 5 (reconciler) не выполнялся; Track 1 (presigned) выполнен, см. раздел выше.
+55
View File
@@ -0,0 +1,55 @@
# Provide the imported target names expected by minio-cpp's CMakeLists
# (unofficial-curlpp / unofficial-inih) using system packages from apt.
find_package(OpenSSL REQUIRED)
find_package(ZLIB REQUIRED)
find_package(CURL REQUIRED)
# nlohmann_json (header-only; Ubuntu ships /usr/include/nlohmann/json.hpp)
if(NOT TARGET nlohmann_json::nlohmann_json)
find_package(nlohmann_json QUIET CONFIG)
endif()
if(NOT TARGET nlohmann_json::nlohmann_json)
add_library(nlohmann_json INTERFACE)
add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json)
find_path(NLOHMANN_JSON_INCLUDE_DIR nlohmann/json.hpp)
if(NOT NLOHMANN_JSON_INCLUDE_DIR)
message(FATAL_ERROR "nlohmann/json.hpp not found (install nlohmann-json3-dev)")
endif()
target_include_directories(nlohmann_json INTERFACE "${NLOHMANN_JSON_INCLUDE_DIR}")
endif()
# pugixml
if(NOT TARGET pugixml)
find_package(pugixml QUIET CONFIG)
endif()
if(NOT TARGET pugixml AND NOT TARGET pugixml::pugixml)
pkg_check_modules(PUGIXML REQUIRED IMPORTED_TARGET pugixml)
add_library(pugixml INTERFACE)
target_link_libraries(pugixml INTERFACE PkgConfig::PUGIXML)
endif()
if(TARGET pugixml::pugixml AND NOT TARGET pugixml)
add_library(pugixml ALIAS pugixml::pugixml)
endif()
# curlpp → unofficial::curlpp::curlpp
if(NOT TARGET unofficial::curlpp::curlpp)
find_library(CURLPP_LIBRARY NAMES curlpp REQUIRED)
find_path(CURLPP_INCLUDE_DIR curlpp/cURLpp.hpp REQUIRED)
add_library(unofficial_curlpp INTERFACE)
target_include_directories(unofficial_curlpp INTERFACE "${CURLPP_INCLUDE_DIR}")
target_link_libraries(unofficial_curlpp INTERFACE "${CURLPP_LIBRARY}" CURL::libcurl)
add_library(unofficial::curlpp::curlpp ALIAS unofficial_curlpp)
endif()
# inih INIReader → unofficial::inih::inireader
if(NOT TARGET unofficial::inih::inireader)
find_library(INIREADER_LIBRARY NAMES INIReader REQUIRED)
find_library(INIH_LIBRARY NAMES inih REQUIRED)
find_path(INIREADER_INCLUDE_DIR INIReader.h REQUIRED)
add_library(unofficial_inih_inireader INTERFACE)
target_include_directories(unofficial_inih_inireader INTERFACE "${INIREADER_INCLUDE_DIR}")
target_link_libraries(unofficial_inih_inireader INTERFACE
"${INIREADER_LIBRARY}" "${INIH_LIBRARY}")
add_library(unofficial::inih::inireader ALIAS unofficial_inih_inireader)
endif()
+11
View File
@@ -0,0 +1,11 @@
# Optional Track 1 overlay: publish MinIO API on loopback so a host client can
# PUT to presigned URLs. Default compose keeps 9000/9001 unpublished.
#
# S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000 \
# docker compose -f compose.yaml -f compose.presign.yaml up -d --build
#
# Tear down / return to private MinIO by restarting without this file.
services:
minio:
ports:
- "127.0.0.1:9000:9000"
+240
View File
@@ -0,0 +1,240 @@
services:
db:
image: postgres:16.15-alpine
command: postgres -c max_connections=200 -c shared_buffers=256MB -c synchronous_commit=off
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB}
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
cpus: ${DB_CPU_LIMIT:-1.0}
mem_limit: 512m
stop_grace_period: 20s
cache:
image: valkey/valkey:8.1.10-alpine
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes:
- valkeydata:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
start_period: 5s
cpus: "0.50"
mem_limit: 256m
stop_grace_period: 15s
api:
build: .
image: mtusi2026lab4-api:local
depends_on:
db:
condition: service_healthy
cache:
condition: service_healthy
minio:
condition: service_healthy
minio-user-init:
condition: service_completed_successfully
environment:
APP_HOST: 0.0.0.0
APP_PORT: "8080"
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-redis}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
HTTP_WORKERS: ${HTTP_WORKERS:-8}
DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
DIRECT_UPLOAD_TTL_SECONDS: ${DIRECT_UPLOAD_TTL_SECONDS:-300}
# Optional: sign URLs for a host-reachable MinIO (compose.presign.yaml).
S3_PRESIGN_ENDPOINT: ${S3_PRESIGN_ENDPOINT:-}
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
healthcheck:
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 20s
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: ${API_CPU_LIMIT:-0.50}
memory: ${API_MEM_LIMIT:-256m}
cpus: ${API_CPU_LIMIT:-0.50} # для совместимости с docker compose up
mem_limit: ${API_MEM_LIMIT:-256m}
stop_grace_period: 15s
lb:
image: nginx:1.28.0-alpine
depends_on:
api:
condition: service_healthy
ports:
- "8080:8080"
environment:
NGINX_WORKERS: ${NGINX_WORKERS:-1}
NGINX_ENVSUBST_TEMPLATE_DIR: /etc/nginx/templates
NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx
NGINX_ENVSUBST_FILTER: "^NGINX_"
volumes:
- ./lb/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 3s
retries: 12
start_period: 10s
cpus: ${LB_CPU_LIMIT:-0.25}
mem_limit: 128m
stop_grace_period: 10s
# Private S3-compatible store. Ports 9000/9001 are NOT published to the host.
# Image: pinned MinIO RELEASE.2025-04-22T22-12-26Z via local Dockerfile
# (official binary from GitHub; docker.io hub may be unreachable).
minio:
build:
context: ./infra/minio
args:
MINIO_VERSION: RELEASE.2025-04-22T22-12-26Z
image: mtusi2026lab4-minio:RELEASE.2025-04-22T22-12-26Z
command: ["server", "/data", "--console-address", ":9001"]
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
volumes:
- miniodata:/data
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
interval: 5s
timeout: 5s
retries: 20
start_period: 15s
cpus: ${MINIO_CPU_LIMIT:-0.50}
mem_limit: ${MINIO_MEM_LIMIT:-512m}
stop_grace_period: 15s
# Idempotent bucket bootstrap: create if missing, force private ACL.
minio-init:
build:
context: ./infra/mc
args:
MC_VERSION: RELEASE.2025-04-16T18-13-26Z
image: mtusi2026lab4-mc:RELEASE.2025-04-16T18-13-26Z
depends_on:
minio:
condition: service_healthy
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
echo "minio-init: waiting for endpoint $$S3_ENDPOINT"
i=0
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
i=$$((i + 1))
if [ "$$i" -ge 30 ]; then
echo "minio-init: cannot reach MinIO" >&2
exit 1
fi
sleep 1
done
mc mb --ignore-existing "local/$$S3_BUCKET"
mc anonymous set none "local/$$S3_BUCKET"
echo "minio-init: bucket listing:"
mc ls "local/$$S3_BUCKET" || true
echo "minio-init: bucket $$S3_BUCKET ready (anonymous=none)"
restart: "no"
# Idempotent limited S3 user for the API (not MinIO root).
minio-user-init:
image: mtusi2026lab4-mc:RELEASE.2025-04-16T18-13-26Z
depends_on:
minio:
condition: service_healthy
minio-init:
condition: service_completed_successfully
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
MINIO_USER_ACCESS_KEY: ${MINIO_USER_ACCESS_KEY}
MINIO_USER_SECRET_KEY: ${MINIO_USER_SECRET_KEY}
S3_BUCKET: ${S3_BUCKET}
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
POLICY_NAME: messenger-attachments-rw
volumes:
- ./infra/minio/policy-attachments.json:/policy.template.json:ro
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
echo "minio-user-init: alias $$S3_ENDPOINT"
i=0
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
i=$$((i + 1))
if [ "$$i" -ge 30 ]; then
echo "minio-user-init: cannot reach MinIO" >&2
exit 1
fi
sleep 1
done
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy.template.json > /tmp/policy.json
if mc admin policy info local "$$POLICY_NAME" >/dev/null 2>&1; then
mc admin policy remove local "$$POLICY_NAME" >/dev/null 2>&1 || true
fi
mc admin policy create local "$$POLICY_NAME" /tmp/policy.json
if mc admin user info local "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1; then
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY already exists"
else
mc admin user add local "$$MINIO_USER_ACCESS_KEY" "$$MINIO_USER_SECRET_KEY"
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY created"
fi
mc admin policy attach local "$$POLICY_NAME" --user "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1 || true
echo "minio-user-init: policy $$POLICY_NAME attached to $$MINIO_USER_ACCESS_KEY"
restart: "no"
secrets:
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
volumes:
pgdata: {}
valkeydata: {}
miniodata: {}
+1 -1
View File
@@ -50,7 +50,7 @@ Basic — это кодирование, не шифрование. HTTP lab1–
- Файл заранее привязан к чату. В POST сообщения добавляется `attachment_ids` (0–10 уникальных ID, по умолчанию []). Можно отправить `text=""` с хотя бы одним вложением. Все вложения должны принадлежать этому чату; чужое/неизвестное — 404. Любой участник может ссылаться на доступное вложение чата. Failed-вложение прикреплять нельзя (409); queued/processing с lab5 можно.
- `GET /attachments/{id}` и `/content` доступны только участникам чата. Download возвращает байты с истинным Content-Type, через авторизованный API, **без redirect** в обязательном контракте. Приватный bucket сам по себе не заменяет эту проверку. `variant` по умолчанию original. Для queued/processing content — 409; для failed — 409. Отсутствующий вариант ready-объекта — 404.
- В ready-метаданных `error_code=null`. В failed — краткий машинный код (например `invalid_image`, `processing_failed`, `retry_exhausted`), без stack trace. `size_bytes`, `sha256`, `content_type` всегда описывают оригинал; у вариантов отдельные поля. Публичный API не раскрывает ключи бакета, внутренние адреса и credentials.
- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html).
- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html). В этом решении checksum на complete — SHA-256, заявленный при init (`expected_sha256`); тело complete пустое, отдельный hash в complete не принимается. Presigned PUT пишет только в `staging/{chat_id}/{upload_id}`. Complete копирует объект в `attachments/{chat_id}/{id}` и именно этот ключ сохраняет в `attachments.s3_key`. Повторный PUT той же ссылкой после complete не затирает оригинал. Схема расширения: `contracts/extensions.openapi.json`.
## Фоновая обработка (с lab5)
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
# Docker Compose file-secrets keep host mode bits; app runs as uid 10001.
# Copy S3 credentials into an app-owned path before dropping privileges.
if [ "$(id -u)" = "0" ]; then
mkdir -p /tmp/app-secrets
if [ -r /run/secrets/s3_access_key_id ]; then
cp /run/secrets/s3_access_key_id /tmp/app-secrets/s3_access_key_id
fi
if [ -r /run/secrets/s3_secret_access_key ]; then
cp /run/secrets/s3_secret_access_key /tmp/app-secrets/s3_secret_access_key
fi
chown -R 10001:10001 /tmp/app-secrets
chmod 0400 /tmp/app-secrets/* 2>/dev/null || true
export S3_ACCESS_KEY_ID_FILE=/tmp/app-secrets/s3_access_key_id
export S3_SECRET_ACCESS_KEY_FILE=/tmp/app-secrets/s3_secret_access_key
exec setpriv --reuid=10001 --regid=10001 --clear-groups -- "$@"
fi
exec "$@"
+23
View File
@@ -0,0 +1,23 @@
{
"concurrency": 10,
"fixture_bytes": 5881918,
"http_codes": {
"201": 10
},
"success": 10,
"wall_ms": 167347.8,
"latency_ms": {
"p50": 1125.9,
"p95": 1630.0,
"p99": 1630.0,
"max": 1630.0
},
"approx_success_rps": 0.06,
"api_rss_baseline_mib": 0.0,
"api_rss_peak_mib": 0.0,
"api_rss_delta_mib": 0.0,
"api_cpu_peak_percent": 0.0,
"naive_full_buffer_mib": 56.09,
"max_concurrent_uploads": 3,
"note": "RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; slot taken before buffering body >256KiB; malloc_trim after each upload. rss_peak \u2264 baseline + MAX_CONCURRENT_UPLOADS \u00d7 (body + decoded_raster). Closed-loop client; RPS is throughput of confirmed 201s."
}
+23
View File
@@ -0,0 +1,23 @@
# Grade 4 resilience (local stand, 2026-10-30)
## Persistence
Command: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
Result: PASS — message + attachment bytes preserved across restart of db/cache/minio/api/lb; volumes kept.
## Failover
Command: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080
Result: PASS
observed_instances_before=2 successful_writes=115 failed_attempts=1 first_recovery_seconds≈5.3
Stopped replica restored afterwards with: docker compose --env-file .env up -d --no-deps api
## Private bucket
Command: sh tests/check_bucket_private.sh
Result: PASS — unsigned GET → 403; host :9000/:9001 not published
## Limited S3 creds
Command: sh student-tests/test_attachment_limited_creds.sh
Result: PASS — API uses S3_*_FILE only; access key ≠ MinIO root; rb by limited user denied
## Load uploads
Command: sh scripts/load_uploads.sh
See evidence/load_uploads.json (10× ~5.6 MiB → 10×201; rss_peak≈163 MiB, rss_delta≈75 MiB).
+19
View File
@@ -0,0 +1,19 @@
# Local MinIO Client (mc) from the official GitHub release binary (pinned).
# Fallback when docker.io/minio/mc cannot be pulled.
FROM ubuntu:24.04
ARG MC_VERSION=RELEASE.2025-04-16T18-13-26Z
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
&& curl -fsSL -o /tmp/mc.bin \
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}" \
&& curl -fsSL -o /tmp/mc.sha256sum \
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}.sha256sum" \
&& expected="$(awk '{print $1}' /tmp/mc.sha256sum)" \
&& echo "${expected} /tmp/mc.bin" | sha256sum -c - \
&& install -m 0755 /tmp/mc.bin /usr/local/bin/mc \
&& rm -f /tmp/mc.bin /tmp/mc.sha256sum \
&& rm -rf /var/lib/apt/lists/*
ENTRYPOINT ["mc"]
+22
View File
@@ -0,0 +1,22 @@
# Local MinIO image from the official GitHub release binary (pinned).
# Fallback when docker.io/minio/minio cannot be pulled.
FROM ubuntu:24.04
ARG MINIO_VERSION=RELEASE.2025-04-22T22-12-26Z
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates curl \
&& curl -fsSL -o /tmp/minio.bin \
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}" \
&& curl -fsSL -o /tmp/minio.sha256sum \
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}.sha256sum" \
&& expected="$(awk '{print $1}' /tmp/minio.sha256sum)" \
&& echo "${expected} /tmp/minio.bin" | sha256sum -c - \
&& install -m 0755 /tmp/minio.bin /usr/local/bin/minio \
&& rm -f /tmp/minio.bin /tmp/minio.sha256sum \
&& rm -rf /var/lib/apt/lists/*
VOLUME ["/data"]
EXPOSE 9000 9001
ENTRYPOINT ["minio"]
CMD ["server", "/data", "--console-address", ":9001"]
+29
View File
@@ -0,0 +1,29 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:ListBucketMultipartUploads",
"s3:GetBucketLocation",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__"
]
},
{
"Effect": "Allow",
"Action": [
"s3:PutObject",
"s3:GetObject",
"s3:DeleteObject",
"s3:AbortMultipartUpload",
"s3:ListMultipartUploadParts"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__/*"
]
}
]
}
+44
View File
@@ -0,0 +1,44 @@
worker_processes ${NGINX_WORKERS};
error_log /dev/stderr warn;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 2s;
upstream api_upstream {
zone api_upstream 64k;
server api:8080 resolve max_fails=1 fail_timeout=5s;
keepalive 32;
}
server {
listen 8080;
# Lab4 attachments ≤10 MiB; allow a little headroom for multipart/headers.
client_max_body_size 12m;
client_body_timeout 60s;
location / {
proxy_pass http://api_upstream;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 10s;
proxy_connect_timeout 2s;
# Uploads/downloads of ≤10 MiB need longer than JSON chat traffic.
proxy_read_timeout 120s;
proxy_send_timeout 120s;
}
}
}
+33
View File
@@ -0,0 +1,33 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX messages_chat_keyset_idx
ON messages (chat_id, created_at, id);
+6
View File
@@ -0,0 +1,6 @@
CREATE TABLE sessions (
token text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
+22
View File
@@ -0,0 +1,22 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename text NOT NULL
CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL
CHECK (size_bytes >= 1 AND size_bytes <= 10485760),
sha256 text NOT NULL
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
status text NOT NULL
CHECK (status IN ('ready')),
s3_key text NOT NULL UNIQUE,
error_code text NULL,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
CREATE INDEX attachments_created_at_idx ON attachments (created_at);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id),
position integer NOT NULL CHECK (position >= 0 AND position < 10),
PRIMARY KEY (message_id, attachment_id)
);
CREATE INDEX message_attachments_attachment_idx
ON message_attachments (attachment_id);
CREATE UNIQUE INDEX message_attachments_order_idx
ON message_attachments (message_id, position);
+26
View File
@@ -0,0 +1,26 @@
-- Track 1 (grade 5): pending direct/presigned upload sessions.
-- Attachment rows stay status=ready only; incomplete uploads live here.
CREATE TABLE direct_uploads (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
filename text NOT NULL
CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CHECK (content_type IN ('image/png', 'image/jpeg')),
expected_size integer NOT NULL
CHECK (expected_size >= 1 AND expected_size <= 10485760),
expected_sha256 text NOT NULL
CHECK (expected_sha256 ~ '^[a-f0-9]{64}$'),
s3_key text NOT NULL UNIQUE,
status text NOT NULL
CHECK (status IN ('pending', 'completed', 'aborted', 'expired')),
attachment_id uuid NULL REFERENCES attachments(id),
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX direct_uploads_status_expires_idx
ON direct_uploads (status, expires_at);
CREATE INDEX direct_uploads_owner_idx ON direct_uploads (owner_id);
+12
View File
@@ -0,0 +1,12 @@
-- Track 1: presigned PUT lands in staging/; complete copies to attachments/.
-- Existing rows keep the old key as staging so in-flight sessions still resolve.
ALTER TABLE direct_uploads ADD COLUMN staging_key text;
UPDATE direct_uploads
SET staging_key = s3_key
WHERE staging_key IS NULL;
ALTER TABLE direct_uploads ALTER COLUMN staging_key SET NOT NULL;
ALTER TABLE direct_uploads
ADD CONSTRAINT direct_uploads_staging_key_key UNIQUE (staging_key);
+254
View File
@@ -0,0 +1,254 @@
#!/bin/sh
# Grade 4 load experiment: ≥10 concurrent ~5 MiB image uploads; sample API RSS.
# Writes evidence/load_uploads.json (no tokens/secrets).
set -eu
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
BASE_URL="${BASE_URL:-http://127.0.0.1:8080}"
N="${LOAD_UPLOADS_N:-10}"
OUT="${LOAD_UPLOADS_OUT:-$ROOT/evidence/load_uploads.json}"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
. "$ROOT/student-tests/lib.sh"
mkdir -p "$(dirname "$OUT")"
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT
echo "load_uploads: generating ~5 MiB PNG fixture..."
python3 - "$TMP/load.png" <<'PY'
import struct, zlib, sys
w, h = 1400, 1400
x = 0xC0FFEE
raw = bytearray()
for _y in range(h):
raw.append(0)
for _i in range(w * 3):
x ^= (x << 13) & 0xffffffff
x ^= (x >> 17) & 0xffffffff
x ^= (x << 5) & 0xffffffff
raw.append(x & 0xff)
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(
">I", zlib.crc32(tag + data) & 0xffffffff)
comp = zlib.compress(bytes(raw), 0)
ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", comp) + chunk(b"IEND", b"")
open(sys.argv[1], "wb").write(png)
print(len(png))
PY
SIZE=$(wc -c < "$TMP/load.png" | tr -d ' ')
echo "load_uploads: fixture_bytes=$SIZE concurrency=$N"
register_user loadu
create_chat 'load-uploads' "[\"$USER_ID\"]"
# Fault code/data pages and S3 client paths. Fresh ubuntu:24.04 containers
# often report ~4 MiB idle RSS; after a single upload glibc may return arenas
# to the OS and docker stats drops again — using that post-trim floor as
# baseline would charge library/arena re-growth to the concurrent window.
# Sample RSS *during* warm uploads and keep the warm peak as baseline floor.
http_json GET /health/ready || true
: > "$TMP/baseline.txt"
(
i=0
while [ "$i" -lt 40 ]; do
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
$($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true
i=$((i + 1))
sleep 0.25
done
) &
warm_stats_pid=$!
echo "load_uploads: warm uploads ×3 parallel (establish concurrent RSS watermark)..."
# Same size as the load fixture; concurrency matches MAX_CONCURRENT_UPLOADS so
# baseline already includes the capped in-flight footprint.
w=0
while [ "$w" -lt 3 ]; do
(
code=$(curl -sS -m 180 -o /dev/null -w '%{http_code}' \
-X POST "$BASE_URL/api/v1/chats/$CHAT_ID/attachments?filename=warm_$w.png" \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: image/png' \
--data-binary @"$TMP/load.png" || printf '000')
echo "$code" > "$TMP/warm_$w.code"
) &
w=$((w + 1))
done
wait
w=0
while [ "$w" -lt 3 ]; do
[ "$(cat "$TMP/warm_$w.code")" = "201" ] || die "warm upload $w failed: $(cat "$TMP/warm_$w.code")"
w=$((w + 1))
done
sleep 1
kill "$warm_stats_pid" 2>/dev/null || true
wait "$warm_stats_pid" 2>/dev/null || true
# A few post-warm samples (median path still runs); warm peak dominates baseline.
b=0
while [ "$b" -lt 4 ]; do
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
$($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true
b=$((b + 1))
sleep 0.3
done
# Sample RSS while uploads run.
(
i=0
while [ "$i" -lt 80 ]; do
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
$($COMPOSE ps -q api) >> "$TMP/stats.txt" 2>/dev/null || true
i=$((i + 1))
sleep 0.35
done
) &
stats_pid=$!
t0=$(python3 -c 'import time; print(time.time())')
i=0
while [ "$i" -lt "$N" ]; do
(
python3 - "$BASE_URL" "$CHAT_ID" "$TOKEN" "$TMP/load.png" "$TMP/res_$i.json" <<'PY'
import json, sys, time, urllib.request
base, chat, token, path, out = sys.argv[1:6]
url = f"{base}/api/v1/chats/{chat}/attachments?filename=load.png"
data = open(path, "rb").read()
req = urllib.request.Request(
url, data=data, method="POST",
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png",
"Content-Length": str(len(data))})
t0 = time.time()
try:
with urllib.request.urlopen(req, timeout=180) as resp:
code = resp.status
body = resp.read()
except Exception as e:
code = 0
body = str(e).encode()
dt = (time.time() - t0) * 1000.0
open(out, "w").write(json.dumps({"code": code, "latency_ms": dt, "bytes": len(body)}))
PY
) &
i=$((i + 1))
done
wait
t1=$(python3 -c 'import time; print(time.time())')
kill "$stats_pid" 2>/dev/null || true
wait "$stats_pid" 2>/dev/null || true
python3 - "$TMP" "$OUT" "$N" "$SIZE" "$t0" "$t1" <<'PY'
import json, os, re, sys
tmpdir, out_path, n, size = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
wall = (float(sys.argv[6]) - float(sys.argv[5])) * 1000.0
def line_mib(line):
m = re.search(r"([\d.]+)(MiB|GiB)", line)
if not m:
return None
v = float(m.group(1))
return v * 1024.0 if m.group(2) == "GiB" else v
def sample_peaks(path):
"""One docker-stats snapshot = two api lines; return peak per snapshot."""
peaks = []
cur = 0.0
nlines = 0
if not os.path.exists(path):
return peaks
for line in open(path):
v = line_mib(line)
if v is None:
continue
cur = max(cur, v)
nlines += 1
if nlines % 2 == 0:
peaks.append(cur)
cur = 0.0
if nlines % 2 == 1 and cur > 0:
peaks.append(cur)
return peaks
def cpu_peak(path):
peak = 0.0
if not os.path.exists(path):
return peak
for line in open(path):
c = re.search(r"([\d.]+)%", line)
if c:
peak = max(peak, float(c.group(1)))
return peak
base_peaks = sample_peaks(os.path.join(tmpdir, "baseline.txt"))
load_peaks = sample_peaks(os.path.join(tmpdir, "stats.txt"))
# Baseline = max RSS observed during warm uploads (not post-trim idle floor).
base_mib = max(base_peaks) if base_peaks else 0.0
peak_mib = max(load_peaks) if load_peaks else 0.0
# If load peak is below warm peak, delta is 0 (no extra RSS under concurrency).
cpu_p = cpu_peak(os.path.join(tmpdir, "stats.txt"))
delta = max(0.0, peak_mib - base_mib)
ok = 0
lat = []
codes = {}
for i in range(n):
row = json.load(open(os.path.join(tmpdir, f"res_{i}.json")))
code = str(row["code"])
codes[code] = codes.get(code, 0) + 1
if code == "201":
ok += 1
lat.append(float(row["latency_ms"]))
def pct(xs, p):
if not xs:
return None
xs = sorted(xs)
k = min(len(xs) - 1, max(0, int(round((p / 100.0) * (len(xs) - 1)))))
return round(xs[k], 1)
linear_mib = (n * size) / (1024 * 1024)
rps = (ok / (wall / 1000.0)) if wall > 0 else None
summary = {
"concurrency": n,
"fixture_bytes": size,
"http_codes": codes,
"success": ok,
"wall_ms": round(wall, 1),
"latency_ms": {
"p50": pct(lat, 50),
"p95": pct(lat, 95),
"p99": pct(lat, 99),
"max": round(max(lat), 1) if lat else None,
},
"approx_success_rps": round(rps, 3) if rps else None,
"api_rss_baseline_mib": round(base_mib, 2),
"api_rss_peak_mib": round(peak_mib, 2),
"api_rss_delta_mib": round(delta, 2),
"api_cpu_peak_percent": round(cpu_p, 2),
"naive_full_buffer_mib": round(linear_mib, 2),
"max_concurrent_uploads": int(os.environ.get("MAX_CONCURRENT_UPLOADS", "3")),
"note": (
"RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; "
"slot taken before buffering body >256KiB; malloc_trim after each upload. "
"rss_peak ≤ baseline + MAX_CONCURRENT_UPLOADS × (body + decoded_raster). "
"Closed-loop client; RPS is throughput of confirmed 201s."
),
}
open(out_path, "w").write(json.dumps(summary, indent=2) + "\n")
print(json.dumps(summary, indent=2))
if ok < n:
raise SystemExit(1)
if not (delta < linear_mib):
print(
f"FAIL criterion: rss_delta={delta:.1f}MiB >= naive={linear_mib:.1f}MiB",
file=sys.stderr,
)
raise SystemExit(2)
PY
echo "load_uploads: wrote $OUT"
echo "PASS load_uploads"
+39
View File
@@ -0,0 +1,39 @@
#!/bin/sh
set -eu
# Persistence hook: restart API and all state stores; keep volumes.
cd "$(dirname "$0")/.."
compose() {
docker compose --env-file .env "$@"
}
deadline=$(( $(date +%s) + 70 ))
wait_healthy() {
svc=$1
want=1
if [ "$svc" = "api" ]; then
want=${API_REPLICAS:-2}
fi
while [ "$(date +%s)" -lt "$deadline" ]; do
states=$(compose ps "$svc" --format '{{.Health}}' || true)
n=$(printf '%s\n' "$states" | grep -cx healthy || true)
if [ "$n" -ge "$want" ]; then
return 0
fi
sleep 1
done
echo "timed out waiting for $svc" >&2
return 1
}
compose restart db cache minio
wait_healthy db
wait_healthy cache
wait_healthy minio
compose start api
compose restart api
wait_healthy api
compose restart lb
wait_healthy lb
exit 0
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
# Failover hook: stop ONE API replica matching TARGET_INSTANCE_ID (hostname).
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
cd "$(dirname "$0")/.."
found=""
ids=$(docker compose --env-file .env ps -q api)
for id in $ids; do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
if [ -z "$found" ]; then
echo "api replica not found" >&2
exit 1
fi
docker stop -t 8 "$found" >/dev/null
exit 0
+1
View File
@@ -0,0 +1 @@
change_me_minio_root_locally
+1
View File
@@ -0,0 +1 @@
minioadmin
+1
View File
@@ -0,0 +1 @@
messenger-s3
+1
View File
@@ -0,0 +1 @@
change_me_s3_user_locally
+125
View File
@@ -0,0 +1,125 @@
#include "attachment_repo.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <sstream>
namespace {
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch())
.count();
return std::to_string(secs);
}
Attachment row_to_attachment(const pqxx::row& r) {
Attachment a;
a.id = r["id"].as<std::string>();
a.chat_id = r["chat_id"].as<std::string>();
a.owner_id = r["owner_id"].as<std::string>();
a.filename = r["filename"].as<std::string>();
a.content_type = r["content_type"].as<std::string>();
a.size_bytes = r["size_bytes"].as<int>();
a.sha256 = r["sha256"].as<std::string>();
a.status = r["status"].as<std::string>();
a.s3_key = r["s3_key"].as<std::string>();
if (r["error_code"].is_null()) {
a.error_code = std::nullopt;
} else {
a.error_code = r["error_code"].as<std::string>();
}
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
a.variants.clear();
return a;
}
constexpr const char* kAttachmentColumns =
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, error_code, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
} // namespace
Attachment AttachmentRepo::insert(const Attachment& in) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
pqxx::result r;
if (in.error_code) {
r = tx.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, "
" to_timestamp($11::bigint), to_timestamp($12::bigint)"
") RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, *in.error_code, ts, ts});
} else {
r = tx.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
") RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, ts, ts});
}
tx.commit();
return row_to_attachment(r[0]);
}
std::optional<Attachment> AttachmentRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kAttachmentColumns) +
" FROM attachments WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to_attachment(r[0]);
}
bool AttachmentRepo::all_belong_to_chat(const std::string& chat_id,
const std::vector<std::string>& ids) {
if (ids.empty()) {
return true;
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
for (const auto& id : ids) {
auto r = tx.exec(
"SELECT 1 FROM attachments "
"WHERE id = $1::uuid AND chat_id = $2::uuid AND status = 'ready'",
pqxx::params{id, chat_id});
if (r.empty()) {
return false;
}
}
return true;
}
+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
class AttachmentRepo {
public:
explicit AttachmentRepo(PgPool& pool) : pool_(pool) {}
Attachment insert(const Attachment& a);
std::optional<Attachment> find_by_id(const std::string& id);
// All IDs must exist, belong to chat_id, and have status=ready.
// Returns false if any id is missing/foreign/failed.
bool all_belong_to_chat(const std::string& chat_id,
const std::vector<std::string>& ids);
private:
PgPool& pool_;
};
+39
View File
@@ -0,0 +1,39 @@
#include "auth.hpp"
#include <sodium.h>
#include <stdexcept>
namespace {
struct SodiumInit {
SodiumInit() {
if (sodium_init() < 0) {
throw std::runtime_error("sodium_init failed");
}
}
};
const SodiumInit g_sodium_init;
}
std::string hash_password(const std::string& plain) {
char hashed[crypto_pwhash_STRBYTES];
if (crypto_pwhash_str(
hashed,
plain.data(), plain.size(),
crypto_pwhash_OPSLIMIT_INTERACTIVE,
crypto_pwhash_MEMLIMIT_INTERACTIVE) != 0) {
throw std::runtime_error("crypto_pwhash_str failed (out of memory?)");
}
return std::string(hashed);
}
bool verify_password(const std::string& plain, const std::string& hash) {
if (hash.empty() || hash.size() >= crypto_pwhash_STRBYTES) {
return false;
}
return crypto_pwhash_str_verify(
hash.c_str(),
plain.data(), plain.size()) == 0;
}
+7
View File
@@ -0,0 +1,7 @@
#pragma once
#include <string>
std::string hash_password(const std::string& plain);
bool verify_password(const std::string& plain, const std::string& hash);
+186
View File
@@ -0,0 +1,186 @@
#include "chat_repo.hpp"
#include "cursor.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <random>
#include <sstream>
#include <string>
namespace {
std::string make_uuid() {
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
Chat row_to_chat(const pqxx::row& r) {
Chat c;
c.id = r["id"].as<std::string>();
c.title = r["title"].as<std::string>();
c.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return c;
}
std::vector<std::string> load_member_ids(pqxx::work& tx,
const std::string& chat_id) {
std::vector<std::string> ids;
auto r = tx.exec(
"SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id",
pqxx::params{chat_id});
ids.reserve(r.size());
for (const auto& row : r) {
ids.push_back(row["user_id"].as<std::string>());
}
return ids;
}
constexpr const char* kChatSelect =
"id, title, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
}
std::optional<Chat> ChatRepo::create(const std::string& title,
const std::vector<std::string>& member_ids) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string chat_id = make_uuid();
const std::string created = now_epoch_str();
try {
tx.exec(
"INSERT INTO chats (id, title, created_at) "
"VALUES ($1, $2, to_timestamp($3::bigint))",
pqxx::params{chat_id, title, created});
for (const auto& uid : member_ids) {
tx.exec(
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
pqxx::params{chat_id, uid});
}
} catch (const pqxx::foreign_key_violation&) {
tx.abort();
return std::nullopt;
} catch (const pqxx::unique_violation&) {
tx.abort();
return std::nullopt;
}
auto r = tx.exec(
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
pqxx::params{chat_id});
Chat c = row_to_chat(r[0]);
c.member_ids = load_member_ids(tx, chat_id);
tx.commit();
return c;
}
std::optional<Chat> ChatRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
pqxx::params{id});
if (r.empty()) {
tx.commit();
return std::nullopt;
}
Chat c = row_to_chat(r[0]);
c.member_ids = load_member_ids(tx, id);
tx.commit();
return c;
}
std::optional<ChatPage> ChatRepo::list_for_user(
const std::string& user_id,
int limit,
const std::optional<std::string>& cursor) {
std::optional<Cursor> cur;
if (cursor) {
cur = parse_cursor(*cursor);
if (!cur) {
return std::nullopt;
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
if (cur) {
auto owner_check = tx.exec(
"SELECT 1 FROM chats WHERE id = $1 AND id IN "
"(SELECT chat_id FROM chat_members WHERE user_id = $2)",
pqxx::params{cur->id, user_id});
if (owner_check.empty()) {
tx.commit();
return std::nullopt;
}
}
std::string sql =
"SELECT " + std::string(kChatSelect) + ", " + kCreatedUsSql + " "
"FROM chats "
"WHERE id IN (SELECT chat_id FROM chat_members WHERE user_id = $1) ";
if (cur) {
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
}
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
pqxx::result r;
if (cur) {
r = tx.exec(sql, pqxx::params{user_id, limit + 1, cur->created_us, cur->id});
} else {
r = tx.exec(sql, pqxx::params{user_id, limit + 1});
}
ChatPage page;
const bool has_more = (static_cast<int>(r.size()) > limit);
const int take = has_more ? limit : static_cast<int>(r.size());
page.items.reserve(take);
for (int i = 0; i < take; ++i) {
Chat c = row_to_chat(r[static_cast<std::size_t>(i)]);
c.member_ids = load_member_ids(tx, c.id);
page.items.push_back(std::move(c));
}
if (has_more && take > 0) {
const auto& last = r[static_cast<std::size_t>(take - 1)];
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
last["id"].as<std::string>());
}
tx.commit();
return page;
}
+23
View File
@@ -0,0 +1,23 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
class ChatRepo {
public:
explicit ChatRepo(PgPool& pool) : pool_(pool) {}
std::optional<Chat> create(const std::string& title,
const std::vector<std::string>& member_ids);
std::optional<Chat> find_by_id(const std::string& id);
std::optional<ChatPage> list_for_user(const std::string& user_id, int limit, const std::optional<std::string>& cursor);
private:
PgPool& pool_;
};
+47
View File
@@ -0,0 +1,47 @@
#pragma once
#include <algorithm>
#include <cctype>
#include <optional>
#include <regex>
#include <string>
// Непрозрачный для клиента курсор keyset-пагинации: "<created_at в мкс от epoch>|<uuid>".
// Микросекунды сохраняют точный created_at строки, даже если в JSON время округлено до секунд.
struct Cursor {
std::string created_us;
std::string id;
};
// Выражение SQL, восстанавливающее timestamptz из created_us без потери точности.
constexpr const char* kCursorTimestampSql =
"(timestamptz 'epoch' + $3::bigint * interval '1 microsecond')";
constexpr const char* kCreatedUsSql =
"(EXTRACT(EPOCH FROM created_at) * 1000000)::bigint AS created_us";
inline bool is_uuid(const std::string& s) {
static const std::regex re(
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
return std::regex_match(s, re);
}
inline std::optional<Cursor> parse_cursor(const std::string& raw) {
const auto bar = raw.find('|');
if (bar == std::string::npos) {
return std::nullopt;
}
Cursor c{raw.substr(0, bar), raw.substr(bar + 1)};
const bool digits = !c.created_us.empty() && c.created_us.size() <= 18 &&
std::all_of(c.created_us.begin(), c.created_us.end(),
[](unsigned char ch) { return std::isdigit(ch); });
if (!digits || !is_uuid(c.id)) {
return std::nullopt;
}
return c;
}
inline std::string make_cursor(const std::string& created_us, const std::string& id) {
return created_us + "|" + id;
}
+360
View File
@@ -0,0 +1,360 @@
#include "db.hpp"
#include "errors.hpp"
#include <chrono>
#include <fstream>
#include <future>
#include <thread>
#include <sys/socket.h>
#include <sys/time.h>
#include <stdexcept>
#include <algorithm>
#include <filesystem>
#include <vector>
#include <set>
#include <iostream>
#include <nlohmann/json.hpp>
namespace fs = std::filesystem;
namespace {
constexpr int kConnectTimeoutSeconds = 2;
constexpr int kTcpUserTimeoutMs = 5000;
constexpr int kStatementTimeoutMs = 5000;
constexpr int kClientIoTimeoutSeconds = 5;
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
std::string add_conn_param(std::string conninfo, const std::string& key,
const std::string& value) {
if (conninfo.find(key + "=") != std::string::npos) {
return conninfo;
}
const bool is_url = conninfo.rfind("postgres://", 0) == 0 ||
conninfo.rfind("postgresql://", 0) == 0;
if (is_url) {
conninfo += (conninfo.find('?') == std::string::npos ? "?" : "&");
} else {
conninfo += " ";
}
return conninfo + key + "=" + value;
}
bool is_alive(pqxx::connection* conn) {
std::promise<void> done;
auto fut = done.get_future();
std::thread th([conn, &done]() {
try {
pqxx::nontransaction tx(*conn);
tx.exec("SELECT 1");
done.set_value();
} catch (...) {
done.set_exception(std::current_exception());
}
});
if (fut.wait_for(std::chrono::seconds(kClientIoTimeoutSeconds)) !=
std::future_status::ready) {
const int fd = conn->sock();
if (fd >= 0) {
::shutdown(fd, SHUT_RDWR);
}
th.join();
return false;
}
th.join();
try {
fut.get();
return true;
} catch (...) {
return false;
}
}
struct Migration {
int version;
std::string path;
};
constexpr int kMigrationsAdvisoryLockKey = 0x6d747573;
Migration parse_migration_name(const fs::path& p) {
const std::string name = p.filename().string();
const auto dot = name.find('.');
if (dot == std::string::npos || name.substr(dot) != ".sql") {
throw std::runtime_error("migration file must end with .sql: " + name);
}
const auto us = name.find('_');
if (us == std::string::npos || us == 0) {
throw std::runtime_error(
"migration file must be named NNN_name.sql: " + name);
}
const std::string num = name.substr(0, us);
if (!std::all_of(num.begin(), num.end(),
[](unsigned char c){ return std::isdigit(c); })) {
throw std::runtime_error(
"migration file prefix must be digits: " + name);
}
Migration m;
m.version = std::stoi(num);
m.path = p.string();
return m;
}}
void apply_migrations(pqxx::connection& conn, const std::string& dir) {
{
pqxx::work tx(conn);
tx.exec("SELECT pg_advisory_lock($1)",
pqxx::params{kMigrationsAdvisoryLockKey});
tx.commit();
}
std::clog << "migration advisory lock acquired\n";
struct Unlock {
pqxx::connection& c;
~Unlock() {
try {
pqxx::work tx(c);
tx.exec("SELECT pg_advisory_unlock($1)",
pqxx::params{kMigrationsAdvisoryLockKey});
tx.commit();
} catch (...) {}
}
} unlock_guard{conn};
{
pqxx::work tx(conn);
tx.exec(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version int PRIMARY KEY,"
" applied_at timestamptz NOT NULL DEFAULT now()"
")");
tx.commit();
}
if (!fs::exists(dir) || !fs::is_directory(dir)) {
throw std::runtime_error("migrations dir not found: " + dir);
}
std::vector<Migration> files;
for (const auto& entry : fs::directory_iterator(dir)) {
if (entry.is_regular_file()) {
files.push_back(parse_migration_name(entry.path()));
}
}
std::sort(files.begin(), files.end(),
[](const Migration& a, const Migration& b){
return a.version < b.version;
});
for (std::size_t i = 1; i < files.size(); ++i) {
if (files[i].version == files[i-1].version) {
throw std::runtime_error(
"duplicate migration version: " + std::to_string(files[i].version));
}
}
std::set<int> applied;
{
pqxx::work tx(conn);
for (const auto& row : tx.exec("SELECT version FROM schema_migrations")) {
applied.insert(row[0].as<int>());
}
tx.commit();
}
for (const auto& m : files) {
if (applied.count(m.version)) {
continue;
}
std::ifstream f(m.path, std::ios::binary);
if (!f) {
throw std::runtime_error("cannot open migration: " + m.path);
}
std::stringstream buf;
buf << f.rdbuf();
const std::string sql = buf.str();
pqxx::work tx(conn);
tx.exec(sql);
tx.exec("INSERT INTO schema_migrations(version) VALUES ($1)",
m.version);
tx.commit();
std::clog << "applied migration " << m.version
<< " (" << m.path << ")\n";
}
}
std::string read_secret_file(const char* env_name) {
const char* path = std::getenv(env_name);
if (path == nullptr || *path == '\0') {
throw std::runtime_error(
std::string("environment variable not set: ") + env_name);
}
std::ifstream f(path, std::ios::binary);
if (!f) {
throw std::runtime_error(
std::string("cannot open secret file: ") + path);
}
std::string s;
std::getline(f, s);
if (!s.empty() && s.back() == '\r') {
s.pop_back();
}
if (s.empty()) {
throw std::runtime_error(
std::string("secret file is empty: ") + path);
}
return s;
}
bool is_dependency_failure(const std::exception& e) {
if (dynamic_cast<const DependencyUnavailable*>(&e) != nullptr ||
dynamic_cast<const pqxx::broken_connection*>(&e) != nullptr ||
dynamic_cast<const pqxx::in_doubt_error*>(&e) != nullptr) {
return true;
}
if (const auto* sql = dynamic_cast<const pqxx::sql_error*>(&e)) {
// 08xxx — ошибки соединения, 57xxx — отмена (statement_timeout) и остановка сервера.
const std::string& state = sql->sqlstate();
return state.rfind("08", 0) == 0 || state.rfind("57", 0) == 0;
}
return false;
}
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
bool statement_timeout) {
std::string full = add_conn_param(conninfo, "connect_timeout",
std::to_string(kConnectTimeoutSeconds));
full = add_conn_param(full, "tcp_user_timeout", std::to_string(kTcpUserTimeoutMs));
auto conn = std::make_unique<pqxx::connection>(full);
const int fd = conn->sock();
if (fd >= 0) {
timeval tv{};
tv.tv_sec = kClientIoTimeoutSeconds;
tv.tv_usec = 0;
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
}
if (statement_timeout) {
pqxx::nontransaction tx(*conn);
tx.exec("SET statement_timeout = " + std::to_string(kStatementTimeoutMs));
}
return conn;
}
PgPool::PgPool(std::string conninfo, std::size_t n)
: conninfo_(std::move(conninfo)) {
if (n == 0) {
throw std::runtime_error("PgPool: n must be > 0");
}
all_.reserve(n);
idle_.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
auto conn = open_pg_connection(conninfo_, true);
idle_.push_back(conn.get());
all_.push_back(std::move(conn));
}
}
PgPool::~PgPool() {
{
std::lock_guard<std::mutex> lk(mu_);
shutting_down_ = true;
}
cv_.notify_all();
}
pqxx::connection* PgPool::take() {
std::unique_lock<std::mutex> lk(mu_);
if (!cv_.wait_for(lk, kAcquireTimeout,
[&]() { return shutting_down_ || !idle_.empty(); })) {
throw DependencyUnavailable("postgres pool: acquire timeout");
}
if (shutting_down_) {
throw DependencyUnavailable("postgres pool is shutting down");
}
pqxx::connection* conn = idle_.back();
idle_.pop_back();
lk.unlock();
if (is_alive(conn)) {
return conn;
}
std::unique_ptr<pqxx::connection> fresh;
try {
fresh = open_pg_connection(conninfo_, true);
} catch (const std::exception& e) {
// Мёртвое соединение возвращается в пул: следующий take() снова
// попробует переподключиться, и пул не теряет слот.
give_back(conn);
throw DependencyUnavailable(std::string("postgres reconnect failed: ") + e.what());
}
std::lock_guard<std::mutex> lk2(mu_);
for (auto& up : all_) {
if (up.get() == conn) {
up = std::move(fresh);
return up.get();
}
}
throw std::logic_error("PgPool: connection not owned by pool");
}
void PgPool::give_back(pqxx::connection* conn) {
{
std::lock_guard<std::mutex> lk(mu_);
idle_.push_back(conn);
}
cv_.notify_one();
}
PgPool::Lease::Lease(PgPool& pool) : pool_(&pool), conn_(pool.take()) {}
PgPool::Lease::~Lease() {
if (pool_ != nullptr && conn_ != nullptr) {
pool_->give_back(conn_);
}
}
PgPool::Lease::Lease(Lease&& other) noexcept
: pool_(other.pool_), conn_(other.conn_) {
other.pool_ = nullptr;
other.conn_ = nullptr;
}
PgPool::Lease PgPool::acquire() {
return Lease(*this);
}
bool PgPool::ping() {
try {
auto lease = acquire();
pqxx::work tx(*lease);
tx.exec("SELECT 1");
tx.commit();
return true;
} catch (...) {
return false;
}
}
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include <condition_variable>
#include <pqxx/pqxx>
#include <cstddef>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
std::string read_secret_file(const char* env_name);
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
bool statement_timeout);
void apply_migrations(pqxx::connection& conn, const std::string& dir);
class PgPool {
public:
bool ping();
PgPool(std::string conninfo, std::size_t n);
~PgPool();
PgPool(const PgPool&) = delete;
PgPool& operator=(const PgPool&) = delete;
class Lease {
public:
explicit Lease(PgPool& pool);
~Lease();
Lease(Lease&& other) noexcept;
Lease& operator=(Lease&&) = delete;
Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
pqxx::connection& operator*() const { return *conn_; }
pqxx::connection* operator->() const { return conn_; }
private:
PgPool* pool_;
pqxx::connection* conn_;
};
Lease acquire();
private:
friend class Lease;
pqxx::connection* take();
void give_back(pqxx::connection* conn);
std::string conninfo_;
std::vector<std::unique_ptr<pqxx::connection>> all_;
std::vector<pqxx::connection*> idle_;
std::mutex mu_;
std::condition_variable cv_;
bool shutting_down_ = false;
};
+282
View File
@@ -0,0 +1,282 @@
#include "direct_upload_repo.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <sstream>
namespace {
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch())
.count();
return std::to_string(secs);
}
DirectUpload row_to(const pqxx::row& r) {
DirectUpload d;
d.id = r["id"].as<std::string>();
d.chat_id = r["chat_id"].as<std::string>();
d.owner_id = r["owner_id"].as<std::string>();
d.filename = r["filename"].as<std::string>();
d.content_type = r["content_type"].as<std::string>();
d.expected_size = r["expected_size"].as<int>();
d.expected_sha256 = r["expected_sha256"].as<std::string>();
d.s3_key = r["s3_key"].as<std::string>();
d.staging_key = r["staging_key"].as<std::string>();
d.status = r["status"].as<std::string>();
d.overdue = r["overdue"].as<bool>();
if (r["attachment_id"].is_null()) {
d.attachment_id = std::nullopt;
} else {
d.attachment_id = r["attachment_id"].as<std::string>();
}
d.expires_at = epoch_to_rfc3339(r["expires_at_epoch"].as<long long>());
d.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
d.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
return d;
}
constexpr const char* kCols =
"id, chat_id, owner_id, filename, content_type, expected_size, "
"expected_sha256, s3_key, staging_key, status, attachment_id, "
"(status = 'pending' AND expires_at < now()) AS overdue, "
"EXTRACT(EPOCH FROM expires_at)::bigint AS expires_at_epoch, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
constexpr const char* kAttachmentColumns =
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
"status, s3_key, error_code, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
Attachment attachment_from_row(const pqxx::row& r) {
Attachment a;
a.id = r["id"].as<std::string>();
a.chat_id = r["chat_id"].as<std::string>();
a.owner_id = r["owner_id"].as<std::string>();
a.filename = r["filename"].as<std::string>();
a.content_type = r["content_type"].as<std::string>();
a.size_bytes = r["size_bytes"].as<int>();
a.sha256 = r["sha256"].as<std::string>();
a.status = r["status"].as<std::string>();
a.s3_key = r["s3_key"].as<std::string>();
if (r["error_code"].is_null()) {
a.error_code = std::nullopt;
} else {
a.error_code = r["error_code"].as<std::string>();
}
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
a.variants.clear();
return a;
}
} // namespace
DirectUpload DirectUploadRepo::insert(const DirectUpload& in) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
// expires_at is already an absolute epoch string from the caller.
auto r = tx.exec(
"INSERT INTO direct_uploads ("
" id, chat_id, owner_id, filename, content_type, expected_size, "
" expected_sha256, s3_key, staging_key, status, attachment_id, "
" expires_at, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, NULL, "
" to_timestamp($11::bigint), to_timestamp($12::bigint), "
" to_timestamp($13::bigint)"
") RETURNING " +
std::string(kCols),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.expected_size, in.expected_sha256,
in.s3_key, in.staging_key, in.status, in.expires_at_epoch,
ts, ts});
tx.commit();
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::find_by_id(
const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kCols) +
" FROM direct_uploads WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::mark_completed(
const std::string& id, const std::string& attachment_id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'completed', "
" attachment_id = $2::uuid, "
" updated_at = to_timestamp($3::bigint) "
"WHERE id = $1::uuid AND status = 'pending' "
"RETURNING " +
std::string(kCols),
pqxx::params{id, attachment_id, ts});
if (r.empty()) {
return std::nullopt;
}
tx.commit();
return row_to(r[0]);
}
std::optional<DirectUpload> DirectUploadRepo::mark_aborted(
const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'aborted', "
" updated_at = to_timestamp($2::bigint) "
"WHERE id = $1::uuid AND status = 'pending' "
"RETURNING " +
std::string(kCols),
pqxx::params{id, ts});
if (r.empty()) {
return std::nullopt;
}
tx.commit();
return row_to(r[0]);
}
std::vector<DirectUpload> DirectUploadRepo::expire_stale() {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
const std::string ts = now_epoch_str();
auto r = tx.exec(
"UPDATE direct_uploads SET status = 'expired', "
" updated_at = to_timestamp($1::bigint) "
"WHERE status = 'pending' AND expires_at < now() "
"RETURNING " +
std::string(kCols),
pqxx::params{ts});
tx.commit();
std::vector<DirectUpload> out;
out.reserve(r.size());
for (const auto& row : r) {
out.push_back(row_to(row));
}
return out;
}
std::vector<std::string> DirectUploadRepo::pending_s3_keys() {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT staging_key FROM direct_uploads WHERE status = 'pending'");
std::vector<std::string> out;
out.reserve(r.size());
for (const auto& row : r) {
out.push_back(row["staging_key"].as<std::string>());
}
return out;
}
void DirectUploadRepo::with_lock(const std::function<void(UploadLock&)>& fn) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
UploadLock lock(tx);
fn(lock);
}
std::optional<DirectUpload> UploadLock::lock(const std::string& id) {
auto r = tx_.exec(
"SELECT " + std::string(kCols) +
" FROM direct_uploads WHERE id = $1::uuid FOR UPDATE",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return row_to(r[0]);
}
std::optional<Attachment> UploadLock::find_attachment(const std::string& id) {
auto r = tx_.exec(
"SELECT " + std::string(kAttachmentColumns) +
" FROM attachments WHERE id = $1::uuid",
pqxx::params{id});
if (r.empty()) {
return std::nullopt;
}
return attachment_from_row(r[0]);
}
std::optional<Attachment> UploadLock::insert_ready(const Attachment& in,
bool& inserted) {
inserted = false;
const std::string ts = now_epoch_str();
auto r = tx_.exec(
"INSERT INTO attachments ("
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
" status, s3_key, error_code, created_at, updated_at"
") VALUES ("
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
") ON CONFLICT (id) DO NOTHING RETURNING " +
std::string(kAttachmentColumns),
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
in.content_type, in.size_bytes, in.sha256, in.status,
in.s3_key, ts, ts});
if (r.empty()) {
return std::nullopt;
}
inserted = true;
return attachment_from_row(r[0]);
}
void UploadLock::mark_completed(const std::string& id,
const std::string& attachment_id) {
const std::string ts = now_epoch_str();
auto r = tx_.exec(
"UPDATE direct_uploads SET status = 'completed', "
" attachment_id = $2::uuid, "
" updated_at = to_timestamp($3::bigint) "
"WHERE id = $1::uuid AND status = 'pending'",
pqxx::params{id, attachment_id, ts});
if (r.affected_rows() != 1) {
throw std::runtime_error("direct upload complete lost the pending row");
}
}
void UploadLock::mark_expired(const std::string& id) {
const std::string ts = now_epoch_str();
tx_.exec(
"UPDATE direct_uploads SET status = 'expired', "
" updated_at = to_timestamp($2::bigint) "
"WHERE id = $1::uuid AND status = 'pending'",
pqxx::params{id, ts});
}
void UploadLock::commit() {
if (committed_) {
return;
}
tx_.commit();
committed_ = true;
}
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <functional>
#include <optional>
#include <string>
#include <vector>
class UploadLock;
class DirectUploadRepo {
public:
explicit DirectUploadRepo(PgPool& pool) : pool_(pool) {}
DirectUpload insert(const DirectUpload& in);
std::optional<DirectUpload> find_by_id(const std::string& id);
// Mark completed and link attachment_id. Returns nullopt if not pending
// (or already finished / wrong owner handled by caller).
std::optional<DirectUpload> mark_completed(const std::string& id,
const std::string& attachment_id);
std::optional<DirectUpload> mark_aborted(const std::string& id);
// pending → expired where expires_at < now; returns expired rows.
std::vector<DirectUpload> expire_stale();
// Staging keys of still-pending sessions (orphan GC must keep these).
std::vector<std::string> pending_s3_keys();
// One PG transaction. Callback may call S3 while the row lock is held.
// Commit explicitly via UploadLock::commit(); otherwise the work rolls back.
void with_lock(const std::function<void(UploadLock&)>& fn);
private:
PgPool& pool_;
};
// Row lock for a single complete(). Not safe to use after with_lock returns.
class UploadLock {
public:
std::optional<DirectUpload> lock(const std::string& id);
std::optional<Attachment> find_attachment(const std::string& id);
// ON CONFLICT (id) DO NOTHING. inserted=false and nullopt → row already exists.
std::optional<Attachment> insert_ready(const Attachment& in, bool& inserted);
void mark_completed(const std::string& id, const std::string& attachment_id);
void mark_expired(const std::string& id);
void commit();
private:
friend class DirectUploadRepo;
explicit UploadLock(pqxx::work& tx) : tx_(tx) {}
pqxx::work& tx_;
bool committed_ = false;
};
+11
View File
@@ -0,0 +1,11 @@
#pragma once
#include <exception>
#include <stdexcept>
// PostgreSQL или Valkey недоступны либо не ответили вовремя: клиент получает 503.
struct DependencyUnavailable : std::runtime_error {
using std::runtime_error::runtime_error;
};
bool is_dependency_failure(const std::exception& e);
+90
View File
@@ -0,0 +1,90 @@
#include "image_util.hpp"
#define STB_IMAGE_IMPLEMENTATION
#define STBI_NO_STDIO
#define STBI_ONLY_JPEG
#define STBI_ONLY_PNG
#include "stb_image.h"
#include <cstring>
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
std::size_t len) {
if (!data || len < 3) {
return std::nullopt;
}
// JPEG: FF D8 FF
if (data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) {
return std::string("image/jpeg");
}
// PNG: 89 50 4E 47
if (len >= 4 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E &&
data[3] == 0x47) {
return std::string("image/png");
}
return std::nullopt;
}
std::string_view image_reject_code(ImageReject r) {
switch (r) {
case ImageReject::unsupported_media_type:
return "unsupported_media_type";
case ImageReject::payload_too_large:
return "payload_too_large";
case ImageReject::invalid_image:
return "invalid_image";
}
return "invalid_image";
}
std::optional<ImageReject> validate_image(const std::uint8_t* data,
std::size_t len,
const ImageLimits& limits,
ImageInfo& out) {
if (!data || len == 0) {
return ImageReject::unsupported_media_type;
}
if (static_cast<std::int64_t>(len) > limits.max_bytes) {
return ImageReject::payload_too_large;
}
auto sniffed = sniff_content_type(data, len);
if (!sniffed) {
return ImageReject::unsupported_media_type;
}
int w = 0;
int h = 0;
int comp = 0;
if (!stbi_info_from_memory(reinterpret_cast<const stbi_uc*>(data),
static_cast<int>(len), &w, &h, &comp)) {
return ImageReject::invalid_image;
}
if (w <= 0 || h <= 0) {
return ImageReject::invalid_image;
}
const std::int64_t pixels =
static_cast<std::int64_t>(w) * static_cast<std::int64_t>(h);
if (pixels > limits.max_pixels) {
return ImageReject::payload_too_large;
}
int load_w = 0;
int load_h = 0;
int load_comp = 0;
stbi_uc* decoded = stbi_load_from_memory(
reinterpret_cast<const stbi_uc*>(data), static_cast<int>(len), &load_w,
&load_h, &load_comp, 0);
if (!decoded) {
return ImageReject::invalid_image;
}
// Integrity only — do not retain decoded raster.
stbi_image_free(decoded);
out.content_type = *sniffed;
out.width = w;
out.height = h;
out.pixel_count = pixels;
return std::nullopt;
}
+41
View File
@@ -0,0 +1,41 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <string_view>
struct ImageLimits {
std::int64_t max_bytes = 10'485'760; // 10 MiB
std::int64_t max_pixels = 40'000'000; // decoded width*height
};
enum class ImageReject {
unsupported_media_type, // → 415
payload_too_large, // → 413 (bytes or pixels)
invalid_image, // → 415 (corrupt / undecodable)
};
struct ImageInfo {
std::string content_type; // image/png | image/jpeg
int width = 0;
int height = 0;
std::int64_t pixel_count = 0;
};
// Magic sniff only (does not decode). Returns nullopt if neither PNG nor JPEG.
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
std::size_t len);
// Full validation pipeline from contracts:
// sniff → stbi_info (dims) → pixel limit → stbi_load (integrity) → free.
// On success fills `out`. On failure returns ImageReject; decoded pixels are
// never retained after return.
std::optional<ImageReject> validate_image(const std::uint8_t* data,
std::size_t len,
const ImageLimits& limits,
ImageInfo& out);
// Human-readable error_code for Attachment failed path / API body.
std::string_view image_reject_code(ImageReject r);
+2329
View File
File diff suppressed because it is too large Load Diff
+374
View File
@@ -0,0 +1,374 @@
#include "message_repo.hpp"
#include "cursor.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <cstdlib>
#include <ctime>
#include <iomanip>
#include <mutex>
#include <random>
#include <sstream>
#include <stdexcept>
#include <string>
#include <sodium.h>
#include <vector>
namespace {
std::string make_uuid() {
static std::mutex rng_mu;
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::lock_guard<std::mutex> lock(rng_mu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string sha256_hex(const std::string& s) {
unsigned char out[crypto_hash_sha256_BYTES];
crypto_hash_sha256(out, reinterpret_cast<const unsigned char*>(s.data()), s.size());
static const char hex[] = "0123456789abcdef";
std::string r;
r.resize(sizeof(out) * 2);
for (std::size_t i = 0; i < sizeof(out); ++i) {
r[i * 2] = hex[out[i] >> 4];
r[i * 2 + 1] = hex[out[i] & 0x0f];
}
return r;
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
Message row_to_message(const pqxx::row& r) {
Message m;
m.id = r["id"].as<std::string>();
m.chat_id = r["chat_id"].as<std::string>();
m.sender_id = r["sender_id"].as<std::string>();
m.text = r["text"].as<std::string>();
m.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return m;
}
constexpr const char* kMessageSelect =
"id, chat_id, sender_id, text, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
std::string canonical_message_body(const std::string& text,
const std::vector<std::string>& attachment_ids) {
std::ostringstream out;
out << "text=" << text << "\nattachment_ids=";
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
if (i > 0) {
out << ',';
}
out << attachment_ids[i];
}
return out.str();
}
int idempotency_ttl_seconds() {
const char* env = std::getenv("IDEMPOTENCY_TTL_SECONDS");
if (env == nullptr || *env == '\0') {
return 86400;
}
try {
const int v = std::stoi(env);
if (v <= 0) {
return 86400;
}
return v;
} catch (...) {
return 86400;
}
}
bool attachments_belong_to_chat(pqxx::work& tx,
const std::string& chat_id,
const std::vector<std::string>& attachment_ids) {
if (attachment_ids.empty()) {
return true;
}
for (const auto& id : attachment_ids) {
auto r = tx.exec(
"SELECT 1 FROM attachments "
"WHERE id = $1 AND chat_id = $2 AND status = 'ready'",
pqxx::params{id, chat_id});
if (r.empty()) {
return false;
}
}
return true;
}
void link_attachments(pqxx::work& tx,
const std::string& message_id,
const std::vector<std::string>& attachment_ids) {
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
tx.exec(
"INSERT INTO message_attachments (message_id, attachment_id, position) "
"VALUES ($1, $2, $3)",
pqxx::params{message_id, attachment_ids[i], static_cast<int>(i)});
}
}
std::vector<std::string> load_attachment_ids(pqxx::work& tx,
const std::string& message_id) {
auto r = tx.exec(
"SELECT attachment_id FROM message_attachments "
"WHERE message_id = $1 ORDER BY position ASC",
pqxx::params{message_id});
std::vector<std::string> ids;
ids.reserve(r.size());
for (const auto& row : r) {
ids.push_back(row["attachment_id"].as<std::string>());
}
return ids;
}
void fill_attachment_ids(pqxx::work& tx, Message& m) {
m.attachment_ids = load_attachment_ids(tx, m.id);
}
void fill_attachment_ids(pqxx::work& tx, std::vector<Message>& items) {
for (auto& m : items) {
fill_attachment_ids(tx, m);
}
}
Message insert_message(pqxx::work& tx,
const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids) {
const std::string id = make_uuid();
const std::string created = now_epoch_str();
auto r = tx.exec(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
"RETURNING " + std::string(kMessageSelect),
pqxx::params{id, chat_id, sender_id, text, created});
Message m = row_to_message(r[0]);
link_attachments(tx, m.id, attachment_ids);
m.attachment_ids = attachment_ids;
return m;
}
bool insert_idempotency_key(pqxx::work& tx,
const std::string& user_id,
const std::string& chat_id,
const std::string& idem_key,
const std::string& request_hash,
const std::string& message_id) {
auto inserted = tx.exec(
"INSERT INTO idempotency_keys "
"(user_id, chat_id, key, request_hash, message_id) "
"VALUES ($1, $2, $3, $4, $5) "
"ON CONFLICT (user_id, chat_id, key) DO NOTHING "
"RETURNING message_id",
pqxx::params{user_id, chat_id, idem_key, request_hash, message_id});
return !inserted.empty();
}
Message load_message(pqxx::work& tx, const std::string& message_id) {
auto r = tx.exec(
"SELECT " + std::string(kMessageSelect) + " FROM messages WHERE id = $1",
pqxx::params{message_id});
if (r.empty()) {
throw std::runtime_error("idempotent message row is missing");
}
Message m = row_to_message(r[0]);
fill_attachment_ids(tx, m);
return m;
}
} // namespace
std::optional<Message> MessageRepo::create(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto member_check = tx.exec(
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
pqxx::params{chat_id, sender_id});
if (member_check.empty()) {
tx.commit();
return std::nullopt;
}
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return std::nullopt;
}
Message created = insert_message(tx, chat_id, sender_id, text, attachment_ids);
tx.commit();
return created;
}
std::optional<MessagePage> MessageRepo::list_page(
const std::string& chat_id,
int limit,
const std::optional<std::string>& cursor) {
std::optional<Cursor> cur;
if (cursor) {
cur = parse_cursor(*cursor);
if (!cur) {
return std::nullopt;
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
if (cur) {
auto owner_check = tx.exec(
"SELECT 1 FROM messages WHERE id = $1 AND chat_id = $2",
pqxx::params{cur->id, chat_id});
if (owner_check.empty()) {
tx.commit();
return std::nullopt;
}
}
std::string sql =
"SELECT " + std::string(kMessageSelect) + ", " + kCreatedUsSql + " "
"FROM messages "
"WHERE chat_id = $1 ";
if (cur) {
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
}
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
pqxx::result r;
if (cur) {
r = tx.exec(sql, pqxx::params{chat_id, limit + 1, cur->created_us, cur->id});
} else {
r = tx.exec(sql, pqxx::params{chat_id, limit + 1});
}
MessagePage page;
const bool has_more = (static_cast<int>(r.size()) > limit);
const int take = has_more ? limit : static_cast<int>(r.size());
page.items.reserve(take);
for (int i = 0; i < take; ++i) {
page.items.push_back(row_to_message(r[static_cast<std::size_t>(i)]));
}
fill_attachment_ids(tx, page.items);
if (has_more && take > 0) {
const auto& last = r[static_cast<std::size_t>(take - 1)];
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
last["id"].as<std::string>());
}
tx.commit();
return page;
}
CreateResult MessageRepo::create_idempotent(
const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids,
const std::string& idem_key) {
const int ttl = idempotency_ttl_seconds();
const std::string hash = sha256_hex(canonical_message_body(text, attachment_ids));
for (int attempt = 0; attempt < 3; ++attempt) {
{
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto member_check = tx.exec(
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
pqxx::params{chat_id, sender_id});
if (member_check.empty()) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
const Message created =
insert_message(tx, chat_id, sender_id, text, attachment_ids);
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
created.id)) {
tx.commit();
return CreateResult{CreateResult::Status::Created, created};
}
}
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto existing = tx.exec(
"SELECT request_hash, message_id, "
"EXTRACT(EPOCH FROM (now() - created_at))::bigint AS age_seconds "
"FROM idempotency_keys "
"WHERE user_id = $1 AND chat_id = $2 AND key = $3 "
"FOR UPDATE",
pqxx::params{sender_id, chat_id, idem_key});
if (existing.empty()) {
tx.commit();
continue;
}
const std::string existing_hash = existing[0]["request_hash"].as<std::string>();
const long long age = existing[0]["age_seconds"].as<long long>();
const std::string message_id = existing[0]["message_id"].as<std::string>();
if (age < ttl) {
if (existing_hash != hash) {
tx.commit();
return CreateResult{CreateResult::Status::KeyConflict, std::nullopt};
}
const Message replayed = load_message(tx, message_id);
tx.commit();
return CreateResult{CreateResult::Status::Replayed, replayed};
}
tx.exec(
"DELETE FROM idempotency_keys "
"WHERE user_id = $1 AND chat_id = $2 AND key = $3",
pqxx::params{sender_id, chat_id, idem_key});
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
tx.commit();
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
}
const Message created =
insert_message(tx, chat_id, sender_id, text, attachment_ids);
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
created.id)) {
tx.commit();
return CreateResult{CreateResult::Status::Created, created};
}
}
throw std::runtime_error("idempotency retry exhausted");
}
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
#include <vector>
struct CreateResult {
enum class Status { Created, Replayed, NotFound, KeyConflict };
Status status;
std::optional<Message> message;
};
class MessageRepo {
public:
explicit MessageRepo(PgPool& pool) : pool_(pool) {}
std::optional<Message> create(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids);
std::optional<MessagePage> list_page(
const std::string& chat_id,
int limit,
const std::optional<std::string>& cursor);
CreateResult create_idempotent(const std::string& chat_id,
const std::string& sender_id,
const std::string& text,
const std::vector<std::string>& attachment_ids,
const std::string& idem_key);
private:
PgPool& pool_;
};
+83
View File
@@ -0,0 +1,83 @@
#pragma once
#include <optional>
#include <string>
#include <vector>
struct User {
std::string id;
std::string username;
std::string display_name;
std::string password_hash;
std::string created_at;
};
struct Chat {
std::string id;
std::string title;
std::vector<std::string> member_ids;
std::string created_at;
};
struct Message {
std::string id;
std::string chat_id;
std::string sender_id;
std::string text;
std::string created_at;
std::vector<std::string> attachment_ids;
};
struct MessagePage {
std::vector<Message> items;
std::optional<std::string> next_cursor;
};
struct ChatPage {
std::vector<Chat> items;
std::optional<std::string> next_cursor;
};
// Lab4: variants stay empty ([]); lab5 fills thumbnail/crop/optimized.
struct Variant {
std::string name;
std::string content_type;
int size_bytes = 0;
std::string sha256;
};
struct Attachment {
std::string id;
std::string chat_id;
std::string owner_id;
std::string filename;
std::string content_type;
int size_bytes = 0;
std::string sha256;
std::string status; // lab4: "ready"
std::vector<Variant> variants; // lab4: always empty
std::optional<std::string> error_code;
std::string s3_key; // internal; never exposed in API JSON
std::string created_at;
std::string updated_at;
};
// Track 1: pending presigned PUT session (not a public attachment until complete).
struct DirectUpload {
std::string id;
std::string chat_id;
std::string owner_id;
std::string filename;
std::string content_type;
int expected_size = 0;
std::string expected_sha256;
std::string s3_key; // final key: attachments/{chat}/{id}
std::string staging_key; // presigned PUT target: staging/{chat}/{id}
std::string status; // pending|completed|aborted|expired
bool overdue = false; // pending && expires_at < now(); not a column
std::optional<std::string> attachment_id;
std::string expires_at; // RFC3339 for JSON
long long expires_at_epoch = 0; // for INSERT
std::string created_at;
std::string updated_at;
};
+66
View File
@@ -0,0 +1,66 @@
#include "pg_session_repo.hpp"
#include <pqxx/pqxx>
#include <sodium.h>
#include <ctime>
#include <stdexcept>
#include <string>
namespace {
std::string random_token_hex() {
unsigned char buf[32];
randombytes_buf(buf, sizeof(buf));
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(sizeof(buf) * 2);
for (std::size_t i = 0; i < sizeof(buf); ++i) {
out[i * 2] = hex[buf[i] >> 4];
out[i * 2 + 1] = hex[buf[i] & 0x0f];
}
return out;
}
std::int64_t now_unix() {
return static_cast<std::int64_t>(std::time(nullptr));
}
}
ISessionRepo::Created PgSessionRepo::create(const std::string& user_id) {
const std::string token = random_token_hex();
const std::int64_t exp = now_unix() + ttl_;
auto lease = pool_.acquire();
pqxx::work tx(*lease);
tx.exec(
"INSERT INTO sessions (token, user_id, expires_at) "
"VALUES ($1, $2, to_timestamp($3::bigint))",
pqxx::params{token, user_id, exp});
tx.commit();
return Created{token, exp};
}
std::optional<std::string> PgSessionRepo::lookup(const std::string& token) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT user_id FROM sessions "
"WHERE token = $1 AND expires_at > now()",
pqxx::params{token});
tx.commit();
if (r.empty()) return std::nullopt;
return r[0]["user_id"].as<std::string>();
}
bool PgSessionRepo::revoke(const std::string& token) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"DELETE FROM sessions WHERE token = $1",
pqxx::params{token});
tx.commit();
return r.affected_rows() > 0;
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "db.hpp"
#include "session_repo_base.hpp"
class PgSessionRepo : public ISessionRepo {
public:
PgSessionRepo(PgPool& pool, int ttl_seconds)
: pool_(pool), ttl_(ttl_seconds) {}
Created create(const std::string& user_id) override;
std::optional<std::string> lookup(const std::string& token) override;
bool revoke(const std::string& token) override;
private:
PgPool& pool_;
int ttl_;
};
+216
View File
@@ -0,0 +1,216 @@
#include "redis_pool.hpp"
#include "errors.hpp"
#include <chrono>
#include <cstdlib>
#include <stdexcept>
#include <string>
namespace {
constexpr timeval kConnectTimeout{2, 0};
constexpr timeval kCommandTimeout{2, 0};
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
struct RedisUrl {
std::string host = "127.0.0.1";
int port = 6379;
std::string password;
int db = 0;
};
RedisUrl parse_redis_url(const std::string& url) {
RedisUrl out;
const std::string prefix = "redis://";
if (url.compare(0, prefix.size(), prefix) != 0) {
throw std::runtime_error("redis url must start with redis://");
}
std::string rest = url.substr(prefix.size());
const auto slash = rest.find('/');
if (slash != std::string::npos) {
const std::string db_str = rest.substr(slash + 1);
rest.resize(slash);
if (!db_str.empty()) {
out.db = std::atoi(db_str.c_str());
}
}
const auto at = rest.find('@');
std::string hostport = rest;
if (at != std::string::npos) {
const std::string creds = rest.substr(0, at);
hostport = rest.substr(at + 1);
const auto colon = creds.find(':');
if (colon != std::string::npos) {
out.password = creds.substr(colon + 1);
} else {
out.password = creds;
}
}
const auto colon = hostport.find(':');
if (colon == std::string::npos) {
out.host = hostport;
} else {
out.host = hostport.substr(0, colon);
out.port = std::atoi(hostport.substr(colon + 1).c_str());
}
return out;
}
redisContext* make_connection(const RedisUrl& u) {
redisContext* c = redisConnectWithTimeout(u.host.c_str(), u.port, kConnectTimeout);
if (c == nullptr) {
throw std::runtime_error("redisConnect: allocation failed");
}
if (c->err) {
const std::string err = c->errstr;
redisFree(c);
throw DependencyUnavailable("redisConnect failed: " + err);
}
if (redisSetTimeout(c, kCommandTimeout) != REDIS_OK) {
redisFree(c);
throw DependencyUnavailable("redisSetTimeout failed");
}
if (!u.password.empty()) {
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "AUTH %s", u.password.c_str()));
if (r == nullptr) {
redisFree(c);
throw std::runtime_error("redis AUTH: no reply");
}
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
redisFree(c);
throw std::runtime_error("redis AUTH failed");
}
}
if (u.db != 0) {
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "SELECT %d", u.db));
if (r == nullptr) {
redisFree(c);
throw std::runtime_error("redis SELECT: no reply");
}
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
redisFree(c);
throw std::runtime_error("redis SELECT failed");
}
}
return c;
}
}
RedisPool::RedisPool(std::string url, std::size_t n)
: url_(std::move(url)) {
if (n == 0) {
throw std::runtime_error("RedisPool: n must be > 0");
}
const RedisUrl parsed = parse_redis_url(url_);
all_.reserve(n);
idle_.reserve(n);
for (std::size_t i = 0; i < n; ++i) {
ConnPtr c(make_connection(parsed));
idle_.push_back(c.get());
all_.push_back(std::move(c));
}
}
RedisPool::~RedisPool() {
{
std::lock_guard<std::mutex> lk(mu_);
shutting_down_ = true;
}
cv_.notify_all();
}
RedisPool::Lease::Lease(RedisPool& pool)
: pool_(&pool), ctx_(pool.take()) {}
RedisPool::Lease::~Lease() {
if (pool_ != nullptr && ctx_ != nullptr) {
pool_->give_back(ctx_);
}
}
RedisPool::Lease::Lease(Lease&& other) noexcept
: pool_(other.pool_), ctx_(other.ctx_) {
other.pool_ = nullptr;
other.ctx_ = nullptr;
}
RedisPool::Lease RedisPool::acquire() {
return Lease(*this);
}
redisContext* RedisPool::take() {
std::unique_lock<std::mutex> lk(mu_);
if (!cv_.wait_for(lk, kAcquireTimeout,
[&]() { return shutting_down_ || !idle_.empty(); })) {
throw DependencyUnavailable("redis pool: acquire timeout");
}
if (shutting_down_) {
throw DependencyUnavailable("redis pool is shutting down");
}
redisContext* c = idle_.back();
idle_.pop_back();
lk.unlock();
bool alive = false;
if (!c->err) {
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
if (r != nullptr && r->type == REDIS_REPLY_STATUS) {
alive = true;
}
if (r != nullptr) freeReplyObject(r);
}
if (alive) {
return c;
}
ConnPtr fresh;
try {
fresh.reset(make_connection(parse_redis_url(url_)));
} catch (const std::exception& e) {
// Слот возвращается в пул, иначе после нескольких неудач пул опустеет навсегда.
give_back(c);
throw DependencyUnavailable(std::string("redis reconnect failed: ") + e.what());
}
std::lock_guard<std::mutex> lk2(mu_);
for (auto& up : all_) {
if (up.get() == c) {
up = std::move(fresh);
return up.get();
}
}
throw std::logic_error("RedisPool: connection not owned by pool");
}
void RedisPool::give_back(redisContext* c) {
{
std::lock_guard<std::mutex> lk(mu_);
idle_.push_back(c);
}
cv_.notify_one();
}
bool RedisPool::ping() {
try {
auto lease = acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
if (r == nullptr) return false;
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
return ok;
} catch (...) {
return false;
}
}
+59
View File
@@ -0,0 +1,59 @@
#pragma once
#include <hiredis/hiredis.h>
#include <condition_variable>
#include <cstddef>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
class RedisPool {
public:
RedisPool(std::string url, std::size_t n);
~RedisPool();
RedisPool(const RedisPool&) = delete;
RedisPool& operator=(const RedisPool&) = delete;
class Lease {
public:
explicit Lease(RedisPool& pool);
~Lease();
Lease(Lease&& other) noexcept;
Lease& operator=(Lease&&) = delete;
Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
redisContext* ctx() const { return ctx_; }
private:
RedisPool* pool_;
redisContext* ctx_;
};
Lease acquire();
bool ping();
private:
friend class Lease;
struct RedisDeleter {
void operator()(redisContext* c) const noexcept {
if (c != nullptr) ::redisFree(c);
}
};
using ConnPtr = std::unique_ptr<redisContext, RedisDeleter>;
redisContext* take();
void give_back(redisContext* c);
std::string url_;
std::vector<ConnPtr> all_;
std::vector<redisContext*> idle_;
std::mutex mu_;
std::condition_variable cv_;
bool shutting_down_ = false;
};
+254
View File
@@ -0,0 +1,254 @@
#include "s3_client.hpp"
#include <miniocpp/args.h>
#include <miniocpp/client.h>
#include <miniocpp/providers.h>
#include <miniocpp/request.h>
#include <miniocpp/response.h>
#include <cstdlib>
#include <fstream>
#include <stdexcept>
namespace {
std::string read_file_trim(const std::string& path) {
std::ifstream in(path);
if (!in) {
throw std::runtime_error("cannot open secret/file: " + path);
}
std::string s((std::istreambuf_iterator<char>(in)),
std::istreambuf_iterator<char>());
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) {
s.pop_back();
}
return s;
}
std::string env_or(const char* key, const char* fallback = "") {
const char* v = std::getenv(key);
return (v && *v) ? std::string(v) : std::string(fallback);
}
void parse_endpoint(const std::string& endpoint, std::string& host, unsigned& port,
bool& https) {
https = true;
host = endpoint;
port = 0;
if (host.rfind("http://", 0) == 0) {
https = false;
host = host.substr(7);
} else if (host.rfind("https://", 0) == 0) {
https = true;
host = host.substr(8);
}
// strip path
const auto slash = host.find('/');
if (slash != std::string::npos) {
host = host.substr(0, slash);
}
const auto colon = host.rfind(':');
if (colon != std::string::npos) {
port = static_cast<unsigned>(std::stoul(host.substr(colon + 1)));
host = host.substr(0, colon);
}
if (port == 0) {
port = https ? 443u : 80u;
}
}
} // namespace
struct S3Client::Impl {
minio::s3::BaseUrl base_url;
minio::s3::BaseUrl presign_url;
std::unique_ptr<minio::creds::StaticProvider> provider;
std::unique_ptr<minio::s3::Client> client;
std::unique_ptr<minio::s3::Client> presign_client; // null → use client
};
S3Client::Config S3Client::from_env() {
Config c;
// Ops endpoint (Stat/Get/Put/Delete). Presign may use S3_PRESIGN_ENDPOINT
// (read in ctor) so signed Host matches the client-reachable address.
c.endpoint = env_or("S3_ENDPOINT", "http://minio:9000");
c.region = env_or("S3_REGION", "us-east-1");
c.bucket = env_or("S3_BUCKET");
if (c.bucket.empty()) {
throw std::runtime_error("S3_BUCKET is required");
}
const std::string key_file = env_or("S3_ACCESS_KEY_ID_FILE");
const std::string secret_file = env_or("S3_SECRET_ACCESS_KEY_FILE");
if (key_file.empty() || secret_file.empty()) {
throw std::runtime_error(
"S3_ACCESS_KEY_ID_FILE and S3_SECRET_ACCESS_KEY_FILE are required");
}
c.access_key = read_file_trim(key_file);
c.secret_key = read_file_trim(secret_file);
if (c.access_key.empty() || c.secret_key.empty()) {
throw std::runtime_error("S3 credentials files are empty");
}
return c;
}
S3Client::S3Client(Config cfg) : cfg_(std::move(cfg)), impl_(std::make_unique<Impl>()) {
if (cfg_.endpoint.empty() || cfg_.bucket.empty() ||
cfg_.access_key.empty() || cfg_.secret_key.empty()) {
throw std::runtime_error("S3Client: incomplete config");
}
std::string host;
unsigned port = 0;
bool https = false;
parse_endpoint(cfg_.endpoint, host, port, https);
impl_->base_url = minio::s3::BaseUrl(host, https, cfg_.region);
impl_->base_url.port = port;
// MinIO requires path-style addressing.
impl_->base_url.virtual_style = false;
impl_->provider = std::make_unique<minio::creds::StaticProvider>(
cfg_.access_key, cfg_.secret_key);
impl_->client =
std::make_unique<minio::s3::Client>(impl_->base_url, impl_->provider.get());
// Separate BaseUrl for presigning when S3_PRESIGN_ENDPOINT differs.
const std::string presign_ep = env_or("S3_PRESIGN_ENDPOINT");
if (!presign_ep.empty() && presign_ep != cfg_.endpoint) {
std::string ph;
unsigned pp = 0;
bool phttps = false;
parse_endpoint(presign_ep, ph, pp, phttps);
impl_->presign_url = minio::s3::BaseUrl(ph, phttps, cfg_.region);
impl_->presign_url.port = pp;
impl_->presign_url.virtual_style = false;
impl_->presign_client = std::make_unique<minio::s3::Client>(
impl_->presign_url, impl_->provider.get());
}
}
S3Client::~S3Client() = default;
void S3Client::put_object(const std::string& key,
std::istream& body,
std::int64_t size_bytes,
const std::string& content_type) {
if (size_bytes < 0) {
throw std::invalid_argument("put_object: negative size");
}
// part_size=0 → library chooses multipart sizing; streams from `body`.
minio::s3::PutObjectArgs args(body, static_cast<long>(size_bytes), 0);
args.bucket = cfg_.bucket;
args.object = key;
if (!content_type.empty()) {
args.content_type = content_type;
}
minio::s3::PutObjectResponse resp = impl_->client->PutObject(args);
if (!resp) {
throw std::runtime_error("S3 PutObject failed: " + resp.Error().String());
}
}
void S3Client::get_object(const std::string& key, std::ostream& out) {
get_object(key, [&](const char* data, std::size_t n) {
out.write(data, static_cast<std::streamsize>(n));
if (!out) {
throw std::runtime_error("get_object: write failed");
}
});
}
std::int64_t S3Client::get_object(
const std::string& key,
const std::function<void(const char*, std::size_t)>& on_chunk) {
std::int64_t total = 0;
minio::s3::GetObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
args.datafunc = [&](minio::http::DataFunctionArgs da) -> bool {
on_chunk(da.datachunk.data(), da.datachunk.size());
total += static_cast<std::int64_t>(da.datachunk.size());
return true;
};
minio::s3::GetObjectResponse resp = impl_->client->GetObject(args);
if (!resp) {
throw std::runtime_error("S3 GetObject failed: " + resp.Error().String());
}
return total;
}
void S3Client::copy_object(const std::string& src_key,
const std::string& dst_key) {
if (src_key.empty() || dst_key.empty() || src_key == dst_key) {
throw std::invalid_argument("copy_object: bad keys");
}
minio::s3::CopyObjectArgs args;
args.bucket = cfg_.bucket;
args.object = dst_key;
args.source.bucket = cfg_.bucket;
args.source.object = src_key;
minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args);
if (!resp) {
throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String());
}
}
void S3Client::delete_object(const std::string& key) {
minio::s3::RemoveObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
minio::s3::RemoveObjectResponse resp = impl_->client->RemoveObject(args);
if (!resp) {
throw std::runtime_error("S3 DeleteObject failed: " + resp.Error().String());
}
}
bool S3Client::stat_object(const std::string& key, std::int64_t& size_bytes,
std::string& etag) {
minio::s3::StatObjectArgs args;
args.bucket = cfg_.bucket;
args.object = key;
minio::s3::StatObjectResponse resp = impl_->client->StatObject(args);
if (!resp) {
// NoSuchKey / Not Found → absent
const std::string err = resp.Error().String();
if (resp.code == "NoSuchKey" || resp.status_code == 404) {
return false;
}
throw std::runtime_error("S3 StatObject failed: " + err);
}
size_bytes = static_cast<std::int64_t>(resp.size);
etag = resp.etag;
return true;
}
std::string S3Client::presign_put(const std::string& key,
unsigned expiry_seconds,
const std::string& content_type) {
if (expiry_seconds < 1 || expiry_seconds > 7 * 24 * 3600u) {
throw std::invalid_argument("presign_put: bad expiry");
}
minio::s3::Client* c =
impl_->presign_client ? impl_->presign_client.get() : impl_->client.get();
// Note: minio-cpp query-presign signs method/host/path/query only — it does
// not bind Content-Type into the signature. Clients should still send
// Content-Type so the object metadata is correct; complete() re-validates
// magic/sha256 against the session.
(void)content_type;
minio::s3::GetPresignedObjectUrlArgs args;
args.bucket = cfg_.bucket;
args.object = key;
args.method = minio::http::Method::kPut;
args.expiry_seconds = expiry_seconds;
minio::s3::GetPresignedObjectUrlResponse resp = c->GetPresignedObjectUrl(args);
if (!resp) {
throw std::runtime_error("S3 PresignPut failed: " + resp.Error().String());
}
return resp.url;
}
+66
View File
@@ -0,0 +1,66 @@
#pragma once
#include <cstdint>
#include <functional>
#include <istream>
#include <memory>
#include <ostream>
#include <string>
// Thin wrapper over AWS SDK for C++ (S3 only). Credentials and SigV4 come from
// the SDK — no hand-rolled signing.
class S3Client {
public:
struct Config {
std::string endpoint; // e.g. http://minio:9000
std::string region; // e.g. us-east-1
std::string bucket;
std::string access_key;
std::string secret_key;
};
explicit S3Client(Config cfg);
~S3Client();
S3Client(const S3Client&) = delete;
S3Client& operator=(const S3Client&) = delete;
static Config from_env();
const std::string& bucket() const { return cfg_.bucket; }
// Stream upload. `body` must yield exactly `size_bytes` bytes.
void put_object(const std::string& key,
std::istream& body,
std::int64_t size_bytes,
const std::string& content_type);
// Stream download into `out` (chunked reads from SDK body stream).
void get_object(const std::string& key, std::ostream& out);
// Stream download with a consumer callback (chunk, size). Returns total bytes.
std::int64_t get_object(const std::string& key,
const std::function<void(const char*, std::size_t)>& on_chunk);
void delete_object(const std::string& key);
// Server-side copy inside the bucket (complete: staging → final).
void copy_object(const std::string& src_key, const std::string& dst_key);
// HeadObject-style metadata. Returns false if the object does not exist.
bool stat_object(const std::string& key, std::int64_t& size_bytes,
std::string& etag);
// Presigned PUT URL via minio-cpp (SigV4). Optional required Content-Type
// is bound into the signature when non-empty.
// expiry_seconds: typically 60–600 for Track 1.
std::string presign_put(const std::string& key,
unsigned expiry_seconds,
const std::string& content_type);
private:
struct Impl;
Config cfg_;
std::unique_ptr<Impl> impl_;
};
+87
View File
@@ -0,0 +1,87 @@
#include "session_repo.hpp"
#include "errors.hpp"
#include <sodium.h>
#include <ctime>
#include <stdexcept>
#include <string>
#include <vector>
namespace {
std::string random_token_hex() {
unsigned char buf[32];
randombytes_buf(buf, sizeof(buf));
static const char hex[] = "0123456789abcdef";
std::string out;
out.resize(sizeof(buf) * 2);
for (std::size_t i = 0; i < sizeof(buf); ++i) {
out[i * 2] = hex[buf[i] >> 4];
out[i * 2 + 1] = hex[buf[i] & 0x0f];
}
return out;
}
std::int64_t now_unix() {
return static_cast<std::int64_t>(std::time(nullptr));
}
redisReply* check_reply(redisContext* c, redisReply* r) {
if (r == nullptr) {
const std::string err = c->err ? c->errstr : "no reply";
throw DependencyUnavailable("redis command failed: " + err);
}
return r;
}
}
ISessionRepo::Created SessionRepo::create(const std::string& user_id) {
const std::string token = random_token_hex();
const std::int64_t exp = now_unix() + ttl_;
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "SET session:%s %s EX %d",
token.c_str(), user_id.c_str(), ttl_));
check_reply(c, r);
const bool ok = (r->type == REDIS_REPLY_STATUS);
freeReplyObject(r);
if (!ok) {
throw std::runtime_error("redis SET returned non-status");
}
return Created{token, exp};
}
std::optional<std::string> SessionRepo::lookup(const std::string& token) {
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "GET session:%s", token.c_str()));
check_reply(c, r);
std::optional<std::string> out;
if (r->type == REDIS_REPLY_STRING) {
out = std::string(r->str, r->len);
}
freeReplyObject(r);
return out;
}
bool SessionRepo::revoke(const std::string& token) {
auto lease = pool_.acquire();
redisContext* c = lease.ctx();
redisReply* r = static_cast<redisReply*>(
redisCommand(c, "DEL session:%s", token.c_str()));
check_reply(c, r);
const bool existed = (r->type == REDIS_REPLY_INTEGER && r->integer == 1);
freeReplyObject(r);
return existed;
}
+18
View File
@@ -0,0 +1,18 @@
#pragma once
#include "redis_pool.hpp"
#include "session_repo_base.hpp"
class SessionRepo : public ISessionRepo {
public:
SessionRepo(RedisPool& pool, int ttl_seconds)
: pool_(pool), ttl_(ttl_seconds) {}
Created create(const std::string& user_id) override;
std::optional<std::string> lookup(const std::string& token) override;
bool revoke(const std::string& token) override;
private:
RedisPool& pool_;
int ttl_;
};
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string>
class ISessionRepo {
public:
virtual ~ISessionRepo() = default;
struct Created {
std::string token;
std::int64_t expires_at; // unix-время в секундах UTC
};
virtual Created create(const std::string& user_id) = 0;
virtual std::optional<std::string> lookup(const std::string& token) = 0;
virtual bool revoke(const std::string& token) = 0;
};
+100
View File
@@ -0,0 +1,100 @@
#include "user_repo.hpp"
#include <pqxx/pqxx>
#include <chrono>
#include <ctime>
#include <iomanip>
#include <random>
#include <sstream>
#include <string>
namespace {
std::string make_uuid() {
static std::mt19937_64 rng{std::random_device{}()};
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
std::ostringstream out;
out << std::hex << std::setfill('0');
out << std::setw(8) << d32(rng);
out << '-' << std::setw(4) << d16(rng);
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
return out.str();
}
std::string epoch_to_rfc3339(long long epoch) {
const std::time_t t = static_cast<std::time_t>(epoch);
std::tm tm{};
gmtime_r(&t, &tm);
std::ostringstream out;
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
return out.str();
}
std::string now_epoch_str() {
const auto now = std::chrono::system_clock::now();
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
now.time_since_epoch()).count();
return std::to_string(secs);
}
User row_to_user(const pqxx::row& r) {
User u;
u.id = r["id"].as<std::string>();
u.username = r["username"].as<std::string>();
u.display_name = r["display_name"].as<std::string>();
u.password_hash = r["password_hash"].as<std::string>();
u.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
return u;
}
constexpr const char* kUserColumns =
"id, username, display_name, password_hash, "
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
}
std::optional<User> UserRepo::create(const std::string& username,
const std::string& display_name,
const std::string& password_hash) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
try {
const std::string id = make_uuid();
auto r = tx.exec(
"INSERT INTO users (id, username, display_name, password_hash, created_at) "
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
"RETURNING " + std::string(kUserColumns),
pqxx::params{id, username, display_name, password_hash, now_epoch_str()});
tx.commit();
return row_to_user(r[0]);
} catch (const pqxx::unique_violation&) {
tx.abort();
return std::nullopt;
}
}
std::optional<User> UserRepo::find_by_id(const std::string& id) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kUserColumns) + " FROM users WHERE id = $1",
pqxx::params{id});
tx.commit();
if (r.empty()) return std::nullopt;
return row_to_user(r[0]);
}
std::optional<User> UserRepo::find_by_username(const std::string& username) {
auto lease = pool_.acquire();
pqxx::work tx(*lease);
auto r = tx.exec(
"SELECT " + std::string(kUserColumns) + " FROM users WHERE username = $1",
pqxx::params{username});
tx.commit();
if (r.empty()) return std::nullopt;
return row_to_user(r[0]);
}
+22
View File
@@ -0,0 +1,22 @@
#pragma once
#include "db.hpp"
#include "models.hpp"
#include <optional>
#include <string>
class UserRepo {
public:
explicit UserRepo(PgPool& pool) : pool_(pool) {}
std::optional<User> create(const std::string& username,
const std::string& display_name,
const std::string& password_hash);
std::optional<User> find_by_id(const std::string& id);
std::optional<User> find_by_username(const std::string& username);
private:
PgPool& pool_;
};
+100
View File
@@ -0,0 +1,100 @@
#!/bin/sh
# Shared helpers for student-tests/*.sh (curl + jq only).
# Usage: . "$(dirname "$0")/lib.sh"
set -eu
BASE_URL="${BASE_URL:-http://127.0.0.1:8080}"
PASS="${STUDENT_TEST_PASSWORD:-Student-test-password-1}"
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
FIXTURES="$ROOT/tools/fixtures"
die() {
printf '%s\n' "$*" >&2
exit 1
}
need() {
command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"
}
need curl
need jq
need sha256sum
need python3
uniq_name() {
printf '%s_%s' "$1" "$(python3 -c 'import uuid; print(uuid.uuid4().hex[:12])')"
}
# http_json METHOD PATH [curl args...]
# Sets: HTTP_CODE, HTTP_BODY, HTTP_HDRS
http_json() {
method=$1
path=$2
shift 2
hdr=$(mktemp)
body=$(mktemp)
code=$(curl -sS -m 60 -o "$body" -D "$hdr" -w '%{http_code}' \
-X "$method" "$BASE_URL$path" "$@") || {
rm -f "$hdr" "$body"
die "curl failed: $method $path"
}
HTTP_CODE=$code
HTTP_BODY=$(cat "$body")
HTTP_HDRS=$(cat "$hdr")
rm -f "$hdr" "$body"
}
expect_code() {
want=$1
[ "$HTTP_CODE" = "$want" ] || die "expected HTTP $want got $HTTP_CODE body=$HTTP_BODY"
}
register_user() {
label=$1
user=$(uniq_name "$label")
http_json POST /api/v1/users \
-H 'Content-Type: application/json' \
-d "{\"username\":\"$user\",\"display_name\":\"$label\",\"password\":\"$PASS\"}"
expect_code 201
USER_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
USERNAME=$user
cred=$(printf '%s' "$USERNAME:$PASS" | python3 -c 'import sys,base64; print(base64.b64encode(sys.stdin.buffer.read()).decode())')
http_json POST /api/v1/auth/sessions -H "Authorization: Basic $cred"
expect_code 201
TOKEN=$(printf '%s' "$HTTP_BODY" | jq -r .token)
[ -n "$TOKEN" ] && [ "$TOKEN" != null ] || die "empty token"
}
create_chat() {
title=${1:-test}
member_json=$2
http_json POST /api/v1/chats \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"title\":\"$title\",\"member_ids\":$member_json}"
expect_code 201
CHAT_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
}
upload_file() {
chat_id=$1
file=$2
filename=$3
ctype=$4
http_json POST "/api/v1/chats/$chat_id/attachments?filename=$filename" \
-H "Authorization: Bearer $TOKEN" \
-H "Content-Type: $ctype" \
--data-binary @"$file"
}
sha256_file() {
sha256sum "$1" | awk '{print $1}'
}
assert_eq() {
[ "$1" = "$2" ] || die "assert_eq failed: '$1' != '$2'${3:+ ($3)}"
}
printf 'lib.sh: BASE_URL=%s\n' "$BASE_URL" >&2
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
# Run grade-3 attachment student tests (+ drain / compensation dry-run).
set -eu
DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
ROOT=$(CDPATH= cd -- "$DIR/.." && pwd)
for t in \
test_attachment_upload_ok.sh \
test_attachment_too_large.sh \
test_attachment_corrupted.sh \
test_attachment_mime_spoof.sh \
test_attachment_foreign.sh \
test_attachment_wrong_chat.sh \
test_graceful_drain.sh
do
echo "=== $t ==="
sh "$DIR/$t"
done
echo "=== tools/gc_orphans.sh (dry-run) ==="
sh "$ROOT/tools/gc_orphans.sh"
echo "ALL grade3 student-tests PASS"
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
ROOT=$(CDPATH= cd -- "$DIR/.." && pwd)
echo "=== tests/check_bucket_private.sh ==="
sh "$ROOT/tests/check_bucket_private.sh"
echo "=== test_attachment_limited_creds.sh ==="
sh "$DIR/test_attachment_limited_creds.sh"
# Load before persistence: restart cools cgroup RSS accounting and makes
# docker-stats delta noisy right after recreate.
echo "=== scripts/load_uploads.sh ==="
sh "$ROOT/scripts/load_uploads.sh"
echo "=== test_attachment_persistence.sh ==="
sh "$DIR/test_attachment_persistence.sh"
echo "ALL grade4 student-tests PASS"
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Grade 5 / Track 1: direct (presigned) upload + proxy still green.
set -eu
DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
echo "=== test_direct_upload_ok.sh ==="
sh "$DIR/test_direct_upload_ok.sh"
echo "=== test_presigned_complete_concurrent.sh ==="
sh "$DIR/test_presigned_complete_concurrent.sh"
echo "=== test_direct_upload_negatives.sh ==="
sh "$DIR/test_direct_upload_negatives.sh"
echo "=== tools/gc_direct_uploads.sh (dry-run) ==="
sh "$DIR/../tools/gc_direct_uploads.sh"
echo "ALL grade5 track1 student-tests PASS"
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
# Truncated / corrupt PNG → 415 (not 500).
set -eu
. "$(dirname "$0")/lib.sh"
register_user alice
create_chat 'corrupt' "[\"$USER_ID\"]"
bad=$(mktemp)
# PNG magic + IHDR-looking garbage, truncated — fails stbi decode.
printf '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x01\x00\x00\x00\x01\x00\x08\x02\x00\x00\x00' > "$bad"
upload_file "$CHAT_ID" "$bad" 'broken.png' 'image/png'
rm -f "$bad"
expect_code 415
code=$(printf '%s' "$HTTP_BODY" | jq -r .error.code)
assert_eq "$code" unsupported_media_type error_code
printf 'PASS test_attachment_corrupted\n'
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
# Non-member → 404 on GET metadata and /content (no leak).
set -eu
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/tiny.png"
[ -f "$png" ] || die "missing fixture $png"
register_user alice
ALICE_ID=$USER_ID
ALICE_TOK=$TOKEN
register_user eve
EVE_TOK=$TOKEN
TOKEN=$ALICE_TOK
create_chat 'foreign' "[\"$ALICE_ID\"]"
upload_file "$CHAT_ID" "$png" 'tiny.png' 'image/png'
expect_code 201
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
http_json GET "/api/v1/attachments/$AID" -H "Authorization: Bearer $EVE_TOK"
expect_code 404
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_found meta
http_json GET "/api/v1/attachments/$AID/content" -H "Authorization: Bearer $EVE_TOK"
expect_code 404
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_found content
http_json GET "/api/v1/attachments/$AID"
expect_code 401
printf 'PASS test_attachment_foreign\n'
+55
View File
@@ -0,0 +1,55 @@
#!/bin/sh
# Grade 4: API uses limited S3 user via *_FILE secrets, not MinIO root.
set -eu
. "$(dirname "$0")/lib.sh"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
# API process env must expose limited-key secret paths, never root password vars.
env_dump=$($COMPOSE exec -T api sh -c 'env' | sort)
printf '%s\n' "$env_dump" | grep -q '^S3_ACCESS_KEY_ID_FILE=' \
|| die "API missing S3_ACCESS_KEY_ID_FILE"
printf '%s\n' "$env_dump" | grep -q '^S3_SECRET_ACCESS_KEY_FILE=' \
|| die "API missing S3_SECRET_ACCESS_KEY_FILE"
# Root MinIO credentials must not be injected into the API container.
if printf '%s\n' "$env_dump" | grep -E '^(MINIO_ROOT_USER|MINIO_ROOT_PASSWORD)=' >/dev/null; then
die "API container has MinIO root credentials in env"
fi
# Secret file contents must differ from root user (compare access key id only).
api_key=$($COMPOSE exec -T api sh -c 'cat "$S3_ACCESS_KEY_ID_FILE"' | tr -d '\r\n')
root_user=$(grep -E '^MINIO_ROOT_USER=' "$ROOT/.env" | head -n1 | cut -d= -f2- | tr -d '\r\n"')
[ -n "$api_key" ] || die "empty limited access key"
[ -n "$root_user" ] || die "MINIO_ROOT_USER missing in .env"
[ "$api_key" != "$root_user" ] || die "API S3 access key equals MinIO root user"
# Limited user cannot delete the bucket (admin action) — policy is object-scoped.
# Probe via mc with the limited credentials from secrets (same values as API).
lim_key=$api_key
lim_secret=$($COMPOSE exec -T api sh -c 'cat "$S3_SECRET_ACCESS_KEY_FILE"' | tr -d '\r\n')
bucket=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
out=$($COMPOSE run --rm --no-deps \
-e LIM_KEY="$lim_key" -e LIM_SECRET="$lim_secret" -e S3_BUCKET="$bucket" \
--entrypoint /bin/sh minio-init -c '
set +e
mc alias set lim "$S3_ENDPOINT" "$LIM_KEY" "$LIM_SECRET" >/dev/null
mc rb --force "lim/$S3_BUCKET" 2>/tmp/rb.err
rc=$?
cat /tmp/rb.err
exit $rc
' 2>&1) || true
printf '%s\n' "$out" | grep -qiE 'AccessDenied|denied|not allowed|InvalidAccessKeyId|error' \
|| die "expected AccessDenied on bucket delete by limited user; out=$out"
# Positive: limited user can still PutObject (smoke via API upload).
png="$FIXTURES/tiny.png"
register_user alice
create_chat 'creds' "[\"$USER_ID\"]"
upload_file "$CHAT_ID" "$png" 'tiny.png' 'image/png'
expect_code 201
printf 'PASS test_attachment_limited_creds\n'
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
# JPEG bytes with Content-Type: image/png → 415 (content wins over header).
set -eu
. "$(dirname "$0")/lib.sh"
jpg="$FIXTURES/tiny.jpg"
[ -f "$jpg" ] || die "missing fixture $jpg"
register_user alice
create_chat 'mime-spoof' "[\"$USER_ID\"]"
upload_file "$CHAT_ID" "$jpg" 'spoof.png' 'image/png'
expect_code 415
code=$(printf '%s' "$HTTP_BODY" | jq -r .error.code)
assert_eq "$code" unsupported_media_type error_code
printf 'PASS test_attachment_mime_spoof\n'
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
# Grade 4: upload → restart API replicas (volumes kept) → download same bytes.
set -eu
. "$(dirname "$0")/lib.sh"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
png="$FIXTURES/320x200.png"
[ -f "$png" ] || die "missing $png"
want_sha=$(sha256_file "$png")
register_user alice
ALICE_ID=$USER_ID
create_chat 'persist' "[\"$ALICE_ID\"]"
upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png'
expect_code 201
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" upload_sha
# Restart only API (and LB); keep pgdata + miniodata.
$COMPOSE restart api
# Wait for both replicas healthy + LB ready.
i=0
while [ "$i" -lt 60 ]; do
http_json GET /health/ready || true
if [ "${HTTP_CODE:-}" = "200" ]; then
break
fi
i=$((i + 1))
sleep 1
done
[ "${HTTP_CODE:-}" = "200" ] || die "stack not ready after api restart"
# Session may still be in Redis (AOF); reuse token.
http_json GET "/api/v1/attachments/$AID" -H "Authorization: Bearer $TOKEN"
expect_code 200
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" meta_sha
out=$(mktemp)
code=$(curl -sS -m 60 -o "$out" -w '%{http_code}' \
-H "Authorization: Bearer $TOKEN" \
"$BASE_URL/api/v1/attachments/$AID/content")
assert_eq "$code" 200 content_status
assert_eq "$(sha256_file "$out")" "$want_sha" content_sha
rm -f "$out"
printf 'PASS test_attachment_persistence\n'
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Grade 4 criteria (negative): 11 MiB body → 413 payload_too_large (not 500).
set -eu
. "$(dirname "$0")/lib.sh"
register_user alice
create_chat 'too-large' "[\"$USER_ID\"]"
big=$(mktemp)
# Valid PNG magic + padding to 11 MiB so Content-Length triggers early reject.
dd if=/dev/zero of="$big" bs=1M count=11 status=none
# Prepend PNG signature so it's not rejected as empty before size check.
printf '\x89PNG\r\n\x1a\n' | dd of="$big" conv=notrunc status=none 2>/dev/null || true
upload_file "$CHAT_ID" "$big" 'huge.png' 'image/png'
rm -f "$big"
expect_code 413
code=$(printf '%s' "$HTTP_BODY" | jq -r .error.code)
assert_eq "$code" payload_too_large error_code
printf 'PASS test_attachment_too_large\n'
+48
View File
@@ -0,0 +1,48 @@
#!/bin/sh
# Grade 3: PNG 320×200 → 201 ready → download bytes match sha256; message with attachment_ids.
set -eu
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/320x200.png"
[ -f "$png" ] || die "missing fixture $png"
want_sha=$(sha256_file "$png")
want_size=$(wc -c < "$png" | tr -d ' ')
register_user alice
ALICE_ID=$USER_ID
ALICE_TOK=$TOKEN
register_user bob
BOB_ID=$USER_ID
BOB_TOK=$TOKEN
TOKEN=$ALICE_TOK
create_chat 'upload-ok' "[\"$ALICE_ID\",\"$BOB_ID\"]"
upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png'
expect_code 201
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" ready status
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .content_type)" image/png ctype
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" sha256
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .size_bytes)" "$want_size" size
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error_code)" null error_code
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -c .variants)" '[]' variants
http_json POST "/api/v1/chats/$CHAT_ID/messages" \
-H "Authorization: Bearer $ALICE_TOK" \
-H 'Content-Type: application/json' \
-H "Idempotency-Key: $(uniq_name idem)" \
-d "{\"text\":\"\",\"attachment_ids\":[\"$AID\"]}"
expect_code 201
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -c .attachment_ids)" "[\"$AID\"]" msg_atts
out=$(mktemp)
curl -sS -m 60 -o "$out" -w '%{http_code}' \
-H "Authorization: Bearer $BOB_TOK" \
"$BASE_URL/api/v1/attachments/$AID/content" > /tmp/att_dl.code
assert_eq "$(cat /tmp/att_dl.code)" 200 download_status
got_sha=$(sha256_file "$out")
assert_eq "$got_sha" "$want_sha" download_sha256
rm -f "$out" /tmp/att_dl.code
printf 'PASS test_attachment_upload_ok\n'
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
# attachment_id from another chat in POST messages → 404.
set -eu
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/tiny.png"
[ -f "$png" ] || die "missing fixture $png"
register_user alice
ALICE_ID=$USER_ID
create_chat 'chat-a' "[\"$ALICE_ID\"]"
CHAT_A=$CHAT_ID
upload_file "$CHAT_A" "$png" 'tiny.png' 'image/png'
expect_code 201
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
create_chat 'chat-b' "[\"$ALICE_ID\"]"
CHAT_B=$CHAT_ID
http_json POST "/api/v1/chats/$CHAT_B/messages" \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' \
-H "Idempotency-Key: $(uniq_name idem)" \
-d "{\"text\":\"hi\",\"attachment_ids\":[\"$AID\"]}"
expect_code 404
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_found
# Empty/whitespace text without attachments → 400
http_json POST "/api/v1/chats/$CHAT_B/messages" \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' \
-H "Idempotency-Key: $(uniq_name idem)" \
-d '{"text":" "}'
expect_code 400
printf 'PASS test_attachment_wrong_chat\n'
+103
View File
@@ -0,0 +1,103 @@
#!/bin/sh
# Track 1 negatives: bad checksum complete, abort, foreign complete, no PUT.
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/320x200.png"
[ -f "$png" ] || die "missing fixture $png"
want_sha=$(sha256_file "$png")
want_size=$(wc -c < "$png" | tr -d ' ')
bad_sha=$(printf '%s' "$want_sha" | tr 'a-f0-9' '0-9a-f' | head -c 64)
# ensure different
[ "$bad_sha" != "$want_sha" ] || bad_sha="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
register_user alice
ALICE_TOKEN=$TOKEN
ALICE_ID=$USER_ID
create_chat "du-neg" "[\"$ALICE_ID\"]"
register_user bob
BOB_TOKEN=$TOKEN
TOKEN=$ALICE_TOKEN
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
put_via_net() {
url=$1
ct=$2
file=$3
code=$($COMPOSE run --rm --no-deps \
-v "$file:/tmp/up.bin:ro" \
--entrypoint curl api \
-sS -m 60 -o /dev/null -w '%{http_code}' \
-X PUT -H "Content-Type: $ct" --data-binary @/tmp/up.bin \
"$url")
printf '%s' "$code"
}
# --- complete without PUT ---
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"a.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
expect_code 201
ID1=$(printf '%s' "$HTTP_BODY" | jq -r .id)
http_json POST "/api/v1/direct-uploads/$ID1/complete" \
-H "Authorization: Bearer $ALICE_TOKEN"
expect_code 409
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_ready
# --- abort ---
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"b.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
expect_code 201
ID2=$(printf '%s' "$HTTP_BODY" | jq -r .id)
URL2=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
CT2=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
put_code=$(put_via_net "$URL2" "$CT2" "$png")
[ "$put_code" = "200" ] || die "PUT abort-case got $put_code"
http_json DELETE "/api/v1/direct-uploads/$ID2" \
-H "Authorization: Bearer $ALICE_TOKEN"
expect_code 204
http_json POST "/api/v1/direct-uploads/$ID2/complete" \
-H "Authorization: Bearer $ALICE_TOKEN"
expect_code 409
# --- foreign owner ---
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"c.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
expect_code 201
ID3=$(printf '%s' "$HTTP_BODY" | jq -r .id)
URL3=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
CT3=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
put_code=$(put_via_net "$URL3" "$CT3" "$png")
[ "$put_code" = "200" ] || die "PUT foreign-case got $put_code"
http_json POST "/api/v1/direct-uploads/$ID3/complete" \
-H "Authorization: Bearer $BOB_TOKEN"
expect_code 404
# --- checksum mismatch (declared sha ≠ bytes) ---
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"d.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$bad_sha\"}"
expect_code 201
ID4=$(printf '%s' "$HTTP_BODY" | jq -r .id)
URL4=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
CT4=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
put_code=$(put_via_net "$URL4" "$CT4" "$png")
[ "$put_code" = "200" ] || die "PUT checksum-case got $put_code"
http_json POST "/api/v1/direct-uploads/$ID4/complete" \
-H "Authorization: Bearer $ALICE_TOKEN"
expect_code 409
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" hash_mismatch
# Proxy path still works
upload_file "$CHAT_ID" "$png" "proxy.png" "image/png"
expect_code 201
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" "ready" proxy_still_ok
echo "PASS test_direct_upload_negatives"
+58
View File
@@ -0,0 +1,58 @@
#!/bin/sh
# Track 1: initiate → PUT to MinIO (compose network) → complete → download.
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/320x200.png"
[ -f "$png" ] || die "missing fixture $png"
want_sha=$(sha256_file "$png")
want_size=$(wc -c < "$png" | tr -d ' ')
register_user alice
ALICE_TOKEN=$TOKEN
ALICE_ID=$USER_ID
create_chat "du-ok" "[\"$ALICE_ID\"]"
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"photo.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
expect_code 201
UPLOAD_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
UPLOAD_URL=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
CT=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
[ -n "$UPLOAD_ID" ] && [ "$UPLOAD_ID" != null ] || die "empty upload id"
[ -n "$UPLOAD_URL" ] && [ "$UPLOAD_URL" != null ] || die "empty upload_url"
assert_eq "$CT" "image/png" content_type_header
# PUT bytes on the compose network (MinIO ports stay unpublished by default).
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
put_code=$($COMPOSE run --rm --no-deps \
-v "$png:/tmp/up.bin:ro" \
--entrypoint curl api \
-sS -m 60 -o /dev/null -w '%{http_code}' \
-X PUT -H "Content-Type: $CT" --data-binary @/tmp/up.bin \
"$UPLOAD_URL")
[ "$put_code" = "200" ] || die "presigned PUT expected 200 got $put_code"
http_json POST "/api/v1/direct-uploads/$UPLOAD_ID/complete" \
-H "Authorization: Bearer $ALICE_TOKEN"
expect_code 201
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" "ready" status
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" sha256
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .id)" "$UPLOAD_ID" attachment_id
out=$(mktemp)
replay_code=$($COMPOSE run --rm --no-deps \
--entrypoint curl api \
-sS -m 60 -o /dev/null -w '%{http_code}' \
-X PUT -H "Content-Type: $CT" --data-binary 'replay-content-12345' \
"$UPLOAD_URL")
[ "$replay_code" = "200" ] || die "replay PUT expected 200 got $replay_code"
curl -sS -m 60 -o "$out" -H "Authorization: Bearer $ALICE_TOKEN" \
"$BASE_URL/api/v1/attachments/$UPLOAD_ID/content"
assert_eq "$(sha256_file "$out")" "$want_sha" download_sha256
rm -f "$out"
echo "PASS test_direct_upload_ok"
@@ -0,0 +1,59 @@
#!/bin/sh
# A missing S3 object must fail before the content response headers are sent.
set -eu
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/320x200.png"
[ -f "$png" ] || die "missing fixture $png"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
response_body=$(mktemp)
object_deleted=0
restore_object() {
action=$1
$COMPOSE run --rm --no-deps -T \
-e "CHAT_ID=$CHAT_ID" -e "ATTACHMENT_ID=$AID" \
-e "OBJECT_ACTION=$action" \
-v "$png:/tmp/restore.png:ro" \
--entrypoint /bin/sh minio-init -c '
set -eu
mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
object="local/$S3_BUCKET/attachments/$CHAT_ID/$ATTACHMENT_ID"
if [ "$OBJECT_ACTION" = delete ]; then
mc rm --force "$object"
else
mc cp /tmp/restore.png "$object"
fi
'
}
cleanup() {
if [ "$object_deleted" = 1 ]; then
restore_object restore || printf 'WARNING: failed to restore test object\n' >&2
fi
rm -f "$response_body"
}
trap cleanup 0 HUP INT TERM
want_sha=$(sha256_file "$png")
TOKEN=""
register_user alice
create_chat 'missing-object' "[\"$USER_ID\"]"
upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png'
expect_code 201
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" metadata_sha
restore_object delete
object_deleted=1
code=$(curl -sS -m 60 -o "$response_body" -w '%{http_code}' \
-H "Authorization: Bearer $TOKEN" \
"$BASE_URL/api/v1/attachments/$AID/content")
case "$code" in
404|5??) ;;
*) die "expected 404/5xx for missing object, got HTTP $code body=$(cat "$response_body")" ;;
esac
restore_object restore
object_deleted=0
printf 'PASS test_download_missing_object\n'
+50
View File
@@ -0,0 +1,50 @@
#!/bin/sh
# Graceful shutdown: SIGTERM → /health/ready becomes 503 while process drains.
# Targets one API container; restores it at the end.
set -eu
. "$(dirname "$0")/lib.sh"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
cid=$($COMPOSE ps -q api | head -n1)
[ -n "$cid" ] || die "no api container"
# Confirm ready before drain.
http_json GET /health/ready
expect_code 200
# docker stop → SIGTERM to PID1 (messenger after entrypoint exec).
# Probe the draining instance directly (LB may still hit the other replica).
docker stop -t 15 "$cid" >/dev/null &
stop_pid=$!
saw_drain=0
i=0
while [ "$i" -lt 40 ]; do
code=$(docker exec "$cid" curl -sS -m 2 -o /dev/null -w '%{http_code}' \
http://127.0.0.1:8080/health/ready 2>/dev/null || printf '000')
if [ "$code" = "503" ] || [ "$code" = "000" ]; then
saw_drain=1
break
fi
i=$((i + 1))
sleep 0.25
done
wait "$stop_pid" || true
[ "$saw_drain" = "1" ] || die "did not observe drain/stop (last code=${code:-none})"
# Restore replicas.
$COMPOSE up -d --no-deps api >/dev/null
j=0
HTTP_CODE=000
while [ "$j" -lt 40 ]; do
http_json GET /health/ready || true
if [ "${HTTP_CODE:-}" = "200" ]; then
break
fi
j=$((j + 1))
sleep 1
done
[ "${HTTP_CODE:-}" = "200" ] || die "stand not ready after recreate"
printf 'PASS test_graceful_drain\n'
@@ -0,0 +1,73 @@
#!/bin/sh
# Track 1: concurrent complete calls publish one intact attachment.
. "$(dirname "$0")/lib.sh"
png="$FIXTURES/320x200.png"
[ -f "$png" ] || die "missing fixture $png"
want_sha=$(sha256_file "$png")
want_size=$(wc -c < "$png" | tr -d ' ')
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' 0 HUP INT TERM
register_user alice
ALICE_TOKEN=$TOKEN
ALICE_ID=$USER_ID
create_chat "du-concurrent" "[\"$ALICE_ID\"]"
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"concurrent.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
expect_code 201
UPLOAD_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
UPLOAD_URL=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
CT=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
[ -n "$UPLOAD_ID" ] && [ "$UPLOAD_ID" != null ] || die "empty upload id"
[ -n "$UPLOAD_URL" ] && [ "$UPLOAD_URL" != null ] || die "empty upload_url"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
put_code=$($COMPOSE run --rm --no-deps \
-v "$png:/tmp/up.bin:ro" \
--entrypoint curl api \
-sS -m 60 -o /dev/null -w '%{http_code}' \
-X PUT -H "Content-Type: $CT" --data-binary @/tmp/up.bin \
"$UPLOAD_URL")
[ "$put_code" = "200" ] || die "presigned PUT expected 200 got $put_code"
pids=
for i in 1 2 3 4; do
curl -sS -m 60 -o "$tmpdir/body.$i" -w '%{http_code}' \
-X POST \
-H "Authorization: Bearer $ALICE_TOKEN" \
"$BASE_URL/api/v1/direct-uploads/$UPLOAD_ID/complete" \
> "$tmpdir/code.$i" &
pids="$pids $!"
done
for pid in $pids; do
wait "$pid" || die "complete request failed"
done
count_201=0
for i in 1 2 3 4; do
code=$(cat "$tmpdir/code.$i")
case "$code" in
201) count_201=$((count_201 + 1)) ;;
200|409) ;;
*) die "unexpected concurrent complete HTTP $code: $(cat "$tmpdir/body.$i")" ;;
esac
done
assert_eq "$count_201" 1 created_count
http_json GET "/api/v1/attachments/$UPLOAD_ID" \
-H "Authorization: Bearer $ALICE_TOKEN"
expect_code 200
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" ready metadata_status
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" metadata_sha
code=$(curl -sS -m 60 -o "$tmpdir/content" -w '%{http_code}' \
-H "Authorization: Bearer $ALICE_TOKEN" \
"$BASE_URL/api/v1/attachments/$UPLOAD_ID/content")
assert_eq "$code" 200 content_status
assert_eq "$(sha256_file "$tmpdir/content")" "$want_sha" content_sha
printf 'PASS test_presigned_complete_concurrent\n'
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Grade 4: private bucket + MinIO ports not published on the host.
set -eu
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
[ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; }
KEY=$($COMPOSE exec -T db sh -c \
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT s3_key FROM attachments LIMIT 1"' \
| tr -d '\r')
if [ -z "$KEY" ]; then
KEY="attachments/does-not-exist/probe"
fi
echo "check_bucket_private: unsigned GET s3://$S3_BUCKET/$KEY"
code=$($COMPOSE run --rm --no-deps --entrypoint /bin/sh minio-init -c \
"curl -sS -o /tmp/body -w '%{http_code}' -m 10 \"http://minio:9000/$S3_BUCKET/$KEY\" || echo 000")
echo "unsigned GET → HTTP $code"
[ "$code" != "200" ] || { echo "FAIL: bucket appears public" >&2; exit 1; }
[ "$code" != "000" ] || { echo "FAIL: cannot reach minio on compose network" >&2; exit 1; }
for port in 9000 9001; do
if curl -sS -m 2 -o /dev/null "http://127.0.0.1:$port/" 2>/dev/null; then
echo "FAIL: host :$port is reachable" >&2
exit 1
fi
echo "ok: host :$port not published"
done
echo "PASS check_bucket_private"
+5
View File
@@ -0,0 +1,5 @@
# stb (vendored)
`stb_image.h` from [nothings/stb](https://github.com/nothings/stb).
Pinned commit: see `VERSION`.
+1
View File
@@ -0,0 +1 @@
013ac3beddff3dbffafd5177e7972067cd2b5083
+7988
View File
File diff suppressed because it is too large Load Diff
+154
View File
@@ -0,0 +1,154 @@
#!/bin/sh
# Track 1 experiment: API RSS during transfer + approx body bytes through API.
# Direct PUT: client→MinIO (compose net). Complete measured separately.
#
# ./tools/exp_direct_vs_proxy.sh
# Optional: SIZES_MB="1 4" CONCURRENCY=3
set -eu
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
. "$ROOT/student-tests/lib.sh"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
SIZES_MB="${SIZES_MB:-1 4}"
CONCURRENCY="${CONCURRENCY:-3}"
OUT_DIR="${OUT_DIR:-$ROOT/artifacts/track1}"
mkdir -p "$OUT_DIR"
need docker
rss_sum() {
ids=$($COMPOSE ps -q api)
total=0
for id in $ids; do
[ -n "$id" ] || continue
rss=$(docker exec "$id" sh -c "awk '/^VmRSS:/ {print \$2}' /proc/1/status" 2>/dev/null || echo 0)
total=$((total + rss))
done
printf '%s' "$total"
}
sample_peak() {
dur=${1:-3}
peak=$(rss_sum)
end=$(( $(date +%s) + dur ))
while [ "$(date +%s)" -lt "$end" ]; do
cur=$(rss_sum)
[ "$cur" -gt "$peak" ] && peak=$cur
sleep 0.15
done
printf '%s' "$peak"
}
make_png() {
mb=$1
out=$2
python3 - "$mb" "$out" <<'PY'
import struct, zlib, sys
mb = float(sys.argv[1])
path = sys.argv[2]
target = int(mb * 1024 * 1024)
w, h = 320, 200
raw = b"".join(b"\x00" + bytes(w * 3) for _ in range(h))
def chunk(tag, data):
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xffffffff)
ihdr = chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
idat = chunk(b"IDAT", zlib.compress(raw, 9))
sig = b"\x89PNG\r\n\x1a\n"
base = sig + ihdr + idat + chunk(b"IEND", b"")
pad_need = max(0, target - len(base) - 12)
text = b"Comment\0" + (b"x" * pad_need)
open(path, "wb").write(sig + ihdr + idat + chunk(b"tEXt", text) + chunk(b"IEND", b""))
PY
}
put_direct() {
url=$1
ct=$2
file=$3
$COMPOSE run --rm --no-deps \
-v "$file:/tmp/up.bin:ro" \
--entrypoint curl api \
-sS -m 120 -o /dev/null \
-X PUT -H "Content-Type: $ct" --data-binary @/tmp/up.bin \
"$url" >/dev/null
}
register_user exp_alice3
ALICE_TOKEN=$TOKEN
ALICE_ID=$USER_ID
create_chat "exp-track1c" "[\"$ALICE_ID\"]"
RESULT="$OUT_DIR/results.tsv"
printf 'mode\tphase\tsize_mb\trss_before_kb\trss_peak_kb\trss_delta_kb\tapi_body_bytes_approx\n' > "$RESULT"
for mb in $SIZES_MB; do
f="$OUT_DIR/payload_${mb}mb.png"
make_png "$mb" "$f"
sha=$(sha256_file "$f")
sz=$(wc -c < "$f" | tr -d ' ')
api_bytes=$((CONCURRENCY * sz))
# --- proxy transfer ---
sleep 1
before=$(rss_sum)
i=0
pids=""
while [ "$i" -lt "$CONCURRENCY" ]; do
( upload_file "$CHAT_ID" "$f" "p_${mb}_$i.png" "image/png" >/dev/null 2>&1 || true ) &
pids="$pids $!"
i=$((i + 1))
done
peak=$(sample_peak 10)
for p in $pids; do wait "$p" || true; done
[ "$peak" -lt "$before" ] && peak=$before
delta=$((peak - before))
printf 'proxy\ttransfer\t%s\t%s\t%s\t%s\t%s\n' "$mb" "$before" "$peak" "$delta" "$api_bytes" | tee -a "$RESULT"
# --- direct: create sessions, then PUT-only window ---
sess="$OUT_DIR/sess_${mb}.tsv"
: > "$sess"
i=0
while [ "$i" -lt "$CONCURRENCY" ]; do
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
-H "Authorization: Bearer $ALICE_TOKEN" \
-H 'Content-Type: application/json' \
-d "{\"filename\":\"d_${mb}_$i.png\",\"content_type\":\"image/png\",\"size_bytes\":$sz,\"sha256\":\"$sha\"}"
[ "$HTTP_CODE" = "201" ] || die "init failed: $HTTP_BODY"
printf '%s\t%s\t%s\n' \
"$(printf '%s' "$HTTP_BODY" | jq -r .id)" \
"$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)" \
"$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')" >> "$sess"
i=$((i + 1))
done
sleep 1
before=$(rss_sum)
pids=""
while IFS="$(printf '\t')" read -r uid url ct; do
[ -n "$url" ] || continue
( put_direct "$url" "$ct" "$f" ) &
pids="$pids $!"
done < "$sess"
peak=$(sample_peak 15)
for p in $pids; do wait "$p" || true; done
[ "$peak" -lt "$before" ] && peak=$before
delta=$((peak - before))
printf 'direct\ttransfer\t%s\t%s\t%s\t%s\t%s\n' "$mb" "$before" "$peak" "$delta" "0" | tee -a "$RESULT"
# --- direct complete ---
before=$(rss_sum)
while IFS="$(printf '\t')" read -r uid url ct; do
[ -n "$uid" ] || continue
http_json POST "/api/v1/direct-uploads/$uid/complete" \
-H "Authorization: Bearer $ALICE_TOKEN"
[ "$HTTP_CODE" = "201" ] || die "complete failed id=$uid body=$HTTP_BODY"
done < "$sess"
peak=$(rss_sum)
[ "$peak" -lt "$before" ] && peak=$before
delta=$((peak - before))
printf 'direct\tcomplete\t%s\t%s\t%s\t%s\t%s\n' "$mb" "$before" "$peak" "$delta" "$api_bytes" | tee -a "$RESULT"
done
echo "wrote $RESULT"
cat "$RESULT"
Binary file not shown.

After

Width:  |  Height:  |  Size: 502 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 644 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 633 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 69 B

+61
View File
@@ -0,0 +1,61 @@
#!/bin/sh
# Expire pending direct-upload sessions past expires_at and delete their S3
# objects (if any). Dry-run by default.
#
# ./tools/gc_direct_uploads.sh
# APPLY=1 ./tools/gc_direct_uploads.sh
set -eu
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
APPLY="${APPLY:-0}"
S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
[ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; }
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT
echo "gc_direct_uploads: apply=$APPLY"
$COMPOSE exec -T db sh -c \
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "
SELECT id || E'\''\t'\'' || s3_key FROM direct_uploads
WHERE status = '\''pending'\'' AND expires_at < now()
"' > "$TMP/stale.tsv" || true
n=$(wc -l < "$TMP/stale.tsv" | tr -d ' ')
echo "gc_direct_uploads: stale_pending=$n"
if [ ! -s "$TMP/stale.tsv" ]; then
echo "gc_direct_uploads: nothing to do"
exit 0
fi
while IFS="$(printf '\t')" read -r id key; do
[ -n "$id" ] || continue
echo " STALE id=$id key=$key"
done < "$TMP/stale.tsv"
if [ "$APPLY" != "1" ]; then
echo "gc_direct_uploads: dry-run; set APPLY=1 to expire + delete objects"
exit 0
fi
$COMPOSE exec -T db sh -c \
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "
UPDATE direct_uploads SET status = '\''expired'\'', updated_at = now()
WHERE status = '\''pending'\'' AND expires_at < now()
"' >/dev/null
cut -f2 "$TMP/stale.tsv" | $COMPOSE run --rm --no-deps --entrypoint /bin/sh minio-init -c "
set -eu
mc alias set local \"\$S3_ENDPOINT\" \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" >/dev/null
while IFS= read -r key; do
[ -n \"\$key\" ] || continue
mc rm --force \"local/\$S3_BUCKET/\$key\" 2>/dev/null || true
echo \"deleted \$key\"
done
"
echo "gc_direct_uploads: done"
+115
View File
@@ -0,0 +1,115 @@
#!/bin/sh
# Reconcile S3 orphans: bucket objects with no matching attachments.s3_key in PG.
#
# Upload path compensation: S3 PUT → PG INSERT; on INSERT failure the API
# best-effort DeleteObject. This script cleans leftovers after crashes between
# PUT and compensate (or a failed DeleteObject).
#
# Dry-run by default. Fresh objects inside GRACE_SECONDS are never deleted.
#
# ./tools/gc_orphans.sh
# APPLY=1 GRACE_SECONDS=3600 ./tools/gc_orphans.sh
set -eu
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
GRACE_SECONDS="${GRACE_SECONDS:-3600}"
APPLY="${APPLY:-0}"
PREFIX="${ORPHAN_PREFIX:-}"
S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
[ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; }
TMP=$(mktemp -d)
trap 'rm -rf "$TMP"' EXIT
echo "gc_orphans: bucket=$S3_BUCKET grace=${GRACE_SECONDS}s apply=$APPLY prefix=$PREFIX"
s3_listing=$($COMPOSE run --rm --no-deps -T --entrypoint /bin/sh minio-init -c "
set -eu
mc alias set local \"\$S3_ENDPOINT\" \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" >/dev/null
mc ls --recursive --json \"local/\$S3_BUCKET/$PREFIX\" || true
")
printf '%s\n' "$s3_listing" > "$TMP/s3.json"
# Keep published attachments and staging objects for active direct uploads.
pg_keys=$($COMPOSE exec -T db sh -c \
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "
SELECT s3_key FROM attachments
UNION
SELECT staging_key FROM direct_uploads WHERE status = '\''pending'\''
"')
printf '%s\n' "$pg_keys" > "$TMP/pg.keys"
python3 - "$TMP/s3.json" "$TMP/pg.keys" "$GRACE_SECONDS" "$TMP/del.keys" "$PREFIX" <<'PY'
import json, sys, time
from datetime import datetime
s3_path, pg_path, grace_s, out_path, prefix = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4], sys.argv[5]
now = time.time()
known = {line.strip() for line in open(pg_path) if line.strip()}
orphans = []
for line in open(s3_path):
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
key = obj.get("key") or obj.get("Key")
if not key:
continue
key = key.lstrip("/")
if prefix and not key.startswith(prefix):
key = prefix.rstrip("/") + "/" + key
if not key.startswith(("attachments/", "staging/")):
continue
if key in known:
continue
age = None
lm = obj.get("lastModified") or obj.get("last_modified")
if lm:
try:
age = now - datetime.fromisoformat(lm.replace("Z", "+00:00")).timestamp()
except Exception:
age = None
orphans.append((key, age, obj.get("size")))
print(f"gc_orphans: scanned orphans={len(orphans)} known_pg={len(known)}")
to_delete = []
for key, age, size in orphans:
age_s = f"{int(age)}s" if age is not None else "?"
if age is not None and age < grace_s:
print(f" SKIP (grace) key={key} age={age_s} size={size}")
continue
print(f" ORPHAN key={key} age={age_s} size={size}")
to_delete.append(key)
with open(out_path, "w") as f:
f.write("\n".join(to_delete) + ("\n" if to_delete else ""))
print(f"gc_orphans: candidates={len(to_delete)}")
PY
if [ "$APPLY" != "1" ]; then
echo "gc_orphans: dry-run; set APPLY=1 to delete candidates older than grace."
exit 0
fi
if [ ! -s "$TMP/del.keys" ]; then
echo "gc_orphans: nothing to delete"
exit 0
fi
while IFS= read -r key; do
[ -n "$key" ] || continue
echo "deleting $S3_BUCKET/$key"
$COMPOSE run --rm --no-deps -T -e "GC_OBJECT_KEY=$key" \
--entrypoint /bin/sh minio-init -c '
set -eu
mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
mc rm --force "local/$S3_BUCKET/$GC_OBJECT_KEY"
'
done < "$TMP/del.keys"
echo "gc_orphans: delete done"
+111
View File
@@ -0,0 +1,111 @@
// Step 5 smoke: PNG / JPEG / truncated PNG / non-image / size limit.
#include "image_util.hpp"
#include <cstdint>
#include <fstream>
#include <iostream>
#include <stdexcept>
#include <string>
#include <vector>
#ifndef IMAGE_SMOKE_FIXTURES
#define IMAGE_SMOKE_FIXTURES "tools/fixtures"
#endif
namespace {
std::vector<std::uint8_t> read_all(const std::string& path) {
std::ifstream in(path, std::ios::binary);
if (!in) {
throw std::runtime_error("cannot open " + path);
}
return std::vector<std::uint8_t>((std::istreambuf_iterator<char>(in)),
std::istreambuf_iterator<char>());
}
bool expect_ok(const char* name, const std::vector<std::uint8_t>& bytes,
const char* want_ct) {
ImageLimits lim;
ImageInfo info;
auto rej = validate_image(bytes.data(), bytes.size(), lim, info);
if (rej) {
std::cerr << name << ": expected OK, got " << image_reject_code(*rej)
<< "\n";
return false;
}
if (info.content_type != want_ct) {
std::cerr << name << ": content_type " << info.content_type << "\n";
return false;
}
std::cout << name << ": OK " << info.content_type << " " << info.width
<< "x" << info.height << "\n";
return true;
}
bool expect_reject(const char* name, const std::vector<std::uint8_t>& bytes,
ImageReject want) {
ImageLimits lim;
ImageInfo info;
auto rej = validate_image(bytes.data(), bytes.size(), lim, info);
if (!rej || *rej != want) {
std::cerr << name << ": expected " << image_reject_code(want) << ", got "
<< (rej ? image_reject_code(*rej) : "OK") << "\n";
return false;
}
std::cout << name << ": OK reject=" << image_reject_code(want) << "\n";
return true;
}
} // namespace
int main(int argc, char** argv) {
try {
const std::string dir =
(argc > 1) ? argv[1] : std::string(IMAGE_SMOKE_FIXTURES);
const auto png = read_all(dir + "/tiny.png");
const auto jpg = read_all(dir + "/tiny.jpg");
bool ok = true;
ok &= expect_ok("png", png, "image/png");
ok &= expect_ok("jpeg", jpg, "image/jpeg");
std::vector<std::uint8_t> trunc(png.begin(),
png.begin() + std::min<std::size_t>(16, png.size()));
ok &= expect_reject("truncated_png", trunc, ImageReject::invalid_image);
std::vector<std::uint8_t> spoof = jpg;
spoof[0] = 'G';
spoof[1] = 'I';
spoof[2] = 'F';
ok &= expect_reject("not_image", spoof, ImageReject::unsupported_media_type);
ImageLimits tiny_lim;
tiny_lim.max_bytes = 10;
ImageInfo info;
auto rej = validate_image(png.data(), png.size(), tiny_lim, info);
if (!rej || *rej != ImageReject::payload_too_large) {
std::cerr << "max_bytes: FAIL\n";
ok = false;
} else {
std::cout << "max_bytes: OK\n";
}
// JPEG bytes sniffed as jpeg regardless of any HTTP Content-Type.
auto sniffed = sniff_content_type(jpg.data(), jpg.size());
if (!sniffed || *sniffed != "image/jpeg") {
std::cerr << "sniff_jpeg: FAIL\n";
ok = false;
} else {
std::cout << "sniff_jpeg: OK (MIME spoof would be ignored)\n";
}
if (!ok) {
return 1;
}
std::cout << "image_smoke: OK\n";
return 0;
} catch (const std::exception& e) {
std::cerr << "image_smoke: FAIL: " << e.what() << "\n";
return 1;
}
}
+35
View File
@@ -0,0 +1,35 @@
// Small smoke tool for step 4: put → get → delete against MinIO.
#include "s3_client.hpp"
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <sstream>
#include <string>
int main() {
try {
auto cfg = S3Client::from_env();
S3Client s3(cfg);
const std::string key = "smoke/step4.txt";
const std::string payload = "lab4-s3-smoke-ok";
std::istringstream in(payload);
s3.put_object(key, in, static_cast<std::int64_t>(payload.size()),
"text/plain");
std::ostringstream out;
s3.get_object(key, out);
if (out.str() != payload) {
std::cerr << "mismatch: got [" << out.str() << "]\n";
return 2;
}
s3.delete_object(key);
std::cout << "s3_smoke: OK bucket=" << cfg.bucket << " key=" << key << "\n";
return 0;
} catch (const std::exception& e) {
std::cerr << "s3_smoke: FAIL: " << e.what() << "\n";
return 1;
}
}