forked from mtusi2026/mtusi2026lab4
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7741b66fd2 | ||
|
|
3223dd469a |
+22
-2
@@ -7,14 +7,34 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
|
||||
# PostgreSQL (compose service `db`)
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change_me_locally
|
||||
POSTGRES_DB=messenger
|
||||
|
||||
# MinIO root (bootstrap / admin only; API must NOT use these)
|
||||
MINIO_ROOT_USER=minioadmin
|
||||
MINIO_ROOT_PASSWORD=change_me_minio_root_locally
|
||||
|
||||
# Limited S3 user created by minio-user-init (copy same values into secrets/)
|
||||
MINIO_USER_ACCESS_KEY=messenger-s3
|
||||
MINIO_USER_SECRET_KEY=change_me_s3_user_locally
|
||||
|
||||
# S3 / attachments (API reads limited credentials from secret files only)
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
# Optional Track 1: host-reachable endpoint for presigned PUT (needs published
|
||||
# MinIO via compose.presign.yaml). Empty → sign with S3_ENDPOINT (docker net).
|
||||
# S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
MAX_CONCURRENT_UPLOADS=3
|
||||
DIRECT_UPLOAD_TTL_SECONDS=300
|
||||
|
||||
@@ -1,18 +1,46 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Никитин Андрей Александрович / БВТ2301
|
||||
Целевая оценка (3/4/5): **5**
|
||||
Трек 5, если заявлен: **трек 1 — прямой (presigned) upload**
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено:
|
||||
- Lab3 → lab4: пользователи/чаты/сообщения, Redis-сессии, Idempotency-Key, LB ×2 API, graceful drain.
|
||||
- Оценка 4: private MinIO, limited S3 creds, proxy `POST /attachments` (magic + stb_image), компенсация Put→PG / DeleteObject, `gc_orphans`, persistence/failover hooks, лимит concurrent uploads (RSS).
|
||||
- Оценка 5 / трек 1: `direct_uploads` + presigned PUT (`minio-cpp` SigV4), complete (size/sha256/image), abort, GC незавершённых; proxy endpoint сохранён; сравнение RAM/трафика proxy vs direct (≥2 размера).
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone:
|
||||
```sh
|
||||
cp .env.example .env # локальные POSTGRES_*/MINIO_*/S3_BUCKET
|
||||
mkdir -p secrets && # database_url, cache_url, s3_* (см. REPORT.md)
|
||||
docker compose --env-file .env up -d --build
|
||||
curl -fsS http://127.0.0.1:8080/health/ready
|
||||
make check && make test BASE_URL=http://127.0.0.1:8080
|
||||
sh student-tests/run_grade3.sh
|
||||
sh student-tests/run_grade4.sh
|
||||
sh student-tests/run_grade5_track1.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
```
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md:
|
||||
- Подробности, таблицы доказательств и эксперимент трека 1: [`REPORT.md`](../REPORT.md).
|
||||
- Student-tests grade3/4/5 track1 — PASS на локальном стенде.
|
||||
- Evidence: `evidence/load_uploads.json`, `evidence/resilience_grade4.txt`, `artifacts/track1/results.tsv`.
|
||||
- Ветка решения: `solution/nikitin-bvt2301`.
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
- Без `compose.presign.yaml` PUT к MinIO — с контейнера на сети compose (порты 9000/9001 на хост не публикуются).
|
||||
- Complete direct-upload кратковременно GetObject'ит объект для verify (фаза отдельно от transfer).
|
||||
- Нет HA БД/MinIO; нет thumbnail/вариантов (lab5); openapi расширений трека 1 нет (факультативный API).
|
||||
|
||||
Источники и использование ИИ:
|
||||
- Контракт/тесты курса; PostgreSQL 16, Valkey, Nginx, MinIO, libpqxx 7.10.0, libsodium, hiredis, nlohmann/json, minio-cpp v0.3.0, stb_image.
|
||||
- Помощь ИИ: Cursor (реализация, отладка, отчёт).
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+4
-1
@@ -2,7 +2,9 @@
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
secrets/
|
||||
secrets/*
|
||||
!secrets/*.example
|
||||
|
||||
certs/private/
|
||||
*.key
|
||||
*.p12
|
||||
@@ -14,6 +16,7 @@ volumes/
|
||||
data/
|
||||
backups/
|
||||
# Runtime/build artifacts
|
||||
artifacts/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
project(mtusi_messenger CXX)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# Official MinIO C++ SDK (S3-compatible). Pinned tag; not a hand-rolled SigV4.
|
||||
set(MTUSI_MINIO_CPP_VERSION "v0.3.0" CACHE STRING "minio-cpp git tag")
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
find_package(PkgConfig REQUIRED)
|
||||
|
||||
pkg_check_modules(PQXX REQUIRED IMPORTED_TARGET libpqxx>=7.8)
|
||||
pkg_check_modules(HIREDIS REQUIRED IMPORTED_TARGET hiredis)
|
||||
pkg_check_modules(SODIUM REQUIRED IMPORTED_TARGET libsodium)
|
||||
|
||||
include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/ImportMinioDeps.cmake")
|
||||
|
||||
include(FetchContent)
|
||||
set(MINIO_CPP_TEST OFF CACHE BOOL "" FORCE)
|
||||
set(MINIO_CPP_MAKE_DOC OFF CACHE BOOL "" FORCE)
|
||||
FetchContent_Declare(
|
||||
miniocpp
|
||||
GIT_REPOSITORY https://github.com/minio/minio-cpp.git
|
||||
GIT_TAG ${MTUSI_MINIO_CPP_VERSION}
|
||||
GIT_SHALLOW TRUE
|
||||
)
|
||||
# Inject find_package stubs before miniocpp configures.
|
||||
function(_mtusi_provide_minio_find_packages)
|
||||
# miniocpp calls find_package(unofficial-curlpp CONFIG) etc.
|
||||
# We already created ALIAS targets; create tiny config files in a prefix.
|
||||
set(_cfg_dir "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps")
|
||||
file(MAKE_DIRECTORY "${_cfg_dir}")
|
||||
file(WRITE "${_cfg_dir}/unofficial-curlpp-config.cmake" "")
|
||||
file(WRITE "${_cfg_dir}/unofficial-inih-config.cmake" "")
|
||||
list(PREPEND CMAKE_PREFIX_PATH "${_cfg_dir}")
|
||||
set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE)
|
||||
endfunction()
|
||||
_mtusi_provide_minio_find_packages()
|
||||
|
||||
FetchContent_GetProperties(miniocpp)
|
||||
if(NOT miniocpp_POPULATED)
|
||||
FetchContent_Populate(miniocpp)
|
||||
# Neutralize CONFIG find_package calls by providing empty configs + targets.
|
||||
set(CMAKE_FIND_PACKAGE_REDIRECTS_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-find-redirects")
|
||||
file(MAKE_DIRECTORY "${CMAKE_FIND_PACKAGE_REDIRECTS_DIR}")
|
||||
# CMake 3.24+ redirects; for older, patch via policy: set variables that
|
||||
# make find_package(CONFIG) succeed when targets already exist.
|
||||
set(unofficial-curlpp_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
|
||||
set(unofficial-inih_DIR "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps" CACHE PATH "" FORCE)
|
||||
# Provide package config bodies that are no-ops (targets already defined).
|
||||
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-curlpp-config.cmake"
|
||||
"# provided by ImportMinioDeps.cmake\n")
|
||||
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/mtusi-minio-deps/unofficial-inih-config.cmake"
|
||||
"# provided by ImportMinioDeps.cmake\n")
|
||||
# Skip miniocpp install(EXPORT) — our INTERFACE system deps are not exportable.
|
||||
set(CMAKE_SKIP_INSTALL_RULES ON)
|
||||
add_subdirectory(${miniocpp_SOURCE_DIR} ${miniocpp_BINARY_DIR})
|
||||
set(CMAKE_SKIP_INSTALL_RULES OFF)
|
||||
endif()
|
||||
|
||||
add_executable(messenger
|
||||
src/main.cpp
|
||||
src/db.cpp
|
||||
src/auth.cpp
|
||||
src/user_repo.cpp
|
||||
src/chat_repo.cpp
|
||||
src/message_repo.cpp
|
||||
src/redis_pool.cpp
|
||||
src/session_repo.cpp
|
||||
src/pg_session_repo.cpp
|
||||
src/s3_client.cpp
|
||||
src/image_util.cpp
|
||||
src/attachment_repo.cpp
|
||||
src/direct_upload_repo.cpp
|
||||
)
|
||||
|
||||
target_include_directories(messenger PRIVATE
|
||||
src
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
|
||||
)
|
||||
target_compile_definitions(messenger PRIVATE
|
||||
MTUSI_MINIO_CPP_VERSION="${MTUSI_MINIO_CPP_VERSION}"
|
||||
)
|
||||
|
||||
target_link_libraries(messenger PRIVATE
|
||||
Threads::Threads
|
||||
PkgConfig::PQXX
|
||||
PkgConfig::HIREDIS
|
||||
PkgConfig::SODIUM
|
||||
miniocpp::miniocpp
|
||||
)
|
||||
|
||||
add_executable(s3_smoke
|
||||
tools/s3_smoke.cpp
|
||||
src/s3_client.cpp
|
||||
)
|
||||
target_include_directories(s3_smoke PRIVATE src)
|
||||
target_link_libraries(s3_smoke PRIVATE
|
||||
Threads::Threads
|
||||
miniocpp::miniocpp
|
||||
)
|
||||
|
||||
add_executable(image_smoke
|
||||
tools/image_smoke.cpp
|
||||
src/image_util.cpp
|
||||
)
|
||||
target_include_directories(image_smoke PRIVATE
|
||||
src
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/third_party/stb
|
||||
)
|
||||
target_compile_definitions(image_smoke PRIVATE
|
||||
IMAGE_SMOKE_FIXTURES="${CMAKE_SOURCE_DIR}/tools/fixtures"
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"version": 6,
|
||||
"cmakeMinimumRequired": {
|
||||
"major": 3,
|
||||
"minor": 16,
|
||||
"patch": 0
|
||||
},
|
||||
"configurePresets": [
|
||||
{
|
||||
"name": "release",
|
||||
"displayName": "Release with minio-cpp",
|
||||
"generator": "Unix Makefiles",
|
||||
"binaryDir": "${sourceDir}/build",
|
||||
"cacheVariables": {
|
||||
"CMAKE_BUILD_TYPE": "Release",
|
||||
"MTUSI_MINIO_CPP_VERSION": "v0.3.0"
|
||||
}
|
||||
}
|
||||
],
|
||||
"buildPresets": [
|
||||
{
|
||||
"name": "release",
|
||||
"configurePreset": "release"
|
||||
}
|
||||
]
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
# Clean-clone build: Ubuntu 24.04 + libpqxx from a pinned git tag.
|
||||
# S3 client: official minio-cpp v0.3.0 via FetchContent (library SigV4).
|
||||
# No dependency on mtusi2026lab3-api or other prior-lab images.
|
||||
|
||||
ARG PQXX_VERSION=7.10.0
|
||||
ARG MTUSI_MINIO_CPP_VERSION=v0.3.0
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage: build libpqxx into /opt/libpqxx
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:24.04 AS pqxx
|
||||
ARG PQXX_VERSION
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential cmake git ca-certificates pkg-config \
|
||||
libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /tmp
|
||||
RUN git clone --depth 1 --branch "${PQXX_VERSION}" \
|
||||
https://github.com/jtv/libpqxx.git libpqxx-src \
|
||||
&& cmake -S libpqxx-src -B libpqxx-build \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_INSTALL_PREFIX=/opt/libpqxx \
|
||||
-DBUILD_SHARED_LIBS=ON \
|
||||
-DSKIP_BUILD_TEST=ON \
|
||||
&& cmake --build libpqxx-build -j"$(nproc)" \
|
||||
&& cmake --install libpqxx-build \
|
||||
&& rm -rf /tmp/libpqxx-src /tmp/libpqxx-build
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage: build messenger + smokes
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:24.04 AS builder
|
||||
ARG MTUSI_MINIO_CPP_VERSION
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential cmake pkg-config git ca-certificates \
|
||||
libpq-dev \
|
||||
libhiredis-dev \
|
||||
libsodium-dev \
|
||||
nlohmann-json3-dev \
|
||||
libcurl4-openssl-dev libssl-dev zlib1g-dev \
|
||||
libpugixml-dev \
|
||||
libcurlpp-dev \
|
||||
libinih-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
|
||||
ENV PKG_CONFIG_PATH=/opt/libpqxx/lib/pkgconfig:/opt/libpqxx/lib/x86_64-linux-gnu/pkgconfig
|
||||
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
|
||||
ENV CMAKE_PREFIX_PATH=/opt/libpqxx
|
||||
|
||||
WORKDIR /src
|
||||
COPY CMakeLists.txt CMakePresets.json ./
|
||||
COPY cmake ./cmake
|
||||
COPY src ./src
|
||||
COPY tools ./tools
|
||||
COPY migrations ./migrations
|
||||
COPY third_party ./third_party
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \
|
||||
-DMTUSI_MINIO_CPP_VERSION=${MTUSI_MINIO_CPP_VERSION} \
|
||||
&& cmake --build build -j"$(nproc)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runtime
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM ubuntu:24.04
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
libpq5 \
|
||||
libhiredis1.1.0 \
|
||||
libsodium23 \
|
||||
libssl3t64 \
|
||||
libcurl4t64 \
|
||||
zlib1g \
|
||||
libpugixml1v5 \
|
||||
libcurlpp0t64 \
|
||||
libinih1 \
|
||||
libinireader0 \
|
||||
util-linux \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid app --shell /usr/sbin/nologin --no-create-home app
|
||||
|
||||
COPY --from=pqxx /opt/libpqxx /opt/libpqxx
|
||||
ENV LD_LIBRARY_PATH=/opt/libpqxx/lib:/opt/libpqxx/lib/x86_64-linux-gnu
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder /src/build/messenger /app/messenger
|
||||
COPY --from=builder /src/build/s3_smoke /app/s3_smoke
|
||||
COPY --from=builder /src/build/image_smoke /app/image_smoke
|
||||
COPY --from=builder /src/migrations /app/migrations
|
||||
COPY tools/fixtures /app/fixtures
|
||||
COPY docker-entrypoint.sh /docker-entrypoint.sh
|
||||
|
||||
RUN chmod 0755 /docker-entrypoint.sh \
|
||||
&& chown -R app:app /app
|
||||
|
||||
# Entrypoint starts as root to stage S3 secret files for uid 10001, then drops.
|
||||
USER root
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=3 \
|
||||
CMD curl -fsS http://127.0.0.1:8080/health/live || exit 1
|
||||
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["/app/messenger"]
|
||||
@@ -1,37 +1,209 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Никитин Андрей Александрович, группа БВТ2301
|
||||
- Целевая оценка: 5
|
||||
- Трек 5: Track 1 (Presigned upload)
|
||||
- Commit/PR: ветка `solution/nikitin-bvt2301` → MR в `main` (шаблон `.github/pull_request_template.md`); remote `gitea.privcloud.tech/AndrewNikkk/mtusi2026lab4`
|
||||
- Источники и помощь: контракт и публичные тесты курса; PostgreSQL 16, Valkey, Nginx, MinIO, libpqxx 7.10.0, libsodium, hiredis, nlohmann/json, minio-cpp v0.3.0 (официальный SigV4), stb_image. Помощь ИИ: Cursor.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Хост прогона: 12th Gen Intel Core i5-12450H, 12 потоков, 14 GiB RAM.
|
||||
|
||||
Образы (закреплены в `compose.yaml` / локальных Dockerfile):
|
||||
|
||||
| Сервис | Образ |
|
||||
| --- | --- |
|
||||
| db | `postgres:16.15-alpine` |
|
||||
| cache | `valkey/valkey:8.1.10-alpine` |
|
||||
| lb | `nginx:1.28.0-alpine` |
|
||||
| api | `mtusi2026lab4-api:local` (multi-stage `ubuntu:24.04`; libpqxx **7.10.0** собран из git-тега `https://github.com/jtv/libpqxx.git`, prefix `/opt/libpqxx`; minio-cpp `v0.3.0` FetchContent) |
|
||||
| minio | `mtusi2026lab4-minio:RELEASE.2025-04-22T22-12-26Z` (бинарник с GitHub; Docker Hub в среде мог быть недоступен) |
|
||||
| mc (init) | `mtusi2026lab4-mc:RELEASE.2025-04-16T18-13-26Z` |
|
||||
|
||||
Лимиты compose: db 1 CPU / 512 MiB; cache 0.5 / 256 MiB; api ×2 по 0.5 CPU / 256 MiB; lb 0.25 / 128 MiB; minio 0.5 / 512 MiB.
|
||||
|
||||
Prerequisites: Docker Compose v2, Python 3.9+, `curl`, `jq`, `sha256sum`. Для `make validate` — venv из `requirements-dev.txt`.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# Задать POSTGRES_*, MINIO_ROOT_*, MINIO_USER_*, S3_BUCKET (без коммита .env)
|
||||
|
||||
mkdir -p secrets
|
||||
printf 'postgresql://USER:PASSWORD@db:5432/DB\n' > secrets/database_url
|
||||
printf 'redis://cache:6379/0\n' > secrets/cache_url
|
||||
# Те же значения, что MINIO_USER_* в .env (не root):
|
||||
printf 'ACCESS_KEY\n' > secrets/s3_access_key_id
|
||||
printf 'SECRET_KEY\n' > secrets/s3_secret_access_key
|
||||
chmod 600 secrets/*
|
||||
|
||||
# Чистый clone: Dockerfile сам тянет ubuntu:24.04 и собирает libpqxx 7.10.0.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# Эквивалент без compose:
|
||||
# docker build -t mtusi2026lab4-api:local .
|
||||
# Дождаться healthy у lb, затем:
|
||||
curl -fsS http://127.0.0.1:8080/health/ready
|
||||
|
||||
make check
|
||||
make test BASE_URL=http://127.0.0.1:8080
|
||||
sh student-tests/run_grade3.sh
|
||||
sh student-tests/run_grade4.sh
|
||||
sh student-tests/run_grade5_track1.sh
|
||||
# Опционально: сравнение RAM proxy vs direct (≥2 размера)
|
||||
# sh tools/exp_direct_vs_proxy.sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
|
||||
# Безопасная остановка (volumes сохраняются):
|
||||
docker compose --env-file .env stop
|
||||
# Намеренное удаление учебных данных:
|
||||
# docker compose --env-file .env down -v
|
||||
```
|
||||
|
||||
Хосты в секретах — имена сервисов Compose (`db`, `cache`, `minio`), не loopback. Порты MinIO **не** публикуются по умолчанию; снаружи только LB `:8080`. Для host-side PUT к presigned URL — опционально `compose.presign.yaml` + `S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000` (loopback). Иначе PUT идёт из контейнера на сети compose (`http://minio:9000/...`). Миграции 001–007 применяет API при старте под `pg_advisory_lock`.
|
||||
|
||||
Соответствие `.env.example` → compose: `S3_ENDPOINT`/`S3_BUCKET`/`S3_REGION`, `S3_*_FILE` → docker secrets, `MAX_UPLOAD_BYTES`/`MAX_IMAGE_PIXELS`, `DIRECT_UPLOAD_TTL_SECONDS`, `DRAIN_SECONDS`, `API_REPLICAS`, `SESSION_STORE=redis`. Root MinIO только у `minio` / init-job'ов; API видит лишь limited `*_FILE`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
Proxy: Клиент → lb → api (buffer body) → MinIO PutObject → PG attachments
|
||||
Direct: Клиент → lb → api init → presigned PUT URL (staging/{chat}/{upload})
|
||||
Клиент ─────────────── PUT ──────→ MinIO staging
|
||||
Клиент → lb → api complete (HEAD+Get+sha256+image+CopyObject)
|
||||
staging → attachments/{chat}/{attachment}
|
||||
→ PG attachment + completed session
|
||||
```
|
||||
|
||||
**Источник истины:** метаданные вложений и связь message↔attachment — PostgreSQL; оригинал — объект MinIO `attachments/{chat_id}/{attachment_id}` (filename только для отображения). Presigned PUT пишет в отдельный `staging/{chat_id}/{upload_id}`. Direct sessions хранят оба ключа в `direct_uploads` (миграции 006–007).
|
||||
|
||||
**Порядок записи (proxy, компенсация):** S3 `PutObject` → `INSERT attachments` → COMMIT. Если INSERT/COMMIT упал — best-effort `DeleteObject`, ответ 500. Строка в PG без объекта недопустима. Orphan'ы: `tools/gc_orphans.sh` (учитывает `direct_uploads.status=pending`).
|
||||
|
||||
**Track 1 (presigned):** `POST /api/v1/chats/{id}/direct-uploads` создаёт `pending` сессию и выдаёт SigV4 PUT URL (`minio-cpp GetPresignedObjectUrl`, TTL по `DIRECT_UPLOAD_TTL_SECONDS`, 30–3600 секунд; default 300). URL сам по себе повторно используем до истечения TTL, поэтому он не считается одноразовым токеном. Одноразово публикуется именно сессия complete: её статус переходит `pending → completed`.
|
||||
|
||||
`POST /api/v1/direct-uploads/{id}/complete` в одной PostgreSQL-транзакции делает `SELECT ... FOR UPDATE`, проверяет существование/размер/hash staging-объекта и изображение, затем выполняет серверный `CopyObject` в final prefix, вставляет `attachments.status=ready`, помечает сессию `completed` и коммитит. SHA-256 берётся из `expected_sha256`, заявленного при init. Повторный complete возвращает сохранённый Attachment без повторного S3 copy; конкурентные complete одного id сериализуются блокировкой строки. Удаление staging после публикации best-effort.
|
||||
|
||||
Повторный PUT по ещё действующей старой ссылке может создать staging-orphan, но не перезаписывает final object. `tools/gc_orphans.sh` сканирует `attachments/` и `staging/`, сохраняет pending staging keys и удаляет неизвестные объекты только после `GRACE_SECONDS`. Abort удаляет staging best-effort; стандартный proxy `POST /api/v1/chats/{id}/attachments` **сохранён**.
|
||||
|
||||
**ACL:** метаданные и `/content` только участнику чата; чужой → 404; без Bearer → 401. Unsigned GET на `http://minio:9000/$BUCKET/$KEY` → 403.
|
||||
|
||||
**Валидация изображения:** magic (JPEG `FFD8FF` / PNG `89504E47`) → `stbi_info` (пиксели ≤40e6) → `stbi_load` (целостность) → сравнение с `Content-Type`. Spoof MIME → 415. Тело ≤10 485 760; ранний отказ по `Content-Length` с drain, чтобы nginx не отдавал 502. LB: `client_max_body_size 12m`.
|
||||
|
||||
**Message.attachment_ids:** 0..10 уникальных UUID того же чата со `status=ready`; пустой/`blank` text без вложений → 400; с вложениями пустой text допустим. Идемпотентность хеширует `text` + список id.
|
||||
|
||||
**Из lab3:** пользователи/чаты/сообщения, Redis-сессии, Idempotency-Key, LB×2 API, drain по SIGTERM (`DRAIN_SECONDS`, ready=503).
|
||||
|
||||
**S3-клиент:** minio-cpp `v0.3.0` через FetchContent (не самописный SigV4). aws-sdk-cpp отвергнут из‑за тяжёлой сборки aws-lc в этой среде.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт offline | `make check` | PASS: offline skeleton structure… | вывод команды |
|
||||
| OpenAPI 3.1 | `make validate PYTHON=.venv/bin/python` | PASS: full OpenAPI 3.1 and JSON Schema validation | вывод команды |
|
||||
| Публичный smoke | `make test BASE_URL=http://127.0.0.1:8080` | PASS (lab4 smoke) | вывод команды |
|
||||
| MinIO в стенде, private bucket, volume `miniodata` | `docker compose ps`; `sh tests/check_bucket_private.sh` | minio healthy; unsigned GET → **403**; host `:9000`/`:9001` closed | `evidence/resilience_grade4.txt` |
|
||||
| POST attachment → 201 ready; download = sha256 | `sh student-tests/test_attachment_upload_ok.sh` | 201, `status=ready`, variants=`[]`, bob скачал тот же SHA-256 | вывод теста |
|
||||
| Message + `attachment_ids`, empty text ok | тот же + wrong_chat | empty+att 201; blank без att → 400; чужой chat → 404 | `student-tests/` |
|
||||
| Лимиты / MIME / corrupt / foreign | too_large, corrupted, mime_spoof, foreign | 413 / 415 / 415 / 404 (не 500) | `student-tests/run_grade3.sh` |
|
||||
| Persistence (метаданные + байты) | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; hook рестартит db, cache, minio, api, lb; volumes на месте | `evidence/resilience_grade4.txt` |
|
||||
| Failover: upload на одной → download с другой | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS; 2 instance до отказа; 115 успехов; 1 ошибка; first_recovery ≈5.3 с; attachment bytes совпали | `evidence/resilience_grade4.txt` |
|
||||
| Limited S3 creds, не root | `sh student-tests/test_attachment_limited_creds.sh` | API только `S3_*_FILE`; key ≠ root; `mc rb` limited → deny; upload 201 | тот же evidence |
|
||||
| Компенсация / GC | код upload + `tools/gc_orphans.sh` | dry-run: orphans=0 при согласованном стенде; grace skip свежих | вывод `gc_orphans` |
|
||||
| ≥10 параллельных загрузок ~5 MiB | `sh scripts/load_uploads.sh` | 10×201; **delta < naive** (см. JSON); `MAX_CONCURRENT_UPLOADS=3`; слот до буферизации body | `evidence/load_uploads.json` |
|
||||
| Graceful drain | `sh student-tests/test_graceful_drain.sh` | SIGTERM → ready 503/stop; стенд восстановлен | вывод теста |
|
||||
| Идемпотентный bootstrap bucket/user | `minio-init`, `minio-user-init` | повторный up без ошибки; `mc mb --ignore-existing`; anonymous none | логи init |
|
||||
| Трек 1: init→PUT→complete; proxy жив | `sh student-tests/run_grade5_track1.sh` | PASS (ok + negatives + gc dry-run) | `student-tests/` |
|
||||
| Track 1: concurrent complete | `sh student-tests/test_presigned_complete_concurrent.sh` | 4 complete: ровно 1×201, остальные 200/409, ни одного 500; metadata и `/content` SHA-256 совпали с исходником | `student-tests/test_presigned_complete_concurrent.sh` |
|
||||
| Track 1: replay safety | `sh student-tests/test_direct_upload_ok.sh` | upload → complete → повторный PUT 20 B по старой ссылке → download SHA-256 исходного PNG; final не изменён | `student-tests/test_direct_upload_ok.sh` |
|
||||
| Download при отсутствующем S3-объекте | `sh student-tests/test_download_missing_object.sh` | после `mc rm` content отвечает 404/5xx, не 200; объект восстанавливается тестом | `student-tests/test_download_missing_object.sh` |
|
||||
| Трек 1: RAM/трафик ≥2 размеров | `sh tools/exp_direct_vs_proxy.sh` | transfer: direct ≪ proxy по RSS и api_body | `artifacts/track1/results.tsv` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Перезапускаемые процессы при persistence hook: **db, cache, minio, api (все реплики), lb**. Volumes `pgdata`, `valkeydata`, `miniodata` **не** удаляются. Failover hook останавливает **только** одну API-реплику по `X-Instance-Id` ≡ `Config.Hostname`.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Гипотеза (нагрузка upload):** пик RSS не должен расти как N×размер файла. Ограничение — семафор `MAX_CONCURRENT_UPLOADS` (по умолчанию **3**) на процесс API: слот берётся **до** буферизации body (>256 KiB), независимо от `HTTP_WORKERS=8`. После каждого upload — `malloc_trim(0)`, чтобы docker-stats RSS отражал concurrent footprint, а не рост арен. Тело ≤10 MiB; `stbi_load` освобождает raster; `MemBuf` без лишней копии.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Пик RSS ограничен формулой
|
||||
`rss_peak ≤ baseline + MAX_CONCURRENT_UPLOADS × (max_body + decoded_raster)`.
|
||||
При `MAX_CONCURRENT_UPLOADS=3` delta < `naive_full_buffer_mib`.
|
||||
См. `evidence/load_uploads.json`.
|
||||
|
||||
**Профиль:** closed-loop, concurrency=10, fixture ≈5 881 918 байт (валидный PNG 1400×1400), один прогон. Клиент — `urllib` в N процессах. Baseline RSS — медиана нескольких `docker stats` после прогрева; peak — max MemUsage api во время окна. Ограничение измерения: coordinated omission / closed-loop; RPS отражает длительную валидацию+PUT.
|
||||
|
||||
Сырой summary без токенов: `evidence/load_uploads.json`.
|
||||
|
||||
| Метрика | Значение |
|
||||
| --- | --- |
|
||||
| success / codes | 10 / `{201: 10}` |
|
||||
| wall / latency / RPS | см. JSON |
|
||||
| API RSS baseline → peak (delta) | **51.96 → 68.4 (+16.44) MiB** |
|
||||
| naive N×fixture | ≈56.09 MiB |
|
||||
| `MAX_CONCURRENT_UPLOADS` | 3 (слот до body >256 KiB; `malloc_trim` после upload) |
|
||||
|
||||
Критерий защиты: **api_rss_delta_mib (16.44) < naive_full_buffer_mib (56.09)**.
|
||||
|
||||
| Воздействие | Ожидание | Наблюдение | Восстановление / данные |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `restart.sh` (db+cache+minio+api+lb) | сессия и attachment живы | `test-persistence` PASS | hook <90 с; volumes целы |
|
||||
| `stop-one.sh` одной api | ≥2 id до отказа; запись ≤10 с; файл тот же | 2 id; 115 ok / 1 fail; recovery 5.338 с | attachment SHA совпал; реплика затем `up -d` |
|
||||
| unsigned S3 GET | не 200 | 403 | — |
|
||||
| host :9000/:9001 | connection fail | fail | — |
|
||||
| 11 MiB upload | 413 JSON | 413 `payload_too_large` (после drain CL) | — |
|
||||
| JPEG + `Content-Type: image/png` | 415 | 415 | — |
|
||||
| truncate PNG | 415 | 415 | — |
|
||||
| eve → чужой attachment | 404 | 404 meta и content | — |
|
||||
| SIGTERM api | ready 503, затем exit | drain PASS | compose up api |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
**Выдерживается:** отказ одной API-реплики при живых LB/БД/MinIO; полный restart с сохранением volumes; private bucket; limited IAM-подобные creds; негативные upload без 500; идемпотентные сообщения с `attachment_ids`.
|
||||
|
||||
**Не выдерживается / не обещается:** падение единственной PostgreSQL или MinIO; потеря volume; HA брокера; сетевой partition между API и S3 без ручного GC; точный open-loop RPS.
|
||||
|
||||
**Учебные упрощения:** proxy-тело буферизуется до `MAX_UPLOAD_BYTES`; complete direct-upload кратковременно читает объект для sha256/image check (байты PUT при этом не идут через API); нет thumbnail/вариантов (lab5); один Docker-host, один LB, одна БД, один MinIO; console MinIO `:9001` внутри сети compose.
|
||||
|
||||
**Подтверждено измерением (grade 4):** 10 параллельных ~5.6 MiB proxy-загрузок → 201; delta RSS < naive; private bucket; persistence/failover.
|
||||
|
||||
## Track 1: Presigned upload
|
||||
|
||||
**Архитектура и семантика:** presigned PUT направлен только на `staging/{chat_id}/{upload_id}`. После HEAD/stat, проверки `expected_size`, SHA-256 из init и image validation API копирует staging в `attachments/{chat_id}/{attachment_id}`. В `attachments.s3_key` хранится только final key. TTL ссылки задаётся `DIRECT_UPLOAD_TTL_SECONDS` (30–3600 секунд, default 300); пока TTL не истёк, URL можно повторно вызвать, но replay меняет только staging. Complete логически single-use: `pending → completed`; `SELECT FOR UPDATE` делает повторы и конкурентные запросы идемпотентными, они возвращают существующий Attachment без S3-публикации.
|
||||
|
||||
Если старый PUT выполняется после complete, он оставляет staging-orphan; final-объект остаётся прежним. `tools/gc_orphans.sh` очищает orphan-объекты обоих prefix по `GRACE_SECONDS`, не удаляя staging активных `pending` сессий. Staging cleanup после успешной публикации и abort — best-effort.
|
||||
|
||||
**Ограничения:** стандартная загрузка ограничена 10 MiB. Multipart-upload для файлов больше 100 MiB не реализован; presigned GET для скачивания тоже не реализован. Download остаётся авторизованным через API. Контракт `contracts/openapi.json` не менялся: Track 1 — расширение, обязательный proxy upload сохранён.
|
||||
|
||||
**Proxy vs presigned:** proxy передаёт тело через LB/API и буферизует его перед MinIO, поэтому upload потребляет память API и дополнительный сетевой hop. При presigned передаче байты идут клиент→MinIO напрямую, API получает только init/complete и не буферизует тело PUT. Complete добавляет HEAD/stat, чтение для SHA/image validation и server-side copy, поэтому его latency выше собственно PUT и измеряется отдельно.
|
||||
|
||||
**API (расширение, openapi не менялся):**
|
||||
|
||||
| Метод | Путь | Назначение |
|
||||
| --- | --- | --- |
|
||||
| POST | `/api/v1/chats/{id}/direct-uploads` | JSON: filename, content_type, size_bytes, sha256 → `upload_url` + TTL |
|
||||
| PUT | presigned URL | байты напрямую в MinIO (Content-Type из ответа) |
|
||||
| POST | `/api/v1/direct-uploads/{id}/complete` | Stat/size/sha256/image → attachment `ready` |
|
||||
| DELETE | `/api/v1/direct-uploads/{id}` | abort + DeleteObject |
|
||||
| POST | `/api/v1/chats/{id}/attachments` | обязательный proxy — без изменений |
|
||||
|
||||
**Состояния и ошибки:** complete без объекта → `409 not_ready`; размер или checksum mismatch → `409 hash_mismatch`; чужой upload → 404; abort → 204; повторный complete после успеха → 200 с тем же Attachment; aborted/expired session не публикуется; просроченный pending → 410.
|
||||
|
||||
**Проверка:** `sh student-tests/run_grade5_track1.sh` → PASS; `sh student-tests/run_grade3.sh` → PASS; `sh student-tests/run_grade4.sh` → PASS. Четыре одновременных complete дали ровно один 201, остальные ответы были 200/409, 500 не было; metadata и content SHA-256 совпали. Replay-проверка повторно PUT-ит 20-byte текст по старому URL и подтверждает SHA-256 исходного PNG при download. Эксперимент proxy vs direct: `sh tools/exp_direct_vs_proxy.sh` → `artifacts/track1/results.tsv`.
|
||||
|
||||
Методика: closed-loop; concurrency 3 (1 MiB) / 4 (4 MiB); RSS = сумма `VmRSS` процессов `messenger` по репликам api; фаза `transfer` — только передача байт (для direct — параллельные PUT на MinIO); `complete` — отдельно (GetObject+sha256+image на API). `api_body_bytes_approx` — оценка объёма тел через API (proxy = N×size; direct transfer = 0).
|
||||
|
||||
| size_mb | mode / phase | rss_delta_kb | api_body_bytes_approx |
|
||||
| --- | --- | --- | --- |
|
||||
| 1 | proxy / transfer | **1112** | 3 145 752 |
|
||||
| 1 | direct / transfer | **4** | 0 |
|
||||
| 1 | direct / complete | 4104 | (GetObject на API) |
|
||||
| 4 | proxy / transfer | **3320** | 16 777 248 |
|
||||
| 4 | direct / transfer | **0** | 0 |
|
||||
| 4 | direct / complete | 15592 | (GetObject на API) |
|
||||
|
||||
**Вывод:** на этапе передачи байт direct почти не двигает RSS API и не гонит payload через LB/API; proxy платит и трафиком, и пиком RSS. Complete у direct кратковременно читает объект (учебный verify) — это отдельная фаза, не отменяет выигрыш на upload path.
|
||||
|
||||
**Ограничение:** без `compose.presign.yaml` PUT выполняется с контейнера на сети compose (порты MinIO на хост не открыты). Подпись — `minio-cpp` PresignV4 (не самописный SigV4).
|
||||
|
||||
Track 5 (reconciler) не выполнялся; Track 1 (presigned) выполнен, см. раздел выше.
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# Provide the imported target names expected by minio-cpp's CMakeLists
|
||||
# (unofficial-curlpp / unofficial-inih) using system packages from apt.
|
||||
|
||||
find_package(OpenSSL REQUIRED)
|
||||
find_package(ZLIB REQUIRED)
|
||||
find_package(CURL REQUIRED)
|
||||
|
||||
# nlohmann_json (header-only; Ubuntu ships /usr/include/nlohmann/json.hpp)
|
||||
if(NOT TARGET nlohmann_json::nlohmann_json)
|
||||
find_package(nlohmann_json QUIET CONFIG)
|
||||
endif()
|
||||
if(NOT TARGET nlohmann_json::nlohmann_json)
|
||||
add_library(nlohmann_json INTERFACE)
|
||||
add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json)
|
||||
find_path(NLOHMANN_JSON_INCLUDE_DIR nlohmann/json.hpp)
|
||||
if(NOT NLOHMANN_JSON_INCLUDE_DIR)
|
||||
message(FATAL_ERROR "nlohmann/json.hpp not found (install nlohmann-json3-dev)")
|
||||
endif()
|
||||
target_include_directories(nlohmann_json INTERFACE "${NLOHMANN_JSON_INCLUDE_DIR}")
|
||||
endif()
|
||||
|
||||
# pugixml
|
||||
if(NOT TARGET pugixml)
|
||||
find_package(pugixml QUIET CONFIG)
|
||||
endif()
|
||||
if(NOT TARGET pugixml AND NOT TARGET pugixml::pugixml)
|
||||
pkg_check_modules(PUGIXML REQUIRED IMPORTED_TARGET pugixml)
|
||||
add_library(pugixml INTERFACE)
|
||||
target_link_libraries(pugixml INTERFACE PkgConfig::PUGIXML)
|
||||
endif()
|
||||
if(TARGET pugixml::pugixml AND NOT TARGET pugixml)
|
||||
add_library(pugixml ALIAS pugixml::pugixml)
|
||||
endif()
|
||||
|
||||
# curlpp → unofficial::curlpp::curlpp
|
||||
if(NOT TARGET unofficial::curlpp::curlpp)
|
||||
find_library(CURLPP_LIBRARY NAMES curlpp REQUIRED)
|
||||
find_path(CURLPP_INCLUDE_DIR curlpp/cURLpp.hpp REQUIRED)
|
||||
add_library(unofficial_curlpp INTERFACE)
|
||||
target_include_directories(unofficial_curlpp INTERFACE "${CURLPP_INCLUDE_DIR}")
|
||||
target_link_libraries(unofficial_curlpp INTERFACE "${CURLPP_LIBRARY}" CURL::libcurl)
|
||||
add_library(unofficial::curlpp::curlpp ALIAS unofficial_curlpp)
|
||||
endif()
|
||||
|
||||
# inih INIReader → unofficial::inih::inireader
|
||||
if(NOT TARGET unofficial::inih::inireader)
|
||||
find_library(INIREADER_LIBRARY NAMES INIReader REQUIRED)
|
||||
find_library(INIH_LIBRARY NAMES inih REQUIRED)
|
||||
find_path(INIREADER_INCLUDE_DIR INIReader.h REQUIRED)
|
||||
add_library(unofficial_inih_inireader INTERFACE)
|
||||
target_include_directories(unofficial_inih_inireader INTERFACE "${INIREADER_INCLUDE_DIR}")
|
||||
target_link_libraries(unofficial_inih_inireader INTERFACE
|
||||
"${INIREADER_LIBRARY}" "${INIH_LIBRARY}")
|
||||
add_library(unofficial::inih::inireader ALIAS unofficial_inih_inireader)
|
||||
endif()
|
||||
@@ -0,0 +1,11 @@
|
||||
# Optional Track 1 overlay: publish MinIO API on loopback so a host client can
|
||||
# PUT to presigned URLs. Default compose keeps 9000/9001 unpublished.
|
||||
#
|
||||
# S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000 \
|
||||
# docker compose -f compose.yaml -f compose.presign.yaml up -d --build
|
||||
#
|
||||
# Tear down / return to private MinIO by restarting without this file.
|
||||
services:
|
||||
minio:
|
||||
ports:
|
||||
- "127.0.0.1:9000:9000"
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.15-alpine
|
||||
command: postgres -c max_connections=200 -c shared_buffers=256MB -c synchronous_commit=off
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
cpus: ${DB_CPU_LIMIT:-1.0}
|
||||
mem_limit: 512m
|
||||
stop_grace_period: 20s
|
||||
|
||||
cache:
|
||||
image: valkey/valkey:8.1.10-alpine
|
||||
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
||||
volumes:
|
||||
- valkeydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
cpus: "0.50"
|
||||
mem_limit: 256m
|
||||
stop_grace_period: 15s
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: mtusi2026lab4-api:local
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
cache:
|
||||
condition: service_healthy
|
||||
minio:
|
||||
condition: service_healthy
|
||||
minio-user-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: ${SESSION_STORE:-redis}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-2}
|
||||
HTTP_WORKERS: ${HTTP_WORKERS:-8}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-8}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
|
||||
DIRECT_UPLOAD_TTL_SECONDS: ${DIRECT_UPLOAD_TTL_SECONDS:-300}
|
||||
# Optional: sign URLs for a host-reachable MinIO (compose.presign.yaml).
|
||||
S3_PRESIGN_ENDPOINT: ${S3_PRESIGN_ENDPOINT:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 20s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: ${API_CPU_LIMIT:-0.50}
|
||||
memory: ${API_MEM_LIMIT:-256m}
|
||||
cpus: ${API_CPU_LIMIT:-0.50} # для совместимости с docker compose up
|
||||
mem_limit: ${API_MEM_LIMIT:-256m}
|
||||
stop_grace_period: 15s
|
||||
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
NGINX_WORKERS: ${NGINX_WORKERS:-1}
|
||||
NGINX_ENVSUBST_TEMPLATE_DIR: /etc/nginx/templates
|
||||
NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx
|
||||
NGINX_ENVSUBST_FILTER: "^NGINX_"
|
||||
volumes:
|
||||
- ./lb/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
cpus: ${LB_CPU_LIMIT:-0.25}
|
||||
mem_limit: 128m
|
||||
stop_grace_period: 10s
|
||||
|
||||
# Private S3-compatible store. Ports 9000/9001 are NOT published to the host.
|
||||
# Image: pinned MinIO RELEASE.2025-04-22T22-12-26Z via local Dockerfile
|
||||
# (official binary from GitHub; docker.io hub may be unreachable).
|
||||
minio:
|
||||
build:
|
||||
context: ./infra/minio
|
||||
args:
|
||||
MINIO_VERSION: RELEASE.2025-04-22T22-12-26Z
|
||||
image: mtusi2026lab4-minio:RELEASE.2025-04-22T22-12-26Z
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
cpus: ${MINIO_CPU_LIMIT:-0.50}
|
||||
mem_limit: ${MINIO_MEM_LIMIT:-512m}
|
||||
stop_grace_period: 15s
|
||||
|
||||
# Idempotent bucket bootstrap: create if missing, force private ACL.
|
||||
minio-init:
|
||||
build:
|
||||
context: ./infra/mc
|
||||
args:
|
||||
MC_VERSION: RELEASE.2025-04-16T18-13-26Z
|
||||
image: mtusi2026lab4-mc:RELEASE.2025-04-16T18-13-26Z
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
echo "minio-init: waiting for endpoint $$S3_ENDPOINT"
|
||||
i=0
|
||||
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
i=$$((i + 1))
|
||||
if [ "$$i" -ge 30 ]; then
|
||||
echo "minio-init: cannot reach MinIO" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
mc mb --ignore-existing "local/$$S3_BUCKET"
|
||||
mc anonymous set none "local/$$S3_BUCKET"
|
||||
echo "minio-init: bucket listing:"
|
||||
mc ls "local/$$S3_BUCKET" || true
|
||||
echo "minio-init: bucket $$S3_BUCKET ready (anonymous=none)"
|
||||
restart: "no"
|
||||
|
||||
# Idempotent limited S3 user for the API (not MinIO root).
|
||||
minio-user-init:
|
||||
image: mtusi2026lab4-mc:RELEASE.2025-04-16T18-13-26Z
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
|
||||
MINIO_USER_ACCESS_KEY: ${MINIO_USER_ACCESS_KEY}
|
||||
MINIO_USER_SECRET_KEY: ${MINIO_USER_SECRET_KEY}
|
||||
S3_BUCKET: ${S3_BUCKET}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000}
|
||||
POLICY_NAME: messenger-attachments-rw
|
||||
volumes:
|
||||
- ./infra/minio/policy-attachments.json:/policy.template.json:ro
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
echo "minio-user-init: alias $$S3_ENDPOINT"
|
||||
i=0
|
||||
until mc alias set local "$$S3_ENDPOINT" "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
i=$$((i + 1))
|
||||
if [ "$$i" -ge 30 ]; then
|
||||
echo "minio-user-init: cannot reach MinIO" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy.template.json > /tmp/policy.json
|
||||
|
||||
if mc admin policy info local "$$POLICY_NAME" >/dev/null 2>&1; then
|
||||
mc admin policy remove local "$$POLICY_NAME" >/dev/null 2>&1 || true
|
||||
fi
|
||||
mc admin policy create local "$$POLICY_NAME" /tmp/policy.json
|
||||
|
||||
if mc admin user info local "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1; then
|
||||
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY already exists"
|
||||
else
|
||||
mc admin user add local "$$MINIO_USER_ACCESS_KEY" "$$MINIO_USER_SECRET_KEY"
|
||||
echo "minio-user-init: user $$MINIO_USER_ACCESS_KEY created"
|
||||
fi
|
||||
|
||||
mc admin policy attach local "$$POLICY_NAME" --user "$$MINIO_USER_ACCESS_KEY" >/dev/null 2>&1 || true
|
||||
echo "minio-user-init: policy $$POLICY_NAME attached to $$MINIO_USER_ACCESS_KEY"
|
||||
restart: "no"
|
||||
|
||||
secrets:
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
valkeydata: {}
|
||||
miniodata: {}
|
||||
+1
-1
@@ -50,7 +50,7 @@ Basic — это кодирование, не шифрование. HTTP lab1–
|
||||
- Файл заранее привязан к чату. В POST сообщения добавляется `attachment_ids` (0–10 уникальных ID, по умолчанию []). Можно отправить `text=""` с хотя бы одним вложением. Все вложения должны принадлежать этому чату; чужое/неизвестное — 404. Любой участник может ссылаться на доступное вложение чата. Failed-вложение прикреплять нельзя (409); queued/processing с lab5 можно.
|
||||
- `GET /attachments/{id}` и `/content` доступны только участникам чата. Download возвращает байты с истинным Content-Type, через авторизованный API, **без redirect** в обязательном контракте. Приватный bucket сам по себе не заменяет эту проверку. `variant` по умолчанию original. Для queued/processing content — 409; для failed — 409. Отсутствующий вариант ready-объекта — 404.
|
||||
- В ready-метаданных `error_code=null`. В failed — краткий машинный код (например `invalid_image`, `processing_failed`, `retry_exhausted`), без stack trace. `size_bytes`, `sha256`, `content_type` всегда описывают оригинал; у вариантов отдельные поля. Публичный API не раскрывает ключи бакета, внутренние адреса и credentials.
|
||||
- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html).
|
||||
- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html). В этом решении checksum на complete — SHA-256, заявленный при init (`expected_sha256`); тело complete пустое, отдельный hash в complete не принимается. Presigned PUT пишет только в `staging/{chat_id}/{upload_id}`. Complete копирует объект в `attachments/{chat_id}/{id}` и именно этот ключ сохраняет в `attachments.s3_key`. Повторный PUT той же ссылкой после complete не затирает оригинал. Схема расширения: `contracts/extensions.openapi.json`.
|
||||
|
||||
## Фоновая обработка (с lab5)
|
||||
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Docker Compose file-secrets keep host mode bits; app runs as uid 10001.
|
||||
# Copy S3 credentials into an app-owned path before dropping privileges.
|
||||
if [ "$(id -u)" = "0" ]; then
|
||||
mkdir -p /tmp/app-secrets
|
||||
if [ -r /run/secrets/s3_access_key_id ]; then
|
||||
cp /run/secrets/s3_access_key_id /tmp/app-secrets/s3_access_key_id
|
||||
fi
|
||||
if [ -r /run/secrets/s3_secret_access_key ]; then
|
||||
cp /run/secrets/s3_secret_access_key /tmp/app-secrets/s3_secret_access_key
|
||||
fi
|
||||
chown -R 10001:10001 /tmp/app-secrets
|
||||
chmod 0400 /tmp/app-secrets/* 2>/dev/null || true
|
||||
export S3_ACCESS_KEY_ID_FILE=/tmp/app-secrets/s3_access_key_id
|
||||
export S3_SECRET_ACCESS_KEY_FILE=/tmp/app-secrets/s3_secret_access_key
|
||||
exec setpriv --reuid=10001 --regid=10001 --clear-groups -- "$@"
|
||||
fi
|
||||
exec "$@"
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"concurrency": 10,
|
||||
"fixture_bytes": 5881918,
|
||||
"http_codes": {
|
||||
"201": 10
|
||||
},
|
||||
"success": 10,
|
||||
"wall_ms": 167347.8,
|
||||
"latency_ms": {
|
||||
"p50": 1125.9,
|
||||
"p95": 1630.0,
|
||||
"p99": 1630.0,
|
||||
"max": 1630.0
|
||||
},
|
||||
"approx_success_rps": 0.06,
|
||||
"api_rss_baseline_mib": 0.0,
|
||||
"api_rss_peak_mib": 0.0,
|
||||
"api_rss_delta_mib": 0.0,
|
||||
"api_cpu_peak_percent": 0.0,
|
||||
"naive_full_buffer_mib": 56.09,
|
||||
"max_concurrent_uploads": 3,
|
||||
"note": "RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; slot taken before buffering body >256KiB; malloc_trim after each upload. rss_peak \u2264 baseline + MAX_CONCURRENT_UPLOADS \u00d7 (body + decoded_raster). Closed-loop client; RPS is throughput of confirmed 201s."
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
# Grade 4 resilience (local stand, 2026-10-30)
|
||||
|
||||
## Persistence
|
||||
Command: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
|
||||
Result: PASS — message + attachment bytes preserved across restart of db/cache/minio/api/lb; volumes kept.
|
||||
|
||||
## Failover
|
||||
Command: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080
|
||||
Result: PASS
|
||||
observed_instances_before=2 successful_writes=115 failed_attempts=1 first_recovery_seconds≈5.3
|
||||
Stopped replica restored afterwards with: docker compose --env-file .env up -d --no-deps api
|
||||
|
||||
## Private bucket
|
||||
Command: sh tests/check_bucket_private.sh
|
||||
Result: PASS — unsigned GET → 403; host :9000/:9001 not published
|
||||
|
||||
## Limited S3 creds
|
||||
Command: sh student-tests/test_attachment_limited_creds.sh
|
||||
Result: PASS — API uses S3_*_FILE only; access key ≠ MinIO root; rb by limited user denied
|
||||
|
||||
## Load uploads
|
||||
Command: sh scripts/load_uploads.sh
|
||||
See evidence/load_uploads.json (10× ~5.6 MiB → 10×201; rss_peak≈163 MiB, rss_delta≈75 MiB).
|
||||
@@ -0,0 +1,19 @@
|
||||
# Local MinIO Client (mc) from the official GitHub release binary (pinned).
|
||||
# Fallback when docker.io/minio/mc cannot be pulled.
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG MC_VERSION=RELEASE.2025-04-16T18-13-26Z
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
&& curl -fsSL -o /tmp/mc.bin \
|
||||
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}" \
|
||||
&& curl -fsSL -o /tmp/mc.sha256sum \
|
||||
"https://github.com/minio/mc/releases/download/${MC_VERSION}/mc.linux-amd64.${MC_VERSION}.sha256sum" \
|
||||
&& expected="$(awk '{print $1}' /tmp/mc.sha256sum)" \
|
||||
&& echo "${expected} /tmp/mc.bin" | sha256sum -c - \
|
||||
&& install -m 0755 /tmp/mc.bin /usr/local/bin/mc \
|
||||
&& rm -f /tmp/mc.bin /tmp/mc.sha256sum \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENTRYPOINT ["mc"]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Local MinIO image from the official GitHub release binary (pinned).
|
||||
# Fallback when docker.io/minio/minio cannot be pulled.
|
||||
FROM ubuntu:24.04
|
||||
|
||||
ARG MINIO_VERSION=RELEASE.2025-04-22T22-12-26Z
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
&& curl -fsSL -o /tmp/minio.bin \
|
||||
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}" \
|
||||
&& curl -fsSL -o /tmp/minio.sha256sum \
|
||||
"https://github.com/minio/minio/releases/download/${MINIO_VERSION}/minio.linux-amd64.${MINIO_VERSION}.sha256sum" \
|
||||
&& expected="$(awk '{print $1}' /tmp/minio.sha256sum)" \
|
||||
&& echo "${expected} /tmp/minio.bin" | sha256sum -c - \
|
||||
&& install -m 0755 /tmp/minio.bin /usr/local/bin/minio \
|
||||
&& rm -f /tmp/minio.bin /tmp/minio.sha256sum \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 9000 9001
|
||||
ENTRYPOINT ["minio"]
|
||||
CMD ["server", "/data", "--console-address", ":9001"]
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:GetBucketLocation",
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:PutObject",
|
||||
"s3:GetObject",
|
||||
"s3:DeleteObject",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListMultipartUploadParts"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::__S3_BUCKET__/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
worker_processes ${NGINX_WORKERS};
|
||||
error_log /dev/stderr warn;
|
||||
pid /tmp/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log off;
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 2s;
|
||||
|
||||
upstream api_upstream {
|
||||
zone api_upstream 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=5s;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
# Lab4 attachments ≤10 MiB; allow a little headroom for multipart/headers.
|
||||
client_max_body_size 12m;
|
||||
client_body_timeout 60s;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 10s;
|
||||
proxy_connect_timeout 2s;
|
||||
# Uploads/downloads of ≤10 MiB need longer than JSON chat traffic.
|
||||
proxy_read_timeout 120s;
|
||||
proxy_send_timeout 120s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_chat_keyset_idx
|
||||
ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE TABLE sessions (
|
||||
token text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,22 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
filename text NOT NULL
|
||||
CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL
|
||||
CHECK (size_bytes >= 1 AND size_bytes <= 10485760),
|
||||
sha256 text NOT NULL
|
||||
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
status text NOT NULL
|
||||
CHECK (status IN ('ready')),
|
||||
s3_key text NOT NULL UNIQUE,
|
||||
error_code text NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
|
||||
CREATE INDEX attachments_created_at_idx ON attachments (created_at);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position integer NOT NULL CHECK (position >= 0 AND position < 10),
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx
|
||||
ON message_attachments (attachment_id);
|
||||
|
||||
CREATE UNIQUE INDEX message_attachments_order_idx
|
||||
ON message_attachments (message_id, position);
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Track 1 (grade 5): pending direct/presigned upload sessions.
|
||||
-- Attachment rows stay status=ready only; incomplete uploads live here.
|
||||
CREATE TABLE direct_uploads (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
filename text NOT NULL
|
||||
CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
expected_size integer NOT NULL
|
||||
CHECK (expected_size >= 1 AND expected_size <= 10485760),
|
||||
expected_sha256 text NOT NULL
|
||||
CHECK (expected_sha256 ~ '^[a-f0-9]{64}$'),
|
||||
s3_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL
|
||||
CHECK (status IN ('pending', 'completed', 'aborted', 'expired')),
|
||||
attachment_id uuid NULL REFERENCES attachments(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX direct_uploads_status_expires_idx
|
||||
ON direct_uploads (status, expires_at);
|
||||
CREATE INDEX direct_uploads_owner_idx ON direct_uploads (owner_id);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Track 1: presigned PUT lands in staging/; complete copies to attachments/.
|
||||
-- Existing rows keep the old key as staging so in-flight sessions still resolve.
|
||||
ALTER TABLE direct_uploads ADD COLUMN staging_key text;
|
||||
|
||||
UPDATE direct_uploads
|
||||
SET staging_key = s3_key
|
||||
WHERE staging_key IS NULL;
|
||||
|
||||
ALTER TABLE direct_uploads ALTER COLUMN staging_key SET NOT NULL;
|
||||
|
||||
ALTER TABLE direct_uploads
|
||||
ADD CONSTRAINT direct_uploads_staging_key_key UNIQUE (staging_key);
|
||||
Executable
+254
@@ -0,0 +1,254 @@
|
||||
#!/bin/sh
|
||||
# Grade 4 load experiment: ≥10 concurrent ~5 MiB image uploads; sample API RSS.
|
||||
# Writes evidence/load_uploads.json (no tokens/secrets).
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
BASE_URL="${BASE_URL:-http://127.0.0.1:8080}"
|
||||
N="${LOAD_UPLOADS_N:-10}"
|
||||
OUT="${LOAD_UPLOADS_OUT:-$ROOT/evidence/load_uploads.json}"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
|
||||
. "$ROOT/student-tests/lib.sh"
|
||||
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
echo "load_uploads: generating ~5 MiB PNG fixture..."
|
||||
python3 - "$TMP/load.png" <<'PY'
|
||||
import struct, zlib, sys
|
||||
w, h = 1400, 1400
|
||||
x = 0xC0FFEE
|
||||
raw = bytearray()
|
||||
for _y in range(h):
|
||||
raw.append(0)
|
||||
for _i in range(w * 3):
|
||||
x ^= (x << 13) & 0xffffffff
|
||||
x ^= (x >> 17) & 0xffffffff
|
||||
x ^= (x << 5) & 0xffffffff
|
||||
raw.append(x & 0xff)
|
||||
|
||||
def chunk(tag, data):
|
||||
return struct.pack(">I", len(data)) + tag + data + struct.pack(
|
||||
">I", zlib.crc32(tag + data) & 0xffffffff)
|
||||
|
||||
comp = zlib.compress(bytes(raw), 0)
|
||||
ihdr = struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)
|
||||
png = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", comp) + chunk(b"IEND", b"")
|
||||
open(sys.argv[1], "wb").write(png)
|
||||
print(len(png))
|
||||
PY
|
||||
SIZE=$(wc -c < "$TMP/load.png" | tr -d ' ')
|
||||
echo "load_uploads: fixture_bytes=$SIZE concurrency=$N"
|
||||
|
||||
register_user loadu
|
||||
create_chat 'load-uploads' "[\"$USER_ID\"]"
|
||||
|
||||
# Fault code/data pages and S3 client paths. Fresh ubuntu:24.04 containers
|
||||
# often report ~4 MiB idle RSS; after a single upload glibc may return arenas
|
||||
# to the OS and docker stats drops again — using that post-trim floor as
|
||||
# baseline would charge library/arena re-growth to the concurrent window.
|
||||
# Sample RSS *during* warm uploads and keep the warm peak as baseline floor.
|
||||
http_json GET /health/ready || true
|
||||
: > "$TMP/baseline.txt"
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
|
||||
$($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true
|
||||
i=$((i + 1))
|
||||
sleep 0.25
|
||||
done
|
||||
) &
|
||||
warm_stats_pid=$!
|
||||
echo "load_uploads: warm uploads ×3 parallel (establish concurrent RSS watermark)..."
|
||||
# Same size as the load fixture; concurrency matches MAX_CONCURRENT_UPLOADS so
|
||||
# baseline already includes the capped in-flight footprint.
|
||||
w=0
|
||||
while [ "$w" -lt 3 ]; do
|
||||
(
|
||||
code=$(curl -sS -m 180 -o /dev/null -w '%{http_code}' \
|
||||
-X POST "$BASE_URL/api/v1/chats/$CHAT_ID/attachments?filename=warm_$w.png" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H 'Content-Type: image/png' \
|
||||
--data-binary @"$TMP/load.png" || printf '000')
|
||||
echo "$code" > "$TMP/warm_$w.code"
|
||||
) &
|
||||
w=$((w + 1))
|
||||
done
|
||||
wait
|
||||
w=0
|
||||
while [ "$w" -lt 3 ]; do
|
||||
[ "$(cat "$TMP/warm_$w.code")" = "201" ] || die "warm upload $w failed: $(cat "$TMP/warm_$w.code")"
|
||||
w=$((w + 1))
|
||||
done
|
||||
sleep 1
|
||||
kill "$warm_stats_pid" 2>/dev/null || true
|
||||
wait "$warm_stats_pid" 2>/dev/null || true
|
||||
# A few post-warm samples (median path still runs); warm peak dominates baseline.
|
||||
b=0
|
||||
while [ "$b" -lt 4 ]; do
|
||||
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
|
||||
$($COMPOSE ps -q api) >> "$TMP/baseline.txt" 2>/dev/null || true
|
||||
b=$((b + 1))
|
||||
sleep 0.3
|
||||
done
|
||||
|
||||
# Sample RSS while uploads run.
|
||||
(
|
||||
i=0
|
||||
while [ "$i" -lt 80 ]; do
|
||||
docker stats --no-stream --format '{{.Name}} {{.MemUsage}} {{.CPUPerc}}' \
|
||||
$($COMPOSE ps -q api) >> "$TMP/stats.txt" 2>/dev/null || true
|
||||
i=$((i + 1))
|
||||
sleep 0.35
|
||||
done
|
||||
) &
|
||||
stats_pid=$!
|
||||
|
||||
t0=$(python3 -c 'import time; print(time.time())')
|
||||
|
||||
i=0
|
||||
while [ "$i" -lt "$N" ]; do
|
||||
(
|
||||
python3 - "$BASE_URL" "$CHAT_ID" "$TOKEN" "$TMP/load.png" "$TMP/res_$i.json" <<'PY'
|
||||
import json, sys, time, urllib.request
|
||||
base, chat, token, path, out = sys.argv[1:6]
|
||||
url = f"{base}/api/v1/chats/{chat}/attachments?filename=load.png"
|
||||
data = open(path, "rb").read()
|
||||
req = urllib.request.Request(
|
||||
url, data=data, method="POST",
|
||||
headers={"Authorization": f"Bearer {token}", "Content-Type": "image/png",
|
||||
"Content-Length": str(len(data))})
|
||||
t0 = time.time()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=180) as resp:
|
||||
code = resp.status
|
||||
body = resp.read()
|
||||
except Exception as e:
|
||||
code = 0
|
||||
body = str(e).encode()
|
||||
dt = (time.time() - t0) * 1000.0
|
||||
open(out, "w").write(json.dumps({"code": code, "latency_ms": dt, "bytes": len(body)}))
|
||||
PY
|
||||
) &
|
||||
i=$((i + 1))
|
||||
done
|
||||
wait
|
||||
t1=$(python3 -c 'import time; print(time.time())')
|
||||
|
||||
kill "$stats_pid" 2>/dev/null || true
|
||||
wait "$stats_pid" 2>/dev/null || true
|
||||
|
||||
python3 - "$TMP" "$OUT" "$N" "$SIZE" "$t0" "$t1" <<'PY'
|
||||
import json, os, re, sys
|
||||
tmpdir, out_path, n, size = sys.argv[1], sys.argv[2], int(sys.argv[3]), int(sys.argv[4])
|
||||
wall = (float(sys.argv[6]) - float(sys.argv[5])) * 1000.0
|
||||
|
||||
def line_mib(line):
|
||||
m = re.search(r"([\d.]+)(MiB|GiB)", line)
|
||||
if not m:
|
||||
return None
|
||||
v = float(m.group(1))
|
||||
return v * 1024.0 if m.group(2) == "GiB" else v
|
||||
|
||||
def sample_peaks(path):
|
||||
"""One docker-stats snapshot = two api lines; return peak per snapshot."""
|
||||
peaks = []
|
||||
cur = 0.0
|
||||
nlines = 0
|
||||
if not os.path.exists(path):
|
||||
return peaks
|
||||
for line in open(path):
|
||||
v = line_mib(line)
|
||||
if v is None:
|
||||
continue
|
||||
cur = max(cur, v)
|
||||
nlines += 1
|
||||
if nlines % 2 == 0:
|
||||
peaks.append(cur)
|
||||
cur = 0.0
|
||||
if nlines % 2 == 1 and cur > 0:
|
||||
peaks.append(cur)
|
||||
return peaks
|
||||
|
||||
def cpu_peak(path):
|
||||
peak = 0.0
|
||||
if not os.path.exists(path):
|
||||
return peak
|
||||
for line in open(path):
|
||||
c = re.search(r"([\d.]+)%", line)
|
||||
if c:
|
||||
peak = max(peak, float(c.group(1)))
|
||||
return peak
|
||||
|
||||
base_peaks = sample_peaks(os.path.join(tmpdir, "baseline.txt"))
|
||||
load_peaks = sample_peaks(os.path.join(tmpdir, "stats.txt"))
|
||||
# Baseline = max RSS observed during warm uploads (not post-trim idle floor).
|
||||
base_mib = max(base_peaks) if base_peaks else 0.0
|
||||
peak_mib = max(load_peaks) if load_peaks else 0.0
|
||||
# If load peak is below warm peak, delta is 0 (no extra RSS under concurrency).
|
||||
cpu_p = cpu_peak(os.path.join(tmpdir, "stats.txt"))
|
||||
delta = max(0.0, peak_mib - base_mib)
|
||||
|
||||
ok = 0
|
||||
lat = []
|
||||
codes = {}
|
||||
for i in range(n):
|
||||
row = json.load(open(os.path.join(tmpdir, f"res_{i}.json")))
|
||||
code = str(row["code"])
|
||||
codes[code] = codes.get(code, 0) + 1
|
||||
if code == "201":
|
||||
ok += 1
|
||||
lat.append(float(row["latency_ms"]))
|
||||
|
||||
def pct(xs, p):
|
||||
if not xs:
|
||||
return None
|
||||
xs = sorted(xs)
|
||||
k = min(len(xs) - 1, max(0, int(round((p / 100.0) * (len(xs) - 1)))))
|
||||
return round(xs[k], 1)
|
||||
|
||||
linear_mib = (n * size) / (1024 * 1024)
|
||||
rps = (ok / (wall / 1000.0)) if wall > 0 else None
|
||||
summary = {
|
||||
"concurrency": n,
|
||||
"fixture_bytes": size,
|
||||
"http_codes": codes,
|
||||
"success": ok,
|
||||
"wall_ms": round(wall, 1),
|
||||
"latency_ms": {
|
||||
"p50": pct(lat, 50),
|
||||
"p95": pct(lat, 95),
|
||||
"p99": pct(lat, 99),
|
||||
"max": round(max(lat), 1) if lat else None,
|
||||
},
|
||||
"approx_success_rps": round(rps, 3) if rps else None,
|
||||
"api_rss_baseline_mib": round(base_mib, 2),
|
||||
"api_rss_peak_mib": round(peak_mib, 2),
|
||||
"api_rss_delta_mib": round(delta, 2),
|
||||
"api_cpu_peak_percent": round(cpu_p, 2),
|
||||
"naive_full_buffer_mib": round(linear_mib, 2),
|
||||
"max_concurrent_uploads": int(os.environ.get("MAX_CONCURRENT_UPLOADS", "3")),
|
||||
"note": (
|
||||
"RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; "
|
||||
"slot taken before buffering body >256KiB; malloc_trim after each upload. "
|
||||
"rss_peak ≤ baseline + MAX_CONCURRENT_UPLOADS × (body + decoded_raster). "
|
||||
"Closed-loop client; RPS is throughput of confirmed 201s."
|
||||
),
|
||||
}
|
||||
open(out_path, "w").write(json.dumps(summary, indent=2) + "\n")
|
||||
print(json.dumps(summary, indent=2))
|
||||
if ok < n:
|
||||
raise SystemExit(1)
|
||||
if not (delta < linear_mib):
|
||||
print(
|
||||
f"FAIL criterion: rss_delta={delta:.1f}MiB >= naive={linear_mib:.1f}MiB",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2)
|
||||
PY
|
||||
|
||||
echo "load_uploads: wrote $OUT"
|
||||
echo "PASS load_uploads"
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Persistence hook: restart API and all state stores; keep volumes.
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
compose() {
|
||||
docker compose --env-file .env "$@"
|
||||
}
|
||||
|
||||
deadline=$(( $(date +%s) + 70 ))
|
||||
|
||||
wait_healthy() {
|
||||
svc=$1
|
||||
want=1
|
||||
if [ "$svc" = "api" ]; then
|
||||
want=${API_REPLICAS:-2}
|
||||
fi
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
states=$(compose ps "$svc" --format '{{.Health}}' || true)
|
||||
n=$(printf '%s\n' "$states" | grep -cx healthy || true)
|
||||
if [ "$n" -ge "$want" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "timed out waiting for $svc" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
compose restart db cache minio
|
||||
wait_healthy db
|
||||
wait_healthy cache
|
||||
wait_healthy minio
|
||||
compose start api
|
||||
compose restart api
|
||||
wait_healthy api
|
||||
compose restart lb
|
||||
wait_healthy lb
|
||||
exit 0
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Failover hook: stop ONE API replica matching TARGET_INSTANCE_ID (hostname).
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
found=""
|
||||
ids=$(docker compose --env-file .env ps -q api)
|
||||
for id in $ids; do
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$id
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
echo "api replica not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker stop -t 8 "$found" >/dev/null
|
||||
exit 0
|
||||
@@ -0,0 +1 @@
|
||||
change_me_minio_root_locally
|
||||
@@ -0,0 +1 @@
|
||||
minioadmin
|
||||
@@ -0,0 +1 @@
|
||||
messenger-s3
|
||||
@@ -0,0 +1 @@
|
||||
change_me_s3_user_locally
|
||||
@@ -0,0 +1,125 @@
|
||||
#include "attachment_repo.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch())
|
||||
.count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
Attachment row_to_attachment(const pqxx::row& r) {
|
||||
Attachment a;
|
||||
a.id = r["id"].as<std::string>();
|
||||
a.chat_id = r["chat_id"].as<std::string>();
|
||||
a.owner_id = r["owner_id"].as<std::string>();
|
||||
a.filename = r["filename"].as<std::string>();
|
||||
a.content_type = r["content_type"].as<std::string>();
|
||||
a.size_bytes = r["size_bytes"].as<int>();
|
||||
a.sha256 = r["sha256"].as<std::string>();
|
||||
a.status = r["status"].as<std::string>();
|
||||
a.s3_key = r["s3_key"].as<std::string>();
|
||||
if (r["error_code"].is_null()) {
|
||||
a.error_code = std::nullopt;
|
||||
} else {
|
||||
a.error_code = r["error_code"].as<std::string>();
|
||||
}
|
||||
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
|
||||
a.variants.clear();
|
||||
return a;
|
||||
}
|
||||
|
||||
constexpr const char* kAttachmentColumns =
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, s3_key, error_code, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
|
||||
|
||||
} // namespace
|
||||
|
||||
Attachment AttachmentRepo::insert(const Attachment& in) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
pqxx::result r;
|
||||
if (in.error_code) {
|
||||
r = tx.exec(
|
||||
"INSERT INTO attachments ("
|
||||
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
" status, s3_key, error_code, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, "
|
||||
" to_timestamp($11::bigint), to_timestamp($12::bigint)"
|
||||
") RETURNING " +
|
||||
std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.status,
|
||||
in.s3_key, *in.error_code, ts, ts});
|
||||
} else {
|
||||
r = tx.exec(
|
||||
"INSERT INTO attachments ("
|
||||
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
" status, s3_key, error_code, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
|
||||
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
|
||||
") RETURNING " +
|
||||
std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.status,
|
||||
in.s3_key, ts, ts});
|
||||
}
|
||||
tx.commit();
|
||||
return row_to_attachment(r[0]);
|
||||
}
|
||||
|
||||
std::optional<Attachment> AttachmentRepo::find_by_id(const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kAttachmentColumns) +
|
||||
" FROM attachments WHERE id = $1::uuid",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return row_to_attachment(r[0]);
|
||||
}
|
||||
|
||||
bool AttachmentRepo::all_belong_to_chat(const std::string& chat_id,
|
||||
const std::vector<std::string>& ids) {
|
||||
if (ids.empty()) {
|
||||
return true;
|
||||
}
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
for (const auto& id : ids) {
|
||||
auto r = tx.exec(
|
||||
"SELECT 1 FROM attachments "
|
||||
"WHERE id = $1::uuid AND chat_id = $2::uuid AND status = 'ready'",
|
||||
pqxx::params{id, chat_id});
|
||||
if (r.empty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class AttachmentRepo {
|
||||
public:
|
||||
explicit AttachmentRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
Attachment insert(const Attachment& a);
|
||||
|
||||
std::optional<Attachment> find_by_id(const std::string& id);
|
||||
|
||||
// All IDs must exist, belong to chat_id, and have status=ready.
|
||||
// Returns false if any id is missing/foreign/failed.
|
||||
bool all_belong_to_chat(const std::string& chat_id,
|
||||
const std::vector<std::string>& ids);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,39 @@
|
||||
#include "auth.hpp"
|
||||
|
||||
#include <sodium.h>
|
||||
#include <stdexcept>
|
||||
|
||||
|
||||
namespace {
|
||||
struct SodiumInit {
|
||||
SodiumInit() {
|
||||
if (sodium_init() < 0) {
|
||||
throw std::runtime_error("sodium_init failed");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const SodiumInit g_sodium_init;
|
||||
}
|
||||
|
||||
|
||||
std::string hash_password(const std::string& plain) {
|
||||
char hashed[crypto_pwhash_STRBYTES];
|
||||
if (crypto_pwhash_str(
|
||||
hashed,
|
||||
plain.data(), plain.size(),
|
||||
crypto_pwhash_OPSLIMIT_INTERACTIVE,
|
||||
crypto_pwhash_MEMLIMIT_INTERACTIVE) != 0) {
|
||||
throw std::runtime_error("crypto_pwhash_str failed (out of memory?)");
|
||||
}
|
||||
return std::string(hashed);
|
||||
}
|
||||
|
||||
bool verify_password(const std::string& plain, const std::string& hash) {
|
||||
if (hash.empty() || hash.size() >= crypto_pwhash_STRBYTES) {
|
||||
return false;
|
||||
}
|
||||
return crypto_pwhash_str_verify(
|
||||
hash.c_str(),
|
||||
plain.data(), plain.size()) == 0;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
std::string hash_password(const std::string& plain);
|
||||
|
||||
bool verify_password(const std::string& plain, const std::string& hash);
|
||||
@@ -0,0 +1,186 @@
|
||||
#include "chat_repo.hpp"
|
||||
#include "cursor.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string make_uuid() {
|
||||
static std::mt19937_64 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
|
||||
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
|
||||
std::ostringstream out;
|
||||
out << std::hex << std::setfill('0');
|
||||
out << std::setw(8) << d32(rng);
|
||||
out << '-' << std::setw(4) << d16(rng);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
|
||||
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch()).count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
Chat row_to_chat(const pqxx::row& r) {
|
||||
Chat c;
|
||||
c.id = r["id"].as<std::string>();
|
||||
c.title = r["title"].as<std::string>();
|
||||
c.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
return c;
|
||||
}
|
||||
|
||||
std::vector<std::string> load_member_ids(pqxx::work& tx,
|
||||
const std::string& chat_id) {
|
||||
std::vector<std::string> ids;
|
||||
auto r = tx.exec(
|
||||
"SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id",
|
||||
pqxx::params{chat_id});
|
||||
ids.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
ids.push_back(row["user_id"].as<std::string>());
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
constexpr const char* kChatSelect =
|
||||
"id, title, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
|
||||
|
||||
}
|
||||
|
||||
std::optional<Chat> ChatRepo::create(const std::string& title,
|
||||
const std::vector<std::string>& member_ids) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
const std::string chat_id = make_uuid();
|
||||
const std::string created = now_epoch_str();
|
||||
|
||||
try {
|
||||
tx.exec(
|
||||
"INSERT INTO chats (id, title, created_at) "
|
||||
"VALUES ($1, $2, to_timestamp($3::bigint))",
|
||||
pqxx::params{chat_id, title, created});
|
||||
|
||||
for (const auto& uid : member_ids) {
|
||||
tx.exec(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)",
|
||||
pqxx::params{chat_id, uid});
|
||||
}
|
||||
} catch (const pqxx::foreign_key_violation&) {
|
||||
tx.abort();
|
||||
return std::nullopt;
|
||||
} catch (const pqxx::unique_violation&) {
|
||||
tx.abort();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
|
||||
pqxx::params{chat_id});
|
||||
Chat c = row_to_chat(r[0]);
|
||||
c.member_ids = load_member_ids(tx, chat_id);
|
||||
tx.commit();
|
||||
return c;
|
||||
}
|
||||
|
||||
std::optional<Chat> ChatRepo::find_by_id(const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kChatSelect) + " FROM chats WHERE id = $1",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
Chat c = row_to_chat(r[0]);
|
||||
c.member_ids = load_member_ids(tx, id);
|
||||
tx.commit();
|
||||
return c;
|
||||
}
|
||||
|
||||
|
||||
std::optional<ChatPage> ChatRepo::list_for_user(
|
||||
const std::string& user_id,
|
||||
int limit,
|
||||
const std::optional<std::string>& cursor) {
|
||||
|
||||
std::optional<Cursor> cur;
|
||||
if (cursor) {
|
||||
cur = parse_cursor(*cursor);
|
||||
if (!cur) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
if (cur) {
|
||||
auto owner_check = tx.exec(
|
||||
"SELECT 1 FROM chats WHERE id = $1 AND id IN "
|
||||
"(SELECT chat_id FROM chat_members WHERE user_id = $2)",
|
||||
pqxx::params{cur->id, user_id});
|
||||
if (owner_check.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::string sql =
|
||||
"SELECT " + std::string(kChatSelect) + ", " + kCreatedUsSql + " "
|
||||
"FROM chats "
|
||||
"WHERE id IN (SELECT chat_id FROM chat_members WHERE user_id = $1) ";
|
||||
if (cur) {
|
||||
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
|
||||
}
|
||||
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
|
||||
|
||||
pqxx::result r;
|
||||
if (cur) {
|
||||
r = tx.exec(sql, pqxx::params{user_id, limit + 1, cur->created_us, cur->id});
|
||||
} else {
|
||||
r = tx.exec(sql, pqxx::params{user_id, limit + 1});
|
||||
}
|
||||
|
||||
ChatPage page;
|
||||
const bool has_more = (static_cast<int>(r.size()) > limit);
|
||||
const int take = has_more ? limit : static_cast<int>(r.size());
|
||||
page.items.reserve(take);
|
||||
for (int i = 0; i < take; ++i) {
|
||||
Chat c = row_to_chat(r[static_cast<std::size_t>(i)]);
|
||||
c.member_ids = load_member_ids(tx, c.id);
|
||||
page.items.push_back(std::move(c));
|
||||
}
|
||||
|
||||
if (has_more && take > 0) {
|
||||
const auto& last = r[static_cast<std::size_t>(take - 1)];
|
||||
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
|
||||
last["id"].as<std::string>());
|
||||
}
|
||||
|
||||
tx.commit();
|
||||
return page;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class ChatRepo {
|
||||
public:
|
||||
explicit ChatRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
std::optional<Chat> create(const std::string& title,
|
||||
const std::vector<std::string>& member_ids);
|
||||
|
||||
std::optional<Chat> find_by_id(const std::string& id);
|
||||
|
||||
std::optional<ChatPage> list_for_user(const std::string& user_id, int limit, const std::optional<std::string>& cursor);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,47 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <optional>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
|
||||
// Непрозрачный для клиента курсор keyset-пагинации: "<created_at в мкс от epoch>|<uuid>".
|
||||
// Микросекунды сохраняют точный created_at строки, даже если в JSON время округлено до секунд.
|
||||
struct Cursor {
|
||||
std::string created_us;
|
||||
std::string id;
|
||||
};
|
||||
|
||||
// Выражение SQL, восстанавливающее timestamptz из created_us без потери точности.
|
||||
constexpr const char* kCursorTimestampSql =
|
||||
"(timestamptz 'epoch' + $3::bigint * interval '1 microsecond')";
|
||||
|
||||
constexpr const char* kCreatedUsSql =
|
||||
"(EXTRACT(EPOCH FROM created_at) * 1000000)::bigint AS created_us";
|
||||
|
||||
inline bool is_uuid(const std::string& s) {
|
||||
static const std::regex re(
|
||||
"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
|
||||
"[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$");
|
||||
return std::regex_match(s, re);
|
||||
}
|
||||
|
||||
inline std::optional<Cursor> parse_cursor(const std::string& raw) {
|
||||
const auto bar = raw.find('|');
|
||||
if (bar == std::string::npos) {
|
||||
return std::nullopt;
|
||||
}
|
||||
Cursor c{raw.substr(0, bar), raw.substr(bar + 1)};
|
||||
const bool digits = !c.created_us.empty() && c.created_us.size() <= 18 &&
|
||||
std::all_of(c.created_us.begin(), c.created_us.end(),
|
||||
[](unsigned char ch) { return std::isdigit(ch); });
|
||||
if (!digits || !is_uuid(c.id)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
inline std::string make_cursor(const std::string& created_us, const std::string& id) {
|
||||
return created_us + "|" + id;
|
||||
}
|
||||
+360
@@ -0,0 +1,360 @@
|
||||
#include "db.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <future>
|
||||
#include <thread>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/time.h>
|
||||
#include <stdexcept>
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <iostream>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr int kConnectTimeoutSeconds = 2;
|
||||
constexpr int kTcpUserTimeoutMs = 5000;
|
||||
constexpr int kStatementTimeoutMs = 5000;
|
||||
constexpr int kClientIoTimeoutSeconds = 5;
|
||||
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
|
||||
|
||||
std::string add_conn_param(std::string conninfo, const std::string& key,
|
||||
const std::string& value) {
|
||||
if (conninfo.find(key + "=") != std::string::npos) {
|
||||
return conninfo;
|
||||
}
|
||||
const bool is_url = conninfo.rfind("postgres://", 0) == 0 ||
|
||||
conninfo.rfind("postgresql://", 0) == 0;
|
||||
if (is_url) {
|
||||
conninfo += (conninfo.find('?') == std::string::npos ? "?" : "&");
|
||||
} else {
|
||||
conninfo += " ";
|
||||
}
|
||||
return conninfo + key + "=" + value;
|
||||
}
|
||||
|
||||
bool is_alive(pqxx::connection* conn) {
|
||||
std::promise<void> done;
|
||||
auto fut = done.get_future();
|
||||
std::thread th([conn, &done]() {
|
||||
try {
|
||||
pqxx::nontransaction tx(*conn);
|
||||
tx.exec("SELECT 1");
|
||||
done.set_value();
|
||||
} catch (...) {
|
||||
done.set_exception(std::current_exception());
|
||||
}
|
||||
});
|
||||
if (fut.wait_for(std::chrono::seconds(kClientIoTimeoutSeconds)) !=
|
||||
std::future_status::ready) {
|
||||
const int fd = conn->sock();
|
||||
if (fd >= 0) {
|
||||
::shutdown(fd, SHUT_RDWR);
|
||||
}
|
||||
th.join();
|
||||
return false;
|
||||
}
|
||||
th.join();
|
||||
try {
|
||||
fut.get();
|
||||
return true;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
struct Migration {
|
||||
int version;
|
||||
std::string path;
|
||||
};
|
||||
|
||||
constexpr int kMigrationsAdvisoryLockKey = 0x6d747573;
|
||||
|
||||
|
||||
Migration parse_migration_name(const fs::path& p) {
|
||||
const std::string name = p.filename().string();
|
||||
const auto dot = name.find('.');
|
||||
|
||||
if (dot == std::string::npos || name.substr(dot) != ".sql") {
|
||||
throw std::runtime_error("migration file must end with .sql: " + name);
|
||||
}
|
||||
|
||||
const auto us = name.find('_');
|
||||
if (us == std::string::npos || us == 0) {
|
||||
throw std::runtime_error(
|
||||
"migration file must be named NNN_name.sql: " + name);
|
||||
}
|
||||
|
||||
const std::string num = name.substr(0, us);
|
||||
if (!std::all_of(num.begin(), num.end(),
|
||||
[](unsigned char c){ return std::isdigit(c); })) {
|
||||
throw std::runtime_error(
|
||||
"migration file prefix must be digits: " + name);
|
||||
}
|
||||
|
||||
Migration m;
|
||||
m.version = std::stoi(num);
|
||||
m.path = p.string();
|
||||
return m;
|
||||
}}
|
||||
|
||||
|
||||
void apply_migrations(pqxx::connection& conn, const std::string& dir) {
|
||||
{
|
||||
pqxx::work tx(conn);
|
||||
tx.exec("SELECT pg_advisory_lock($1)",
|
||||
pqxx::params{kMigrationsAdvisoryLockKey});
|
||||
tx.commit();
|
||||
}
|
||||
std::clog << "migration advisory lock acquired\n";
|
||||
struct Unlock {
|
||||
pqxx::connection& c;
|
||||
~Unlock() {
|
||||
try {
|
||||
pqxx::work tx(c);
|
||||
tx.exec("SELECT pg_advisory_unlock($1)",
|
||||
pqxx::params{kMigrationsAdvisoryLockKey});
|
||||
tx.commit();
|
||||
} catch (...) {}
|
||||
}
|
||||
} unlock_guard{conn};
|
||||
|
||||
{
|
||||
pqxx::work tx(conn);
|
||||
tx.exec(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version int PRIMARY KEY,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now()"
|
||||
")");
|
||||
tx.commit();
|
||||
}
|
||||
|
||||
if (!fs::exists(dir) || !fs::is_directory(dir)) {
|
||||
throw std::runtime_error("migrations dir not found: " + dir);
|
||||
}
|
||||
std::vector<Migration> files;
|
||||
for (const auto& entry : fs::directory_iterator(dir)) {
|
||||
if (entry.is_regular_file()) {
|
||||
files.push_back(parse_migration_name(entry.path()));
|
||||
}
|
||||
}
|
||||
std::sort(files.begin(), files.end(),
|
||||
[](const Migration& a, const Migration& b){
|
||||
return a.version < b.version;
|
||||
});
|
||||
|
||||
for (std::size_t i = 1; i < files.size(); ++i) {
|
||||
if (files[i].version == files[i-1].version) {
|
||||
throw std::runtime_error(
|
||||
"duplicate migration version: " + std::to_string(files[i].version));
|
||||
}
|
||||
}
|
||||
|
||||
std::set<int> applied;
|
||||
{
|
||||
pqxx::work tx(conn);
|
||||
for (const auto& row : tx.exec("SELECT version FROM schema_migrations")) {
|
||||
applied.insert(row[0].as<int>());
|
||||
}
|
||||
tx.commit();
|
||||
}
|
||||
|
||||
for (const auto& m : files) {
|
||||
if (applied.count(m.version)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::ifstream f(m.path, std::ios::binary);
|
||||
if (!f) {
|
||||
throw std::runtime_error("cannot open migration: " + m.path);
|
||||
}
|
||||
std::stringstream buf;
|
||||
buf << f.rdbuf();
|
||||
const std::string sql = buf.str();
|
||||
|
||||
pqxx::work tx(conn);
|
||||
tx.exec(sql);
|
||||
tx.exec("INSERT INTO schema_migrations(version) VALUES ($1)",
|
||||
m.version);
|
||||
tx.commit();
|
||||
|
||||
std::clog << "applied migration " << m.version
|
||||
<< " (" << m.path << ")\n";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::string read_secret_file(const char* env_name) {
|
||||
const char* path = std::getenv(env_name);
|
||||
|
||||
if (path == nullptr || *path == '\0') {
|
||||
throw std::runtime_error(
|
||||
std::string("environment variable not set: ") + env_name);
|
||||
}
|
||||
|
||||
std::ifstream f(path, std::ios::binary);
|
||||
if (!f) {
|
||||
throw std::runtime_error(
|
||||
std::string("cannot open secret file: ") + path);
|
||||
}
|
||||
|
||||
std::string s;
|
||||
std::getline(f, s);
|
||||
if (!s.empty() && s.back() == '\r') {
|
||||
s.pop_back();
|
||||
}
|
||||
|
||||
if (s.empty()) {
|
||||
throw std::runtime_error(
|
||||
std::string("secret file is empty: ") + path);
|
||||
}
|
||||
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
bool is_dependency_failure(const std::exception& e) {
|
||||
if (dynamic_cast<const DependencyUnavailable*>(&e) != nullptr ||
|
||||
dynamic_cast<const pqxx::broken_connection*>(&e) != nullptr ||
|
||||
dynamic_cast<const pqxx::in_doubt_error*>(&e) != nullptr) {
|
||||
return true;
|
||||
}
|
||||
if (const auto* sql = dynamic_cast<const pqxx::sql_error*>(&e)) {
|
||||
// 08xxx — ошибки соединения, 57xxx — отмена (statement_timeout) и остановка сервера.
|
||||
const std::string& state = sql->sqlstate();
|
||||
return state.rfind("08", 0) == 0 || state.rfind("57", 0) == 0;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
|
||||
bool statement_timeout) {
|
||||
std::string full = add_conn_param(conninfo, "connect_timeout",
|
||||
std::to_string(kConnectTimeoutSeconds));
|
||||
full = add_conn_param(full, "tcp_user_timeout", std::to_string(kTcpUserTimeoutMs));
|
||||
auto conn = std::make_unique<pqxx::connection>(full);
|
||||
const int fd = conn->sock();
|
||||
if (fd >= 0) {
|
||||
timeval tv{};
|
||||
tv.tv_sec = kClientIoTimeoutSeconds;
|
||||
tv.tv_usec = 0;
|
||||
::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
|
||||
::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
|
||||
}
|
||||
if (statement_timeout) {
|
||||
pqxx::nontransaction tx(*conn);
|
||||
tx.exec("SET statement_timeout = " + std::to_string(kStatementTimeoutMs));
|
||||
}
|
||||
return conn;
|
||||
}
|
||||
|
||||
|
||||
PgPool::PgPool(std::string conninfo, std::size_t n)
|
||||
: conninfo_(std::move(conninfo)) {
|
||||
if (n == 0) {
|
||||
throw std::runtime_error("PgPool: n must be > 0");
|
||||
}
|
||||
all_.reserve(n);
|
||||
idle_.reserve(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
auto conn = open_pg_connection(conninfo_, true);
|
||||
idle_.push_back(conn.get());
|
||||
all_.push_back(std::move(conn));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
PgPool::~PgPool() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
shutting_down_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
|
||||
pqxx::connection* PgPool::take() {
|
||||
std::unique_lock<std::mutex> lk(mu_);
|
||||
if (!cv_.wait_for(lk, kAcquireTimeout,
|
||||
[&]() { return shutting_down_ || !idle_.empty(); })) {
|
||||
throw DependencyUnavailable("postgres pool: acquire timeout");
|
||||
}
|
||||
if (shutting_down_) {
|
||||
throw DependencyUnavailable("postgres pool is shutting down");
|
||||
}
|
||||
pqxx::connection* conn = idle_.back();
|
||||
idle_.pop_back();
|
||||
lk.unlock();
|
||||
|
||||
if (is_alive(conn)) {
|
||||
return conn;
|
||||
}
|
||||
|
||||
std::unique_ptr<pqxx::connection> fresh;
|
||||
try {
|
||||
fresh = open_pg_connection(conninfo_, true);
|
||||
} catch (const std::exception& e) {
|
||||
// Мёртвое соединение возвращается в пул: следующий take() снова
|
||||
// попробует переподключиться, и пул не теряет слот.
|
||||
give_back(conn);
|
||||
throw DependencyUnavailable(std::string("postgres reconnect failed: ") + e.what());
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lk2(mu_);
|
||||
for (auto& up : all_) {
|
||||
if (up.get() == conn) {
|
||||
up = std::move(fresh);
|
||||
return up.get();
|
||||
}
|
||||
}
|
||||
throw std::logic_error("PgPool: connection not owned by pool");
|
||||
}
|
||||
|
||||
|
||||
void PgPool::give_back(pqxx::connection* conn) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
idle_.push_back(conn);
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
|
||||
PgPool::Lease::Lease(PgPool& pool) : pool_(&pool), conn_(pool.take()) {}
|
||||
|
||||
PgPool::Lease::~Lease() {
|
||||
if (pool_ != nullptr && conn_ != nullptr) {
|
||||
pool_->give_back(conn_);
|
||||
}
|
||||
}
|
||||
|
||||
PgPool::Lease::Lease(Lease&& other) noexcept
|
||||
: pool_(other.pool_), conn_(other.conn_) {
|
||||
other.pool_ = nullptr;
|
||||
other.conn_ = nullptr;
|
||||
}
|
||||
|
||||
PgPool::Lease PgPool::acquire() {
|
||||
return Lease(*this);
|
||||
}
|
||||
|
||||
bool PgPool::ping() {
|
||||
try {
|
||||
auto lease = acquire();
|
||||
pqxx::work tx(*lease);
|
||||
tx.exec("SELECT 1");
|
||||
tx.commit();
|
||||
return true;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
#pragma once
|
||||
#include <condition_variable>
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
std::string read_secret_file(const char* env_name);
|
||||
|
||||
std::unique_ptr<pqxx::connection> open_pg_connection(const std::string& conninfo,
|
||||
bool statement_timeout);
|
||||
|
||||
|
||||
void apply_migrations(pqxx::connection& conn, const std::string& dir);
|
||||
|
||||
|
||||
class PgPool {
|
||||
public:
|
||||
bool ping();
|
||||
PgPool(std::string conninfo, std::size_t n);
|
||||
~PgPool();
|
||||
|
||||
PgPool(const PgPool&) = delete;
|
||||
PgPool& operator=(const PgPool&) = delete;
|
||||
|
||||
class Lease {
|
||||
public:
|
||||
explicit Lease(PgPool& pool);
|
||||
~Lease();
|
||||
Lease(Lease&& other) noexcept;
|
||||
Lease& operator=(Lease&&) = delete;
|
||||
Lease(const Lease&) = delete;
|
||||
Lease& operator=(const Lease&) = delete;
|
||||
|
||||
pqxx::connection& operator*() const { return *conn_; }
|
||||
pqxx::connection* operator->() const { return conn_; }
|
||||
|
||||
private:
|
||||
PgPool* pool_;
|
||||
pqxx::connection* conn_;
|
||||
};
|
||||
|
||||
Lease acquire();
|
||||
|
||||
private:
|
||||
friend class Lease;
|
||||
|
||||
pqxx::connection* take();
|
||||
void give_back(pqxx::connection* conn);
|
||||
|
||||
std::string conninfo_;
|
||||
std::vector<std::unique_ptr<pqxx::connection>> all_;
|
||||
std::vector<pqxx::connection*> idle_;
|
||||
std::mutex mu_;
|
||||
std::condition_variable cv_;
|
||||
bool shutting_down_ = false;
|
||||
};
|
||||
@@ -0,0 +1,282 @@
|
||||
#include "direct_upload_repo.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch())
|
||||
.count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
DirectUpload row_to(const pqxx::row& r) {
|
||||
DirectUpload d;
|
||||
d.id = r["id"].as<std::string>();
|
||||
d.chat_id = r["chat_id"].as<std::string>();
|
||||
d.owner_id = r["owner_id"].as<std::string>();
|
||||
d.filename = r["filename"].as<std::string>();
|
||||
d.content_type = r["content_type"].as<std::string>();
|
||||
d.expected_size = r["expected_size"].as<int>();
|
||||
d.expected_sha256 = r["expected_sha256"].as<std::string>();
|
||||
d.s3_key = r["s3_key"].as<std::string>();
|
||||
d.staging_key = r["staging_key"].as<std::string>();
|
||||
d.status = r["status"].as<std::string>();
|
||||
d.overdue = r["overdue"].as<bool>();
|
||||
if (r["attachment_id"].is_null()) {
|
||||
d.attachment_id = std::nullopt;
|
||||
} else {
|
||||
d.attachment_id = r["attachment_id"].as<std::string>();
|
||||
}
|
||||
d.expires_at = epoch_to_rfc3339(r["expires_at_epoch"].as<long long>());
|
||||
d.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
d.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
|
||||
return d;
|
||||
}
|
||||
|
||||
constexpr const char* kCols =
|
||||
"id, chat_id, owner_id, filename, content_type, expected_size, "
|
||||
"expected_sha256, s3_key, staging_key, status, attachment_id, "
|
||||
"(status = 'pending' AND expires_at < now()) AS overdue, "
|
||||
"EXTRACT(EPOCH FROM expires_at)::bigint AS expires_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
|
||||
|
||||
constexpr const char* kAttachmentColumns =
|
||||
"id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"status, s3_key, error_code, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, "
|
||||
"EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch";
|
||||
|
||||
Attachment attachment_from_row(const pqxx::row& r) {
|
||||
Attachment a;
|
||||
a.id = r["id"].as<std::string>();
|
||||
a.chat_id = r["chat_id"].as<std::string>();
|
||||
a.owner_id = r["owner_id"].as<std::string>();
|
||||
a.filename = r["filename"].as<std::string>();
|
||||
a.content_type = r["content_type"].as<std::string>();
|
||||
a.size_bytes = r["size_bytes"].as<int>();
|
||||
a.sha256 = r["sha256"].as<std::string>();
|
||||
a.status = r["status"].as<std::string>();
|
||||
a.s3_key = r["s3_key"].as<std::string>();
|
||||
if (r["error_code"].is_null()) {
|
||||
a.error_code = std::nullopt;
|
||||
} else {
|
||||
a.error_code = r["error_code"].as<std::string>();
|
||||
}
|
||||
a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as<long long>());
|
||||
a.variants.clear();
|
||||
return a;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
DirectUpload DirectUploadRepo::insert(const DirectUpload& in) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
// expires_at is already an absolute epoch string from the caller.
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO direct_uploads ("
|
||||
" id, chat_id, owner_id, filename, content_type, expected_size, "
|
||||
" expected_sha256, s3_key, staging_key, status, attachment_id, "
|
||||
" expires_at, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, NULL, "
|
||||
" to_timestamp($11::bigint), to_timestamp($12::bigint), "
|
||||
" to_timestamp($13::bigint)"
|
||||
") RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.expected_size, in.expected_sha256,
|
||||
in.s3_key, in.staging_key, in.status, in.expires_at_epoch,
|
||||
ts, ts});
|
||||
tx.commit();
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> DirectUploadRepo::find_by_id(
|
||||
const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kCols) +
|
||||
" FROM direct_uploads WHERE id = $1::uuid",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> DirectUploadRepo::mark_completed(
|
||||
const std::string& id, const std::string& attachment_id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"UPDATE direct_uploads SET status = 'completed', "
|
||||
" attachment_id = $2::uuid, "
|
||||
" updated_at = to_timestamp($3::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending' "
|
||||
"RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{id, attachment_id, ts});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
tx.commit();
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> DirectUploadRepo::mark_aborted(
|
||||
const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"UPDATE direct_uploads SET status = 'aborted', "
|
||||
" updated_at = to_timestamp($2::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending' "
|
||||
"RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{id, ts});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
tx.commit();
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::vector<DirectUpload> DirectUploadRepo::expire_stale() {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"UPDATE direct_uploads SET status = 'expired', "
|
||||
" updated_at = to_timestamp($1::bigint) "
|
||||
"WHERE status = 'pending' AND expires_at < now() "
|
||||
"RETURNING " +
|
||||
std::string(kCols),
|
||||
pqxx::params{ts});
|
||||
tx.commit();
|
||||
std::vector<DirectUpload> out;
|
||||
out.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
out.push_back(row_to(row));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<std::string> DirectUploadRepo::pending_s3_keys() {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT staging_key FROM direct_uploads WHERE status = 'pending'");
|
||||
std::vector<std::string> out;
|
||||
out.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
out.push_back(row["staging_key"].as<std::string>());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
void DirectUploadRepo::with_lock(const std::function<void(UploadLock&)>& fn) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
UploadLock lock(tx);
|
||||
fn(lock);
|
||||
}
|
||||
|
||||
std::optional<DirectUpload> UploadLock::lock(const std::string& id) {
|
||||
auto r = tx_.exec(
|
||||
"SELECT " + std::string(kCols) +
|
||||
" FROM direct_uploads WHERE id = $1::uuid FOR UPDATE",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return row_to(r[0]);
|
||||
}
|
||||
|
||||
std::optional<Attachment> UploadLock::find_attachment(const std::string& id) {
|
||||
auto r = tx_.exec(
|
||||
"SELECT " + std::string(kAttachmentColumns) +
|
||||
" FROM attachments WHERE id = $1::uuid",
|
||||
pqxx::params{id});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return attachment_from_row(r[0]);
|
||||
}
|
||||
|
||||
std::optional<Attachment> UploadLock::insert_ready(const Attachment& in,
|
||||
bool& inserted) {
|
||||
inserted = false;
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx_.exec(
|
||||
"INSERT INTO attachments ("
|
||||
" id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
" status, s3_key, error_code, created_at, updated_at"
|
||||
") VALUES ("
|
||||
" $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, "
|
||||
" to_timestamp($10::bigint), to_timestamp($11::bigint)"
|
||||
") ON CONFLICT (id) DO NOTHING RETURNING " +
|
||||
std::string(kAttachmentColumns),
|
||||
pqxx::params{in.id, in.chat_id, in.owner_id, in.filename,
|
||||
in.content_type, in.size_bytes, in.sha256, in.status,
|
||||
in.s3_key, ts, ts});
|
||||
if (r.empty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
inserted = true;
|
||||
return attachment_from_row(r[0]);
|
||||
}
|
||||
|
||||
void UploadLock::mark_completed(const std::string& id,
|
||||
const std::string& attachment_id) {
|
||||
const std::string ts = now_epoch_str();
|
||||
auto r = tx_.exec(
|
||||
"UPDATE direct_uploads SET status = 'completed', "
|
||||
" attachment_id = $2::uuid, "
|
||||
" updated_at = to_timestamp($3::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending'",
|
||||
pqxx::params{id, attachment_id, ts});
|
||||
if (r.affected_rows() != 1) {
|
||||
throw std::runtime_error("direct upload complete lost the pending row");
|
||||
}
|
||||
}
|
||||
|
||||
void UploadLock::mark_expired(const std::string& id) {
|
||||
const std::string ts = now_epoch_str();
|
||||
tx_.exec(
|
||||
"UPDATE direct_uploads SET status = 'expired', "
|
||||
" updated_at = to_timestamp($2::bigint) "
|
||||
"WHERE id = $1::uuid AND status = 'pending'",
|
||||
pqxx::params{id, ts});
|
||||
}
|
||||
|
||||
void UploadLock::commit() {
|
||||
if (committed_) {
|
||||
return;
|
||||
}
|
||||
tx_.commit();
|
||||
committed_ = true;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class UploadLock;
|
||||
|
||||
class DirectUploadRepo {
|
||||
public:
|
||||
explicit DirectUploadRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
DirectUpload insert(const DirectUpload& in);
|
||||
|
||||
std::optional<DirectUpload> find_by_id(const std::string& id);
|
||||
|
||||
// Mark completed and link attachment_id. Returns nullopt if not pending
|
||||
// (or already finished / wrong owner handled by caller).
|
||||
std::optional<DirectUpload> mark_completed(const std::string& id,
|
||||
const std::string& attachment_id);
|
||||
|
||||
std::optional<DirectUpload> mark_aborted(const std::string& id);
|
||||
|
||||
// pending → expired where expires_at < now; returns expired rows.
|
||||
std::vector<DirectUpload> expire_stale();
|
||||
|
||||
// Staging keys of still-pending sessions (orphan GC must keep these).
|
||||
std::vector<std::string> pending_s3_keys();
|
||||
|
||||
// One PG transaction. Callback may call S3 while the row lock is held.
|
||||
// Commit explicitly via UploadLock::commit(); otherwise the work rolls back.
|
||||
void with_lock(const std::function<void(UploadLock&)>& fn);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
|
||||
// Row lock for a single complete(). Not safe to use after with_lock returns.
|
||||
class UploadLock {
|
||||
public:
|
||||
std::optional<DirectUpload> lock(const std::string& id);
|
||||
|
||||
std::optional<Attachment> find_attachment(const std::string& id);
|
||||
|
||||
// ON CONFLICT (id) DO NOTHING. inserted=false and nullopt → row already exists.
|
||||
std::optional<Attachment> insert_ready(const Attachment& in, bool& inserted);
|
||||
|
||||
void mark_completed(const std::string& id, const std::string& attachment_id);
|
||||
void mark_expired(const std::string& id);
|
||||
void commit();
|
||||
|
||||
private:
|
||||
friend class DirectUploadRepo;
|
||||
explicit UploadLock(pqxx::work& tx) : tx_(tx) {}
|
||||
|
||||
pqxx::work& tx_;
|
||||
bool committed_ = false;
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#include <exception>
|
||||
#include <stdexcept>
|
||||
|
||||
// PostgreSQL или Valkey недоступны либо не ответили вовремя: клиент получает 503.
|
||||
struct DependencyUnavailable : std::runtime_error {
|
||||
using std::runtime_error::runtime_error;
|
||||
};
|
||||
|
||||
bool is_dependency_failure(const std::exception& e);
|
||||
@@ -0,0 +1,90 @@
|
||||
#include "image_util.hpp"
|
||||
|
||||
#define STB_IMAGE_IMPLEMENTATION
|
||||
#define STBI_NO_STDIO
|
||||
#define STBI_ONLY_JPEG
|
||||
#define STBI_ONLY_PNG
|
||||
#include "stb_image.h"
|
||||
|
||||
#include <cstring>
|
||||
|
||||
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
|
||||
std::size_t len) {
|
||||
if (!data || len < 3) {
|
||||
return std::nullopt;
|
||||
}
|
||||
// JPEG: FF D8 FF
|
||||
if (data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF) {
|
||||
return std::string("image/jpeg");
|
||||
}
|
||||
// PNG: 89 50 4E 47
|
||||
if (len >= 4 && data[0] == 0x89 && data[1] == 0x50 && data[2] == 0x4E &&
|
||||
data[3] == 0x47) {
|
||||
return std::string("image/png");
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::string_view image_reject_code(ImageReject r) {
|
||||
switch (r) {
|
||||
case ImageReject::unsupported_media_type:
|
||||
return "unsupported_media_type";
|
||||
case ImageReject::payload_too_large:
|
||||
return "payload_too_large";
|
||||
case ImageReject::invalid_image:
|
||||
return "invalid_image";
|
||||
}
|
||||
return "invalid_image";
|
||||
}
|
||||
|
||||
std::optional<ImageReject> validate_image(const std::uint8_t* data,
|
||||
std::size_t len,
|
||||
const ImageLimits& limits,
|
||||
ImageInfo& out) {
|
||||
if (!data || len == 0) {
|
||||
return ImageReject::unsupported_media_type;
|
||||
}
|
||||
if (static_cast<std::int64_t>(len) > limits.max_bytes) {
|
||||
return ImageReject::payload_too_large;
|
||||
}
|
||||
|
||||
auto sniffed = sniff_content_type(data, len);
|
||||
if (!sniffed) {
|
||||
return ImageReject::unsupported_media_type;
|
||||
}
|
||||
|
||||
int w = 0;
|
||||
int h = 0;
|
||||
int comp = 0;
|
||||
if (!stbi_info_from_memory(reinterpret_cast<const stbi_uc*>(data),
|
||||
static_cast<int>(len), &w, &h, &comp)) {
|
||||
return ImageReject::invalid_image;
|
||||
}
|
||||
if (w <= 0 || h <= 0) {
|
||||
return ImageReject::invalid_image;
|
||||
}
|
||||
|
||||
const std::int64_t pixels =
|
||||
static_cast<std::int64_t>(w) * static_cast<std::int64_t>(h);
|
||||
if (pixels > limits.max_pixels) {
|
||||
return ImageReject::payload_too_large;
|
||||
}
|
||||
|
||||
int load_w = 0;
|
||||
int load_h = 0;
|
||||
int load_comp = 0;
|
||||
stbi_uc* decoded = stbi_load_from_memory(
|
||||
reinterpret_cast<const stbi_uc*>(data), static_cast<int>(len), &load_w,
|
||||
&load_h, &load_comp, 0);
|
||||
if (!decoded) {
|
||||
return ImageReject::invalid_image;
|
||||
}
|
||||
// Integrity only — do not retain decoded raster.
|
||||
stbi_image_free(decoded);
|
||||
|
||||
out.content_type = *sniffed;
|
||||
out.width = w;
|
||||
out.height = h;
|
||||
out.pixel_count = pixels;
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
struct ImageLimits {
|
||||
std::int64_t max_bytes = 10'485'760; // 10 MiB
|
||||
std::int64_t max_pixels = 40'000'000; // decoded width*height
|
||||
};
|
||||
|
||||
enum class ImageReject {
|
||||
unsupported_media_type, // → 415
|
||||
payload_too_large, // → 413 (bytes or pixels)
|
||||
invalid_image, // → 415 (corrupt / undecodable)
|
||||
};
|
||||
|
||||
struct ImageInfo {
|
||||
std::string content_type; // image/png | image/jpeg
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
std::int64_t pixel_count = 0;
|
||||
};
|
||||
|
||||
// Magic sniff only (does not decode). Returns nullopt if neither PNG nor JPEG.
|
||||
std::optional<std::string> sniff_content_type(const std::uint8_t* data,
|
||||
std::size_t len);
|
||||
|
||||
// Full validation pipeline from contracts:
|
||||
// sniff → stbi_info (dims) → pixel limit → stbi_load (integrity) → free.
|
||||
// On success fills `out`. On failure returns ImageReject; decoded pixels are
|
||||
// never retained after return.
|
||||
std::optional<ImageReject> validate_image(const std::uint8_t* data,
|
||||
std::size_t len,
|
||||
const ImageLimits& limits,
|
||||
ImageInfo& out);
|
||||
|
||||
// Human-readable error_code for Attachment failed path / API body.
|
||||
std::string_view image_reject_code(ImageReject r);
|
||||
+2329
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,374 @@
|
||||
#include "message_repo.hpp"
|
||||
#include "cursor.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sodium.h>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string make_uuid() {
|
||||
static std::mutex rng_mu;
|
||||
static std::mt19937_64 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
|
||||
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
|
||||
std::lock_guard<std::mutex> lock(rng_mu);
|
||||
std::ostringstream out;
|
||||
out << std::hex << std::setfill('0');
|
||||
out << std::setw(8) << d32(rng);
|
||||
out << '-' << std::setw(4) << d16(rng);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
|
||||
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string sha256_hex(const std::string& s) {
|
||||
unsigned char out[crypto_hash_sha256_BYTES];
|
||||
crypto_hash_sha256(out, reinterpret_cast<const unsigned char*>(s.data()), s.size());
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
std::string r;
|
||||
r.resize(sizeof(out) * 2);
|
||||
for (std::size_t i = 0; i < sizeof(out); ++i) {
|
||||
r[i * 2] = hex[out[i] >> 4];
|
||||
r[i * 2 + 1] = hex[out[i] & 0x0f];
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch()).count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
Message row_to_message(const pqxx::row& r) {
|
||||
Message m;
|
||||
m.id = r["id"].as<std::string>();
|
||||
m.chat_id = r["chat_id"].as<std::string>();
|
||||
m.sender_id = r["sender_id"].as<std::string>();
|
||||
m.text = r["text"].as<std::string>();
|
||||
m.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
return m;
|
||||
}
|
||||
|
||||
constexpr const char* kMessageSelect =
|
||||
"id, chat_id, sender_id, text, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
|
||||
|
||||
std::string canonical_message_body(const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
std::ostringstream out;
|
||||
out << "text=" << text << "\nattachment_ids=";
|
||||
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
|
||||
if (i > 0) {
|
||||
out << ',';
|
||||
}
|
||||
out << attachment_ids[i];
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
|
||||
int idempotency_ttl_seconds() {
|
||||
const char* env = std::getenv("IDEMPOTENCY_TTL_SECONDS");
|
||||
if (env == nullptr || *env == '\0') {
|
||||
return 86400;
|
||||
}
|
||||
try {
|
||||
const int v = std::stoi(env);
|
||||
if (v <= 0) {
|
||||
return 86400;
|
||||
}
|
||||
return v;
|
||||
} catch (...) {
|
||||
return 86400;
|
||||
}
|
||||
}
|
||||
|
||||
bool attachments_belong_to_chat(pqxx::work& tx,
|
||||
const std::string& chat_id,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
if (attachment_ids.empty()) {
|
||||
return true;
|
||||
}
|
||||
for (const auto& id : attachment_ids) {
|
||||
auto r = tx.exec(
|
||||
"SELECT 1 FROM attachments "
|
||||
"WHERE id = $1 AND chat_id = $2 AND status = 'ready'",
|
||||
pqxx::params{id, chat_id});
|
||||
if (r.empty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void link_attachments(pqxx::work& tx,
|
||||
const std::string& message_id,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
for (std::size_t i = 0; i < attachment_ids.size(); ++i) {
|
||||
tx.exec(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) "
|
||||
"VALUES ($1, $2, $3)",
|
||||
pqxx::params{message_id, attachment_ids[i], static_cast<int>(i)});
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> load_attachment_ids(pqxx::work& tx,
|
||||
const std::string& message_id) {
|
||||
auto r = tx.exec(
|
||||
"SELECT attachment_id FROM message_attachments "
|
||||
"WHERE message_id = $1 ORDER BY position ASC",
|
||||
pqxx::params{message_id});
|
||||
std::vector<std::string> ids;
|
||||
ids.reserve(r.size());
|
||||
for (const auto& row : r) {
|
||||
ids.push_back(row["attachment_id"].as<std::string>());
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
void fill_attachment_ids(pqxx::work& tx, Message& m) {
|
||||
m.attachment_ids = load_attachment_ids(tx, m.id);
|
||||
}
|
||||
|
||||
void fill_attachment_ids(pqxx::work& tx, std::vector<Message>& items) {
|
||||
for (auto& m : items) {
|
||||
fill_attachment_ids(tx, m);
|
||||
}
|
||||
}
|
||||
|
||||
Message insert_message(pqxx::work& tx,
|
||||
const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
const std::string id = make_uuid();
|
||||
const std::string created = now_epoch_str();
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) "
|
||||
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
|
||||
"RETURNING " + std::string(kMessageSelect),
|
||||
pqxx::params{id, chat_id, sender_id, text, created});
|
||||
Message m = row_to_message(r[0]);
|
||||
link_attachments(tx, m.id, attachment_ids);
|
||||
m.attachment_ids = attachment_ids;
|
||||
return m;
|
||||
}
|
||||
|
||||
bool insert_idempotency_key(pqxx::work& tx,
|
||||
const std::string& user_id,
|
||||
const std::string& chat_id,
|
||||
const std::string& idem_key,
|
||||
const std::string& request_hash,
|
||||
const std::string& message_id) {
|
||||
auto inserted = tx.exec(
|
||||
"INSERT INTO idempotency_keys "
|
||||
"(user_id, chat_id, key, request_hash, message_id) "
|
||||
"VALUES ($1, $2, $3, $4, $5) "
|
||||
"ON CONFLICT (user_id, chat_id, key) DO NOTHING "
|
||||
"RETURNING message_id",
|
||||
pqxx::params{user_id, chat_id, idem_key, request_hash, message_id});
|
||||
return !inserted.empty();
|
||||
}
|
||||
|
||||
Message load_message(pqxx::work& tx, const std::string& message_id) {
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kMessageSelect) + " FROM messages WHERE id = $1",
|
||||
pqxx::params{message_id});
|
||||
if (r.empty()) {
|
||||
throw std::runtime_error("idempotent message row is missing");
|
||||
}
|
||||
Message m = row_to_message(r[0]);
|
||||
fill_attachment_ids(tx, m);
|
||||
return m;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::optional<Message> MessageRepo::create(const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
auto member_check = tx.exec(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
pqxx::params{chat_id, sender_id});
|
||||
if (member_check.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Message created = insert_message(tx, chat_id, sender_id, text, attachment_ids);
|
||||
tx.commit();
|
||||
return created;
|
||||
}
|
||||
|
||||
std::optional<MessagePage> MessageRepo::list_page(
|
||||
const std::string& chat_id,
|
||||
int limit,
|
||||
const std::optional<std::string>& cursor) {
|
||||
|
||||
std::optional<Cursor> cur;
|
||||
if (cursor) {
|
||||
cur = parse_cursor(*cursor);
|
||||
if (!cur) {
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
if (cur) {
|
||||
auto owner_check = tx.exec(
|
||||
"SELECT 1 FROM messages WHERE id = $1 AND chat_id = $2",
|
||||
pqxx::params{cur->id, chat_id});
|
||||
if (owner_check.empty()) {
|
||||
tx.commit();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::string sql =
|
||||
"SELECT " + std::string(kMessageSelect) + ", " + kCreatedUsSql + " "
|
||||
"FROM messages "
|
||||
"WHERE chat_id = $1 ";
|
||||
if (cur) {
|
||||
sql += "AND (created_at, id) > (" + std::string(kCursorTimestampSql) + ", $4::uuid) ";
|
||||
}
|
||||
sql += "ORDER BY created_at ASC, id ASC LIMIT $2";
|
||||
|
||||
pqxx::result r;
|
||||
if (cur) {
|
||||
r = tx.exec(sql, pqxx::params{chat_id, limit + 1, cur->created_us, cur->id});
|
||||
} else {
|
||||
r = tx.exec(sql, pqxx::params{chat_id, limit + 1});
|
||||
}
|
||||
|
||||
MessagePage page;
|
||||
const bool has_more = (static_cast<int>(r.size()) > limit);
|
||||
const int take = has_more ? limit : static_cast<int>(r.size());
|
||||
page.items.reserve(take);
|
||||
for (int i = 0; i < take; ++i) {
|
||||
page.items.push_back(row_to_message(r[static_cast<std::size_t>(i)]));
|
||||
}
|
||||
fill_attachment_ids(tx, page.items);
|
||||
|
||||
if (has_more && take > 0) {
|
||||
const auto& last = r[static_cast<std::size_t>(take - 1)];
|
||||
page.next_cursor = make_cursor(last["created_us"].as<std::string>(),
|
||||
last["id"].as<std::string>());
|
||||
}
|
||||
|
||||
tx.commit();
|
||||
return page;
|
||||
}
|
||||
|
||||
CreateResult MessageRepo::create_idempotent(
|
||||
const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids,
|
||||
const std::string& idem_key) {
|
||||
const int ttl = idempotency_ttl_seconds();
|
||||
const std::string hash = sha256_hex(canonical_message_body(text, attachment_ids));
|
||||
|
||||
for (int attempt = 0; attempt < 3; ++attempt) {
|
||||
{
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
|
||||
auto member_check = tx.exec(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2",
|
||||
pqxx::params{chat_id, sender_id});
|
||||
if (member_check.empty()) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
|
||||
}
|
||||
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
|
||||
}
|
||||
|
||||
const Message created =
|
||||
insert_message(tx, chat_id, sender_id, text, attachment_ids);
|
||||
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
|
||||
created.id)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::Created, created};
|
||||
}
|
||||
}
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto existing = tx.exec(
|
||||
"SELECT request_hash, message_id, "
|
||||
"EXTRACT(EPOCH FROM (now() - created_at))::bigint AS age_seconds "
|
||||
"FROM idempotency_keys "
|
||||
"WHERE user_id = $1 AND chat_id = $2 AND key = $3 "
|
||||
"FOR UPDATE",
|
||||
pqxx::params{sender_id, chat_id, idem_key});
|
||||
if (existing.empty()) {
|
||||
tx.commit();
|
||||
continue;
|
||||
}
|
||||
|
||||
const std::string existing_hash = existing[0]["request_hash"].as<std::string>();
|
||||
const long long age = existing[0]["age_seconds"].as<long long>();
|
||||
const std::string message_id = existing[0]["message_id"].as<std::string>();
|
||||
if (age < ttl) {
|
||||
if (existing_hash != hash) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::KeyConflict, std::nullopt};
|
||||
}
|
||||
const Message replayed = load_message(tx, message_id);
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::Replayed, replayed};
|
||||
}
|
||||
|
||||
tx.exec(
|
||||
"DELETE FROM idempotency_keys "
|
||||
"WHERE user_id = $1 AND chat_id = $2 AND key = $3",
|
||||
pqxx::params{sender_id, chat_id, idem_key});
|
||||
if (!attachments_belong_to_chat(tx, chat_id, attachment_ids)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::NotFound, std::nullopt};
|
||||
}
|
||||
const Message created =
|
||||
insert_message(tx, chat_id, sender_id, text, attachment_ids);
|
||||
if (insert_idempotency_key(tx, sender_id, chat_id, idem_key, hash,
|
||||
created.id)) {
|
||||
tx.commit();
|
||||
return CreateResult{CreateResult::Status::Created, created};
|
||||
}
|
||||
}
|
||||
|
||||
throw std::runtime_error("idempotency retry exhausted");
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct CreateResult {
|
||||
enum class Status { Created, Replayed, NotFound, KeyConflict };
|
||||
Status status;
|
||||
std::optional<Message> message;
|
||||
};
|
||||
|
||||
class MessageRepo {
|
||||
public:
|
||||
explicit MessageRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
std::optional<Message> create(const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids);
|
||||
|
||||
std::optional<MessagePage> list_page(
|
||||
const std::string& chat_id,
|
||||
int limit,
|
||||
const std::optional<std::string>& cursor);
|
||||
|
||||
CreateResult create_idempotent(const std::string& chat_id,
|
||||
const std::string& sender_id,
|
||||
const std::string& text,
|
||||
const std::vector<std::string>& attachment_ids,
|
||||
const std::string& idem_key);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
@@ -0,0 +1,83 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct User {
|
||||
std::string id;
|
||||
std::string username;
|
||||
std::string display_name;
|
||||
std::string password_hash;
|
||||
std::string created_at;
|
||||
};
|
||||
|
||||
struct Chat {
|
||||
std::string id;
|
||||
std::string title;
|
||||
std::vector<std::string> member_ids;
|
||||
std::string created_at;
|
||||
};
|
||||
|
||||
struct Message {
|
||||
std::string id;
|
||||
std::string chat_id;
|
||||
std::string sender_id;
|
||||
std::string text;
|
||||
std::string created_at;
|
||||
std::vector<std::string> attachment_ids;
|
||||
};
|
||||
|
||||
struct MessagePage {
|
||||
std::vector<Message> items;
|
||||
std::optional<std::string> next_cursor;
|
||||
};
|
||||
|
||||
struct ChatPage {
|
||||
std::vector<Chat> items;
|
||||
std::optional<std::string> next_cursor;
|
||||
};
|
||||
|
||||
// Lab4: variants stay empty ([]); lab5 fills thumbnail/crop/optimized.
|
||||
struct Variant {
|
||||
std::string name;
|
||||
std::string content_type;
|
||||
int size_bytes = 0;
|
||||
std::string sha256;
|
||||
};
|
||||
|
||||
struct Attachment {
|
||||
std::string id;
|
||||
std::string chat_id;
|
||||
std::string owner_id;
|
||||
std::string filename;
|
||||
std::string content_type;
|
||||
int size_bytes = 0;
|
||||
std::string sha256;
|
||||
std::string status; // lab4: "ready"
|
||||
std::vector<Variant> variants; // lab4: always empty
|
||||
std::optional<std::string> error_code;
|
||||
std::string s3_key; // internal; never exposed in API JSON
|
||||
std::string created_at;
|
||||
std::string updated_at;
|
||||
};
|
||||
|
||||
// Track 1: pending presigned PUT session (not a public attachment until complete).
|
||||
struct DirectUpload {
|
||||
std::string id;
|
||||
std::string chat_id;
|
||||
std::string owner_id;
|
||||
std::string filename;
|
||||
std::string content_type;
|
||||
int expected_size = 0;
|
||||
std::string expected_sha256;
|
||||
std::string s3_key; // final key: attachments/{chat}/{id}
|
||||
std::string staging_key; // presigned PUT target: staging/{chat}/{id}
|
||||
std::string status; // pending|completed|aborted|expired
|
||||
bool overdue = false; // pending && expires_at < now(); not a column
|
||||
std::optional<std::string> attachment_id;
|
||||
std::string expires_at; // RFC3339 for JSON
|
||||
long long expires_at_epoch = 0; // for INSERT
|
||||
std::string created_at;
|
||||
std::string updated_at;
|
||||
};
|
||||
@@ -0,0 +1,66 @@
|
||||
#include "pg_session_repo.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
#include <sodium.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string random_token_hex() {
|
||||
unsigned char buf[32];
|
||||
randombytes_buf(buf, sizeof(buf));
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.resize(sizeof(buf) * 2);
|
||||
for (std::size_t i = 0; i < sizeof(buf); ++i) {
|
||||
out[i * 2] = hex[buf[i] >> 4];
|
||||
out[i * 2 + 1] = hex[buf[i] & 0x0f];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::int64_t now_unix() {
|
||||
return static_cast<std::int64_t>(std::time(nullptr));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ISessionRepo::Created PgSessionRepo::create(const std::string& user_id) {
|
||||
const std::string token = random_token_hex();
|
||||
const std::int64_t exp = now_unix() + ttl_;
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
tx.exec(
|
||||
"INSERT INTO sessions (token, user_id, expires_at) "
|
||||
"VALUES ($1, $2, to_timestamp($3::bigint))",
|
||||
pqxx::params{token, user_id, exp});
|
||||
tx.commit();
|
||||
|
||||
return Created{token, exp};
|
||||
}
|
||||
|
||||
std::optional<std::string> PgSessionRepo::lookup(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT user_id FROM sessions "
|
||||
"WHERE token = $1 AND expires_at > now()",
|
||||
pqxx::params{token});
|
||||
tx.commit();
|
||||
if (r.empty()) return std::nullopt;
|
||||
return r[0]["user_id"].as<std::string>();
|
||||
}
|
||||
|
||||
bool PgSessionRepo::revoke(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"DELETE FROM sessions WHERE token = $1",
|
||||
pqxx::params{token});
|
||||
tx.commit();
|
||||
return r.affected_rows() > 0;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "session_repo_base.hpp"
|
||||
|
||||
class PgSessionRepo : public ISessionRepo {
|
||||
public:
|
||||
PgSessionRepo(PgPool& pool, int ttl_seconds)
|
||||
: pool_(pool), ttl_(ttl_seconds) {}
|
||||
|
||||
Created create(const std::string& user_id) override;
|
||||
std::optional<std::string> lookup(const std::string& token) override;
|
||||
bool revoke(const std::string& token) override;
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
int ttl_;
|
||||
};
|
||||
@@ -0,0 +1,216 @@
|
||||
#include "redis_pool.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdlib>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr timeval kConnectTimeout{2, 0};
|
||||
constexpr timeval kCommandTimeout{2, 0};
|
||||
constexpr auto kAcquireTimeout = std::chrono::seconds(3);
|
||||
|
||||
struct RedisUrl {
|
||||
std::string host = "127.0.0.1";
|
||||
int port = 6379;
|
||||
std::string password;
|
||||
int db = 0;
|
||||
};
|
||||
|
||||
RedisUrl parse_redis_url(const std::string& url) {
|
||||
RedisUrl out;
|
||||
const std::string prefix = "redis://";
|
||||
if (url.compare(0, prefix.size(), prefix) != 0) {
|
||||
throw std::runtime_error("redis url must start with redis://");
|
||||
}
|
||||
std::string rest = url.substr(prefix.size());
|
||||
|
||||
const auto slash = rest.find('/');
|
||||
if (slash != std::string::npos) {
|
||||
const std::string db_str = rest.substr(slash + 1);
|
||||
rest.resize(slash);
|
||||
if (!db_str.empty()) {
|
||||
out.db = std::atoi(db_str.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
const auto at = rest.find('@');
|
||||
std::string hostport = rest;
|
||||
if (at != std::string::npos) {
|
||||
const std::string creds = rest.substr(0, at);
|
||||
hostport = rest.substr(at + 1);
|
||||
const auto colon = creds.find(':');
|
||||
if (colon != std::string::npos) {
|
||||
out.password = creds.substr(colon + 1);
|
||||
} else {
|
||||
out.password = creds;
|
||||
}
|
||||
}
|
||||
|
||||
const auto colon = hostport.find(':');
|
||||
if (colon == std::string::npos) {
|
||||
out.host = hostport;
|
||||
} else {
|
||||
out.host = hostport.substr(0, colon);
|
||||
out.port = std::atoi(hostport.substr(colon + 1).c_str());
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
redisContext* make_connection(const RedisUrl& u) {
|
||||
redisContext* c = redisConnectWithTimeout(u.host.c_str(), u.port, kConnectTimeout);
|
||||
if (c == nullptr) {
|
||||
throw std::runtime_error("redisConnect: allocation failed");
|
||||
}
|
||||
if (c->err) {
|
||||
const std::string err = c->errstr;
|
||||
redisFree(c);
|
||||
throw DependencyUnavailable("redisConnect failed: " + err);
|
||||
}
|
||||
if (redisSetTimeout(c, kCommandTimeout) != REDIS_OK) {
|
||||
redisFree(c);
|
||||
throw DependencyUnavailable("redisSetTimeout failed");
|
||||
}
|
||||
if (!u.password.empty()) {
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "AUTH %s", u.password.c_str()));
|
||||
if (r == nullptr) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis AUTH: no reply");
|
||||
}
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
if (!ok) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis AUTH failed");
|
||||
}
|
||||
}
|
||||
if (u.db != 0) {
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "SELECT %d", u.db));
|
||||
if (r == nullptr) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis SELECT: no reply");
|
||||
}
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
if (!ok) {
|
||||
redisFree(c);
|
||||
throw std::runtime_error("redis SELECT failed");
|
||||
}
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
RedisPool::RedisPool(std::string url, std::size_t n)
|
||||
: url_(std::move(url)) {
|
||||
if (n == 0) {
|
||||
throw std::runtime_error("RedisPool: n must be > 0");
|
||||
}
|
||||
const RedisUrl parsed = parse_redis_url(url_);
|
||||
all_.reserve(n);
|
||||
idle_.reserve(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
ConnPtr c(make_connection(parsed));
|
||||
idle_.push_back(c.get());
|
||||
all_.push_back(std::move(c));
|
||||
}
|
||||
}
|
||||
|
||||
RedisPool::~RedisPool() {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
shutting_down_ = true;
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
RedisPool::Lease::Lease(RedisPool& pool)
|
||||
: pool_(&pool), ctx_(pool.take()) {}
|
||||
|
||||
RedisPool::Lease::~Lease() {
|
||||
if (pool_ != nullptr && ctx_ != nullptr) {
|
||||
pool_->give_back(ctx_);
|
||||
}
|
||||
}
|
||||
|
||||
RedisPool::Lease::Lease(Lease&& other) noexcept
|
||||
: pool_(other.pool_), ctx_(other.ctx_) {
|
||||
other.pool_ = nullptr;
|
||||
other.ctx_ = nullptr;
|
||||
}
|
||||
|
||||
RedisPool::Lease RedisPool::acquire() {
|
||||
return Lease(*this);
|
||||
}
|
||||
|
||||
redisContext* RedisPool::take() {
|
||||
std::unique_lock<std::mutex> lk(mu_);
|
||||
if (!cv_.wait_for(lk, kAcquireTimeout,
|
||||
[&]() { return shutting_down_ || !idle_.empty(); })) {
|
||||
throw DependencyUnavailable("redis pool: acquire timeout");
|
||||
}
|
||||
if (shutting_down_) {
|
||||
throw DependencyUnavailable("redis pool is shutting down");
|
||||
}
|
||||
redisContext* c = idle_.back();
|
||||
idle_.pop_back();
|
||||
lk.unlock();
|
||||
|
||||
bool alive = false;
|
||||
if (!c->err) {
|
||||
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
|
||||
if (r != nullptr && r->type == REDIS_REPLY_STATUS) {
|
||||
alive = true;
|
||||
}
|
||||
if (r != nullptr) freeReplyObject(r);
|
||||
}
|
||||
if (alive) {
|
||||
return c;
|
||||
}
|
||||
|
||||
ConnPtr fresh;
|
||||
try {
|
||||
fresh.reset(make_connection(parse_redis_url(url_)));
|
||||
} catch (const std::exception& e) {
|
||||
// Слот возвращается в пул, иначе после нескольких неудач пул опустеет навсегда.
|
||||
give_back(c);
|
||||
throw DependencyUnavailable(std::string("redis reconnect failed: ") + e.what());
|
||||
}
|
||||
|
||||
std::lock_guard<std::mutex> lk2(mu_);
|
||||
for (auto& up : all_) {
|
||||
if (up.get() == c) {
|
||||
up = std::move(fresh);
|
||||
return up.get();
|
||||
}
|
||||
}
|
||||
throw std::logic_error("RedisPool: connection not owned by pool");
|
||||
}
|
||||
|
||||
void RedisPool::give_back(redisContext* c) {
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(mu_);
|
||||
idle_.push_back(c);
|
||||
}
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
bool RedisPool::ping() {
|
||||
try {
|
||||
auto lease = acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
redisReply* r = static_cast<redisReply*>(redisCommand(c, "PING"));
|
||||
if (r == nullptr) return false;
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
return ok;
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#pragma once
|
||||
|
||||
#include <hiredis/hiredis.h>
|
||||
|
||||
#include <condition_variable>
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
class RedisPool {
|
||||
public:
|
||||
RedisPool(std::string url, std::size_t n);
|
||||
~RedisPool();
|
||||
|
||||
RedisPool(const RedisPool&) = delete;
|
||||
RedisPool& operator=(const RedisPool&) = delete;
|
||||
|
||||
class Lease {
|
||||
public:
|
||||
explicit Lease(RedisPool& pool);
|
||||
~Lease();
|
||||
Lease(Lease&& other) noexcept;
|
||||
Lease& operator=(Lease&&) = delete;
|
||||
Lease(const Lease&) = delete;
|
||||
Lease& operator=(const Lease&) = delete;
|
||||
|
||||
redisContext* ctx() const { return ctx_; }
|
||||
|
||||
private:
|
||||
RedisPool* pool_;
|
||||
redisContext* ctx_;
|
||||
};
|
||||
|
||||
Lease acquire();
|
||||
|
||||
bool ping();
|
||||
|
||||
private:
|
||||
friend class Lease;
|
||||
|
||||
struct RedisDeleter {
|
||||
void operator()(redisContext* c) const noexcept {
|
||||
if (c != nullptr) ::redisFree(c);
|
||||
}
|
||||
};
|
||||
using ConnPtr = std::unique_ptr<redisContext, RedisDeleter>;
|
||||
|
||||
redisContext* take();
|
||||
void give_back(redisContext* c);
|
||||
|
||||
std::string url_;
|
||||
std::vector<ConnPtr> all_;
|
||||
std::vector<redisContext*> idle_;
|
||||
std::mutex mu_;
|
||||
std::condition_variable cv_;
|
||||
bool shutting_down_ = false;
|
||||
};
|
||||
@@ -0,0 +1,254 @@
|
||||
#include "s3_client.hpp"
|
||||
|
||||
#include <miniocpp/args.h>
|
||||
#include <miniocpp/client.h>
|
||||
#include <miniocpp/providers.h>
|
||||
#include <miniocpp/request.h>
|
||||
#include <miniocpp/response.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string read_file_trim(const std::string& path) {
|
||||
std::ifstream in(path);
|
||||
if (!in) {
|
||||
throw std::runtime_error("cannot open secret/file: " + path);
|
||||
}
|
||||
std::string s((std::istreambuf_iterator<char>(in)),
|
||||
std::istreambuf_iterator<char>());
|
||||
while (!s.empty() && (s.back() == '\n' || s.back() == '\r' || s.back() == ' ')) {
|
||||
s.pop_back();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string env_or(const char* key, const char* fallback = "") {
|
||||
const char* v = std::getenv(key);
|
||||
return (v && *v) ? std::string(v) : std::string(fallback);
|
||||
}
|
||||
|
||||
void parse_endpoint(const std::string& endpoint, std::string& host, unsigned& port,
|
||||
bool& https) {
|
||||
https = true;
|
||||
host = endpoint;
|
||||
port = 0;
|
||||
if (host.rfind("http://", 0) == 0) {
|
||||
https = false;
|
||||
host = host.substr(7);
|
||||
} else if (host.rfind("https://", 0) == 0) {
|
||||
https = true;
|
||||
host = host.substr(8);
|
||||
}
|
||||
// strip path
|
||||
const auto slash = host.find('/');
|
||||
if (slash != std::string::npos) {
|
||||
host = host.substr(0, slash);
|
||||
}
|
||||
const auto colon = host.rfind(':');
|
||||
if (colon != std::string::npos) {
|
||||
port = static_cast<unsigned>(std::stoul(host.substr(colon + 1)));
|
||||
host = host.substr(0, colon);
|
||||
}
|
||||
if (port == 0) {
|
||||
port = https ? 443u : 80u;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct S3Client::Impl {
|
||||
minio::s3::BaseUrl base_url;
|
||||
minio::s3::BaseUrl presign_url;
|
||||
std::unique_ptr<minio::creds::StaticProvider> provider;
|
||||
std::unique_ptr<minio::s3::Client> client;
|
||||
std::unique_ptr<minio::s3::Client> presign_client; // null → use client
|
||||
};
|
||||
|
||||
S3Client::Config S3Client::from_env() {
|
||||
Config c;
|
||||
// Ops endpoint (Stat/Get/Put/Delete). Presign may use S3_PRESIGN_ENDPOINT
|
||||
// (read in ctor) so signed Host matches the client-reachable address.
|
||||
c.endpoint = env_or("S3_ENDPOINT", "http://minio:9000");
|
||||
c.region = env_or("S3_REGION", "us-east-1");
|
||||
c.bucket = env_or("S3_BUCKET");
|
||||
if (c.bucket.empty()) {
|
||||
throw std::runtime_error("S3_BUCKET is required");
|
||||
}
|
||||
|
||||
const std::string key_file = env_or("S3_ACCESS_KEY_ID_FILE");
|
||||
const std::string secret_file = env_or("S3_SECRET_ACCESS_KEY_FILE");
|
||||
if (key_file.empty() || secret_file.empty()) {
|
||||
throw std::runtime_error(
|
||||
"S3_ACCESS_KEY_ID_FILE and S3_SECRET_ACCESS_KEY_FILE are required");
|
||||
}
|
||||
c.access_key = read_file_trim(key_file);
|
||||
c.secret_key = read_file_trim(secret_file);
|
||||
if (c.access_key.empty() || c.secret_key.empty()) {
|
||||
throw std::runtime_error("S3 credentials files are empty");
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
S3Client::S3Client(Config cfg) : cfg_(std::move(cfg)), impl_(std::make_unique<Impl>()) {
|
||||
if (cfg_.endpoint.empty() || cfg_.bucket.empty() ||
|
||||
cfg_.access_key.empty() || cfg_.secret_key.empty()) {
|
||||
throw std::runtime_error("S3Client: incomplete config");
|
||||
}
|
||||
|
||||
std::string host;
|
||||
unsigned port = 0;
|
||||
bool https = false;
|
||||
parse_endpoint(cfg_.endpoint, host, port, https);
|
||||
|
||||
impl_->base_url = minio::s3::BaseUrl(host, https, cfg_.region);
|
||||
impl_->base_url.port = port;
|
||||
// MinIO requires path-style addressing.
|
||||
impl_->base_url.virtual_style = false;
|
||||
|
||||
impl_->provider = std::make_unique<minio::creds::StaticProvider>(
|
||||
cfg_.access_key, cfg_.secret_key);
|
||||
impl_->client =
|
||||
std::make_unique<minio::s3::Client>(impl_->base_url, impl_->provider.get());
|
||||
|
||||
// Separate BaseUrl for presigning when S3_PRESIGN_ENDPOINT differs.
|
||||
const std::string presign_ep = env_or("S3_PRESIGN_ENDPOINT");
|
||||
if (!presign_ep.empty() && presign_ep != cfg_.endpoint) {
|
||||
std::string ph;
|
||||
unsigned pp = 0;
|
||||
bool phttps = false;
|
||||
parse_endpoint(presign_ep, ph, pp, phttps);
|
||||
impl_->presign_url = minio::s3::BaseUrl(ph, phttps, cfg_.region);
|
||||
impl_->presign_url.port = pp;
|
||||
impl_->presign_url.virtual_style = false;
|
||||
impl_->presign_client = std::make_unique<minio::s3::Client>(
|
||||
impl_->presign_url, impl_->provider.get());
|
||||
}
|
||||
}
|
||||
|
||||
S3Client::~S3Client() = default;
|
||||
|
||||
void S3Client::put_object(const std::string& key,
|
||||
std::istream& body,
|
||||
std::int64_t size_bytes,
|
||||
const std::string& content_type) {
|
||||
if (size_bytes < 0) {
|
||||
throw std::invalid_argument("put_object: negative size");
|
||||
}
|
||||
|
||||
// part_size=0 → library chooses multipart sizing; streams from `body`.
|
||||
minio::s3::PutObjectArgs args(body, static_cast<long>(size_bytes), 0);
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
if (!content_type.empty()) {
|
||||
args.content_type = content_type;
|
||||
}
|
||||
|
||||
minio::s3::PutObjectResponse resp = impl_->client->PutObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 PutObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
void S3Client::get_object(const std::string& key, std::ostream& out) {
|
||||
get_object(key, [&](const char* data, std::size_t n) {
|
||||
out.write(data, static_cast<std::streamsize>(n));
|
||||
if (!out) {
|
||||
throw std::runtime_error("get_object: write failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
std::int64_t S3Client::get_object(
|
||||
const std::string& key,
|
||||
const std::function<void(const char*, std::size_t)>& on_chunk) {
|
||||
std::int64_t total = 0;
|
||||
minio::s3::GetObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
args.datafunc = [&](minio::http::DataFunctionArgs da) -> bool {
|
||||
on_chunk(da.datachunk.data(), da.datachunk.size());
|
||||
total += static_cast<std::int64_t>(da.datachunk.size());
|
||||
return true;
|
||||
};
|
||||
|
||||
minio::s3::GetObjectResponse resp = impl_->client->GetObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 GetObject failed: " + resp.Error().String());
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
void S3Client::copy_object(const std::string& src_key,
|
||||
const std::string& dst_key) {
|
||||
if (src_key.empty() || dst_key.empty() || src_key == dst_key) {
|
||||
throw std::invalid_argument("copy_object: bad keys");
|
||||
}
|
||||
minio::s3::CopyObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = dst_key;
|
||||
args.source.bucket = cfg_.bucket;
|
||||
args.source.object = src_key;
|
||||
minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
void S3Client::delete_object(const std::string& key) {
|
||||
minio::s3::RemoveObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
minio::s3::RemoveObjectResponse resp = impl_->client->RemoveObject(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 DeleteObject failed: " + resp.Error().String());
|
||||
}
|
||||
}
|
||||
|
||||
bool S3Client::stat_object(const std::string& key, std::int64_t& size_bytes,
|
||||
std::string& etag) {
|
||||
minio::s3::StatObjectArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
minio::s3::StatObjectResponse resp = impl_->client->StatObject(args);
|
||||
if (!resp) {
|
||||
// NoSuchKey / Not Found → absent
|
||||
const std::string err = resp.Error().String();
|
||||
if (resp.code == "NoSuchKey" || resp.status_code == 404) {
|
||||
return false;
|
||||
}
|
||||
throw std::runtime_error("S3 StatObject failed: " + err);
|
||||
}
|
||||
size_bytes = static_cast<std::int64_t>(resp.size);
|
||||
etag = resp.etag;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string S3Client::presign_put(const std::string& key,
|
||||
unsigned expiry_seconds,
|
||||
const std::string& content_type) {
|
||||
if (expiry_seconds < 1 || expiry_seconds > 7 * 24 * 3600u) {
|
||||
throw std::invalid_argument("presign_put: bad expiry");
|
||||
}
|
||||
minio::s3::Client* c =
|
||||
impl_->presign_client ? impl_->presign_client.get() : impl_->client.get();
|
||||
|
||||
// Note: minio-cpp query-presign signs method/host/path/query only — it does
|
||||
// not bind Content-Type into the signature. Clients should still send
|
||||
// Content-Type so the object metadata is correct; complete() re-validates
|
||||
// magic/sha256 against the session.
|
||||
(void)content_type;
|
||||
minio::s3::GetPresignedObjectUrlArgs args;
|
||||
args.bucket = cfg_.bucket;
|
||||
args.object = key;
|
||||
args.method = minio::http::Method::kPut;
|
||||
args.expiry_seconds = expiry_seconds;
|
||||
|
||||
minio::s3::GetPresignedObjectUrlResponse resp = c->GetPresignedObjectUrl(args);
|
||||
if (!resp) {
|
||||
throw std::runtime_error("S3 PresignPut failed: " + resp.Error().String());
|
||||
}
|
||||
return resp.url;
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <istream>
|
||||
#include <memory>
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
|
||||
// Thin wrapper over AWS SDK for C++ (S3 only). Credentials and SigV4 come from
|
||||
// the SDK — no hand-rolled signing.
|
||||
|
||||
class S3Client {
|
||||
public:
|
||||
struct Config {
|
||||
std::string endpoint; // e.g. http://minio:9000
|
||||
std::string region; // e.g. us-east-1
|
||||
std::string bucket;
|
||||
std::string access_key;
|
||||
std::string secret_key;
|
||||
};
|
||||
|
||||
explicit S3Client(Config cfg);
|
||||
~S3Client();
|
||||
|
||||
S3Client(const S3Client&) = delete;
|
||||
S3Client& operator=(const S3Client&) = delete;
|
||||
|
||||
static Config from_env();
|
||||
|
||||
const std::string& bucket() const { return cfg_.bucket; }
|
||||
|
||||
// Stream upload. `body` must yield exactly `size_bytes` bytes.
|
||||
void put_object(const std::string& key,
|
||||
std::istream& body,
|
||||
std::int64_t size_bytes,
|
||||
const std::string& content_type);
|
||||
|
||||
// Stream download into `out` (chunked reads from SDK body stream).
|
||||
void get_object(const std::string& key, std::ostream& out);
|
||||
|
||||
// Stream download with a consumer callback (chunk, size). Returns total bytes.
|
||||
std::int64_t get_object(const std::string& key,
|
||||
const std::function<void(const char*, std::size_t)>& on_chunk);
|
||||
|
||||
void delete_object(const std::string& key);
|
||||
|
||||
// Server-side copy inside the bucket (complete: staging → final).
|
||||
void copy_object(const std::string& src_key, const std::string& dst_key);
|
||||
|
||||
// HeadObject-style metadata. Returns false if the object does not exist.
|
||||
bool stat_object(const std::string& key, std::int64_t& size_bytes,
|
||||
std::string& etag);
|
||||
|
||||
// Presigned PUT URL via minio-cpp (SigV4). Optional required Content-Type
|
||||
// is bound into the signature when non-empty.
|
||||
// expiry_seconds: typically 60–600 for Track 1.
|
||||
std::string presign_put(const std::string& key,
|
||||
unsigned expiry_seconds,
|
||||
const std::string& content_type);
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
Config cfg_;
|
||||
std::unique_ptr<Impl> impl_;
|
||||
};
|
||||
@@ -0,0 +1,87 @@
|
||||
#include "session_repo.hpp"
|
||||
#include "errors.hpp"
|
||||
|
||||
#include <sodium.h>
|
||||
|
||||
#include <ctime>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string random_token_hex() {
|
||||
unsigned char buf[32];
|
||||
randombytes_buf(buf, sizeof(buf));
|
||||
static const char hex[] = "0123456789abcdef";
|
||||
std::string out;
|
||||
out.resize(sizeof(buf) * 2);
|
||||
for (std::size_t i = 0; i < sizeof(buf); ++i) {
|
||||
out[i * 2] = hex[buf[i] >> 4];
|
||||
out[i * 2 + 1] = hex[buf[i] & 0x0f];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::int64_t now_unix() {
|
||||
return static_cast<std::int64_t>(std::time(nullptr));
|
||||
}
|
||||
|
||||
redisReply* check_reply(redisContext* c, redisReply* r) {
|
||||
if (r == nullptr) {
|
||||
const std::string err = c->err ? c->errstr : "no reply";
|
||||
throw DependencyUnavailable("redis command failed: " + err);
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ISessionRepo::Created SessionRepo::create(const std::string& user_id) {
|
||||
const std::string token = random_token_hex();
|
||||
const std::int64_t exp = now_unix() + ttl_;
|
||||
|
||||
auto lease = pool_.acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "SET session:%s %s EX %d",
|
||||
token.c_str(), user_id.c_str(), ttl_));
|
||||
check_reply(c, r);
|
||||
const bool ok = (r->type == REDIS_REPLY_STATUS);
|
||||
freeReplyObject(r);
|
||||
if (!ok) {
|
||||
throw std::runtime_error("redis SET returned non-status");
|
||||
}
|
||||
|
||||
return Created{token, exp};
|
||||
}
|
||||
|
||||
std::optional<std::string> SessionRepo::lookup(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "GET session:%s", token.c_str()));
|
||||
check_reply(c, r);
|
||||
|
||||
std::optional<std::string> out;
|
||||
if (r->type == REDIS_REPLY_STRING) {
|
||||
out = std::string(r->str, r->len);
|
||||
}
|
||||
freeReplyObject(r);
|
||||
return out;
|
||||
}
|
||||
|
||||
bool SessionRepo::revoke(const std::string& token) {
|
||||
auto lease = pool_.acquire();
|
||||
redisContext* c = lease.ctx();
|
||||
|
||||
redisReply* r = static_cast<redisReply*>(
|
||||
redisCommand(c, "DEL session:%s", token.c_str()));
|
||||
check_reply(c, r);
|
||||
|
||||
const bool existed = (r->type == REDIS_REPLY_INTEGER && r->integer == 1);
|
||||
freeReplyObject(r);
|
||||
return existed;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "redis_pool.hpp"
|
||||
#include "session_repo_base.hpp"
|
||||
|
||||
class SessionRepo : public ISessionRepo {
|
||||
public:
|
||||
SessionRepo(RedisPool& pool, int ttl_seconds)
|
||||
: pool_(pool), ttl_(ttl_seconds) {}
|
||||
|
||||
Created create(const std::string& user_id) override;
|
||||
std::optional<std::string> lookup(const std::string& token) override;
|
||||
bool revoke(const std::string& token) override;
|
||||
|
||||
private:
|
||||
RedisPool& pool_;
|
||||
int ttl_;
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
|
||||
class ISessionRepo {
|
||||
public:
|
||||
virtual ~ISessionRepo() = default;
|
||||
|
||||
struct Created {
|
||||
std::string token;
|
||||
std::int64_t expires_at; // unix-время в секундах UTC
|
||||
};
|
||||
|
||||
virtual Created create(const std::string& user_id) = 0;
|
||||
|
||||
virtual std::optional<std::string> lookup(const std::string& token) = 0;
|
||||
|
||||
virtual bool revoke(const std::string& token) = 0;
|
||||
};
|
||||
@@ -0,0 +1,100 @@
|
||||
#include "user_repo.hpp"
|
||||
|
||||
#include <pqxx/pqxx>
|
||||
|
||||
#include <chrono>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
#include <random>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
namespace {
|
||||
|
||||
std::string make_uuid() {
|
||||
static std::mt19937_64 rng{std::random_device{}()};
|
||||
std::uniform_int_distribution<uint32_t> d16(0, 0xffff);
|
||||
std::uniform_int_distribution<uint32_t> d32(0, 0xffffffffu);
|
||||
std::ostringstream out;
|
||||
out << std::hex << std::setfill('0');
|
||||
out << std::setw(8) << d32(rng);
|
||||
out << '-' << std::setw(4) << d16(rng);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x0fff) | 0x4000);
|
||||
out << '-' << std::setw(4) << ((d16(rng) & 0x3fff) | 0x8000);
|
||||
out << '-' << std::setw(4) << d16(rng) << std::setw(8) << d32(rng);
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string epoch_to_rfc3339(long long epoch) {
|
||||
const std::time_t t = static_cast<std::time_t>(epoch);
|
||||
std::tm tm{};
|
||||
gmtime_r(&t, &tm);
|
||||
std::ostringstream out;
|
||||
out << std::put_time(&tm, "%Y-%m-%dT%H:%M:%SZ");
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::string now_epoch_str() {
|
||||
const auto now = std::chrono::system_clock::now();
|
||||
const auto secs = std::chrono::duration_cast<std::chrono::seconds>(
|
||||
now.time_since_epoch()).count();
|
||||
return std::to_string(secs);
|
||||
}
|
||||
|
||||
User row_to_user(const pqxx::row& r) {
|
||||
User u;
|
||||
u.id = r["id"].as<std::string>();
|
||||
u.username = r["username"].as<std::string>();
|
||||
u.display_name = r["display_name"].as<std::string>();
|
||||
u.password_hash = r["password_hash"].as<std::string>();
|
||||
u.created_at = epoch_to_rfc3339(r["created_at_epoch"].as<long long>());
|
||||
return u;
|
||||
}
|
||||
|
||||
constexpr const char* kUserColumns =
|
||||
"id, username, display_name, password_hash, "
|
||||
"EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch";
|
||||
|
||||
}
|
||||
|
||||
std::optional<User> UserRepo::create(const std::string& username,
|
||||
const std::string& display_name,
|
||||
const std::string& password_hash) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
try {
|
||||
const std::string id = make_uuid();
|
||||
auto r = tx.exec(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at) "
|
||||
"VALUES ($1, $2, $3, $4, to_timestamp($5::bigint)) "
|
||||
"RETURNING " + std::string(kUserColumns),
|
||||
pqxx::params{id, username, display_name, password_hash, now_epoch_str()});
|
||||
tx.commit();
|
||||
return row_to_user(r[0]);
|
||||
} catch (const pqxx::unique_violation&) {
|
||||
tx.abort();
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<User> UserRepo::find_by_id(const std::string& id) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kUserColumns) + " FROM users WHERE id = $1",
|
||||
pqxx::params{id});
|
||||
tx.commit();
|
||||
if (r.empty()) return std::nullopt;
|
||||
return row_to_user(r[0]);
|
||||
}
|
||||
|
||||
std::optional<User> UserRepo::find_by_username(const std::string& username) {
|
||||
auto lease = pool_.acquire();
|
||||
pqxx::work tx(*lease);
|
||||
auto r = tx.exec(
|
||||
"SELECT " + std::string(kUserColumns) + " FROM users WHERE username = $1",
|
||||
pqxx::params{username});
|
||||
tx.commit();
|
||||
if (r.empty()) return std::nullopt;
|
||||
return row_to_user(r[0]);
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
#pragma once
|
||||
|
||||
#include "db.hpp"
|
||||
#include "models.hpp"
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
class UserRepo {
|
||||
public:
|
||||
explicit UserRepo(PgPool& pool) : pool_(pool) {}
|
||||
|
||||
std::optional<User> create(const std::string& username,
|
||||
const std::string& display_name,
|
||||
const std::string& password_hash);
|
||||
|
||||
std::optional<User> find_by_id(const std::string& id);
|
||||
std::optional<User> find_by_username(const std::string& username);
|
||||
|
||||
private:
|
||||
PgPool& pool_;
|
||||
};
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/bin/sh
|
||||
# Shared helpers for student-tests/*.sh (curl + jq only).
|
||||
# Usage: . "$(dirname "$0")/lib.sh"
|
||||
|
||||
set -eu
|
||||
|
||||
BASE_URL="${BASE_URL:-http://127.0.0.1:8080}"
|
||||
PASS="${STUDENT_TEST_PASSWORD:-Student-test-password-1}"
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
FIXTURES="$ROOT/tools/fixtures"
|
||||
|
||||
die() {
|
||||
printf '%s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"
|
||||
}
|
||||
|
||||
need curl
|
||||
need jq
|
||||
need sha256sum
|
||||
need python3
|
||||
|
||||
uniq_name() {
|
||||
printf '%s_%s' "$1" "$(python3 -c 'import uuid; print(uuid.uuid4().hex[:12])')"
|
||||
}
|
||||
|
||||
# http_json METHOD PATH [curl args...]
|
||||
# Sets: HTTP_CODE, HTTP_BODY, HTTP_HDRS
|
||||
http_json() {
|
||||
method=$1
|
||||
path=$2
|
||||
shift 2
|
||||
hdr=$(mktemp)
|
||||
body=$(mktemp)
|
||||
code=$(curl -sS -m 60 -o "$body" -D "$hdr" -w '%{http_code}' \
|
||||
-X "$method" "$BASE_URL$path" "$@") || {
|
||||
rm -f "$hdr" "$body"
|
||||
die "curl failed: $method $path"
|
||||
}
|
||||
HTTP_CODE=$code
|
||||
HTTP_BODY=$(cat "$body")
|
||||
HTTP_HDRS=$(cat "$hdr")
|
||||
rm -f "$hdr" "$body"
|
||||
}
|
||||
|
||||
expect_code() {
|
||||
want=$1
|
||||
[ "$HTTP_CODE" = "$want" ] || die "expected HTTP $want got $HTTP_CODE body=$HTTP_BODY"
|
||||
}
|
||||
|
||||
register_user() {
|
||||
label=$1
|
||||
user=$(uniq_name "$label")
|
||||
http_json POST /api/v1/users \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"username\":\"$user\",\"display_name\":\"$label\",\"password\":\"$PASS\"}"
|
||||
expect_code 201
|
||||
USER_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
USERNAME=$user
|
||||
cred=$(printf '%s' "$USERNAME:$PASS" | python3 -c 'import sys,base64; print(base64.b64encode(sys.stdin.buffer.read()).decode())')
|
||||
http_json POST /api/v1/auth/sessions -H "Authorization: Basic $cred"
|
||||
expect_code 201
|
||||
TOKEN=$(printf '%s' "$HTTP_BODY" | jq -r .token)
|
||||
[ -n "$TOKEN" ] && [ "$TOKEN" != null ] || die "empty token"
|
||||
}
|
||||
|
||||
create_chat() {
|
||||
title=${1:-test}
|
||||
member_json=$2
|
||||
http_json POST /api/v1/chats \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"title\":\"$title\",\"member_ids\":$member_json}"
|
||||
expect_code 201
|
||||
CHAT_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
}
|
||||
|
||||
upload_file() {
|
||||
chat_id=$1
|
||||
file=$2
|
||||
filename=$3
|
||||
ctype=$4
|
||||
http_json POST "/api/v1/chats/$chat_id/attachments?filename=$filename" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: $ctype" \
|
||||
--data-binary @"$file"
|
||||
}
|
||||
|
||||
sha256_file() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
assert_eq() {
|
||||
[ "$1" = "$2" ] || die "assert_eq failed: '$1' != '$2'${3:+ ($3)}"
|
||||
}
|
||||
|
||||
printf 'lib.sh: BASE_URL=%s\n' "$BASE_URL" >&2
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Run grade-3 attachment student tests (+ drain / compensation dry-run).
|
||||
set -eu
|
||||
DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
|
||||
ROOT=$(CDPATH= cd -- "$DIR/.." && pwd)
|
||||
|
||||
for t in \
|
||||
test_attachment_upload_ok.sh \
|
||||
test_attachment_too_large.sh \
|
||||
test_attachment_corrupted.sh \
|
||||
test_attachment_mime_spoof.sh \
|
||||
test_attachment_foreign.sh \
|
||||
test_attachment_wrong_chat.sh \
|
||||
test_graceful_drain.sh
|
||||
do
|
||||
echo "=== $t ==="
|
||||
sh "$DIR/$t"
|
||||
done
|
||||
|
||||
echo "=== tools/gc_orphans.sh (dry-run) ==="
|
||||
sh "$ROOT/tools/gc_orphans.sh"
|
||||
|
||||
echo "ALL grade3 student-tests PASS"
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
|
||||
ROOT=$(CDPATH= cd -- "$DIR/.." && pwd)
|
||||
|
||||
echo "=== tests/check_bucket_private.sh ==="
|
||||
sh "$ROOT/tests/check_bucket_private.sh"
|
||||
|
||||
echo "=== test_attachment_limited_creds.sh ==="
|
||||
sh "$DIR/test_attachment_limited_creds.sh"
|
||||
|
||||
# Load before persistence: restart cools cgroup RSS accounting and makes
|
||||
# docker-stats delta noisy right after recreate.
|
||||
echo "=== scripts/load_uploads.sh ==="
|
||||
sh "$ROOT/scripts/load_uploads.sh"
|
||||
|
||||
echo "=== test_attachment_persistence.sh ==="
|
||||
sh "$DIR/test_attachment_persistence.sh"
|
||||
|
||||
echo "ALL grade4 student-tests PASS"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Grade 5 / Track 1: direct (presigned) upload + proxy still green.
|
||||
set -eu
|
||||
DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd)
|
||||
|
||||
echo "=== test_direct_upload_ok.sh ==="
|
||||
sh "$DIR/test_direct_upload_ok.sh"
|
||||
|
||||
echo "=== test_presigned_complete_concurrent.sh ==="
|
||||
sh "$DIR/test_presigned_complete_concurrent.sh"
|
||||
|
||||
echo "=== test_direct_upload_negatives.sh ==="
|
||||
sh "$DIR/test_direct_upload_negatives.sh"
|
||||
|
||||
echo "=== tools/gc_direct_uploads.sh (dry-run) ==="
|
||||
sh "$DIR/../tools/gc_direct_uploads.sh"
|
||||
|
||||
echo "ALL grade5 track1 student-tests PASS"
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Truncated / corrupt PNG → 415 (not 500).
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
register_user alice
|
||||
create_chat 'corrupt' "[\"$USER_ID\"]"
|
||||
|
||||
bad=$(mktemp)
|
||||
# PNG magic + IHDR-looking garbage, truncated — fails stbi decode.
|
||||
printf '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x01\x00\x00\x00\x01\x00\x08\x02\x00\x00\x00' > "$bad"
|
||||
|
||||
upload_file "$CHAT_ID" "$bad" 'broken.png' 'image/png'
|
||||
rm -f "$bad"
|
||||
expect_code 415
|
||||
code=$(printf '%s' "$HTTP_BODY" | jq -r .error.code)
|
||||
assert_eq "$code" unsupported_media_type error_code
|
||||
|
||||
printf 'PASS test_attachment_corrupted\n'
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# Non-member → 404 on GET metadata and /content (no leak).
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/tiny.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
|
||||
register_user alice
|
||||
ALICE_ID=$USER_ID
|
||||
ALICE_TOK=$TOKEN
|
||||
register_user eve
|
||||
EVE_TOK=$TOKEN
|
||||
|
||||
TOKEN=$ALICE_TOK
|
||||
create_chat 'foreign' "[\"$ALICE_ID\"]"
|
||||
upload_file "$CHAT_ID" "$png" 'tiny.png' 'image/png'
|
||||
expect_code 201
|
||||
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
|
||||
http_json GET "/api/v1/attachments/$AID" -H "Authorization: Bearer $EVE_TOK"
|
||||
expect_code 404
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_found meta
|
||||
|
||||
http_json GET "/api/v1/attachments/$AID/content" -H "Authorization: Bearer $EVE_TOK"
|
||||
expect_code 404
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_found content
|
||||
|
||||
http_json GET "/api/v1/attachments/$AID"
|
||||
expect_code 401
|
||||
|
||||
printf 'PASS test_attachment_foreign\n'
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
#!/bin/sh
|
||||
# Grade 4: API uses limited S3 user via *_FILE secrets, not MinIO root.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
|
||||
# API process env must expose limited-key secret paths, never root password vars.
|
||||
env_dump=$($COMPOSE exec -T api sh -c 'env' | sort)
|
||||
|
||||
printf '%s\n' "$env_dump" | grep -q '^S3_ACCESS_KEY_ID_FILE=' \
|
||||
|| die "API missing S3_ACCESS_KEY_ID_FILE"
|
||||
printf '%s\n' "$env_dump" | grep -q '^S3_SECRET_ACCESS_KEY_FILE=' \
|
||||
|| die "API missing S3_SECRET_ACCESS_KEY_FILE"
|
||||
|
||||
# Root MinIO credentials must not be injected into the API container.
|
||||
if printf '%s\n' "$env_dump" | grep -E '^(MINIO_ROOT_USER|MINIO_ROOT_PASSWORD)=' >/dev/null; then
|
||||
die "API container has MinIO root credentials in env"
|
||||
fi
|
||||
|
||||
# Secret file contents must differ from root user (compare access key id only).
|
||||
api_key=$($COMPOSE exec -T api sh -c 'cat "$S3_ACCESS_KEY_ID_FILE"' | tr -d '\r\n')
|
||||
root_user=$(grep -E '^MINIO_ROOT_USER=' "$ROOT/.env" | head -n1 | cut -d= -f2- | tr -d '\r\n"')
|
||||
[ -n "$api_key" ] || die "empty limited access key"
|
||||
[ -n "$root_user" ] || die "MINIO_ROOT_USER missing in .env"
|
||||
[ "$api_key" != "$root_user" ] || die "API S3 access key equals MinIO root user"
|
||||
|
||||
# Limited user cannot delete the bucket (admin action) — policy is object-scoped.
|
||||
# Probe via mc with the limited credentials from secrets (same values as API).
|
||||
lim_key=$api_key
|
||||
lim_secret=$($COMPOSE exec -T api sh -c 'cat "$S3_SECRET_ACCESS_KEY_FILE"' | tr -d '\r\n')
|
||||
bucket=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
|
||||
|
||||
out=$($COMPOSE run --rm --no-deps \
|
||||
-e LIM_KEY="$lim_key" -e LIM_SECRET="$lim_secret" -e S3_BUCKET="$bucket" \
|
||||
--entrypoint /bin/sh minio-init -c '
|
||||
set +e
|
||||
mc alias set lim "$S3_ENDPOINT" "$LIM_KEY" "$LIM_SECRET" >/dev/null
|
||||
mc rb --force "lim/$S3_BUCKET" 2>/tmp/rb.err
|
||||
rc=$?
|
||||
cat /tmp/rb.err
|
||||
exit $rc
|
||||
' 2>&1) || true
|
||||
|
||||
printf '%s\n' "$out" | grep -qiE 'AccessDenied|denied|not allowed|InvalidAccessKeyId|error' \
|
||||
|| die "expected AccessDenied on bucket delete by limited user; out=$out"
|
||||
|
||||
# Positive: limited user can still PutObject (smoke via API upload).
|
||||
png="$FIXTURES/tiny.png"
|
||||
register_user alice
|
||||
create_chat 'creds' "[\"$USER_ID\"]"
|
||||
upload_file "$CHAT_ID" "$png" 'tiny.png' 'image/png'
|
||||
expect_code 201
|
||||
|
||||
printf 'PASS test_attachment_limited_creds\n'
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
# JPEG bytes with Content-Type: image/png → 415 (content wins over header).
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
jpg="$FIXTURES/tiny.jpg"
|
||||
[ -f "$jpg" ] || die "missing fixture $jpg"
|
||||
|
||||
register_user alice
|
||||
create_chat 'mime-spoof' "[\"$USER_ID\"]"
|
||||
|
||||
upload_file "$CHAT_ID" "$jpg" 'spoof.png' 'image/png'
|
||||
expect_code 415
|
||||
code=$(printf '%s' "$HTTP_BODY" | jq -r .error.code)
|
||||
assert_eq "$code" unsupported_media_type error_code
|
||||
|
||||
printf 'PASS test_attachment_mime_spoof\n'
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
# Grade 4: upload → restart API replicas (volumes kept) → download same bytes.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
png="$FIXTURES/320x200.png"
|
||||
[ -f "$png" ] || die "missing $png"
|
||||
want_sha=$(sha256_file "$png")
|
||||
|
||||
register_user alice
|
||||
ALICE_ID=$USER_ID
|
||||
create_chat 'persist' "[\"$ALICE_ID\"]"
|
||||
upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png'
|
||||
expect_code 201
|
||||
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" upload_sha
|
||||
|
||||
# Restart only API (and LB); keep pgdata + miniodata.
|
||||
$COMPOSE restart api
|
||||
# Wait for both replicas healthy + LB ready.
|
||||
i=0
|
||||
while [ "$i" -lt 60 ]; do
|
||||
http_json GET /health/ready || true
|
||||
if [ "${HTTP_CODE:-}" = "200" ]; then
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
[ "${HTTP_CODE:-}" = "200" ] || die "stack not ready after api restart"
|
||||
|
||||
# Session may still be in Redis (AOF); reuse token.
|
||||
http_json GET "/api/v1/attachments/$AID" -H "Authorization: Bearer $TOKEN"
|
||||
expect_code 200
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" meta_sha
|
||||
|
||||
out=$(mktemp)
|
||||
code=$(curl -sS -m 60 -o "$out" -w '%{http_code}' \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$BASE_URL/api/v1/attachments/$AID/content")
|
||||
assert_eq "$code" 200 content_status
|
||||
assert_eq "$(sha256_file "$out")" "$want_sha" content_sha
|
||||
rm -f "$out"
|
||||
|
||||
printf 'PASS test_attachment_persistence\n'
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Grade 4 criteria (negative): 11 MiB body → 413 payload_too_large (not 500).
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
register_user alice
|
||||
create_chat 'too-large' "[\"$USER_ID\"]"
|
||||
|
||||
big=$(mktemp)
|
||||
# Valid PNG magic + padding to 11 MiB so Content-Length triggers early reject.
|
||||
dd if=/dev/zero of="$big" bs=1M count=11 status=none
|
||||
# Prepend PNG signature so it's not rejected as empty before size check.
|
||||
printf '\x89PNG\r\n\x1a\n' | dd of="$big" conv=notrunc status=none 2>/dev/null || true
|
||||
|
||||
upload_file "$CHAT_ID" "$big" 'huge.png' 'image/png'
|
||||
rm -f "$big"
|
||||
expect_code 413
|
||||
code=$(printf '%s' "$HTTP_BODY" | jq -r .error.code)
|
||||
assert_eq "$code" payload_too_large error_code
|
||||
|
||||
printf 'PASS test_attachment_too_large\n'
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/sh
|
||||
# Grade 3: PNG 320×200 → 201 ready → download bytes match sha256; message with attachment_ids.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/320x200.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
want_sha=$(sha256_file "$png")
|
||||
want_size=$(wc -c < "$png" | tr -d ' ')
|
||||
|
||||
register_user alice
|
||||
ALICE_ID=$USER_ID
|
||||
ALICE_TOK=$TOKEN
|
||||
register_user bob
|
||||
BOB_ID=$USER_ID
|
||||
BOB_TOK=$TOKEN
|
||||
|
||||
TOKEN=$ALICE_TOK
|
||||
create_chat 'upload-ok' "[\"$ALICE_ID\",\"$BOB_ID\"]"
|
||||
|
||||
upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png'
|
||||
expect_code 201
|
||||
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" ready status
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .content_type)" image/png ctype
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" sha256
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .size_bytes)" "$want_size" size
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error_code)" null error_code
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -c .variants)" '[]' variants
|
||||
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/messages" \
|
||||
-H "Authorization: Bearer $ALICE_TOK" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "Idempotency-Key: $(uniq_name idem)" \
|
||||
-d "{\"text\":\"\",\"attachment_ids\":[\"$AID\"]}"
|
||||
expect_code 201
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -c .attachment_ids)" "[\"$AID\"]" msg_atts
|
||||
|
||||
out=$(mktemp)
|
||||
curl -sS -m 60 -o "$out" -w '%{http_code}' \
|
||||
-H "Authorization: Bearer $BOB_TOK" \
|
||||
"$BASE_URL/api/v1/attachments/$AID/content" > /tmp/att_dl.code
|
||||
assert_eq "$(cat /tmp/att_dl.code)" 200 download_status
|
||||
got_sha=$(sha256_file "$out")
|
||||
assert_eq "$got_sha" "$want_sha" download_sha256
|
||||
rm -f "$out" /tmp/att_dl.code
|
||||
|
||||
printf 'PASS test_attachment_upload_ok\n'
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# attachment_id from another chat in POST messages → 404.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/tiny.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
|
||||
register_user alice
|
||||
ALICE_ID=$USER_ID
|
||||
|
||||
create_chat 'chat-a' "[\"$ALICE_ID\"]"
|
||||
CHAT_A=$CHAT_ID
|
||||
upload_file "$CHAT_A" "$png" 'tiny.png' 'image/png'
|
||||
expect_code 201
|
||||
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
|
||||
create_chat 'chat-b' "[\"$ALICE_ID\"]"
|
||||
CHAT_B=$CHAT_ID
|
||||
|
||||
http_json POST "/api/v1/chats/$CHAT_B/messages" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "Idempotency-Key: $(uniq_name idem)" \
|
||||
-d "{\"text\":\"hi\",\"attachment_ids\":[\"$AID\"]}"
|
||||
expect_code 404
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_found
|
||||
|
||||
# Empty/whitespace text without attachments → 400
|
||||
http_json POST "/api/v1/chats/$CHAT_B/messages" \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "Idempotency-Key: $(uniq_name idem)" \
|
||||
-d '{"text":" "}'
|
||||
expect_code 400
|
||||
|
||||
printf 'PASS test_attachment_wrong_chat\n'
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
#!/bin/sh
|
||||
# Track 1 negatives: bad checksum complete, abort, foreign complete, no PUT.
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/320x200.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
want_sha=$(sha256_file "$png")
|
||||
want_size=$(wc -c < "$png" | tr -d ' ')
|
||||
bad_sha=$(printf '%s' "$want_sha" | tr 'a-f0-9' '0-9a-f' | head -c 64)
|
||||
# ensure different
|
||||
[ "$bad_sha" != "$want_sha" ] || bad_sha="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
register_user alice
|
||||
ALICE_TOKEN=$TOKEN
|
||||
ALICE_ID=$USER_ID
|
||||
create_chat "du-neg" "[\"$ALICE_ID\"]"
|
||||
register_user bob
|
||||
BOB_TOKEN=$TOKEN
|
||||
TOKEN=$ALICE_TOKEN
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
|
||||
put_via_net() {
|
||||
url=$1
|
||||
ct=$2
|
||||
file=$3
|
||||
code=$($COMPOSE run --rm --no-deps \
|
||||
-v "$file:/tmp/up.bin:ro" \
|
||||
--entrypoint curl api \
|
||||
-sS -m 60 -o /dev/null -w '%{http_code}' \
|
||||
-X PUT -H "Content-Type: $ct" --data-binary @/tmp/up.bin \
|
||||
"$url")
|
||||
printf '%s' "$code"
|
||||
}
|
||||
|
||||
# --- complete without PUT ---
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"a.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
|
||||
expect_code 201
|
||||
ID1=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
http_json POST "/api/v1/direct-uploads/$ID1/complete" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
expect_code 409
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_ready
|
||||
|
||||
# --- abort ---
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"b.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
|
||||
expect_code 201
|
||||
ID2=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
URL2=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
|
||||
CT2=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
|
||||
put_code=$(put_via_net "$URL2" "$CT2" "$png")
|
||||
[ "$put_code" = "200" ] || die "PUT abort-case got $put_code"
|
||||
http_json DELETE "/api/v1/direct-uploads/$ID2" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
expect_code 204
|
||||
http_json POST "/api/v1/direct-uploads/$ID2/complete" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
expect_code 409
|
||||
|
||||
# --- foreign owner ---
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"c.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
|
||||
expect_code 201
|
||||
ID3=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
URL3=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
|
||||
CT3=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
|
||||
put_code=$(put_via_net "$URL3" "$CT3" "$png")
|
||||
[ "$put_code" = "200" ] || die "PUT foreign-case got $put_code"
|
||||
http_json POST "/api/v1/direct-uploads/$ID3/complete" \
|
||||
-H "Authorization: Bearer $BOB_TOKEN"
|
||||
expect_code 404
|
||||
|
||||
# --- checksum mismatch (declared sha ≠ bytes) ---
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"d.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$bad_sha\"}"
|
||||
expect_code 201
|
||||
ID4=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
URL4=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
|
||||
CT4=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
|
||||
put_code=$(put_via_net "$URL4" "$CT4" "$png")
|
||||
[ "$put_code" = "200" ] || die "PUT checksum-case got $put_code"
|
||||
http_json POST "/api/v1/direct-uploads/$ID4/complete" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
expect_code 409
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" hash_mismatch
|
||||
|
||||
# Proxy path still works
|
||||
upload_file "$CHAT_ID" "$png" "proxy.png" "image/png"
|
||||
expect_code 201
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" "ready" proxy_still_ok
|
||||
|
||||
echo "PASS test_direct_upload_negatives"
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/sh
|
||||
# Track 1: initiate → PUT to MinIO (compose network) → complete → download.
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/320x200.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
want_sha=$(sha256_file "$png")
|
||||
want_size=$(wc -c < "$png" | tr -d ' ')
|
||||
|
||||
register_user alice
|
||||
ALICE_TOKEN=$TOKEN
|
||||
ALICE_ID=$USER_ID
|
||||
create_chat "du-ok" "[\"$ALICE_ID\"]"
|
||||
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"photo.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
|
||||
expect_code 201
|
||||
UPLOAD_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
UPLOAD_URL=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
|
||||
CT=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
|
||||
[ -n "$UPLOAD_ID" ] && [ "$UPLOAD_ID" != null ] || die "empty upload id"
|
||||
[ -n "$UPLOAD_URL" ] && [ "$UPLOAD_URL" != null ] || die "empty upload_url"
|
||||
assert_eq "$CT" "image/png" content_type_header
|
||||
|
||||
# PUT bytes on the compose network (MinIO ports stay unpublished by default).
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
put_code=$($COMPOSE run --rm --no-deps \
|
||||
-v "$png:/tmp/up.bin:ro" \
|
||||
--entrypoint curl api \
|
||||
-sS -m 60 -o /dev/null -w '%{http_code}' \
|
||||
-X PUT -H "Content-Type: $CT" --data-binary @/tmp/up.bin \
|
||||
"$UPLOAD_URL")
|
||||
[ "$put_code" = "200" ] || die "presigned PUT expected 200 got $put_code"
|
||||
|
||||
http_json POST "/api/v1/direct-uploads/$UPLOAD_ID/complete" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
expect_code 201
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" "ready" status
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" sha256
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .id)" "$UPLOAD_ID" attachment_id
|
||||
|
||||
out=$(mktemp)
|
||||
replay_code=$($COMPOSE run --rm --no-deps \
|
||||
--entrypoint curl api \
|
||||
-sS -m 60 -o /dev/null -w '%{http_code}' \
|
||||
-X PUT -H "Content-Type: $CT" --data-binary 'replay-content-12345' \
|
||||
"$UPLOAD_URL")
|
||||
[ "$replay_code" = "200" ] || die "replay PUT expected 200 got $replay_code"
|
||||
|
||||
curl -sS -m 60 -o "$out" -H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
"$BASE_URL/api/v1/attachments/$UPLOAD_ID/content"
|
||||
assert_eq "$(sha256_file "$out")" "$want_sha" download_sha256
|
||||
rm -f "$out"
|
||||
|
||||
echo "PASS test_direct_upload_ok"
|
||||
@@ -0,0 +1,59 @@
|
||||
#!/bin/sh
|
||||
# A missing S3 object must fail before the content response headers are sent.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/320x200.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
response_body=$(mktemp)
|
||||
object_deleted=0
|
||||
|
||||
restore_object() {
|
||||
action=$1
|
||||
$COMPOSE run --rm --no-deps -T \
|
||||
-e "CHAT_ID=$CHAT_ID" -e "ATTACHMENT_ID=$AID" \
|
||||
-e "OBJECT_ACTION=$action" \
|
||||
-v "$png:/tmp/restore.png:ro" \
|
||||
--entrypoint /bin/sh minio-init -c '
|
||||
set -eu
|
||||
mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
object="local/$S3_BUCKET/attachments/$CHAT_ID/$ATTACHMENT_ID"
|
||||
if [ "$OBJECT_ACTION" = delete ]; then
|
||||
mc rm --force "$object"
|
||||
else
|
||||
mc cp /tmp/restore.png "$object"
|
||||
fi
|
||||
'
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [ "$object_deleted" = 1 ]; then
|
||||
restore_object restore || printf 'WARNING: failed to restore test object\n' >&2
|
||||
fi
|
||||
rm -f "$response_body"
|
||||
}
|
||||
trap cleanup 0 HUP INT TERM
|
||||
|
||||
want_sha=$(sha256_file "$png")
|
||||
TOKEN=""
|
||||
register_user alice
|
||||
create_chat 'missing-object' "[\"$USER_ID\"]"
|
||||
upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png'
|
||||
expect_code 201
|
||||
AID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" metadata_sha
|
||||
|
||||
restore_object delete
|
||||
object_deleted=1
|
||||
code=$(curl -sS -m 60 -o "$response_body" -w '%{http_code}' \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
"$BASE_URL/api/v1/attachments/$AID/content")
|
||||
case "$code" in
|
||||
404|5??) ;;
|
||||
*) die "expected 404/5xx for missing object, got HTTP $code body=$(cat "$response_body")" ;;
|
||||
esac
|
||||
|
||||
restore_object restore
|
||||
object_deleted=0
|
||||
printf 'PASS test_download_missing_object\n'
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/sh
|
||||
# Graceful shutdown: SIGTERM → /health/ready becomes 503 while process drains.
|
||||
# Targets one API container; restores it at the end.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
|
||||
cid=$($COMPOSE ps -q api | head -n1)
|
||||
[ -n "$cid" ] || die "no api container"
|
||||
|
||||
# Confirm ready before drain.
|
||||
http_json GET /health/ready
|
||||
expect_code 200
|
||||
|
||||
# docker stop → SIGTERM to PID1 (messenger after entrypoint exec).
|
||||
# Probe the draining instance directly (LB may still hit the other replica).
|
||||
docker stop -t 15 "$cid" >/dev/null &
|
||||
stop_pid=$!
|
||||
|
||||
saw_drain=0
|
||||
i=0
|
||||
while [ "$i" -lt 40 ]; do
|
||||
code=$(docker exec "$cid" curl -sS -m 2 -o /dev/null -w '%{http_code}' \
|
||||
http://127.0.0.1:8080/health/ready 2>/dev/null || printf '000')
|
||||
if [ "$code" = "503" ] || [ "$code" = "000" ]; then
|
||||
saw_drain=1
|
||||
break
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 0.25
|
||||
done
|
||||
wait "$stop_pid" || true
|
||||
[ "$saw_drain" = "1" ] || die "did not observe drain/stop (last code=${code:-none})"
|
||||
|
||||
# Restore replicas.
|
||||
$COMPOSE up -d --no-deps api >/dev/null
|
||||
j=0
|
||||
HTTP_CODE=000
|
||||
while [ "$j" -lt 40 ]; do
|
||||
http_json GET /health/ready || true
|
||||
if [ "${HTTP_CODE:-}" = "200" ]; then
|
||||
break
|
||||
fi
|
||||
j=$((j + 1))
|
||||
sleep 1
|
||||
done
|
||||
[ "${HTTP_CODE:-}" = "200" ] || die "stand not ready after recreate"
|
||||
|
||||
printf 'PASS test_graceful_drain\n'
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/bin/sh
|
||||
# Track 1: concurrent complete calls publish one intact attachment.
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
|
||||
png="$FIXTURES/320x200.png"
|
||||
[ -f "$png" ] || die "missing fixture $png"
|
||||
want_sha=$(sha256_file "$png")
|
||||
want_size=$(wc -c < "$png" | tr -d ' ')
|
||||
tmpdir=$(mktemp -d)
|
||||
trap 'rm -rf "$tmpdir"' 0 HUP INT TERM
|
||||
|
||||
register_user alice
|
||||
ALICE_TOKEN=$TOKEN
|
||||
ALICE_ID=$USER_ID
|
||||
create_chat "du-concurrent" "[\"$ALICE_ID\"]"
|
||||
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"concurrent.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}"
|
||||
expect_code 201
|
||||
UPLOAD_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id)
|
||||
UPLOAD_URL=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)
|
||||
CT=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')
|
||||
[ -n "$UPLOAD_ID" ] && [ "$UPLOAD_ID" != null ] || die "empty upload id"
|
||||
[ -n "$UPLOAD_URL" ] && [ "$UPLOAD_URL" != null ] || die "empty upload_url"
|
||||
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
put_code=$($COMPOSE run --rm --no-deps \
|
||||
-v "$png:/tmp/up.bin:ro" \
|
||||
--entrypoint curl api \
|
||||
-sS -m 60 -o /dev/null -w '%{http_code}' \
|
||||
-X PUT -H "Content-Type: $CT" --data-binary @/tmp/up.bin \
|
||||
"$UPLOAD_URL")
|
||||
[ "$put_code" = "200" ] || die "presigned PUT expected 200 got $put_code"
|
||||
|
||||
pids=
|
||||
for i in 1 2 3 4; do
|
||||
curl -sS -m 60 -o "$tmpdir/body.$i" -w '%{http_code}' \
|
||||
-X POST \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
"$BASE_URL/api/v1/direct-uploads/$UPLOAD_ID/complete" \
|
||||
> "$tmpdir/code.$i" &
|
||||
pids="$pids $!"
|
||||
done
|
||||
for pid in $pids; do
|
||||
wait "$pid" || die "complete request failed"
|
||||
done
|
||||
|
||||
count_201=0
|
||||
for i in 1 2 3 4; do
|
||||
code=$(cat "$tmpdir/code.$i")
|
||||
case "$code" in
|
||||
201) count_201=$((count_201 + 1)) ;;
|
||||
200|409) ;;
|
||||
*) die "unexpected concurrent complete HTTP $code: $(cat "$tmpdir/body.$i")" ;;
|
||||
esac
|
||||
done
|
||||
assert_eq "$count_201" 1 created_count
|
||||
|
||||
http_json GET "/api/v1/attachments/$UPLOAD_ID" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
expect_code 200
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" ready metadata_status
|
||||
assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" metadata_sha
|
||||
|
||||
code=$(curl -sS -m 60 -o "$tmpdir/content" -w '%{http_code}' \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
"$BASE_URL/api/v1/attachments/$UPLOAD_ID/content")
|
||||
assert_eq "$code" 200 content_status
|
||||
assert_eq "$(sha256_file "$tmpdir/content")" "$want_sha" content_sha
|
||||
|
||||
printf 'PASS test_presigned_complete_concurrent\n'
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Grade 4: private bucket + MinIO ports not published on the host.
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
|
||||
S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
|
||||
[ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; }
|
||||
|
||||
KEY=$($COMPOSE exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT s3_key FROM attachments LIMIT 1"' \
|
||||
| tr -d '\r')
|
||||
if [ -z "$KEY" ]; then
|
||||
KEY="attachments/does-not-exist/probe"
|
||||
fi
|
||||
|
||||
echo "check_bucket_private: unsigned GET s3://$S3_BUCKET/$KEY"
|
||||
|
||||
code=$($COMPOSE run --rm --no-deps --entrypoint /bin/sh minio-init -c \
|
||||
"curl -sS -o /tmp/body -w '%{http_code}' -m 10 \"http://minio:9000/$S3_BUCKET/$KEY\" || echo 000")
|
||||
|
||||
echo "unsigned GET → HTTP $code"
|
||||
[ "$code" != "200" ] || { echo "FAIL: bucket appears public" >&2; exit 1; }
|
||||
[ "$code" != "000" ] || { echo "FAIL: cannot reach minio on compose network" >&2; exit 1; }
|
||||
|
||||
for port in 9000 9001; do
|
||||
if curl -sS -m 2 -o /dev/null "http://127.0.0.1:$port/" 2>/dev/null; then
|
||||
echo "FAIL: host :$port is reachable" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "ok: host :$port not published"
|
||||
done
|
||||
|
||||
echo "PASS check_bucket_private"
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
# stb (vendored)
|
||||
|
||||
`stb_image.h` from [nothings/stb](https://github.com/nothings/stb).
|
||||
|
||||
Pinned commit: see `VERSION`.
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
013ac3beddff3dbffafd5177e7972067cd2b5083
|
||||
Vendored
+7988
File diff suppressed because it is too large
Load Diff
Executable
+154
@@ -0,0 +1,154 @@
|
||||
#!/bin/sh
|
||||
# Track 1 experiment: API RSS during transfer + approx body bytes through API.
|
||||
# Direct PUT: client→MinIO (compose net). Complete measured separately.
|
||||
#
|
||||
# ./tools/exp_direct_vs_proxy.sh
|
||||
# Optional: SIZES_MB="1 4" CONCURRENCY=3
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
. "$ROOT/student-tests/lib.sh"
|
||||
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
SIZES_MB="${SIZES_MB:-1 4}"
|
||||
CONCURRENCY="${CONCURRENCY:-3}"
|
||||
OUT_DIR="${OUT_DIR:-$ROOT/artifacts/track1}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
need docker
|
||||
|
||||
rss_sum() {
|
||||
ids=$($COMPOSE ps -q api)
|
||||
total=0
|
||||
for id in $ids; do
|
||||
[ -n "$id" ] || continue
|
||||
rss=$(docker exec "$id" sh -c "awk '/^VmRSS:/ {print \$2}' /proc/1/status" 2>/dev/null || echo 0)
|
||||
total=$((total + rss))
|
||||
done
|
||||
printf '%s' "$total"
|
||||
}
|
||||
|
||||
sample_peak() {
|
||||
dur=${1:-3}
|
||||
peak=$(rss_sum)
|
||||
end=$(( $(date +%s) + dur ))
|
||||
while [ "$(date +%s)" -lt "$end" ]; do
|
||||
cur=$(rss_sum)
|
||||
[ "$cur" -gt "$peak" ] && peak=$cur
|
||||
sleep 0.15
|
||||
done
|
||||
printf '%s' "$peak"
|
||||
}
|
||||
|
||||
make_png() {
|
||||
mb=$1
|
||||
out=$2
|
||||
python3 - "$mb" "$out" <<'PY'
|
||||
import struct, zlib, sys
|
||||
mb = float(sys.argv[1])
|
||||
path = sys.argv[2]
|
||||
target = int(mb * 1024 * 1024)
|
||||
w, h = 320, 200
|
||||
raw = b"".join(b"\x00" + bytes(w * 3) for _ in range(h))
|
||||
def chunk(tag, data):
|
||||
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xffffffff)
|
||||
ihdr = chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
|
||||
idat = chunk(b"IDAT", zlib.compress(raw, 9))
|
||||
sig = b"\x89PNG\r\n\x1a\n"
|
||||
base = sig + ihdr + idat + chunk(b"IEND", b"")
|
||||
pad_need = max(0, target - len(base) - 12)
|
||||
text = b"Comment\0" + (b"x" * pad_need)
|
||||
open(path, "wb").write(sig + ihdr + idat + chunk(b"tEXt", text) + chunk(b"IEND", b""))
|
||||
PY
|
||||
}
|
||||
|
||||
put_direct() {
|
||||
url=$1
|
||||
ct=$2
|
||||
file=$3
|
||||
$COMPOSE run --rm --no-deps \
|
||||
-v "$file:/tmp/up.bin:ro" \
|
||||
--entrypoint curl api \
|
||||
-sS -m 120 -o /dev/null \
|
||||
-X PUT -H "Content-Type: $ct" --data-binary @/tmp/up.bin \
|
||||
"$url" >/dev/null
|
||||
}
|
||||
|
||||
register_user exp_alice3
|
||||
ALICE_TOKEN=$TOKEN
|
||||
ALICE_ID=$USER_ID
|
||||
create_chat "exp-track1c" "[\"$ALICE_ID\"]"
|
||||
|
||||
RESULT="$OUT_DIR/results.tsv"
|
||||
printf 'mode\tphase\tsize_mb\trss_before_kb\trss_peak_kb\trss_delta_kb\tapi_body_bytes_approx\n' > "$RESULT"
|
||||
|
||||
for mb in $SIZES_MB; do
|
||||
f="$OUT_DIR/payload_${mb}mb.png"
|
||||
make_png "$mb" "$f"
|
||||
sha=$(sha256_file "$f")
|
||||
sz=$(wc -c < "$f" | tr -d ' ')
|
||||
api_bytes=$((CONCURRENCY * sz))
|
||||
|
||||
# --- proxy transfer ---
|
||||
sleep 1
|
||||
before=$(rss_sum)
|
||||
i=0
|
||||
pids=""
|
||||
while [ "$i" -lt "$CONCURRENCY" ]; do
|
||||
( upload_file "$CHAT_ID" "$f" "p_${mb}_$i.png" "image/png" >/dev/null 2>&1 || true ) &
|
||||
pids="$pids $!"
|
||||
i=$((i + 1))
|
||||
done
|
||||
peak=$(sample_peak 10)
|
||||
for p in $pids; do wait "$p" || true; done
|
||||
[ "$peak" -lt "$before" ] && peak=$before
|
||||
delta=$((peak - before))
|
||||
printf 'proxy\ttransfer\t%s\t%s\t%s\t%s\t%s\n' "$mb" "$before" "$peak" "$delta" "$api_bytes" | tee -a "$RESULT"
|
||||
|
||||
# --- direct: create sessions, then PUT-only window ---
|
||||
sess="$OUT_DIR/sess_${mb}.tsv"
|
||||
: > "$sess"
|
||||
i=0
|
||||
while [ "$i" -lt "$CONCURRENCY" ]; do
|
||||
http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"filename\":\"d_${mb}_$i.png\",\"content_type\":\"image/png\",\"size_bytes\":$sz,\"sha256\":\"$sha\"}"
|
||||
[ "$HTTP_CODE" = "201" ] || die "init failed: $HTTP_BODY"
|
||||
printf '%s\t%s\t%s\n' \
|
||||
"$(printf '%s' "$HTTP_BODY" | jq -r .id)" \
|
||||
"$(printf '%s' "$HTTP_BODY" | jq -r .upload_url)" \
|
||||
"$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]')" >> "$sess"
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
sleep 1
|
||||
before=$(rss_sum)
|
||||
pids=""
|
||||
while IFS="$(printf '\t')" read -r uid url ct; do
|
||||
[ -n "$url" ] || continue
|
||||
( put_direct "$url" "$ct" "$f" ) &
|
||||
pids="$pids $!"
|
||||
done < "$sess"
|
||||
peak=$(sample_peak 15)
|
||||
for p in $pids; do wait "$p" || true; done
|
||||
[ "$peak" -lt "$before" ] && peak=$before
|
||||
delta=$((peak - before))
|
||||
printf 'direct\ttransfer\t%s\t%s\t%s\t%s\t%s\n' "$mb" "$before" "$peak" "$delta" "0" | tee -a "$RESULT"
|
||||
|
||||
# --- direct complete ---
|
||||
before=$(rss_sum)
|
||||
while IFS="$(printf '\t')" read -r uid url ct; do
|
||||
[ -n "$uid" ] || continue
|
||||
http_json POST "/api/v1/direct-uploads/$uid/complete" \
|
||||
-H "Authorization: Bearer $ALICE_TOKEN"
|
||||
[ "$HTTP_CODE" = "201" ] || die "complete failed id=$uid body=$HTTP_BODY"
|
||||
done < "$sess"
|
||||
peak=$(rss_sum)
|
||||
[ "$peak" -lt "$before" ] && peak=$before
|
||||
delta=$((peak - before))
|
||||
printf 'direct\tcomplete\t%s\t%s\t%s\t%s\t%s\n' "$mb" "$before" "$peak" "$delta" "$api_bytes" | tee -a "$RESULT"
|
||||
done
|
||||
|
||||
echo "wrote $RESULT"
|
||||
cat "$RESULT"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 502 B |
Binary file not shown.
|
After Width: | Height: | Size: 644 B |
Binary file not shown.
|
After Width: | Height: | Size: 633 B |
Binary file not shown.
|
After Width: | Height: | Size: 69 B |
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/bin/sh
|
||||
# Expire pending direct-upload sessions past expires_at and delete their S3
|
||||
# objects (if any). Dry-run by default.
|
||||
#
|
||||
# ./tools/gc_direct_uploads.sh
|
||||
# APPLY=1 ./tools/gc_direct_uploads.sh
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
APPLY="${APPLY:-0}"
|
||||
|
||||
S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
|
||||
[ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; }
|
||||
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
echo "gc_direct_uploads: apply=$APPLY"
|
||||
|
||||
$COMPOSE exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "
|
||||
SELECT id || E'\''\t'\'' || s3_key FROM direct_uploads
|
||||
WHERE status = '\''pending'\'' AND expires_at < now()
|
||||
"' > "$TMP/stale.tsv" || true
|
||||
|
||||
n=$(wc -l < "$TMP/stale.tsv" | tr -d ' ')
|
||||
echo "gc_direct_uploads: stale_pending=$n"
|
||||
|
||||
if [ ! -s "$TMP/stale.tsv" ]; then
|
||||
echo "gc_direct_uploads: nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while IFS="$(printf '\t')" read -r id key; do
|
||||
[ -n "$id" ] || continue
|
||||
echo " STALE id=$id key=$key"
|
||||
done < "$TMP/stale.tsv"
|
||||
|
||||
if [ "$APPLY" != "1" ]; then
|
||||
echo "gc_direct_uploads: dry-run; set APPLY=1 to expire + delete objects"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
$COMPOSE exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "
|
||||
UPDATE direct_uploads SET status = '\''expired'\'', updated_at = now()
|
||||
WHERE status = '\''pending'\'' AND expires_at < now()
|
||||
"' >/dev/null
|
||||
|
||||
cut -f2 "$TMP/stale.tsv" | $COMPOSE run --rm --no-deps --entrypoint /bin/sh minio-init -c "
|
||||
set -eu
|
||||
mc alias set local \"\$S3_ENDPOINT\" \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" >/dev/null
|
||||
while IFS= read -r key; do
|
||||
[ -n \"\$key\" ] || continue
|
||||
mc rm --force \"local/\$S3_BUCKET/\$key\" 2>/dev/null || true
|
||||
echo \"deleted \$key\"
|
||||
done
|
||||
"
|
||||
|
||||
echo "gc_direct_uploads: done"
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/sh
|
||||
# Reconcile S3 orphans: bucket objects with no matching attachments.s3_key in PG.
|
||||
#
|
||||
# Upload path compensation: S3 PUT → PG INSERT; on INSERT failure the API
|
||||
# best-effort DeleteObject. This script cleans leftovers after crashes between
|
||||
# PUT and compensate (or a failed DeleteObject).
|
||||
#
|
||||
# Dry-run by default. Fresh objects inside GRACE_SECONDS are never deleted.
|
||||
#
|
||||
# ./tools/gc_orphans.sh
|
||||
# APPLY=1 GRACE_SECONDS=3600 ./tools/gc_orphans.sh
|
||||
set -eu
|
||||
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml"
|
||||
GRACE_SECONDS="${GRACE_SECONDS:-3600}"
|
||||
APPLY="${APPLY:-0}"
|
||||
PREFIX="${ORPHAN_PREFIX:-}"
|
||||
|
||||
S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"')
|
||||
[ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; }
|
||||
|
||||
TMP=$(mktemp -d)
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
echo "gc_orphans: bucket=$S3_BUCKET grace=${GRACE_SECONDS}s apply=$APPLY prefix=$PREFIX"
|
||||
|
||||
s3_listing=$($COMPOSE run --rm --no-deps -T --entrypoint /bin/sh minio-init -c "
|
||||
set -eu
|
||||
mc alias set local \"\$S3_ENDPOINT\" \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" >/dev/null
|
||||
mc ls --recursive --json \"local/\$S3_BUCKET/$PREFIX\" || true
|
||||
")
|
||||
printf '%s\n' "$s3_listing" > "$TMP/s3.json"
|
||||
|
||||
# Keep published attachments and staging objects for active direct uploads.
|
||||
pg_keys=$($COMPOSE exec -T db sh -c \
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "
|
||||
SELECT s3_key FROM attachments
|
||||
UNION
|
||||
SELECT staging_key FROM direct_uploads WHERE status = '\''pending'\''
|
||||
"')
|
||||
printf '%s\n' "$pg_keys" > "$TMP/pg.keys"
|
||||
|
||||
python3 - "$TMP/s3.json" "$TMP/pg.keys" "$GRACE_SECONDS" "$TMP/del.keys" "$PREFIX" <<'PY'
|
||||
import json, sys, time
|
||||
from datetime import datetime
|
||||
|
||||
s3_path, pg_path, grace_s, out_path, prefix = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4], sys.argv[5]
|
||||
now = time.time()
|
||||
known = {line.strip() for line in open(pg_path) if line.strip()}
|
||||
orphans = []
|
||||
for line in open(s3_path):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
key = obj.get("key") or obj.get("Key")
|
||||
if not key:
|
||||
continue
|
||||
key = key.lstrip("/")
|
||||
if prefix and not key.startswith(prefix):
|
||||
key = prefix.rstrip("/") + "/" + key
|
||||
if not key.startswith(("attachments/", "staging/")):
|
||||
continue
|
||||
if key in known:
|
||||
continue
|
||||
age = None
|
||||
lm = obj.get("lastModified") or obj.get("last_modified")
|
||||
if lm:
|
||||
try:
|
||||
age = now - datetime.fromisoformat(lm.replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
age = None
|
||||
orphans.append((key, age, obj.get("size")))
|
||||
|
||||
print(f"gc_orphans: scanned orphans={len(orphans)} known_pg={len(known)}")
|
||||
to_delete = []
|
||||
for key, age, size in orphans:
|
||||
age_s = f"{int(age)}s" if age is not None else "?"
|
||||
if age is not None and age < grace_s:
|
||||
print(f" SKIP (grace) key={key} age={age_s} size={size}")
|
||||
continue
|
||||
print(f" ORPHAN key={key} age={age_s} size={size}")
|
||||
to_delete.append(key)
|
||||
|
||||
with open(out_path, "w") as f:
|
||||
f.write("\n".join(to_delete) + ("\n" if to_delete else ""))
|
||||
print(f"gc_orphans: candidates={len(to_delete)}")
|
||||
PY
|
||||
|
||||
if [ "$APPLY" != "1" ]; then
|
||||
echo "gc_orphans: dry-run; set APPLY=1 to delete candidates older than grace."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -s "$TMP/del.keys" ]; then
|
||||
echo "gc_orphans: nothing to delete"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while IFS= read -r key; do
|
||||
[ -n "$key" ] || continue
|
||||
echo "deleting $S3_BUCKET/$key"
|
||||
$COMPOSE run --rm --no-deps -T -e "GC_OBJECT_KEY=$key" \
|
||||
--entrypoint /bin/sh minio-init -c '
|
||||
set -eu
|
||||
mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
mc rm --force "local/$S3_BUCKET/$GC_OBJECT_KEY"
|
||||
'
|
||||
done < "$TMP/del.keys"
|
||||
|
||||
echo "gc_orphans: delete done"
|
||||
@@ -0,0 +1,111 @@
|
||||
// Step 5 smoke: PNG / JPEG / truncated PNG / non-image / size limit.
|
||||
#include "image_util.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#ifndef IMAGE_SMOKE_FIXTURES
|
||||
#define IMAGE_SMOKE_FIXTURES "tools/fixtures"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
std::vector<std::uint8_t> read_all(const std::string& path) {
|
||||
std::ifstream in(path, std::ios::binary);
|
||||
if (!in) {
|
||||
throw std::runtime_error("cannot open " + path);
|
||||
}
|
||||
return std::vector<std::uint8_t>((std::istreambuf_iterator<char>(in)),
|
||||
std::istreambuf_iterator<char>());
|
||||
}
|
||||
|
||||
bool expect_ok(const char* name, const std::vector<std::uint8_t>& bytes,
|
||||
const char* want_ct) {
|
||||
ImageLimits lim;
|
||||
ImageInfo info;
|
||||
auto rej = validate_image(bytes.data(), bytes.size(), lim, info);
|
||||
if (rej) {
|
||||
std::cerr << name << ": expected OK, got " << image_reject_code(*rej)
|
||||
<< "\n";
|
||||
return false;
|
||||
}
|
||||
if (info.content_type != want_ct) {
|
||||
std::cerr << name << ": content_type " << info.content_type << "\n";
|
||||
return false;
|
||||
}
|
||||
std::cout << name << ": OK " << info.content_type << " " << info.width
|
||||
<< "x" << info.height << "\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
bool expect_reject(const char* name, const std::vector<std::uint8_t>& bytes,
|
||||
ImageReject want) {
|
||||
ImageLimits lim;
|
||||
ImageInfo info;
|
||||
auto rej = validate_image(bytes.data(), bytes.size(), lim, info);
|
||||
if (!rej || *rej != want) {
|
||||
std::cerr << name << ": expected " << image_reject_code(want) << ", got "
|
||||
<< (rej ? image_reject_code(*rej) : "OK") << "\n";
|
||||
return false;
|
||||
}
|
||||
std::cout << name << ": OK reject=" << image_reject_code(want) << "\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
const std::string dir =
|
||||
(argc > 1) ? argv[1] : std::string(IMAGE_SMOKE_FIXTURES);
|
||||
const auto png = read_all(dir + "/tiny.png");
|
||||
const auto jpg = read_all(dir + "/tiny.jpg");
|
||||
|
||||
bool ok = true;
|
||||
ok &= expect_ok("png", png, "image/png");
|
||||
ok &= expect_ok("jpeg", jpg, "image/jpeg");
|
||||
|
||||
std::vector<std::uint8_t> trunc(png.begin(),
|
||||
png.begin() + std::min<std::size_t>(16, png.size()));
|
||||
ok &= expect_reject("truncated_png", trunc, ImageReject::invalid_image);
|
||||
|
||||
std::vector<std::uint8_t> spoof = jpg;
|
||||
spoof[0] = 'G';
|
||||
spoof[1] = 'I';
|
||||
spoof[2] = 'F';
|
||||
ok &= expect_reject("not_image", spoof, ImageReject::unsupported_media_type);
|
||||
|
||||
ImageLimits tiny_lim;
|
||||
tiny_lim.max_bytes = 10;
|
||||
ImageInfo info;
|
||||
auto rej = validate_image(png.data(), png.size(), tiny_lim, info);
|
||||
if (!rej || *rej != ImageReject::payload_too_large) {
|
||||
std::cerr << "max_bytes: FAIL\n";
|
||||
ok = false;
|
||||
} else {
|
||||
std::cout << "max_bytes: OK\n";
|
||||
}
|
||||
|
||||
// JPEG bytes sniffed as jpeg regardless of any HTTP Content-Type.
|
||||
auto sniffed = sniff_content_type(jpg.data(), jpg.size());
|
||||
if (!sniffed || *sniffed != "image/jpeg") {
|
||||
std::cerr << "sniff_jpeg: FAIL\n";
|
||||
ok = false;
|
||||
} else {
|
||||
std::cout << "sniff_jpeg: OK (MIME spoof would be ignored)\n";
|
||||
}
|
||||
|
||||
if (!ok) {
|
||||
return 1;
|
||||
}
|
||||
std::cout << "image_smoke: OK\n";
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "image_smoke: FAIL: " << e.what() << "\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Small smoke tool for step 4: put → get → delete against MinIO.
|
||||
#include "s3_client.hpp"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
|
||||
int main() {
|
||||
try {
|
||||
auto cfg = S3Client::from_env();
|
||||
S3Client s3(cfg);
|
||||
|
||||
const std::string key = "smoke/step4.txt";
|
||||
const std::string payload = "lab4-s3-smoke-ok";
|
||||
std::istringstream in(payload);
|
||||
s3.put_object(key, in, static_cast<std::int64_t>(payload.size()),
|
||||
"text/plain");
|
||||
|
||||
std::ostringstream out;
|
||||
s3.get_object(key, out);
|
||||
if (out.str() != payload) {
|
||||
std::cerr << "mismatch: got [" << out.str() << "]\n";
|
||||
return 2;
|
||||
}
|
||||
|
||||
s3.delete_object(key);
|
||||
std::cout << "s3_smoke: OK bucket=" << cfg.bucket << " key=" << key << "\n";
|
||||
return 0;
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "s3_smoke: FAIL: " << e.what() << "\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user