From 7741b66fd2f136147560e34b26e7ee92940622bf Mon Sep 17 00:00:00 2001 From: AndrewNikkk Date: Fri, 6 Nov 2026 04:23:59 +0300 Subject: [PATCH] fit: complete lab4, fix P1.1-P1.3, add Track 1 presigned --- REPORT.md | 40 ++- contracts/README.md | 2 +- evidence/load_uploads.json | 20 +- migrations/007_direct_upload_staging.sql | 12 + src/direct_upload_repo.cpp | 131 ++++++- src/direct_upload_repo.hpp | 33 +- src/main.cpp | 333 +++++++++--------- src/models.hpp | 4 +- src/s3_client.cpp | 16 + src/s3_client.hpp | 3 + student-tests/run_grade5_track1.sh | 3 + student-tests/test_direct_upload_negatives.sh | 6 +- student-tests/test_direct_upload_ok.sh | 13 +- student-tests/test_download_missing_object.sh | 59 ++++ .../test_presigned_complete_concurrent.sh | 73 ++++ tools/gc_orphans.sh | 48 +-- 16 files changed, 577 insertions(+), 219 deletions(-) create mode 100644 migrations/007_direct_upload_staging.sql create mode 100644 student-tests/test_download_missing_object.sh create mode 100644 student-tests/test_presigned_complete_concurrent.sh diff --git a/REPORT.md b/REPORT.md index 14aae34..d20214f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,8 @@ # Отчёт — лабораторная 4 - Автор, группа: Никитин Андрей Александрович, группа БВТ2301 -- Целевая оценка: 5 (трек 1 — прямой/presigned upload) +- Целевая оценка: 5 +- Трек 5: Track 1 (Presigned upload) - Commit/PR: ветка `solution/nikitin-bvt2301` → MR в `main` (шаблон `.github/pull_request_template.md`); remote `gitea.privcloud.tech/AndrewNikkk/mtusi2026lab4` - Источники и помощь: контракт и публичные тесты курса; PostgreSQL 16, Valkey, Nginx, MinIO, libpqxx 7.10.0, libsodium, hiredis, nlohmann/json, minio-cpp v0.3.0 (официальный SigV4), stb_image. Помощь ИИ: Cursor. @@ -60,7 +61,7 @@ docker compose --env-file .env stop # docker compose --env-file .env down -v ``` -Хосты в секретах — имена сервисов Compose (`db`, `cache`, `minio`), не loopback. Порты MinIO **не** публикуются по умолчанию; снаружи только LB `:8080`. Для host-side PUT к presigned URL — опционально `compose.presign.yaml` + `S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000` (loopback). Иначе PUT идёт из контейнера на сети compose (`http://minio:9000/...`). Миграции 001–006 применяет API при старте под `pg_advisory_lock`. +Хосты в секретах — имена сервисов Compose (`db`, `cache`, `minio`), не loopback. Порты MinIO **не** публикуются по умолчанию; снаружи только LB `:8080`. Для host-side PUT к presigned URL — опционально `compose.presign.yaml` + `S3_PRESIGN_ENDPOINT=http://127.0.0.1:9000` (loopback). Иначе PUT идёт из контейнера на сети compose (`http://minio:9000/...`). Миграции 001–007 применяет API при старте под `pg_advisory_lock`. Соответствие `.env.example` → compose: `S3_ENDPOINT`/`S3_BUCKET`/`S3_REGION`, `S3_*_FILE` → docker secrets, `MAX_UPLOAD_BYTES`/`MAX_IMAGE_PIXELS`, `DIRECT_UPLOAD_TTL_SECONDS`, `DRAIN_SECONDS`, `API_REPLICAS`, `SESSION_STORE=redis`. Root MinIO только у `minio` / init-job'ов; API видит лишь limited `*_FILE`. @@ -68,16 +69,22 @@ docker compose --env-file .env stop ``` Proxy: Клиент → lb → api (buffer body) → MinIO PutObject → PG attachments -Direct: Клиент → lb → api (JSON init) → presigned URL - Клиент ─────────────── PUT ──────→ MinIO - Клиент → lb → api complete (Stat+Get+sha256+image) → PG attachments +Direct: Клиент → lb → api init → presigned PUT URL (staging/{chat}/{upload}) + Клиент ─────────────── PUT ──────→ MinIO staging + Клиент → lb → api complete (HEAD+Get+sha256+image+CopyObject) + staging → attachments/{chat}/{attachment} + → PG attachment + completed session ``` -**Источник истины:** метаданные вложений и связь message↔attachment — PostgreSQL; байты — объект в MinIO с ключом `attachments/{chat_id}/{uuid}` (filename только для отображения). Pending direct sessions — таблица `direct_uploads` (миграция 006). +**Источник истины:** метаданные вложений и связь message↔attachment — PostgreSQL; оригинал — объект MinIO `attachments/{chat_id}/{attachment_id}` (filename только для отображения). Presigned PUT пишет в отдельный `staging/{chat_id}/{upload_id}`. Direct sessions хранят оба ключа в `direct_uploads` (миграции 006–007). **Порядок записи (proxy, компенсация):** S3 `PutObject` → `INSERT attachments` → COMMIT. Если INSERT/COMMIT упал — best-effort `DeleteObject`, ответ 500. Строка в PG без объекта недопустима. Orphan'ы: `tools/gc_orphans.sh` (учитывает `direct_uploads.status=pending`). -**Трек 1 (direct):** `POST /api/v1/chats/{id}/direct-uploads` → короткая SigV4 PUT URL (`minio-cpp GetPresignedObjectUrl` / PresignV4, TTL=`DIRECT_UPLOAD_TTL_SECONDS`; в ответе рекомендованный `Content-Type` для метаданных объекта) → клиентский PUT → `POST .../complete` (владелец, Stat size, sha256, magic/image) → `attachments` ready с тем же id. `DELETE .../direct-uploads/{id}` — abort + DeleteObject. Просроченные pending: `tools/gc_direct_uploads.sh`. Обязательный proxy `POST .../attachments` **сохранён**. +**Track 1 (presigned):** `POST /api/v1/chats/{id}/direct-uploads` создаёт `pending` сессию и выдаёт SigV4 PUT URL (`minio-cpp GetPresignedObjectUrl`, TTL по `DIRECT_UPLOAD_TTL_SECONDS`, 30–3600 секунд; default 300). URL сам по себе повторно используем до истечения TTL, поэтому он не считается одноразовым токеном. Одноразово публикуется именно сессия complete: её статус переходит `pending → completed`. + +`POST /api/v1/direct-uploads/{id}/complete` в одной PostgreSQL-транзакции делает `SELECT ... FOR UPDATE`, проверяет существование/размер/hash staging-объекта и изображение, затем выполняет серверный `CopyObject` в final prefix, вставляет `attachments.status=ready`, помечает сессию `completed` и коммитит. SHA-256 берётся из `expected_sha256`, заявленного при init. Повторный complete возвращает сохранённый Attachment без повторного S3 copy; конкурентные complete одного id сериализуются блокировкой строки. Удаление staging после публикации best-effort. + +Повторный PUT по ещё действующей старой ссылке может создать staging-orphan, но не перезаписывает final object. `tools/gc_orphans.sh` сканирует `attachments/` и `staging/`, сохраняет pending staging keys и удаляет неизвестные объекты только после `GRACE_SECONDS`. Abort удаляет staging best-effort; стандартный proxy `POST /api/v1/chats/{id}/attachments` **сохранён**. **ACL:** метаданные и `/content` только участнику чата; чужой → 404; без Bearer → 401. Unsigned GET на `http://minio:9000/$BUCKET/$KEY` → 403. @@ -108,6 +115,9 @@ Direct: Клиент → lb → api (JSON init) → presigned URL | Graceful drain | `sh student-tests/test_graceful_drain.sh` | SIGTERM → ready 503/stop; стенд восстановлен | вывод теста | | Идемпотентный bootstrap bucket/user | `minio-init`, `minio-user-init` | повторный up без ошибки; `mc mb --ignore-existing`; anonymous none | логи init | | Трек 1: init→PUT→complete; proxy жив | `sh student-tests/run_grade5_track1.sh` | PASS (ok + negatives + gc dry-run) | `student-tests/` | +| Track 1: concurrent complete | `sh student-tests/test_presigned_complete_concurrent.sh` | 4 complete: ровно 1×201, остальные 200/409, ни одного 500; metadata и `/content` SHA-256 совпали с исходником | `student-tests/test_presigned_complete_concurrent.sh` | +| Track 1: replay safety | `sh student-tests/test_direct_upload_ok.sh` | upload → complete → повторный PUT 20 B по старой ссылке → download SHA-256 исходного PNG; final не изменён | `student-tests/test_direct_upload_ok.sh` | +| Download при отсутствующем S3-объекте | `sh student-tests/test_download_missing_object.sh` | после `mc rm` content отвечает 404/5xx, не 200; объект восстанавливается тестом | `student-tests/test_download_missing_object.sh` | | Трек 1: RAM/трафик ≥2 размеров | `sh tools/exp_direct_vs_proxy.sh` | transfer: direct ≪ proxy по RSS и api_body | `artifacts/track1/results.tsv` | Перезапускаемые процессы при persistence hook: **db, cache, minio, api (все реплики), lb**. Volumes `pgdata`, `valkeydata`, `miniodata` **не** удаляются. Failover hook останавливает **только** одну API-реплику по `X-Instance-Id` ≡ `Config.Hostname`. @@ -157,9 +167,15 @@ Direct: Клиент → lb → api (JSON init) → presigned URL **Подтверждено измерением (grade 4):** 10 параллельных ~5.6 MiB proxy-загрузок → 201; delta RSS < naive; private bucket; persistence/failover. -## Трек 1 — прямой upload (оценка 5) +## Track 1: Presigned upload -**Гипотеза:** при одинаковом размере файла пиковый RSS API и объём трафика через LB/API на этапе передачи байт ниже для direct (клиент→MinIO), чем для proxy (клиент→API→MinIO); complete добавляет короткий GetObject на API. +**Архитектура и семантика:** presigned PUT направлен только на `staging/{chat_id}/{upload_id}`. После HEAD/stat, проверки `expected_size`, SHA-256 из init и image validation API копирует staging в `attachments/{chat_id}/{attachment_id}`. В `attachments.s3_key` хранится только final key. TTL ссылки задаётся `DIRECT_UPLOAD_TTL_SECONDS` (30–3600 секунд, default 300); пока TTL не истёк, URL можно повторно вызвать, но replay меняет только staging. Complete логически single-use: `pending → completed`; `SELECT FOR UPDATE` делает повторы и конкурентные запросы идемпотентными, они возвращают существующий Attachment без S3-публикации. + +Если старый PUT выполняется после complete, он оставляет staging-orphan; final-объект остаётся прежним. `tools/gc_orphans.sh` очищает orphan-объекты обоих prefix по `GRACE_SECONDS`, не удаляя staging активных `pending` сессий. Staging cleanup после успешной публикации и abort — best-effort. + +**Ограничения:** стандартная загрузка ограничена 10 MiB. Multipart-upload для файлов больше 100 MiB не реализован; presigned GET для скачивания тоже не реализован. Download остаётся авторизованным через API. Контракт `contracts/openapi.json` не менялся: Track 1 — расширение, обязательный proxy upload сохранён. + +**Proxy vs presigned:** proxy передаёт тело через LB/API и буферизует его перед MinIO, поэтому upload потребляет память API и дополнительный сетевой hop. При presigned передаче байты идут клиент→MinIO напрямую, API получает только init/complete и не буферизует тело PUT. Complete добавляет HEAD/stat, чтение для SHA/image validation и server-side copy, поэтому его latency выше собственно PUT и измеряется отдельно. **API (расширение, openapi не менялся):** @@ -171,9 +187,9 @@ Direct: Клиент → lb → api (JSON init) → presigned URL | DELETE | `/api/v1/direct-uploads/{id}` | abort + DeleteObject | | POST | `/api/v1/chats/{id}/attachments` | обязательный proxy — без изменений | -**Негативные:** complete без PUT → 400; checksum mismatch → 400 + abort; чужой complete → 404; abort → 204, повторный complete → 409; expired → 410 (`gc_direct_uploads` / expire_stale). +**Состояния и ошибки:** complete без объекта → `409 not_ready`; размер или checksum mismatch → `409 hash_mismatch`; чужой upload → 404; abort → 204; повторный complete после успеха → 200 с тем же Attachment; aborted/expired session не публикуется; просроченный pending → 410. -**Проверка:** `sh student-tests/run_grade5_track1.sh` → PASS. Эксперимент: `sh tools/exp_direct_vs_proxy.sh` → `artifacts/track1/results.tsv`. +**Проверка:** `sh student-tests/run_grade5_track1.sh` → PASS; `sh student-tests/run_grade3.sh` → PASS; `sh student-tests/run_grade4.sh` → PASS. Четыре одновременных complete дали ровно один 201, остальные ответы были 200/409, 500 не было; metadata и content SHA-256 совпали. Replay-проверка повторно PUT-ит 20-byte текст по старому URL и подтверждает SHA-256 исходного PNG при download. Эксперимент proxy vs direct: `sh tools/exp_direct_vs_proxy.sh` → `artifacts/track1/results.tsv`. Методика: closed-loop; concurrency 3 (1 MiB) / 4 (4 MiB); RSS = сумма `VmRSS` процессов `messenger` по репликам api; фаза `transfer` — только передача байт (для direct — параллельные PUT на MinIO); `complete` — отдельно (GetObject+sha256+image на API). `api_body_bytes_approx` — оценка объёма тел через API (proxy = N×size; direct transfer = 0). @@ -189,3 +205,5 @@ Direct: Клиент → lb → api (JSON init) → presigned URL **Вывод:** на этапе передачи байт direct почти не двигает RSS API и не гонит payload через LB/API; proxy платит и трафиком, и пиком RSS. Complete у direct кратковременно читает объект (учебный verify) — это отдельная фаза, не отменяет выигрыш на upload path. **Ограничение:** без `compose.presign.yaml` PUT выполняется с контейнера на сети compose (порты MinIO на хост не открыты). Подпись — `minio-cpp` PresignV4 (не самописный SigV4). + +Track 5 (reconciler) не выполнялся; Track 1 (presigned) выполнен, см. раздел выше. diff --git a/contracts/README.md b/contracts/README.md index 8c58b32..70cbcad 100644 --- a/contracts/README.md +++ b/contracts/README.md @@ -50,7 +50,7 @@ Basic — это кодирование, не шифрование. HTTP lab1– - Файл заранее привязан к чату. В POST сообщения добавляется `attachment_ids` (0–10 уникальных ID, по умолчанию []). Можно отправить `text=""` с хотя бы одним вложением. Все вложения должны принадлежать этому чату; чужое/неизвестное — 404. Любой участник может ссылаться на доступное вложение чата. Failed-вложение прикреплять нельзя (409); queued/processing с lab5 можно. - `GET /attachments/{id}` и `/content` доступны только участникам чата. Download возвращает байты с истинным Content-Type, через авторизованный API, **без redirect** в обязательном контракте. Приватный bucket сам по себе не заменяет эту проверку. `variant` по умолчанию original. Для queued/processing content — 409; для failed — 409. Отсутствующий вариант ready-объекта — 404. - В ready-метаданных `error_code=null`. В failed — краткий машинный код (например `invalid_image`, `processing_failed`, `retry_exhausted`), без stack trace. `size_bytes`, `sha256`, `content_type` всегда описывают оригинал; у вариантов отдельные поля. Публичный API не раскрывает ключи бакета, внутренние адреса и credentials. -- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html). +- Прямой upload/download по временной подписи — факультативное расширение. Его дополняют ограничение срока/размера, уникальный ключ и подтверждение реально загруженного объекта; он не заменяет стандартный endpoint. Особенности подписанных URL: [S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html). В этом решении checksum на complete — SHA-256, заявленный при init (`expected_sha256`); тело complete пустое, отдельный hash в complete не принимается. Presigned PUT пишет только в `staging/{chat_id}/{upload_id}`. Complete копирует объект в `attachments/{chat_id}/{id}` и именно этот ключ сохраняет в `attachments.s3_key`. Повторный PUT той же ссылкой после complete не затирает оригинал. Схема расширения: `contracts/extensions.openapi.json`. ## Фоновая обработка (с lab5) diff --git a/evidence/load_uploads.json b/evidence/load_uploads.json index 1c7af42..203f8e3 100644 --- a/evidence/load_uploads.json +++ b/evidence/load_uploads.json @@ -5,18 +5,18 @@ "201": 10 }, "success": 10, - "wall_ms": 160918.8, + "wall_ms": 167347.8, "latency_ms": { - "p50": 398.4, - "p95": 669.1, - "p99": 669.1, - "max": 669.1 + "p50": 1125.9, + "p95": 1630.0, + "p99": 1630.0, + "max": 1630.0 }, - "approx_success_rps": 0.062, - "api_rss_baseline_mib": 51.96, - "api_rss_peak_mib": 68.4, - "api_rss_delta_mib": 16.44, - "api_cpu_peak_percent": 4.0, + "approx_success_rps": 0.06, + "api_rss_baseline_mib": 0.0, + "api_rss_peak_mib": 0.0, + "api_rss_delta_mib": 0.0, + "api_cpu_peak_percent": 0.0, "naive_full_buffer_mib": 56.09, "max_concurrent_uploads": 3, "note": "RSS capped by MAX_CONCURRENT_UPLOADS (default 3) per API process; slot taken before buffering body >256KiB; malloc_trim after each upload. rss_peak \u2264 baseline + MAX_CONCURRENT_UPLOADS \u00d7 (body + decoded_raster). Closed-loop client; RPS is throughput of confirmed 201s." diff --git a/migrations/007_direct_upload_staging.sql b/migrations/007_direct_upload_staging.sql new file mode 100644 index 0000000..cd3d422 --- /dev/null +++ b/migrations/007_direct_upload_staging.sql @@ -0,0 +1,12 @@ +-- Track 1: presigned PUT lands in staging/; complete copies to attachments/. +-- Existing rows keep the old key as staging so in-flight sessions still resolve. +ALTER TABLE direct_uploads ADD COLUMN staging_key text; + +UPDATE direct_uploads + SET staging_key = s3_key + WHERE staging_key IS NULL; + +ALTER TABLE direct_uploads ALTER COLUMN staging_key SET NOT NULL; + +ALTER TABLE direct_uploads + ADD CONSTRAINT direct_uploads_staging_key_key UNIQUE (staging_key); diff --git a/src/direct_upload_repo.cpp b/src/direct_upload_repo.cpp index e79354e..e5b925d 100644 --- a/src/direct_upload_repo.cpp +++ b/src/direct_upload_repo.cpp @@ -36,7 +36,9 @@ DirectUpload row_to(const pqxx::row& r) { d.expected_size = r["expected_size"].as(); d.expected_sha256 = r["expected_sha256"].as(); d.s3_key = r["s3_key"].as(); + d.staging_key = r["staging_key"].as(); d.status = r["status"].as(); + d.overdue = r["overdue"].as(); if (r["attachment_id"].is_null()) { d.attachment_id = std::nullopt; } else { @@ -50,11 +52,40 @@ DirectUpload row_to(const pqxx::row& r) { constexpr const char* kCols = "id, chat_id, owner_id, filename, content_type, expected_size, " - "expected_sha256, s3_key, status, attachment_id, " + "expected_sha256, s3_key, staging_key, status, attachment_id, " + "(status = 'pending' AND expires_at < now()) AS overdue, " "EXTRACT(EPOCH FROM expires_at)::bigint AS expires_at_epoch, " "EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, " "EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch"; +constexpr const char* kAttachmentColumns = + "id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "status, s3_key, error_code, " + "EXTRACT(EPOCH FROM created_at)::bigint AS created_at_epoch, " + "EXTRACT(EPOCH FROM updated_at)::bigint AS updated_at_epoch"; + +Attachment attachment_from_row(const pqxx::row& r) { + Attachment a; + a.id = r["id"].as(); + a.chat_id = r["chat_id"].as(); + a.owner_id = r["owner_id"].as(); + a.filename = r["filename"].as(); + a.content_type = r["content_type"].as(); + a.size_bytes = r["size_bytes"].as(); + a.sha256 = r["sha256"].as(); + a.status = r["status"].as(); + a.s3_key = r["s3_key"].as(); + if (r["error_code"].is_null()) { + a.error_code = std::nullopt; + } else { + a.error_code = r["error_code"].as(); + } + a.created_at = epoch_to_rfc3339(r["created_at_epoch"].as()); + a.updated_at = epoch_to_rfc3339(r["updated_at_epoch"].as()); + a.variants.clear(); + return a; +} + } // namespace DirectUpload DirectUploadRepo::insert(const DirectUpload& in) { @@ -65,17 +96,18 @@ DirectUpload DirectUploadRepo::insert(const DirectUpload& in) { auto r = tx.exec( "INSERT INTO direct_uploads (" " id, chat_id, owner_id, filename, content_type, expected_size, " - " expected_sha256, s3_key, status, attachment_id, " + " expected_sha256, s3_key, staging_key, status, attachment_id, " " expires_at, created_at, updated_at" ") VALUES (" - " $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, " - " to_timestamp($10::bigint), to_timestamp($11::bigint), " - " to_timestamp($12::bigint)" + " $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, $10, NULL, " + " to_timestamp($11::bigint), to_timestamp($12::bigint), " + " to_timestamp($13::bigint)" ") RETURNING " + std::string(kCols), pqxx::params{in.id, in.chat_id, in.owner_id, in.filename, in.content_type, in.expected_size, in.expected_sha256, - in.s3_key, in.status, in.expires_at_epoch, ts, ts}); + in.s3_key, in.staging_key, in.status, in.expires_at_epoch, + ts, ts}); tx.commit(); return row_to(r[0]); } @@ -157,11 +189,94 @@ std::vector DirectUploadRepo::pending_s3_keys() { auto lease = pool_.acquire(); pqxx::work tx(*lease); auto r = tx.exec( - "SELECT s3_key FROM direct_uploads WHERE status = 'pending'"); + "SELECT staging_key FROM direct_uploads WHERE status = 'pending'"); std::vector out; out.reserve(r.size()); for (const auto& row : r) { - out.push_back(row["s3_key"].as()); + out.push_back(row["staging_key"].as()); } return out; } + +void DirectUploadRepo::with_lock(const std::function& fn) { + auto lease = pool_.acquire(); + pqxx::work tx(*lease); + UploadLock lock(tx); + fn(lock); +} + +std::optional UploadLock::lock(const std::string& id) { + auto r = tx_.exec( + "SELECT " + std::string(kCols) + + " FROM direct_uploads WHERE id = $1::uuid FOR UPDATE", + pqxx::params{id}); + if (r.empty()) { + return std::nullopt; + } + return row_to(r[0]); +} + +std::optional UploadLock::find_attachment(const std::string& id) { + auto r = tx_.exec( + "SELECT " + std::string(kAttachmentColumns) + + " FROM attachments WHERE id = $1::uuid", + pqxx::params{id}); + if (r.empty()) { + return std::nullopt; + } + return attachment_from_row(r[0]); +} + +std::optional UploadLock::insert_ready(const Attachment& in, + bool& inserted) { + inserted = false; + const std::string ts = now_epoch_str(); + auto r = tx_.exec( + "INSERT INTO attachments (" + " id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + " status, s3_key, error_code, created_at, updated_at" + ") VALUES (" + " $1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9, NULL, " + " to_timestamp($10::bigint), to_timestamp($11::bigint)" + ") ON CONFLICT (id) DO NOTHING RETURNING " + + std::string(kAttachmentColumns), + pqxx::params{in.id, in.chat_id, in.owner_id, in.filename, + in.content_type, in.size_bytes, in.sha256, in.status, + in.s3_key, ts, ts}); + if (r.empty()) { + return std::nullopt; + } + inserted = true; + return attachment_from_row(r[0]); +} + +void UploadLock::mark_completed(const std::string& id, + const std::string& attachment_id) { + const std::string ts = now_epoch_str(); + auto r = tx_.exec( + "UPDATE direct_uploads SET status = 'completed', " + " attachment_id = $2::uuid, " + " updated_at = to_timestamp($3::bigint) " + "WHERE id = $1::uuid AND status = 'pending'", + pqxx::params{id, attachment_id, ts}); + if (r.affected_rows() != 1) { + throw std::runtime_error("direct upload complete lost the pending row"); + } +} + +void UploadLock::mark_expired(const std::string& id) { + const std::string ts = now_epoch_str(); + tx_.exec( + "UPDATE direct_uploads SET status = 'expired', " + " updated_at = to_timestamp($2::bigint) " + "WHERE id = $1::uuid AND status = 'pending'", + pqxx::params{id, ts}); +} + +void UploadLock::commit() { + if (committed_) { + return; + } + tx_.commit(); + committed_ = true; +} diff --git a/src/direct_upload_repo.hpp b/src/direct_upload_repo.hpp index f676b5e..571f4f9 100644 --- a/src/direct_upload_repo.hpp +++ b/src/direct_upload_repo.hpp @@ -3,10 +3,13 @@ #include "db.hpp" #include "models.hpp" +#include #include #include #include +class UploadLock; + class DirectUploadRepo { public: explicit DirectUploadRepo(PgPool& pool) : pool_(pool) {} @@ -22,12 +25,38 @@ public: std::optional mark_aborted(const std::string& id); - // pending → expired where expires_at < now; returns expired rows (s3_key). + // pending → expired where expires_at < now; returns expired rows. std::vector expire_stale(); - // Active pending keys (for orphan GC exclusion). + // Staging keys of still-pending sessions (orphan GC must keep these). std::vector pending_s3_keys(); + // One PG transaction. Callback may call S3 while the row lock is held. + // Commit explicitly via UploadLock::commit(); otherwise the work rolls back. + void with_lock(const std::function& fn); + private: PgPool& pool_; }; + +// Row lock for a single complete(). Not safe to use after with_lock returns. +class UploadLock { +public: + std::optional lock(const std::string& id); + + std::optional find_attachment(const std::string& id); + + // ON CONFLICT (id) DO NOTHING. inserted=false and nullopt → row already exists. + std::optional insert_ready(const Attachment& in, bool& inserted); + + void mark_completed(const std::string& id, const std::string& attachment_id); + void mark_expired(const std::string& id); + void commit(); + +private: + friend class DirectUploadRepo; + explicit UploadLock(pqxx::work& tx) : tx_(tx) {} + + pqxx::work& tx_; + bool committed_ = false; +}; diff --git a/src/main.cpp b/src/main.cpp index 993e439..b9ccba0 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1686,6 +1686,7 @@ static void handle_client(int client, send_all(client, error_response(503, "unavailable", "service unavailable", request_id)); } else { + bool response_headers_sent = false; try { const auto att = attachments.find_by_id(attachment_id); if (!att) { @@ -1705,36 +1706,51 @@ static void handle_client(int client, 409, "conflict", "attachment not ready", request_id)); } else { - std::string head = - "HTTP/1.1 200 OK\r\n" - "Content-Type: " + att->content_type + "\r\n" - "X-Request-Id: " + request_id + "\r\n" - "X-Instance-Id: " + g_instance_id + "\r\n" - "Content-Length: " + - std::to_string(att->size_bytes) + "\r\n" - "Connection: close\r\n" - "\r\n"; - send_all(client, head); - s3->get_object(att->s3_key, [&](const char* data, - std::size_t n) { - size_t sent = 0; - while (sent < n) { - const ssize_t w = ::write( - client, data + sent, n - sent); - if (w <= 0) { - throw std::runtime_error( - "content write failed"); + std::int64_t object_size = 0; + std::string etag; + if (!s3->stat_object(att->s3_key, object_size, etag)) { + send_all(client, error_response( + 404, "not_found", "not found", request_id)); + } else if (object_size != att->size_bytes) { + send_all(client, error_response( + 500, "internal_error", "internal error", + request_id)); + } else { + std::string head = + "HTTP/1.1 200 OK\r\n" + "Content-Type: " + att->content_type + "\r\n" + "X-Request-Id: " + request_id + "\r\n" + "X-Instance-Id: " + g_instance_id + "\r\n" + "Content-Length: " + + std::to_string(att->size_bytes) + "\r\n" + "Connection: close\r\n" + "\r\n"; + response_headers_sent = true; + send_all(client, head); + s3->get_object(att->s3_key, [&](const char* data, + std::size_t n) { + size_t sent = 0; + while (sent < n) { + const ssize_t w = ::write( + client, data + sent, n - sent); + if (w <= 0) { + throw std::runtime_error( + "content write failed"); + } + sent += static_cast(w); } - sent += static_cast(w); - } - }); + }); + } } } } catch (const std::exception& e) { - // If headers were not sent yet, return JSON error; otherwise - // the connection is already mid-body — just close. - send_all(client, failure_response(e, "download_attachment", - request_id)); + if (response_headers_sent) { + std::clog << "download_attachment stream failed: " + << e.what() << "\n"; + } else { + send_all(client, failure_response(e, "download_attachment", + request_id)); + } } } } @@ -1845,6 +1861,7 @@ static void handle_client(int client, d.expected_size = static_cast(*size_b); d.expected_sha256 = *sha; d.s3_key = "attachments/" + chat_id + "/" + d.id; + d.staging_key = "staging/" + chat_id + "/" + d.id; d.status = "pending"; const auto now = std::chrono::system_clock::now(); const auto secs = @@ -1854,7 +1871,7 @@ static void handle_client(int client, d.expires_at_epoch = secs + ttl; const DirectUpload saved = direct_uploads.insert(d); const std::string url = s3->presign_put( - saved.s3_key, static_cast(ttl), + saved.staging_key, static_cast(ttl), saved.content_type); nlohmann::json out; out["id"] = saved.id; @@ -1890,137 +1907,137 @@ static void handle_client(int client, "service unavailable", request_id)); } else { try { - auto sess = direct_uploads.find_by_id(direct_upload_id); - if (!sess || sess->owner_id != user_id) { - send_all(client, error_response(404, "not_found", - "not found", request_id)); - } else if (sess->status != "pending") { - send_all(client, error_response( - 409, "conflict", - "upload session not pending", request_id)); - } else { - // Flip overdue pending → expired, then re-read. - direct_uploads.expire_stale(); - sess = direct_uploads.find_by_id(direct_upload_id); - if (!sess || sess->status != "pending") { - send_all(client, error_response( - 410, "gone", "upload session expired", request_id)); - } else { - std::int64_t obj_size = 0; - std::string etag; - if (!s3->stat_object(sess->s3_key, obj_size, etag)) { - send_all(client, error_response( - 400, "invalid_request", - "object not uploaded", request_id)); - } else if (obj_size != sess->expected_size) { - try { - s3->delete_object(sess->s3_key); - } catch (...) { - } - direct_uploads.mark_aborted(sess->id); - send_all(client, error_response( - 400, "invalid_request", - "size mismatch", request_id)); - } else { - std::vector buf; - buf.reserve(static_cast(obj_size)); - s3->get_object( - sess->s3_key, - [&](const char* data, std::size_t n) { - buf.insert(buf.end(), - reinterpret_cast( - data), - reinterpret_cast( - data) + - n); - }); - if (static_cast(buf.size()) != - sess->expected_size) { - send_all(client, error_response( - 400, "invalid_request", - "size mismatch", request_id)); - } else { - const std::string got_sha = sha256_hex_bytes( - buf.data(), buf.size()); - if (got_sha != sess->expected_sha256) { - try { - s3->delete_object(sess->s3_key); - } catch (...) { - } - direct_uploads.mark_aborted(sess->id); - send_all(client, error_response( - 400, "invalid_request", - "checksum mismatch", request_id)); - } else { - ImageLimits lim; - if (const char* env = - std::getenv("MAX_UPLOAD_BYTES")) { - try { - lim.max_bytes = std::stoll(env); - } catch (...) { - } - } - if (const char* env = - std::getenv("MAX_IMAGE_PIXELS")) { - try { - lim.max_pixels = std::stoll(env); - } catch (...) { - } - } - ImageInfo info; - auto rej = validate_image( - buf.data(), buf.size(), lim, info); - if (rej || - info.content_type != sess->content_type) { - try { - s3->delete_object(sess->s3_key); - } catch (...) { - } - direct_uploads.mark_aborted(sess->id); - send_all(client, error_response( - 415, "unsupported_media_type", - "unsupported media type", - request_id)); - } else { - Attachment a; - a.id = sess->id; // stable id = session id - a.chat_id = sess->chat_id; - a.owner_id = sess->owner_id; - a.filename = sess->filename; - a.content_type = sess->content_type; - a.size_bytes = sess->expected_size; - a.sha256 = sess->expected_sha256; - a.status = "ready"; - a.s3_key = sess->s3_key; - bool inserted = false; - try { - const Attachment saved = - attachments.insert(a); - inserted = true; - direct_uploads.mark_completed( - sess->id, saved.id); - send_all(client, http_response( - 201, "Created", request_id, - attachment_json(saved))); - } catch (const std::exception& e) { - if (!inserted) { - try { - s3->delete_object(a.s3_key); - } catch (...) { - } - direct_uploads.mark_aborted( - sess->id); - } - send_all(client, failure_response( - e, "direct_upload_complete", - request_id)); - } - } - } - } + std::string response; + direct_uploads.with_lock([&](UploadLock& upload) { + const auto sess = upload.lock(direct_upload_id); + if (!sess || sess->owner_id != user_id) { + response = error_response(404, "not_found", "not found", + request_id); + return; + } + if (sess->status == "completed") { + const auto saved = upload.find_attachment( + sess->attachment_id.value_or(sess->id)); + if (!saved) { + throw std::runtime_error( + "completed upload has no attachment"); + } + upload.commit(); + response = http_response(200, "OK", request_id, + attachment_json(*saved)); + return; + } + if (sess->status != "pending") { + response = error_response( + 409, "conflict", "upload session not pending", + request_id); + return; + } + if (sess->overdue) { + upload.mark_expired(sess->id); + upload.commit(); + response = error_response(410, "gone", + "upload session expired", + request_id); + return; + } + + std::int64_t obj_size = 0; + std::string etag; + if (!s3->stat_object(sess->staging_key, obj_size, etag)) { + upload.commit(); + response = error_response(409, "not_ready", + "upload object not ready", + request_id); + return; + } + if (obj_size != sess->expected_size) { + upload.commit(); + response = error_response(409, "hash_mismatch", + "uploaded object mismatch", + request_id); + return; + } + + std::vector buf; + buf.reserve(static_cast(obj_size)); + s3->get_object( + sess->staging_key, + [&](const char* data, std::size_t n) { + buf.insert(buf.end(), + reinterpret_cast(data), + reinterpret_cast(data) + n); + }); + if (static_cast(buf.size()) != + sess->expected_size || + sha256_hex_bytes(buf.data(), buf.size()) != + sess->expected_sha256) { + upload.commit(); + response = error_response(409, "hash_mismatch", + "uploaded object mismatch", + request_id); + return; + } + + ImageLimits lim; + if (const char* env = std::getenv("MAX_UPLOAD_BYTES")) { + try { + lim.max_bytes = std::stoll(env); + } catch (...) { } } - } + if (const char* env = std::getenv("MAX_IMAGE_PIXELS")) { + try { + lim.max_pixels = std::stoll(env); + } catch (...) { + } + } + ImageInfo info; + const auto rejection = + validate_image(buf.data(), buf.size(), lim, info); + if (rejection || info.content_type != sess->content_type) { + upload.commit(); + response = error_response(415, "unsupported_media_type", + "unsupported media type", + request_id); + return; + } + + Attachment a; + a.id = sess->id; + a.chat_id = sess->chat_id; + a.owner_id = sess->owner_id; + a.filename = sess->filename; + a.content_type = sess->content_type; + a.size_bytes = sess->expected_size; + a.sha256 = sess->expected_sha256; + a.status = "ready"; + a.s3_key = sess->s3_key; + + s3->copy_object(sess->staging_key, a.s3_key); + bool inserted = false; + auto saved = upload.insert_ready(a, inserted); + if (!saved) { + saved = upload.find_attachment(a.id); + } + if (!saved) { + throw std::runtime_error( + "attachment insert conflict without existing row"); + } + try { + s3->delete_object(sess->staging_key); + } catch (const std::exception& e) { + std::clog << "direct upload staging cleanup failed: " + << e.what() << "\n"; + } + upload.mark_completed(sess->id, saved->id); + upload.commit(); + response = http_response(inserted ? 201 : 200, + inserted ? "Created" : "OK", + request_id, + attachment_json(*saved)); + }); + send_all(client, response); } catch (const std::exception& e) { send_all(client, failure_response(e, "direct_upload_complete", request_id)); @@ -2050,8 +2067,8 @@ static void handle_client(int client, try { std::int64_t sz = 0; std::string et; - if (s3->stat_object(sess->s3_key, sz, et)) { - s3->delete_object(sess->s3_key); + if (s3->stat_object(sess->staging_key, sz, et)) { + s3->delete_object(sess->staging_key); } } catch (const std::exception& de) { std::clog << "abort DeleteObject failed: " diff --git a/src/models.hpp b/src/models.hpp index f3c3957..9350d86 100644 --- a/src/models.hpp +++ b/src/models.hpp @@ -71,8 +71,10 @@ struct DirectUpload { std::string content_type; int expected_size = 0; std::string expected_sha256; - std::string s3_key; // internal + std::string s3_key; // final key: attachments/{chat}/{id} + std::string staging_key; // presigned PUT target: staging/{chat}/{id} std::string status; // pending|completed|aborted|expired + bool overdue = false; // pending && expires_at < now(); not a column std::optional attachment_id; std::string expires_at; // RFC3339 for JSON long long expires_at_epoch = 0; // for INSERT diff --git a/src/s3_client.cpp b/src/s3_client.cpp index fc2222b..e65b585 100644 --- a/src/s3_client.cpp +++ b/src/s3_client.cpp @@ -181,6 +181,22 @@ std::int64_t S3Client::get_object( return total; } +void S3Client::copy_object(const std::string& src_key, + const std::string& dst_key) { + if (src_key.empty() || dst_key.empty() || src_key == dst_key) { + throw std::invalid_argument("copy_object: bad keys"); + } + minio::s3::CopyObjectArgs args; + args.bucket = cfg_.bucket; + args.object = dst_key; + args.source.bucket = cfg_.bucket; + args.source.object = src_key; + minio::s3::CopyObjectResponse resp = impl_->client->CopyObject(args); + if (!resp) { + throw std::runtime_error("S3 CopyObject failed: " + resp.Error().String()); + } +} + void S3Client::delete_object(const std::string& key) { minio::s3::RemoveObjectArgs args; args.bucket = cfg_.bucket; diff --git a/src/s3_client.hpp b/src/s3_client.hpp index 7babae2..7c48dc4 100644 --- a/src/s3_client.hpp +++ b/src/s3_client.hpp @@ -45,6 +45,9 @@ public: void delete_object(const std::string& key); + // Server-side copy inside the bucket (complete: staging → final). + void copy_object(const std::string& src_key, const std::string& dst_key); + // HeadObject-style metadata. Returns false if the object does not exist. bool stat_object(const std::string& key, std::int64_t& size_bytes, std::string& etag); diff --git a/student-tests/run_grade5_track1.sh b/student-tests/run_grade5_track1.sh index 24144ac..fdab7a6 100755 --- a/student-tests/run_grade5_track1.sh +++ b/student-tests/run_grade5_track1.sh @@ -6,6 +6,9 @@ DIR=$(CDPATH= cd -- "$(dirname "$0")" && pwd) echo "=== test_direct_upload_ok.sh ===" sh "$DIR/test_direct_upload_ok.sh" +echo "=== test_presigned_complete_concurrent.sh ===" +sh "$DIR/test_presigned_complete_concurrent.sh" + echo "=== test_direct_upload_negatives.sh ===" sh "$DIR/test_direct_upload_negatives.sh" diff --git a/student-tests/test_direct_upload_negatives.sh b/student-tests/test_direct_upload_negatives.sh index dc1e930..d4fcdf1 100755 --- a/student-tests/test_direct_upload_negatives.sh +++ b/student-tests/test_direct_upload_negatives.sh @@ -43,7 +43,8 @@ expect_code 201 ID1=$(printf '%s' "$HTTP_BODY" | jq -r .id) http_json POST "/api/v1/direct-uploads/$ID1/complete" \ -H "Authorization: Bearer $ALICE_TOKEN" -expect_code 400 +expect_code 409 +assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" not_ready # --- abort --- http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \ @@ -91,7 +92,8 @@ put_code=$(put_via_net "$URL4" "$CT4" "$png") [ "$put_code" = "200" ] || die "PUT checksum-case got $put_code" http_json POST "/api/v1/direct-uploads/$ID4/complete" \ -H "Authorization: Bearer $ALICE_TOKEN" -expect_code 400 +expect_code 409 +assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .error.code)" hash_mismatch # Proxy path still works upload_file "$CHAT_ID" "$png" "proxy.png" "image/png" diff --git a/student-tests/test_direct_upload_ok.sh b/student-tests/test_direct_upload_ok.sh index 85096aa..712387d 100755 --- a/student-tests/test_direct_upload_ok.sh +++ b/student-tests/test_direct_upload_ok.sh @@ -27,14 +27,12 @@ assert_eq "$CT" "image/png" content_type_header # PUT bytes on the compose network (MinIO ports stay unpublished by default). ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml" -$COMPOSE run --rm --no-deps \ +put_code=$($COMPOSE run --rm --no-deps \ -v "$png:/tmp/up.bin:ro" \ --entrypoint curl api \ -sS -m 60 -o /dev/null -w '%{http_code}' \ -X PUT -H "Content-Type: $CT" --data-binary @/tmp/up.bin \ - "$UPLOAD_URL" > /tmp/du_put_code.$$ -put_code=$(cat /tmp/du_put_code.$$) -rm -f /tmp/du_put_code.$$ + "$UPLOAD_URL") [ "$put_code" = "200" ] || die "presigned PUT expected 200 got $put_code" http_json POST "/api/v1/direct-uploads/$UPLOAD_ID/complete" \ @@ -45,6 +43,13 @@ assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" sha256 assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .id)" "$UPLOAD_ID" attachment_id out=$(mktemp) +replay_code=$($COMPOSE run --rm --no-deps \ + --entrypoint curl api \ + -sS -m 60 -o /dev/null -w '%{http_code}' \ + -X PUT -H "Content-Type: $CT" --data-binary 'replay-content-12345' \ + "$UPLOAD_URL") +[ "$replay_code" = "200" ] || die "replay PUT expected 200 got $replay_code" + curl -sS -m 60 -o "$out" -H "Authorization: Bearer $ALICE_TOKEN" \ "$BASE_URL/api/v1/attachments/$UPLOAD_ID/content" assert_eq "$(sha256_file "$out")" "$want_sha" download_sha256 diff --git a/student-tests/test_download_missing_object.sh b/student-tests/test_download_missing_object.sh new file mode 100644 index 0000000..8ec471f --- /dev/null +++ b/student-tests/test_download_missing_object.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# A missing S3 object must fail before the content response headers are sent. +set -eu +. "$(dirname "$0")/lib.sh" + +png="$FIXTURES/320x200.png" +[ -f "$png" ] || die "missing fixture $png" +COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml" +response_body=$(mktemp) +object_deleted=0 + +restore_object() { + action=$1 + $COMPOSE run --rm --no-deps -T \ + -e "CHAT_ID=$CHAT_ID" -e "ATTACHMENT_ID=$AID" \ + -e "OBJECT_ACTION=$action" \ + -v "$png:/tmp/restore.png:ro" \ + --entrypoint /bin/sh minio-init -c ' + set -eu + mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + object="local/$S3_BUCKET/attachments/$CHAT_ID/$ATTACHMENT_ID" + if [ "$OBJECT_ACTION" = delete ]; then + mc rm --force "$object" + else + mc cp /tmp/restore.png "$object" + fi + ' +} + +cleanup() { + if [ "$object_deleted" = 1 ]; then + restore_object restore || printf 'WARNING: failed to restore test object\n' >&2 + fi + rm -f "$response_body" +} +trap cleanup 0 HUP INT TERM + +want_sha=$(sha256_file "$png") +TOKEN="" +register_user alice +create_chat 'missing-object' "[\"$USER_ID\"]" +upload_file "$CHAT_ID" "$png" 'sample.png' 'image/png' +expect_code 201 +AID=$(printf '%s' "$HTTP_BODY" | jq -r .id) +assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" metadata_sha + +restore_object delete +object_deleted=1 +code=$(curl -sS -m 60 -o "$response_body" -w '%{http_code}' \ + -H "Authorization: Bearer $TOKEN" \ + "$BASE_URL/api/v1/attachments/$AID/content") +case "$code" in + 404|5??) ;; + *) die "expected 404/5xx for missing object, got HTTP $code body=$(cat "$response_body")" ;; +esac + +restore_object restore +object_deleted=0 +printf 'PASS test_download_missing_object\n' \ No newline at end of file diff --git a/student-tests/test_presigned_complete_concurrent.sh b/student-tests/test_presigned_complete_concurrent.sh new file mode 100644 index 0000000..8656de2 --- /dev/null +++ b/student-tests/test_presigned_complete_concurrent.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# Track 1: concurrent complete calls publish one intact attachment. +. "$(dirname "$0")/lib.sh" + +png="$FIXTURES/320x200.png" +[ -f "$png" ] || die "missing fixture $png" +want_sha=$(sha256_file "$png") +want_size=$(wc -c < "$png" | tr -d ' ') +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' 0 HUP INT TERM + +register_user alice +ALICE_TOKEN=$TOKEN +ALICE_ID=$USER_ID +create_chat "du-concurrent" "[\"$ALICE_ID\"]" + +http_json POST "/api/v1/chats/$CHAT_ID/direct-uploads" \ + -H "Authorization: Bearer $ALICE_TOKEN" \ + -H 'Content-Type: application/json' \ + -d "{\"filename\":\"concurrent.png\",\"content_type\":\"image/png\",\"size_bytes\":$want_size,\"sha256\":\"$want_sha\"}" +expect_code 201 +UPLOAD_ID=$(printf '%s' "$HTTP_BODY" | jq -r .id) +UPLOAD_URL=$(printf '%s' "$HTTP_BODY" | jq -r .upload_url) +CT=$(printf '%s' "$HTTP_BODY" | jq -r '.headers["Content-Type"]') +[ -n "$UPLOAD_ID" ] && [ "$UPLOAD_ID" != null ] || die "empty upload id" +[ -n "$UPLOAD_URL" ] && [ "$UPLOAD_URL" != null ] || die "empty upload_url" + +COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml" +put_code=$($COMPOSE run --rm --no-deps \ + -v "$png:/tmp/up.bin:ro" \ + --entrypoint curl api \ + -sS -m 60 -o /dev/null -w '%{http_code}' \ + -X PUT -H "Content-Type: $CT" --data-binary @/tmp/up.bin \ + "$UPLOAD_URL") +[ "$put_code" = "200" ] || die "presigned PUT expected 200 got $put_code" + +pids= +for i in 1 2 3 4; do + curl -sS -m 60 -o "$tmpdir/body.$i" -w '%{http_code}' \ + -X POST \ + -H "Authorization: Bearer $ALICE_TOKEN" \ + "$BASE_URL/api/v1/direct-uploads/$UPLOAD_ID/complete" \ + > "$tmpdir/code.$i" & + pids="$pids $!" +done +for pid in $pids; do + wait "$pid" || die "complete request failed" +done + +count_201=0 +for i in 1 2 3 4; do + code=$(cat "$tmpdir/code.$i") + case "$code" in + 201) count_201=$((count_201 + 1)) ;; + 200|409) ;; + *) die "unexpected concurrent complete HTTP $code: $(cat "$tmpdir/body.$i")" ;; + esac +done +assert_eq "$count_201" 1 created_count + +http_json GET "/api/v1/attachments/$UPLOAD_ID" \ + -H "Authorization: Bearer $ALICE_TOKEN" +expect_code 200 +assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .status)" ready metadata_status +assert_eq "$(printf '%s' "$HTTP_BODY" | jq -r .sha256)" "$want_sha" metadata_sha + +code=$(curl -sS -m 60 -o "$tmpdir/content" -w '%{http_code}' \ + -H "Authorization: Bearer $ALICE_TOKEN" \ + "$BASE_URL/api/v1/attachments/$UPLOAD_ID/content") +assert_eq "$code" 200 content_status +assert_eq "$(sha256_file "$tmpdir/content")" "$want_sha" content_sha + +printf 'PASS test_presigned_complete_concurrent\n' \ No newline at end of file diff --git a/tools/gc_orphans.sh b/tools/gc_orphans.sh index 2e6da61..138a07d 100755 --- a/tools/gc_orphans.sh +++ b/tools/gc_orphans.sh @@ -15,7 +15,7 @@ ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" COMPOSE="docker compose --env-file $ROOT/.env -f $ROOT/compose.yaml" GRACE_SECONDS="${GRACE_SECONDS:-3600}" APPLY="${APPLY:-0}" -PREFIX="${ORPHAN_PREFIX:-attachments/}" +PREFIX="${ORPHAN_PREFIX:-}" S3_BUCKET=$($COMPOSE exec -T api sh -c 'printf %s "$S3_BUCKET"') [ -n "$S3_BUCKET" ] || { echo "S3_BUCKET empty" >&2; exit 1; } @@ -25,26 +25,27 @@ trap 'rm -rf "$TMP"' EXIT echo "gc_orphans: bucket=$S3_BUCKET grace=${GRACE_SECONDS}s apply=$APPLY prefix=$PREFIX" -$COMPOSE run --rm --no-deps --entrypoint /bin/sh minio-init -c " +s3_listing=$($COMPOSE run --rm --no-deps -T --entrypoint /bin/sh minio-init -c " set -eu mc alias set local \"\$S3_ENDPOINT\" \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" >/dev/null mc ls --recursive --json \"local/\$S3_BUCKET/$PREFIX\" || true -" > "$TMP/s3.json" +") +printf '%s\n' "$s3_listing" > "$TMP/s3.json" -# Known keys: ready attachments + still-pending direct uploads (Track 1). -$COMPOSE exec -T db sh -c \ +# Keep published attachments and staging objects for active direct uploads. +pg_keys=$($COMPOSE exec -T db sh -c \ 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc " SELECT s3_key FROM attachments UNION - SELECT s3_key FROM direct_uploads WHERE status = '\''pending'\'' - "' \ - > "$TMP/pg.keys" + SELECT staging_key FROM direct_uploads WHERE status = '\''pending'\'' + "') +printf '%s\n' "$pg_keys" > "$TMP/pg.keys" -python3 - "$TMP/s3.json" "$TMP/pg.keys" "$GRACE_SECONDS" "$TMP/del.keys" <<'PY' +python3 - "$TMP/s3.json" "$TMP/pg.keys" "$GRACE_SECONDS" "$TMP/del.keys" "$PREFIX" <<'PY' import json, sys, time from datetime import datetime -s3_path, pg_path, grace_s, out_path = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4] +s3_path, pg_path, grace_s, out_path, prefix = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4], sys.argv[5] now = time.time() known = {line.strip() for line in open(pg_path) if line.strip()} orphans = [] @@ -59,9 +60,11 @@ for line in open(s3_path): key = obj.get("key") or obj.get("Key") if not key: continue - # mc ls under prefix returns keys relative to that prefix. - if not key.startswith("attachments/"): - key = "attachments/" + key.lstrip("/") + key = key.lstrip("/") + if prefix and not key.startswith(prefix): + key = prefix.rstrip("/") + "/" + key + if not key.startswith(("attachments/", "staging/")): + continue if key in known: continue age = None @@ -98,14 +101,15 @@ if [ ! -s "$TMP/del.keys" ]; then exit 0 fi -$COMPOSE run --rm --no-deps --entrypoint /bin/sh minio-init -c " - set -eu - mc alias set local \"\$S3_ENDPOINT\" \"\$MINIO_ROOT_USER\" \"\$MINIO_ROOT_PASSWORD\" >/dev/null - while IFS= read -r key; do - [ -n \"\$key\" ] || continue - echo \"deleting \$S3_BUCKET/\$key\" - mc rm --force \"local/\$S3_BUCKET/\$key\" - done -" < "$TMP/del.keys" +while IFS= read -r key; do + [ -n "$key" ] || continue + echo "deleting $S3_BUCKET/$key" + $COMPOSE run --rm --no-deps -T -e "GC_OBJECT_KEY=$key" \ + --entrypoint /bin/sh minio-init -c ' + set -eu + mc alias set local "$S3_ENDPOINT" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + mc rm --force "local/$S3_BUCKET/$GC_OBJECT_KEY" + ' +done < "$TMP/del.keys" echo "gc_orphans: delete done"