diff --git a/.env.example b/.env.example index 2e5dbb0b..40f59ee9 100644 --- a/.env.example +++ b/.env.example @@ -20,3 +20,10 @@ MINIO_ROOT_USER=minio-root MINIO_ROOT_PASSWORD=replace-with-a-different-long-random-root-secret MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +MAX_CONCURRENT_UPLOADS=3 +UPLOAD_PERMIT_WAIT_MILLIS=10000 +# Orphans younger than this are never deleted: protects the S3 PUT -> DB COMMIT window. +ATTACHMENT_CLEANUP_GRACE_SECONDS=3600 +ATTACHMENT_CLEANUP_DRY_RUN=false +ATTACHMENT_RECONCILE_INTERVAL_MILLIS=600000 +ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS=60000 diff --git a/REPORT.md b/REPORT.md index ec7b0661..ae41226b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Егор Антипин, БПИ2301 -- Целевая оценка; выбранный трек 5, если нужен: 3 (обязательный lab4) +- Целевая оценка; выбранный трек 5, если нужен: 4 - Commit/PR: TODO после публикации - Использованные источники, библиотеки и помощь ИИ: контракт репозитория; MinIO Java SDK 8.6.0; документация MinIO/S3; помощь Codex при реализации и проверке. @@ -17,6 +17,9 @@ docker compose --env-file .env up -d --build --scale api=2 docker compose --env-file .env ps python3 tests/smoke.py --base-url http://localhost:8080 python3 student_tests/lab4_negative.py --base-url http://localhost:8080 +ENV_FILE=.env ./scripts/check-private-bucket.sh +python3 student_tests/concurrent_upload_memory.py --base-url http://localhost:8080 \ + --compose-env-file .env --concurrency 12 --output evidence/upload-memory.json ``` `migrate` применяет Flyway V1–V4 один раз. `minio-init` идемпотентно создаёт приватный bucket и ограниченного API-user. Обычная безопасная остановка: `docker compose --env-file .env down` — named volumes сохраняются. Намеренное удаление только учебных данных: `docker compose --env-file .env down --volumes`. @@ -27,7 +30,11 @@ python3 student_tests/lab4_negative.py --base-url http://localhost:8080 `attachments` содержит chat/owner, display filename, истинный MIME, size, SHA-256 и непрозрачный object key вида `attachments//original`. `message_attachments` использует составные FK `(message_id, chat_id)` и `(attachment_id, chat_id)`, поэтому база не позволяет связать сообщение с вложением другого чата. -Upload читает raw body порциями с жёсткой границей 10 MiB, проверяет filename, реальный PNG/JPEG через ImageIO, совпадение MIME и ≤40 млн пикселей. После успешного S3 PUT метаданные фиксируются отдельной DB-транзакцией. При ошибке DB выполняется компенсирующий DELETE объекта. Ответ 201 возвращается только после обеих записей. Ошибка MinIO преобразуется в JSON 503 без внутренних деталей. +Upload читает raw body порциями по 8 KiB с жёсткой границей 10 MiB, проверяет filename, реальный PNG/JPEG через ImageIO, совпадение MIME и ≤40 млн пикселей. До чтения body захватывается один из трёх fair semaphore permits на реплику; ожидание ограничено 10 секундами и возвращает 429 с `Retry-After`. Nginx ограничивает body 11 MiB, ожидание тела 10 секундами, запрос к API 20 секундами; MinIO client имеет connect 2 с и общий call timeout 15 с. Поэтому число одновременно удерживаемых byte/decode buffers ограничено. + +После успешного S3 PUT метаданные фиксируются отдельной DB-транзакцией. При ошибке DB выполняется компенсирующий DELETE объекта. Обрыв входного потока происходит до PUT и не создаёт object. Ответ 201 возвращается только после обеих записей. Ошибка MinIO преобразуется в JSON 503 без внутренних деталей. + +Для аварии процесса между S3 PUT и DB COMMIT работает `AttachmentReconciler`: каждые 10 минут он сравнивает `attachments.object_key` с prefix `attachments/` в MinIO. Orphan удаляется только если `lastModified` старше grace period 3600 секунд; свежий объект не трогается, что защищает нормальный commit window. Missing object не удаляет метаданные автоматически, а пишет ERROR для ручного расследования. Есть `ATTACHMENT_CLEANUP_DRY_RUN=true`. Unit-тесты доказывают old/fresh orphan, dry-run и missing-object поведение. Download сначала загружает метаданные и проверяет членство пользователя в чате, затем читает object через API. Redirect/presigned URL не используется. Filename участвует только в `Content-Disposition`. @@ -38,24 +45,26 @@ Compose и Swarm используют отдельный persistent volume `mini | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | | Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:18084` | PASS, включая upload/message/download/ACL | `evidence/lab4-2026-11-11.md` | -| Private bucket/bootstrap | `docker compose logs minio-init` | bucket private, API policy attached | `evidence/lab4-2026-11-11.md` | +| Private bucket/bootstrap | повторный `minio-init`; `scripts/check-private-bucket.cmd` | bootstrap повторяем; anonymous list запрещён; API credentials non-admin | `evidence/lab4-2026-11-11.md` | | Негативные файлы и чужой чат | `python -X utf8 student_tests/lab4_negative.py --base-url http://localhost:18084` | PASS | `evidence/lab4-2026-11-11.md` | +| Обрыв/DB failure/reconciler | `AttachmentServiceTest`, `AttachmentReconcilerTest` | PASS: нет object до PUT; compensating DELETE; grace/dry-run/missing | `evidence/lab4-2026-11-11.md` | +| 12 конкурентных upload + RAM | `student_tests/concurrent_upload_memory.py ... --concurrency 12` | 12/12; peak 839909375 B aggregate; +171442175 B | `evidence/upload-memory-2026-11-11.json` | | Полный restart | `tests/resilience.py --mode persistence ...` | PASS; session/message/original preserved | `evidence/lab4-2026-11-11.md` | | Отказ API-реплики | `tests/resilience.py --mode failover ...` | PASS; download через другую replica | `evidence/lab4-2026-11-11.md` | | Недоступность S3 | stop MinIO + `lab4_negative.py --expect-s3-unavailable` | JSON 503 unavailable; после start smoke PASS | `evidence/lab4-2026-11-11.md` | -Unit-набор `AttachmentServiceTest` отдельно проверяет server-generated key, MIME spoofing, ранний byte limit, ACL до download, чужой chat attachment и отказ ObjectStorage до записи метаданных. +Unit-набор `AttachmentServiceTest` отдельно проверяет server-generated key, MIME spoofing, ранний byte limit, ACL до download, чужой chat attachment, обрыв request body, отказ ObjectStorage и compensating DELETE после DB failure. `AttachmentReconcilerTest` проверяет grace period, dry-run и missing objects. Полный Gradle suite внутри Compose network: `BUILD SUCCESSFUL`. ## Эксперименты и отказы -Проверялась функциональная устойчивость, не максимальная производительность. Стенд: Docker Desktop, 2 API replicas по 768 MiB, MinIO 512 MiB, PostgreSQL 768 MiB. Fixture — PNG 111817 bytes. Нагрузочный эксперимент уровня 4 не заявлен. +Проверялась функциональная устойчивость и ограниченность памяти, не максимальная производительность. Стенд: Docker Desktop, 2 API replicas по 768 MiB, MinIO 512 MiB, PostgreSQL 768 MiB. Конкурентный эксперимент сгенерировал один валидный несжимаемый PNG 1800×1200, 6835833 bytes, и одновременно отправил его 12 раз. Все 12 upload завершились 201 за 10.156 с. По пяти `docker stats` samples aggregate API baseline был 668467200 B, peak 839909375 B, рост 171442175 B. Результат повторяемый, но sampling может пропустить краткий пик и не является профилированием JVM heap. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| Stop обслужившей API replica | Запросы продолжает вторая replica; общий MinIO сохраняет original | 2 instance до отказа, 105 успешных записей, 0 ошибок | 1.636 с до подтверждённого восстановления | +| Stop обслужившей API replica | Запросы продолжает вторая replica; общий MinIO сохраняет original | 2 instance до отказа, 102 успешные записи, 0 ошибок | 1.386 с до подтверждённого восстановления | | Restart nginx/API/Redis/PostgreSQL/MinIO без удаления volumes | Сессия, message, attachment metadata и bytes сохранятся | PASS persistence | <90 с, потерь нет | | Stop MinIO | Upload/download не превращаются в 500 | JSON 503 `unavailable`; после start публичный smoke снова PASS | Восстановление после healthcheck MinIO | ## Ограничения и следующие шаги -Стенд выдерживает отказ одной API-реплики и перезапуск всех сервисов при сохранённых named volumes. MinIO развёрнут одной репликой: во время его отказа attachment upload/download недоступны с 503, но PostgreSQL API остаётся работоспособным. HA самого S3, multipart upload, presigned flow, thumbnail pipeline и автоматический reconciler orphan/missing objects не входят в заявленный уровень. Компенсирующий DELETE закрывает обычную ошибку DB после S3 PUT, но аварийное завершение процесса между двумя действиями потребовало бы отдельного reconciler для уровня 4. +Стенд выдерживает отказ одной API-реплики и перезапуск всех сервисов при сохранённых named volumes. MinIO развёрнут одной репликой: во время его отказа attachment upload/download недоступны с 503, но PostgreSQL API остаётся работоспособным. HA самого S3, multipart upload, presigned flow и thumbnail pipeline не реализованы. Reconciler автоматически удаляет только старые orphan objects; missing objects намеренно не чинятся автоматически, чтобы не скрывать потерю данных. diff --git a/compose.yaml b/compose.yaml index 7a9d1e1f..bc0ebf42 100644 --- a/compose.yaml +++ b/compose.yaml @@ -28,6 +28,12 @@ x-api-environment: &api-environment S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3} + UPLOAD_PERMIT_WAIT_MILLIS: ${UPLOAD_PERMIT_WAIT_MILLIS:-10000} + ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600} + ATTACHMENT_CLEANUP_DRY_RUN: ${ATTACHMENT_CLEANUP_DRY_RUN:-false} + ATTACHMENT_RECONCILE_INTERVAL_MILLIS: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:-600000} + ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:-60000} services: postgres: @@ -107,6 +113,35 @@ services: restart: "no" networks: [backend] + private-bucket-check: + profiles: ["verification"] + image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc + depends_on: + minio-init: + condition: service_completed_successfully + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set anonymous http://minio:9000 + if mc ls "anonymous/$$S3_BUCKET" >/dev/null 2>&1; then + echo "FAIL: anonymous bucket listing succeeded" >&2 + exit 1 + fi + mc alias set restricted http://minio:9000 "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" + mc ls "restricted/$$S3_BUCKET" >/dev/null + if mc admin info restricted >/dev/null 2>&1; then + echo "FAIL: API credentials have administrative access" >&2 + exit 1 + fi + echo "PASS: bucket is private and API credentials are non-admin" + restart: "no" + networks: [backend] + # One-shot Flyway runner. It is invoked explicitly before API replicas. migrate: image: hla-lab4-api:local diff --git a/evidence/lab4-2026-11-11.md b/evidence/lab4-2026-11-11.md index 36be6b87..c386a7aa 100644 --- a/evidence/lab4-2026-11-11.md +++ b/evidence/lab4-2026-11-11.md @@ -12,10 +12,11 @@ ## Проверки ```text -AttachmentServiceTest: BUILD SUCCESSFUL, 6 tests. +AttachmentServiceTest + AttachmentReconcilerTest: BUILD SUCCESSFUL, 10 tests. tests/smoke.py: PASS: lab4 public API scenario. student_tests/lab4_negative.py: -PASS: foreign chat/user, MIME spoof, corruption, byte limit and filename checks. +PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, +byte limit and filename checks. ``` Persistence hook останавливал nginx, обе API, Redis, PostgreSQL и MinIO, затем запускал те же named volumes и повторял idempotent bucket bootstrap: @@ -27,7 +28,7 @@ PASS: persistence preserved confirmed data and existing session. Failover hook остановил replica из `X-Instance-Id`, обслужившую тестовый запрос: ```json -{"observed_instances_before":2,"successful_writes":105,"failed_attempts":0,"first_recovery_seconds":1.636} +{"observed_instances_before":2,"successful_writes":102,"failed_attempts":0,"first_recovery_seconds":1.386} ``` ```text @@ -41,3 +42,38 @@ PASS: unavailable S3 returns contract JSON 503 ``` После запуска того же MinIO volume публичный smoke повторно завершился PASS, checksum скачанного original совпал. + +## Level 4: consistency, memory and private S3 + +Unit tests inject an interrupted request stream before S3 PUT and a repository +failure after S3 PUT. The first leaves storage empty; the second verifies the +compensating DELETE. Reconciler tests prove that an orphan younger than the +one-hour grace period survives, an older orphan is deleted, dry-run never +deletes, and a missing object is reported. + +Bootstrap was executed again against the existing bucket and completed with +exit code 0. The direct S3 probe (without application API) produced: + +```text +PASS: bucket is private and API credentials are non-admin +``` + +Full Gradle suite using the live Compose PostgreSQL/Redis/MinIO network: + +```text +BUILD SUCCESSFUL in 11m 47s +``` + +Concurrent memory experiment (`evidence/upload-memory-2026-11-11.json`): + +```json +{"concurrency":12,"successful_uploads":12,"payload_bytes_each":6835833, + "duration_seconds":10.156,"api_replicas":2,"memory_samples":5, + "baseline_api_memory_bytes":668467200,"peak_api_memory_bytes":839909375, + "peak_increase_bytes":171442175,"configured_max_concurrent_uploads_per_replica":3} +``` + +The memory values are the aggregate of both API containers from `docker stats`. +Sampling can miss a shorter peak, so this is an operational bound observation, +not a JVM heap proof. The code-level bounds are 10 MiB/request, 40M pixels and +three active uploads per replica. diff --git a/evidence/upload-memory-2026-11-11.json b/evidence/upload-memory-2026-11-11.json new file mode 100644 index 00000000..6ee7149e --- /dev/null +++ b/evidence/upload-memory-2026-11-11.json @@ -0,0 +1,14 @@ +{ + "concurrency": 12, + "successful_uploads": 12, + "payload_bytes_each": 6835833, + "duration_seconds": 10.156, + "api_replicas": 2, + "memory_samples": 5, + "baseline_api_memory_bytes": 668467200, + "peak_api_memory_bytes": 839909375, + "peak_increase_bytes": 171442175, + "configured_max_concurrent_uploads_per_replica": 3, + "configured_max_upload_bytes": 10485760, + "configured_max_image_pixels": 40000000 +} diff --git a/infra/README.md b/infra/README.md index 463176ac..3ba78e9c 100644 --- a/infra/README.md +++ b/infra/README.md @@ -25,3 +25,24 @@ Политика содержит placeholder `__S3_BUCKET__`; bootstrap подставляет фактическое значение `S3_BUCKET` перед созданием policy. Повторный запуск безопасен: bucket, user и policy переиспользуются, а private anonymous policy устанавливается снова. + +Проверка вне API: + +```sh +ENV_FILE=.env ./scripts/check-private-bucket.sh +``` + +Одноразовый `private-bucket-check` подключается непосредственно к S3 внутри +backend network без HTTP API приложения. Он требует, чтобы anonymous list bucket +завершился отказом, проверяет list с ограниченными API credentials и требует, +чтобы `mc admin info` с этими credentials также завершился отказом. Сервис имеет +Compose profile `verification` и не работает постоянно. + +## Сверка PostgreSQL и MinIO + +API запускает `AttachmentReconciler` с prefix `attachments/`. Метаданные из +PostgreSQL сравниваются с объектами MinIO. Старый orphan удаляется после +`ATTACHMENT_CLEANUP_GRACE_SECONDS` (по умолчанию 3600), свежий orphan защищён от +гонки PUT/COMMIT. `ATTACHMENT_CLEANUP_DRY_RUN=true` только сообщает кандидатов. +Missing object логируется как ERROR, но metadata автоматически не удаляется. +Интервал и initial delay задаются через `.env.example`. diff --git a/infra/nginx.conf b/infra/nginx.conf index 1ef77ece..7aabc52a 100644 --- a/infra/nginx.conf +++ b/infra/nginx.conf @@ -8,6 +8,7 @@ events { http { # Let the application enforce the exact 10 MiB contract and return JSON. client_max_body_size 11m; + client_body_timeout 10s; access_log /var/log/nginx/access.log; error_log /var/log/nginx/error.log warn; @@ -66,6 +67,22 @@ http { proxy_next_upstream_tries 2; } + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + + proxy_connect_timeout 500ms; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + # Upload is not replayed: it has no idempotency key. The client may retry explicitly. + proxy_next_upstream off; + } + location / { proxy_pass http://api_backend; proxy_http_version 1.1; diff --git a/scripts/check-private-bucket.cmd b/scripts/check-private-bucket.cmd new file mode 100644 index 00000000..1065a92d --- /dev/null +++ b/scripts/check-private-bucket.cmd @@ -0,0 +1,7 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." +if not defined ENV_FILE set "ENV_FILE=.env" + +docker compose --env-file "%ENV_FILE%" --profile verification run --rm private-bucket-check +exit /b %ERRORLEVEL% diff --git a/scripts/check-private-bucket.sh b/scripts/check-private-bucket.sh new file mode 100644 index 00000000..eab94689 --- /dev/null +++ b/scripts/check-private-bucket.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +PROJECT_DIR=$(dirname -- "$SCRIPT_DIR") +cd "$PROJECT_DIR" +: "${ENV_FILE:=.env}" + +docker compose --env-file "$ENV_FILE" --profile verification run --rm private-bucket-check diff --git a/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java b/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java index 32e25b92..4e0eb367 100644 --- a/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java +++ b/src/main/java/com/example/mtusi_messenger/MtusiMessengerApplication.java @@ -3,8 +3,10 @@ package com.example.mtusi_messenger; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication +@EnableScheduling public class MtusiMessengerApplication { public static void main(String[] args) { diff --git a/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java b/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java index 12a46122..202f9116 100644 --- a/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java +++ b/src/main/java/com/example/mtusi_messenger/dto/CreateMessageRequest.java @@ -23,7 +23,7 @@ public record CreateMessageRequest( } @AssertTrue(message = "attachment_ids must be unique and non-null") - public boolean areAttachmentIdsValid() { + public boolean isAttachmentIdsValid() { return attachment_ids != null && attachment_ids.stream().noneMatch(java.util.Objects::isNull) && new HashSet<>(attachment_ids).size() == attachment_ids.size(); diff --git a/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java b/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java index de41f5a6..d7cecee4 100644 --- a/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java +++ b/src/main/java/com/example/mtusi_messenger/exception/ApiExceptions.java @@ -12,5 +12,6 @@ public final class ApiExceptions { public static ApiException conflict(String m) { return new ApiException("conflict", HttpStatus.CONFLICT, m); } public static ApiException payloadTooLarge(String m) { return new ApiException("payload_too_large", HttpStatus.PAYLOAD_TOO_LARGE, m); } public static ApiException unsupportedMediaType(String m) { return new ApiException("unsupported_media_type", HttpStatus.UNSUPPORTED_MEDIA_TYPE, m); } + public static ApiException tooManyRequests(String m) { return new ApiException("too_many_requests", HttpStatus.TOO_MANY_REQUESTS, m); } public static ApiException unavailable(String m) { return new ApiException("unavailable", HttpStatus.SERVICE_UNAVAILABLE, m); } } diff --git a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java index eee186a8..a6b1ce32 100644 --- a/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java +++ b/src/main/java/com/example/mtusi_messenger/repository/AttachmentRepository.java @@ -5,9 +5,11 @@ import com.example.mtusi_messenger.model.Attachment; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.Set; public interface AttachmentRepository { Attachment save(Attachment attachment); Optional findById(UUID id); List findAllByIdsInChat(UUID chatId, List ids); + Set findAllObjectKeys(); } diff --git a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java index 34c9e6a5..e56e4461 100644 --- a/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java +++ b/src/main/java/com/example/mtusi_messenger/repository/PostgresAttachmentRepository.java @@ -9,6 +9,8 @@ import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.Set; +import java.util.HashSet; @Repository public class PostgresAttachmentRepository implements AttachmentRepository { @@ -58,6 +60,12 @@ public class PostgresAttachmentRepository implements AttachmentRepository { (rs, rowNum) -> map(rs), args); } + @Override + public Set findAllObjectKeys() { + return new HashSet<>(jdbc.query("SELECT object_key FROM attachments", + (rs, rowNum) -> rs.getString("object_key"))); + } + private static Attachment map(java.sql.ResultSet rs) throws java.sql.SQLException { return new Attachment( rs.getObject("id", UUID.class), diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java new file mode 100644 index 00000000..4e45794b --- /dev/null +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentReconciler.java @@ -0,0 +1,104 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; + +@Component +public class AttachmentReconciler { + private static final Logger log = LoggerFactory.getLogger(AttachmentReconciler.class); + private static final String PREFIX = "attachments/"; + private final AttachmentRepository attachments; + private final ObjectStorage storage; + private final Duration gracePeriod; + private final boolean dryRun; + private final Clock clock; + + @Autowired + public AttachmentReconciler( + AttachmentRepository attachments, + ObjectStorage storage, + @Value("${messenger.attachments.cleanup-grace-seconds:3600}") long graceSeconds, + @Value("${messenger.attachments.cleanup-dry-run:false}") boolean dryRun + ) { + this(attachments, storage, Duration.ofSeconds(graceSeconds), dryRun, Clock.systemUTC()); + } + + AttachmentReconciler(AttachmentRepository attachments, + ObjectStorage storage, + Duration gracePeriod, + boolean dryRun, + Clock clock) { + if (gracePeriod.isNegative() || gracePeriod.isZero()) { + throw new IllegalArgumentException("Attachment cleanup grace period must be positive"); + } + this.attachments = attachments; + this.storage = storage; + this.gracePeriod = gracePeriod; + this.dryRun = dryRun; + this.clock = clock; + } + + @Scheduled( + fixedDelayString = "${messenger.attachments.reconcile-interval-millis:600000}", + initialDelayString = "${messenger.attachments.reconcile-initial-delay-millis:60000}") + public void scheduledReconcile() { + try { + Report report = reconcile(); + if (report.orphanObjects() > 0 || report.missingObjects() > 0) { + log.warn("Attachment reconciliation: {} orphan, {} deleted, {} missing, dryRun={}", + report.orphanObjects(), report.deletedObjects(), report.missingObjects(), dryRun); + } + } catch (RuntimeException error) { + log.warn("Attachment reconciliation postponed: {}", error.getMessage()); + } + } + + public Report reconcile() { + Set databaseKeys = attachments.findAllObjectKeys(); + var storedObjects = storage.list(PREFIX); + Set storageKeys = new HashSet<>(); + Instant deleteBefore = clock.instant().minus(gracePeriod); + int orphans = 0; + int deleted = 0; + + for (ObjectStorage.StoredObject object : storedObjects) { + storageKeys.add(object.objectKey()); + if (!databaseKeys.contains(object.objectKey()) && object.lastModified().isBefore(deleteBefore)) { + orphans++; + if (!dryRun) { + storage.delete(object.objectKey()); + deleted++; + } + } + } + + int missing = 0; + for (String databaseKey : databaseKeys) { + if (!storageKeys.contains(databaseKey)) { + missing++; + log.error("Attachment metadata references missing object {}", databaseKey); + } + } + return new Report(databaseKeys.size(), storedObjects.size(), orphans, deleted, missing, dryRun); + } + + public record Report(int metadataObjects, + int storageObjects, + int orphanObjects, + int deletedObjects, + int missingObjects, + boolean dryRun) { + } +} diff --git a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java index b612241b..83a15875 100644 --- a/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java +++ b/src/main/java/com/example/mtusi_messenger/service/AttachmentService.java @@ -25,6 +25,8 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.UUID; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; @Service public class AttachmentService { @@ -35,6 +37,8 @@ public class AttachmentService { private final TransactionTemplate transaction; private final long maxUploadBytes; private final long maxImagePixels; + private final Semaphore uploadPermits; + private final long uploadPermitWaitMillis; public AttachmentService( AttachmentRepository attachments, @@ -42,7 +46,9 @@ public class AttachmentService { ObjectStorage storage, TransactionTemplate transaction, @Value("${messenger.attachments.max-upload-bytes:10485760}") long maxUploadBytes, - @Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels + @Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels, + @Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads, + @Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis ) { this.attachments = attachments; this.chats = chats; @@ -50,6 +56,11 @@ public class AttachmentService { this.transaction = transaction; this.maxUploadBytes = maxUploadBytes; this.maxImagePixels = maxImagePixels; + if (maxConcurrentUploads < 1 || uploadPermitWaitMillis < 1) { + throw new IllegalArgumentException("Upload concurrency limits must be positive"); + } + this.uploadPermits = new Semaphore(maxConcurrentUploads, true); + this.uploadPermitWaitMillis = uploadPermitWaitMillis; } public Attachment upload(UUID chatId, @@ -65,6 +76,30 @@ public class AttachmentService { throw ApiExceptions.payloadTooLarge("Attachment exceeds 10 MiB"); } + try { + try { + if (!uploadPermits.tryAcquire(uploadPermitWaitMillis, TimeUnit.MILLISECONDS)) { + throw ApiExceptions.tooManyRequests("Upload capacity is temporarily exhausted"); + } + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw ApiExceptions.unavailable("Upload was interrupted"); + } + try { + return storeValidated(chatId, ownerId, safeFilename, contentType, input); + } finally { + uploadPermits.release(); + } + } catch (com.example.mtusi_messenger.exception.ApiException error) { + throw error; + } + } + + private Attachment storeValidated(UUID chatId, + UUID ownerId, + String safeFilename, + String contentType, + InputStream input) { byte[] bytes = readBounded(input); if (bytes.length == 0) { throw ApiExceptions.unsupportedMediaType("Image is empty"); diff --git a/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java b/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java index 32cbaa75..b50f4b4c 100644 --- a/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java +++ b/src/main/java/com/example/mtusi_messenger/storage/MinioObjectStorage.java @@ -2,6 +2,7 @@ package com.example.mtusi_messenger.storage; import io.minio.GetObjectArgs; import io.minio.MinioClient; +import io.minio.ListObjectsArgs; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; import org.springframework.beans.factory.annotation.Value; @@ -10,6 +11,8 @@ import okhttp3.OkHttpClient; import java.io.ByteArrayInputStream; import java.time.Duration; +import java.util.ArrayList; +import java.util.List; @Component public class MinioObjectStorage implements ObjectStorage { @@ -71,4 +74,19 @@ public class MinioObjectStorage implements ObjectStorage { throw new ObjectStorageException("Cannot remove attachment object", error); } } + + @Override + public List list(String prefix) { + try { + List objects = new ArrayList<>(); + for (var result : client.listObjects(ListObjectsArgs.builder() + .bucket(bucket).prefix(prefix).recursive(true).build())) { + var item = result.get(); + objects.add(new StoredObject(item.objectName(), item.lastModified().toInstant(), item.size())); + } + return List.copyOf(objects); + } catch (Exception error) { + throw new ObjectStorageException("Cannot list attachment objects", error); + } + } } diff --git a/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java b/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java index ec038d70..bb521dd9 100644 --- a/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java +++ b/src/main/java/com/example/mtusi_messenger/storage/ObjectStorage.java @@ -1,7 +1,14 @@ package com.example.mtusi_messenger.storage; +import java.time.Instant; +import java.util.List; + public interface ObjectStorage { void put(String objectKey, byte[] bytes, String contentType); byte[] get(String objectKey); void delete(String objectKey); + List list(String prefix); + + record StoredObject(String objectKey, Instant lastModified, long sizeBytes) { + } } diff --git a/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java b/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java index 77d05700..d9209c3c 100644 --- a/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java +++ b/src/main/java/com/example/mtusi_messenger/web/GlobalExceptionHandler.java @@ -23,6 +23,9 @@ public class GlobalExceptionHandler { if (e.status() == HttpStatus.UNAUTHORIZED && e.authenticate() != null) { builder.header("WWW-Authenticate", e.authenticate()); } + if (e.status() == HttpStatus.TOO_MANY_REQUESTS) { + builder.header("Retry-After", "1"); + } return builder.body(body); } @ExceptionHandler({MethodArgumentNotValidException.class, MethodArgumentTypeMismatchException.class, HttpMessageNotReadableException.class}) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7add237e..6a55c26a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -69,3 +69,9 @@ messenger: attachments: max-upload-bytes: ${MAX_UPLOAD_BYTES:10485760} max-image-pixels: ${MAX_IMAGE_PIXELS:40000000} + max-concurrent-uploads: ${MAX_CONCURRENT_UPLOADS:3} + upload-permit-wait-millis: ${UPLOAD_PERMIT_WAIT_MILLIS:10000} + cleanup-grace-seconds: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:3600} + cleanup-dry-run: ${ATTACHMENT_CLEANUP_DRY_RUN:false} + reconcile-interval-millis: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:600000} + reconcile-initial-delay-millis: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:60000} diff --git a/src/test/java/com/example/mtusi_messenger/service/AttachmentReconcilerTest.java b/src/test/java/com/example/mtusi_messenger/service/AttachmentReconcilerTest.java new file mode 100644 index 00000000..49573d28 --- /dev/null +++ b/src/test/java/com/example/mtusi_messenger/service/AttachmentReconcilerTest.java @@ -0,0 +1,75 @@ +package com.example.mtusi_messenger.service; + +import com.example.mtusi_messenger.repository.AttachmentRepository; +import com.example.mtusi_messenger.storage.ObjectStorage; +import org.junit.jupiter.api.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class AttachmentReconcilerTest { + private static final Instant NOW = Instant.parse("2026-11-11T12:00:00Z"); + + @Test + void deletesOnlyOrphansOlderThanGracePeriodAndReportsMissingObjects() { + AttachmentRepository repository = mock(AttachmentRepository.class); + when(repository.findAllObjectKeys()).thenReturn(Set.of( + "attachments/present/original", "attachments/missing/original")); + FakeStorage storage = new FakeStorage(List.of( + object("attachments/present/original", NOW.minusSeconds(7200)), + object("attachments/old-orphan/original", NOW.minusSeconds(7200)), + object("attachments/fresh-orphan/original", NOW.minusSeconds(30)))); + AttachmentReconciler reconciler = new AttachmentReconciler(repository, storage, + Duration.ofHours(1), false, Clock.fixed(NOW, ZoneOffset.UTC)); + + AttachmentReconciler.Report report = reconciler.reconcile(); + + assertEquals(List.of("attachments/old-orphan/original"), storage.deleted); + assertEquals(1, report.orphanObjects()); + assertEquals(1, report.deletedObjects()); + assertEquals(1, report.missingObjects()); + } + + @Test + void dryRunNeverDeletesOldOrphan() { + AttachmentRepository repository = mock(AttachmentRepository.class); + when(repository.findAllObjectKeys()).thenReturn(Set.of()); + FakeStorage storage = new FakeStorage(List.of( + object("attachments/old-orphan/original", NOW.minusSeconds(7200)))); + AttachmentReconciler reconciler = new AttachmentReconciler(repository, storage, + Duration.ofHours(1), true, Clock.fixed(NOW, ZoneOffset.UTC)); + + AttachmentReconciler.Report report = reconciler.reconcile(); + + assertEquals(List.of(), storage.deleted); + assertEquals(1, report.orphanObjects()); + assertEquals(0, report.deletedObjects()); + } + + private static ObjectStorage.StoredObject object(String key, Instant modified) { + return new ObjectStorage.StoredObject(key, modified, 100); + } + + private static final class FakeStorage implements ObjectStorage { + private final List objects; + private final List deleted = new ArrayList<>(); + + private FakeStorage(List objects) { + this.objects = objects; + } + + @Override public void put(String objectKey, byte[] bytes, String contentType) { } + @Override public byte[] get(String objectKey) { return new byte[0]; } + @Override public void delete(String objectKey) { deleted.add(objectKey); } + @Override public List list(String prefix) { return objects; } + } +} diff --git a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java index 7e32ebf5..8a53cb45 100644 --- a/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java +++ b/src/test/java/com/example/mtusi_messenger/service/AttachmentServiceTest.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.io.IOException; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -51,7 +52,7 @@ class AttachmentServiceTest { new Chat(chatId, "test", List.of(ownerId), Instant.now())); when(repository.save(any())).thenAnswer(invocation -> invocation.getArgument(0)); service = new AttachmentService(repository, chats, storage, transaction, - 10_485_760, 40_000_000); + 10_485_760, 40_000_000, 3, 10_000); png = Files.readAllBytes(Path.of("tests", "fixtures", "sample.png")); } @@ -110,6 +111,39 @@ class AttachmentServiceTest { verify(repository, never()).save(any()); } + @Test + void interruptedRequestDoesNotCreateS3Object() { + java.io.InputStream source = new java.io.InputStream() { + private int position; + + @Override + public int read() throws IOException { + if (position >= 16_384) { + throw new IOException("client disconnected"); + } + return png[position++]; + } + }; + + ApiException error = assertThrows(ApiException.class, () -> service.upload( + chatId, ownerId, "photo.png", "image/png", -1, source)); + + assertEquals(HttpStatus.BAD_REQUEST, error.status()); + assertTrue(storage.objects.isEmpty()); + verify(repository, never()).save(any()); + } + + @Test + void databaseFailureCompensatesAlreadyWrittenS3Object() { + when(repository.save(any())).thenThrow(new IllegalStateException("database commit failed")); + + assertThrows(IllegalStateException.class, () -> service.upload( + chatId, ownerId, "photo.png", "image/png", png.length, + new ByteArrayInputStream(png))); + + assertTrue(storage.objects.isEmpty(), "compensating delete must remove the orphan"); + } + @Test void downloadsOnlyAfterChatAuthorization() { Attachment attachment = new Attachment(UUID.randomUUID(), chatId, ownerId, "photo.png", @@ -144,8 +178,17 @@ class AttachmentServiceTest { public void delete(String objectKey) { objects.remove(objectKey); } + + @Override + public List list(String prefix) { + return objects.entrySet().stream() + .filter(entry -> entry.getKey().startsWith(prefix)) + .map(entry -> new StoredObject(entry.getKey(), Instant.now(), entry.getValue().length)) + .toList(); + } } private static final class IOExceptionForTest extends RuntimeException { } + } diff --git a/student_tests/README.md b/student_tests/README.md index a4b188ec..c08c4d23 100644 --- a/student_tests/README.md +++ b/student_tests/README.md @@ -12,3 +12,7 @@ The course-provided black-box suite remains in `tests/`; Spring tests remain in - `rolling_update.py` — sequential replacement under write load. - `node_failover.py` — hard loss of one worker in the disposable three-node Swarm lab, task relocation and confirmed-data verification. +- `lab4_negative.py` — oversized/corrupt/MIME-spoofed uploads, unsafe filename, + unknown attachment, foreign user and cross-chat binding; every user error is 4xx. +- `concurrent_upload_memory.py` — 12 simultaneous ~6.5 MiB valid PNG uploads and + aggregate API container memory sampling through `docker stats`. diff --git a/student_tests/concurrent_upload_memory.py b/student_tests/concurrent_upload_memory.py new file mode 100644 index 00000000..a268da4d --- /dev/null +++ b/student_tests/concurrent_upload_memory.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run 10+ simultaneous valid PNG uploads and sample aggregate API container RAM.""" +import argparse +import binascii +import concurrent.futures +import json +import random +import re +import struct +import subprocess +import sys +import threading +import time +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run # noqa: E402 + + +def png_chunk(kind, payload): + return (struct.pack('>I', len(payload)) + kind + payload + + struct.pack('>I', binascii.crc32(kind + payload) & 0xffffffff)) + + +def make_png(width=1800, height=1200): + rng = random.Random(20261111) + rows = bytearray() + for _ in range(height): + rows.append(0) + rows.extend(rng.randbytes(width * 3)) + return (b'\x89PNG\r\n\x1a\n' + + png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + png_chunk(b'IDAT', zlib.compress(bytes(rows), 1)) + + png_chunk(b'IEND', b'')) + + +def memory_bytes(value): + match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)', value.strip()) + if not match: + raise ValueError('unrecognized Docker memory value: ' + value) + number, unit = match.groups() + factors = {'B': 1, 'kB': 1000, 'KiB': 1024, 'MB': 1000**2, + 'MiB': 1024**2, 'GB': 1000**3, 'GiB': 1024**3} + return int(float(number) * factors[unit]) + + +def api_containers(env_file): + result = subprocess.run( + ['docker', 'compose', '--env-file', env_file, 'ps', '-q', 'api'], + cwd=ROOT, check=True, capture_output=True, text=True) + ids = [line for line in result.stdout.splitlines() if line] + check(len(ids) >= 2, 'memory experiment requires at least two API containers') + return ids + + +def sample_total_memory(container_ids): + result = subprocess.run( + ['docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *container_ids], + check=True, capture_output=True, text=True) + usages = [] + for line in result.stdout.splitlines(): + if line.strip(): + usages.append(memory_bytes(line.split('/', 1)[0].strip())) + check(len(usages) == len(container_ids), 'docker stats did not return every API container') + return sum(usages) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-env-file', default='.env') + parser.add_argument('--concurrency', type=int, default=12) + parser.add_argument('--output', default='evidence/upload-memory.json') + args = parser.parse_args() + check(args.concurrency >= 10, 'experiment requires concurrency >= 10') + + bootstrap = Client(args.base_url, timeout=30) + alice, bob, _, chat = bootstrap.bootstrap() + payload = make_png() + check(len(payload) <= 10_485_760, 'generated PNG exceeds upload limit') + containers = api_containers(args.compose_env_file) + baseline = sample_total_memory(containers) + peak = baseline + samples = [] + stop = threading.Event() + + def monitor(): + nonlocal peak + while not stop.is_set(): + try: + current = sample_total_memory(containers) + samples.append(current) + peak = max(peak, current) + except (subprocess.SubprocessError, Failure, ValueError): + pass + stop.wait(0.1) + + barrier = threading.Barrier(args.concurrency) + + def upload(index): + client = Client(args.base_url, timeout=30) + barrier.wait(timeout=30) + path = (f"/api/v1/chats/{chat['id']}/attachments" + f"?filename=memory-{index}.png") + return client.call('POST', path, 201, actor=alice, raw=payload, + headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id'] + + monitor_thread = threading.Thread(target=monitor, daemon=True) + monitor_thread.start() + started = time.monotonic() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: + ids = list(pool.map(upload, range(args.concurrency))) + finally: + stop.set() + monitor_thread.join(timeout=5) + duration = time.monotonic() - started + check(len(ids) == args.concurrency and len(set(ids)) == len(ids), + 'not every concurrent upload completed exactly once') + + result = { + 'concurrency': args.concurrency, + 'successful_uploads': len(ids), + 'payload_bytes_each': len(payload), + 'duration_seconds': round(duration, 3), + 'api_replicas': len(containers), + 'memory_samples': len(samples), + 'baseline_api_memory_bytes': baseline, + 'peak_api_memory_bytes': peak, + 'peak_increase_bytes': max(0, peak - baseline), + 'configured_max_concurrent_uploads_per_replica': 3, + 'configured_max_upload_bytes': 10_485_760, + 'configured_max_image_pixels': 40_000_000, + } + output = ROOT / args.output + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print(json.dumps(result, ensure_ascii=False)) + print('PASS: concurrent upload memory experiment') + + +if __name__ == '__main__': + run(main) diff --git a/student_tests/lab4_negative.py b/student_tests/lab4_negative.py index 5e847187..ae285434 100644 --- a/student_tests/lab4_negative.py +++ b/student_tests/lab4_negative.py @@ -39,6 +39,12 @@ def main(): client.call('POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404, actor=alice, headers={'Idempotency-Key': uuid.uuid4().hex}, body={'text': '', 'attachment_ids': [attachment['id']]}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 404, + actor=alice, headers={'Idempotency-Key': uuid.uuid4().hex}, + body={'text': '', 'attachment_ids': [str(uuid.uuid4())]}) + client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 400, + actor=alice, headers={'Idempotency-Key': uuid.uuid4().hex}, + body={'text': '', 'attachment_ids': [attachment['id'], attachment['id']]}) client.call('POST', upload_path, 415, actor=alice, raw=png, headers={'Content-Type': 'image/jpeg'}) @@ -52,7 +58,7 @@ def main(): downloaded = client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob) check(downloaded['bytes'] == original, 'authorized download changed bytes') - print('PASS: foreign chat/user, MIME spoof, corruption, byte limit and filename checks') + print('PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks') if __name__ == '__main__': diff --git a/swarm/nginx.conf b/swarm/nginx.conf index 044151ce..74b00a1d 100644 --- a/swarm/nginx.conf +++ b/swarm/nginx.conf @@ -6,6 +6,8 @@ events { } http { + client_max_body_size 11m; + client_body_timeout 10s; access_log /var/log/nginx/access.log; error_log /var/log/nginx/error.log warn; resolver 127.0.0.11 valid=2s ipv6=off; @@ -53,6 +55,20 @@ http { proxy_next_upstream_tries 2; } + location ~ ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_request_buffering on; + proxy_connect_timeout 500ms; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + proxy_next_upstream off; + } + location / { proxy_pass http://api_backend; proxy_http_version 1.1; diff --git a/swarm/stack.app.yaml b/swarm/stack.app.yaml index bab354e7..5be08131 100644 --- a/swarm/stack.app.yaml +++ b/swarm/stack.app.yaml @@ -27,6 +27,12 @@ services: S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required} MAX_UPLOAD_BYTES: 10485760 MAX_IMAGE_PIXELS: 40000000 + MAX_CONCURRENT_UPLOADS: 3 + UPLOAD_PERMIT_WAIT_MILLIS: 10000 + ATTACHMENT_CLEANUP_GRACE_SECONDS: 3600 + ATTACHMENT_CLEANUP_DRY_RUN: "false" + ATTACHMENT_RECONCILE_INTERVAL_MILLIS: 600000 + ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS: 60000 FLYWAY_ENABLED: "false" healthcheck: test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"]