Author SHA1 Message Date
EgorAnti a4d58b2eec 5 2026-11-19 10:57:39 +03:00
EgorAnti 75d2f78913 4 2026-11-19 07:57:14 +03:00
EgorAnti 77a43fa9cf init 2026-11-18 20:33:27 +03:00
148 changed files with 9645 additions and 40 deletions
+51 -19
View File
@@ -1,25 +1,57 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
# Copy to .env for a local synthetic environment. Never commit .env.
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
DATABASE_NAME=messenger
DATABASE_USER=messenger
DATABASE_PASSWORD=replace-with-a-long-random-database-password
RABBITMQ_USER=messenger
RABBITMQ_PASSWORD=replace-with-a-long-random-rabbitmq-password
RABBITMQ_VHOST=/
ATTACHMENT_JOB_EXCHANGE=attachments.v1
ATTACHMENT_JOB_QUEUE=attachments.process.level4.v1
ATTACHMENT_JOB_ROUTING_KEY=attachment.process.v1
ATTACHMENT_DLQ_EXCHANGE=attachments.dlx.v1
ATTACHMENT_DLQ_QUEUE=attachments.failed.v1
ATTACHMENT_DLQ_ROUTING_KEY=attachment.failed.v1
ATTACHMENT_PIPELINE_VERSION=2
WORKER_CONCURRENCY=2
WORKER_PREFETCH=1
JOB_MAX_ATTEMPTS=5
JOB_RETRY_BASE_DELAY=1s
JOB_RETRY_MAX_DELAY=30s
JOB_TIMEOUT=60s
JOB_LEASE_DURATION=90s
JOB_LEASE_REAPER_INTERVAL=5s
WORKER_MAX_IMAGE_PIXELS=40000000
WORKER_MEMORY_BUDGET_BYTES=536870912
WORKER_MEMORY_WAIT=2s
# Fault injection is opt-in and intended only for an isolated test stand.
FAULT_CRASH_AFTER_OUTBOX_COMMIT=false
FAULT_CRASH_BEFORE_SIDE_EFFECT=false
FAULT_CRASH_AFTER_SIDE_EFFECT=false
FAULT_CRASH_AFTER_RESULT_COMMIT=false
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_ACCESS_KEY_ID=messenger-api
S3_SECRET_ACCESS_KEY=replace-with-a-long-random-api-secret
MINIO_ROOT_USER=minio-root
MINIO_ROOT_PASSWORD=replace-with-a-different-long-random-root-secret
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
MAX_CONCURRENT_UPLOADS=3
UPLOAD_PERMIT_WAIT_MILLIS=10000
# Staging objects are deleted only after the final result is committed.
# A separate lifecycle/cleanup policy may remove abandoned staging objects
# after this retention interval; it must never remove active jobs.
STAGING_RETENTION_SECONDS=86400
ATTACHMENT_CLEANUP_GRACE_SECONDS=3600
ATTACHMENT_CLEANUP_DRY_RUN=false
ATTACHMENT_RECONCILE_INTERVAL_MILLIS=600000
ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS=60000
+1
View File
@@ -15,6 +15,7 @@ data/
backups/
# Runtime/build artifacts
.venv/
.gradle/
__pycache__/
*.py[cod]
node_modules/
+37
View File
@@ -0,0 +1,37 @@
# Stage 1: Build
FROM eclipse-temurin:23-jdk AS build
WORKDIR /app
COPY gradlew .
COPY gradle gradle
COPY build.gradle settings.gradle ./
COPY src src
RUN --mount=type=cache,target=/root/.gradle \
chmod +x gradlew && ./gradlew bootJar --no-daemon
# Stage 2: Runtime
FROM eclipse-temurin:23-jre
WORKDIR /app
COPY --from=build /app/build/libs/*.jar app.jar
RUN apt-get update \
&& apt-get install --yes --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/* \
&& useradd --system --create-home messenger
USER messenger
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \
CMD curl --fail --silent --show-error http://localhost:8080/health/ready || exit 1
ENTRYPOINT ["java", "-jar", "app.jar"]
+190 -21
View File
@@ -1,37 +1,206 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Антипин Егор Иванович
- Целевая оценка: **5**, трек 2 «Доказанная идемпотентность pipeline»
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Библиотеки: Spring AMQP, RabbitMQ Java client, Thumbnailator, webp-imageio, MinIO SDK, PostgreSQL/Flyway
- Помощь ИИ: проектирование versioned pipeline, fencing и fault-injection, реализация, подготовка тестов и черновиков evidence; результат проверен сборкой, unit-, black-box- и аварийными тестами
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Desktop/Engine с Compose v2, Java 23 и Python 3.9+.
## Архитектура и решения
```sh
cp .env.example .env
# Заменить все replace-with-* уникальными локальными значениями.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
python -X utf8 tests/smoke.py --base-url http://localhost:8080
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Автоматические `student_tests/lab5_*.py` используют `.env.example` и рассчитаны
на одноразовый учебный стенд; для них удобно поднимать Compose с тем же файлом:
## Доказательства критериев
```sh
docker compose --env-file .env.example up -d --build
python -X utf8 student_tests/lab5_pipeline_idempotency.py
```
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
`docker compose --env-file .env down` сохраняет volumes. Удаление учебных данных
выполняется только явной командой оператора с `--volumes`.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Pipeline и гарантии
## Эксперименты и отказы
Upload разделён на durable-границы: API проверяет bytes/MIME/dimensions, пишет
неизменённый original в приватный `staging/<attachment_id>/original`, затем одной
транзакцией создаёт `attachments(queued)` и `attachment_jobs(published=false)`.
Запись job является transactional outbox. Фоновый publisher публикует persistent
JSON в durable RabbitMQ queue, ждёт publisher confirm и только потом помечает job
опубликованной. Поэтому COMMIT без publish восстанавливается автоматически, а
HTTP `202` означает, что staging и намерение обработки уже устойчивы.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Отдельный `worker` захватывает job с lease, читает доверенный staging key из
PostgreSQL и создаёт детерминированные S3 keys. В broker message есть только
`schema_version`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`,
`created_at`; bytes, credentials, произвольные URL и клиентские пути отсутствуют.
Эффект идемпотентен по `(attachment_id, pipeline_version, variant)`. Все variant
rows и `ready` фиксируются атомарно; ack выполняется после COMMIT. Redelivery уже
`ready` job — no-op.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Обязательные варианты для активной версии:
## Ограничения и следующие шаги
| Вариант | Правило | Формат/качество |
| --- | --- | --- |
| `original` | bytes и SHA-256 совпадают с upload | исходный MIME |
| `thumbnail` | fit в 256×256, без увеличения, EXIF orientation, alpha на белом | JPEG, quality зависит от версии |
| `crop` | центрированный квадрат 128×128 после EXIF orientation | JPEG, quality зависит от версии |
| `optimized` | fit до 1280×1280, без увеличения, alpha сохраняется | WebP, quality зависит от версии |
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Новые файлы кодируются из пикселей без копирования EXIF/ICC/прочих исходных
metadata.
## Versioned pipeline v1/v2
Миграция `V7__versioned_pipeline_and_fault_points.sql` добавляет:
- `attachments.pipeline_version`;
- уникальность `attachment_jobs(attachment_id, pipeline_version)`;
- состояние `superseded`;
- audit-таблицу `attachment_pipeline_promotions`;
- маркеры fault-injection до и после side effect.
`ImageVariantProcessor` выбирает профиль по `pipeline_version`: v1 и v2 имеют
разные quality для JPEG/WebP, поэтому результаты версий различаются побайтово.
Ключи объектов версионированы: `thumbnail-v1.jpg`, `thumbnail-v2.jpg`,
`optimized-v1.webp` и т.д.; общий `original` неизменяем. Старая версия физически
не может перезаписать объект новой версии. Одна и та же `pipeline_version`
обязана быть детерминированной: изменение quality/алгоритма оформляется новой
версией, иначе `ON CONFLICT DO NOTHING` сохранит уже записанные variant rows.
Повышение версии — одна аудированная транзакция
`scripts/promote-pipeline.ps1`/`.sh`:
1. блокирует attachment;
2. переводит старые `queued/processing` jobs в `superseded` и `published=true`;
3. создаёт новую job версии `N`;
4. обновляет `attachments.pipeline_version` и `status=queued`;
5. пишет `attachment_pipeline_promotions`.
Fencing выполняется в каждом переходе состояния: `claim`, `markReady`,
`markFailed`, retry, expired-lease recovery и replay проверяют одновременно
`job.pipeline_version`, `attachments.pipeline_version`, lease token и terminal
state. Поэтому старая job после promotion не может ни получить lease, ни
записать variants, ни перевести attachment в `ready`.
Если v1 уже удалил staging после безопасного COMMIT, worker версии v2 читает
неизменяемый `attachments/<id>/original`; содержимое и SHA-256 те же.
## Fault injection и инварианты
| Точка | Поведение |
| --- | --- |
| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API падает после COMMIT attachment/outbox, до publish; republisher восстанавливает намерение |
| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker падает до S3 PUT; lease истекает, reaper возвращает job в outbox |
| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker падает после детерминированных S3 PUT, до result COMMIT; повтор сходится к тем же объектам |
| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker падает после atomic `ready` COMMIT, до consumer ack; redelivery — no-op |
Все флаги по умолчанию выключены и предназначены только для изолированного
стенда. Автоматический `student_tests/lab5_pipeline_idempotency.py` проверяет:
- duplicate: две одинаковые v2-доставки дают один результат;
- reorder: v2 доставляется раньше stale v1;
- stale: v1 после v2 не меняет snapshot активного результата;
- superseded: повышение ещё queued v1 job отменяет её без variant rows;
- crash до S3 side effect: 0 объектов, 0 result rows, восстановление через
повторную попытку;
- crash после S3 side effect: 4 объекта до COMMIT, после повтора — те же 4
объекта и ровно 3 variant rows;
- crash после result COMMIT: redelivery не дублирует variants;
- все 5 подтверждённых вложений остаются `ready`.
Зафиксированные инварианты:
1. `ready` активной версии ⇒ ровно три variant rows активной версии и нет
`queued/processing` jobs.
2. Не более одной активной job на attachment.
3. `(attachment_id, pipeline_version, name)` уникален; повтор не создаёт дубль.
4. Versioned object keys не позволяют старой версии перезаписать новую.
5. `pipeline_version` повышается только аудированной транзакцией; старая
`queued/processing` job становится `superseded`.
6. Подтверждённые задачи не теряются после duplicate/reorder/stale/crash.
7. Нет variant rows версии больше активной.
## Почему это не exactly-once доставка
RabbitMQ обеспечивает **at-least-once**: сообщение может быть доставлено
повторно после потери соединения или падения worker до ack; порядок сообщений не
является бизнес-порядком. Publisher confirm подтверждает только приём брокером,
а consumer ack идёт после PostgreSQL COMMIT. Между S3 PUT и COMMIT нет
распределённой транзакции, поэтому crash в этом окне оставляет уже выполненные
side effects и вызывает повторную обработку.
Идемпотентность достигнута на уровне **эффекта**, а не доставки: deterministic
object keys, `ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady`
приводят любые повторы к одному наблюдаемому состоянию активной версии. То же
самое можно назвать **effectively-once** результатом при at-least-once
доставке. Exactly-once доставка потребовала бы транзакционной связи RabbitMQ,
PostgreSQL и S3 или эквивалентного распределённого протокола; в этой учебной
архитектуре её нет, и тест сознательно демонстрирует лишние повторы S3 PUT,
которые не меняют результат.
## Ограничения worker и retry
- Upload: `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000`.
- Worker повторно проверяет header/dimensions до полного decode и применяет тот же pixel cap.
- Оценка памяти на job резервируется общим честным semaphore; бюджет `WORKER_MEMORY_BUDGET_BYTES=536870912`, ожидание `WORKER_MEMORY_WAIT=2s`. Compose дополнительно ограничивает worker `1024m`, JVM использует до 70% container RAM.
- Обработка ограничена `JOB_TIMEOUT=60s`; просроченный task теряет lease и не может зафиксировать результат.
- `WORKER_CONCURRENCY=2`, `WORKER_PREFETCH=1`; lease 90s, reaper каждые 5s.
- Не более `JOB_MAX_ATTEMPTS=5`. Временные ошибки получают exponential backoff с jitter 50–100% и cap 30s. Ошибки decode/format/size постоянные и сразу становятся безопасным `failed/error_code`; exhausted retry — `retry_exhausted`.
- Permanent malformed broker delivery и исчерпанная работа dead-lettered в durable `attachments.failed.v1`. Реестр failed остаётся в PostgreSQL.
Операторский replay выполняется `scripts/replay-failed.ps1 -JobId <uuid>` или
`scripts/replay-failed.sh <uuid>`. Одна транзакция блокирует одновременно failed
job и attachment, пишет `attachment_job_replays`, сбрасывает outbox и состояние.
Не-failed, уже recovered и `ready` jobs дают явный безопасный `no-op`; replay
также требует совпадения `pipeline_version`.
После безопасного COMMIT результата worker best-effort удаляет staging.
`StagingCleaner` удаляет только неактивные объекты старше
`STAGING_RETENTION_SECONDS=86400`; queued/processing staging защищён. Отдельного
GC для orphan-объектов старых версий после promotion «в полёте» нет: active
результат они не перезаписывают, но место могут занимать.
## Проверки
| Критерий | Команда | Результат |
| --- | --- | --- |
| Unit + integration | `gradlew test` | `BUILD SUCCESSFUL`, 30 tests |
| Contract/Compose | `python -X utf8 scripts/check_contract.py`; `docker compose --env-file .env.example config --quiet` | PASS, exit 0 |
| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://localhost:8080` | PASS |
| Трек 5.2 | `python -X utf8 student_tests/lab5_pipeline_idempotency.py` | PASS: version fencing, superseded, duplicate/reorder, pre/post-side-effect crashes |
| Worker stop/restart, redelivery | `python -X utf8 student_tests/lab5_worker_recovery.py` | PASS; original неизменен, 3 variants, duplicate не создаёт rows |
| COMMIT→publish и result COMMIT→ack | `python -X utf8 student_tests/lab5_fault_injection.py` | PASS; обе задачи дошли до ready |
| DLQ, replay, lease | `python -X utf8 student_tests/lab5_dlq_replay_lease.py` | PASS; DLQ +1, одна audit-запись, ready не replayed, expired lease восстановлен |
| Lab3: shared idempotency | `python -X utf8 student_tests/idempotency_concurrent.py` | PASS; 32 запроса, 2 реплики, одно сообщение |
| Lab3: readiness gate | `python -X utf8 student_tests/readiness_gate.py` | PASS; обе реплики 503 на сломанной схеме и восстановились |
| Lab3: failover каждой реплики | `python -X utf8 student_tests/failover_each.py ...` | PASS; 2 реплики, confirmed writes сохранены |
| Lab3: rolling update | `python -X utf8 student_tests/rolling_update.py ...` | PASS; 4 instance ID, 335 успешных записей, 3 переходных отказа |
| Lab4: negative | `python -X utf8 student_tests/lab4_negative.py` | PASS; ownership/MIME/corruption/limit/filename |
| Lab4: upload memory | `python -X utf8 student_tests/concurrent_upload_memory.py ...` | PASS; 12/12 uploads, 2 API реплики |
Подробный журнал с SQL-инвариантами и JSON-агрегатами:
`evidence/lab5-level5-2026-11-19.md`. Предыдущие уровни:
`evidence/lab5-level4-2026-11-19.md`, `evidence/lab4-2026-11-11.md`.
## Ограничения
- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO;
эксперимент не доказывает отказоустойчивость к потере host/диска.
- Exactly-once доставки нет; доказан at-least-once + идемпотентный эффект.
- Versioned orphan-объекты старых версий после promotion «в полёте» не
перезаписывают active результат, но автоматического GC для них нет.
- `promote-pipeline` — операторский скрипт с SQL-транзакцией, не публичный API.
- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант;
на чистом volume и для новых данных действует инвариант трёх вариантов.
- Числа failover/rolling — проверка сохранности на одном Windows-хосте, не
benchmark пропускной способности.
+58
View File
@@ -0,0 +1,58 @@
plugins {
id 'java'
id 'org.springframework.boot' version '4.1.1'
id 'io.spring.dependency-management' version '1.1.7'
}
group = 'com.example'
version = '0.0.1-SNAPSHOT'
java {
toolchain {
languageVersion = JavaLanguageVersion.of(23)
}
}
repositories {
mavenCentral()
}
dependencies {
// implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
// implementation 'org.springframework.boot:spring-boot-starter-security'
implementation 'org.springframework.boot:spring-boot-starter-webmvc'
implementation 'org.springframework.boot:spring-boot-starter-validation'
implementation 'org.springframework.boot:spring-boot-starter-jdbc'
implementation 'org.springframework.boot:spring-boot-starter-data-redis'
implementation 'org.springframework.boot:spring-boot-starter-amqp'
implementation 'org.springframework.boot:spring-boot-flyway'
implementation 'org.flywaydb:flyway-core'
implementation 'org.flywaydb:flyway-database-postgresql'
implementation 'org.springframework.boot:spring-boot-starter-security'
implementation 'org.bouncycastle:bcprov-jdk18on:1.84'
implementation 'org.apache.commons:commons-pool2'
implementation 'io.minio:minio:8.6.0'
implementation 'net.coobird:thumbnailator:0.4.20'
implementation 'org.sejda.imageio:webp-imageio:0.1.6'
runtimeOnly 'org.postgresql:postgresql'
implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310'
implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.3'
compileOnly 'org.projectlombok:lombok'
developmentOnly 'org.springframework.boot:spring-boot-devtools'
annotationProcessor 'org.projectlombok:lombok'
// testImplementation 'org.springframework.boot:spring-boot-starter-data-jpa-test'
// testImplementation 'org.springframework.boot:spring-boot-starter-security-test'
testImplementation 'org.springframework.boot:spring-boot-starter-test'
testImplementation 'org.springframework.boot:spring-boot-starter-webmvc-test'
testCompileOnly 'org.projectlombok:lombok'
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
testAnnotationProcessor 'org.projectlombok:lombok'
implementation("com.github.f4b6a3:uuid-creator:6.1.1")
}
tasks.named('test') {
useJUnitPlatform()
}
+309
View File
@@ -0,0 +1,309 @@
name: hla-lab5
x-api-environment: &api-environment
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL: jdbc:postgresql://postgres:5432/${DATABASE_NAME:-messenger}
DATABASE_USER: ${DATABASE_USER:-messenger}
DATABASE_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env}
DATABASE_POOL_SIZE: ${DATABASE_POOL_SIZE:-10}
DATABASE_MIN_IDLE: ${DATABASE_MIN_IDLE:-2}
DATABASE_QUERY_TIMEOUT_SECONDS: ${DATABASE_QUERY_TIMEOUT_SECONDS:-5}
DATABASE_TRANSACTION_TIMEOUT_SECONDS: ${DATABASE_TRANSACTION_TIMEOUT_SECONDS:-5}
REDIS_HOST: redis
REDIS_PORT: 6379
REDIS_CONNECT_TIMEOUT: ${REDIS_CONNECT_TIMEOUT:-1s}
REDIS_COMMAND_TIMEOUT: ${REDIS_COMMAND_TIMEOUT:-500ms}
REDIS_POOL_MAX_ACTIVE: ${REDIS_POOL_MAX_ACTIVE:-8}
REDIS_POOL_MAX_IDLE: ${REDIS_POOL_MAX_IDLE:-8}
REDIS_POOL_MIN_IDLE: ${REDIS_POOL_MIN_IDLE:-0}
REDIS_POOL_MAX_WAIT: ${REDIS_POOL_MAX_WAIT:-500ms}
USER_CACHE_TTL_SECONDS: ${USER_CACHE_TTL_SECONDS:-300}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
S3_ENDPOINT: http://minio:9000
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_CONCURRENT_UPLOADS: ${MAX_CONCURRENT_UPLOADS:-3}
UPLOAD_PERMIT_WAIT_MILLIS: ${UPLOAD_PERMIT_WAIT_MILLIS:-10000}
RABBITMQ_HOST: rabbitmq
RABBITMQ_PORT: 5672
RABBITMQ_USER: ${RABBITMQ_USER:-messenger}
RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
RABBITMQ_VHOST: ${RABBITMQ_VHOST:-/}
ATTACHMENT_JOB_EXCHANGE: ${ATTACHMENT_JOB_EXCHANGE:-attachments.v1}
ATTACHMENT_JOB_QUEUE: ${ATTACHMENT_JOB_QUEUE:-attachments.process.level4.v1}
ATTACHMENT_JOB_ROUTING_KEY: ${ATTACHMENT_JOB_ROUTING_KEY:-attachment.process.v1}
ATTACHMENT_DLQ_EXCHANGE: ${ATTACHMENT_DLQ_EXCHANGE:-attachments.dlx.v1}
ATTACHMENT_DLQ_QUEUE: ${ATTACHMENT_DLQ_QUEUE:-attachments.failed.v1}
ATTACHMENT_DLQ_ROUTING_KEY: ${ATTACHMENT_DLQ_ROUTING_KEY:-attachment.failed.v1}
ATTACHMENT_PIPELINE_VERSION: ${ATTACHMENT_PIPELINE_VERSION:-2}
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_RETRY_BASE_DELAY: ${JOB_RETRY_BASE_DELAY:-1s}
JOB_RETRY_MAX_DELAY: ${JOB_RETRY_MAX_DELAY:-30s}
JOB_TIMEOUT: ${JOB_TIMEOUT:-60s}
JOB_LEASE_DURATION: ${JOB_LEASE_DURATION:-90s}
JOB_LEASE_REAPER_INTERVAL: ${JOB_LEASE_REAPER_INTERVAL:-5s}
WORKER_MAX_IMAGE_PIXELS: ${WORKER_MAX_IMAGE_PIXELS:-40000000}
WORKER_MEMORY_BUDGET_BYTES: ${WORKER_MEMORY_BUDGET_BYTES:-536870912}
WORKER_MEMORY_WAIT: ${WORKER_MEMORY_WAIT:-2s}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
ATTACHMENT_CLEANUP_GRACE_SECONDS: ${ATTACHMENT_CLEANUP_GRACE_SECONDS:-3600}
ATTACHMENT_CLEANUP_DRY_RUN: ${ATTACHMENT_CLEANUP_DRY_RUN:-false}
ATTACHMENT_RECONCILE_INTERVAL_MILLIS: ${ATTACHMENT_RECONCILE_INTERVAL_MILLIS:-600000}
ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS: ${ATTACHMENT_RECONCILE_INITIAL_DELAY_MILLIS:-60000}
services:
postgres:
image: postgres:16.4-bookworm
environment:
POSTGRES_DB: ${DATABASE_NAME:-messenger}
POSTGRES_USER: ${DATABASE_USER:-messenger}
POSTGRES_PASSWORD: ${DATABASE_PASSWORD:?Set DATABASE_PASSWORD in .env}
volumes:
- postgres-data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 2s
timeout: 2s
retries: 30
restart: unless-stopped
cpus: 1.0
mem_limit: 768m
networks: [backend]
redis:
image: redis:7.4-bookworm
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes:
- redis-data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 30
restart: unless-stopped
cpus: 0.25
mem_limit: 256m
networks: [backend]
rabbitmq:
image: rabbitmq:4.1.4-management-alpine
environment:
RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-messenger}
RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?Set RABBITMQ_PASSWORD in .env}
RABBITMQ_DEFAULT_VHOST: ${RABBITMQ_VHOST:-/}
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
volumes:
- rabbitmq-data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
interval: 10s
timeout: 10s
start_period: 60s
retries: 12
restart: unless-stopped
cpus: 0.5
mem_limit: 1024m
networks: [backend]
minio:
image: bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd0e172c4cc0870f4bdc3142018e2a37be9acf04d68f386600daad427e0cab
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
volumes:
- minio-data:/bitnami/minio/data
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "http://localhost:9000/minio/health/live"]
interval: 2s
timeout: 2s
retries: 30
restart: unless-stopped
cpus: 0.75
mem_limit: 512m
networks: [backend]
minio-init:
image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc
depends_on:
minio:
condition: service_healthy
environment:
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
volumes:
- ./infra/minio-api-policy.json:/policy/minio-api-policy.json:ro
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
sed "s/__S3_BUCKET__/$$S3_BUCKET/g" /policy/minio-api-policy.json > /tmp/minio-api-policy.json
attempt=0
until mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" \
&& mc mb --ignore-existing "local/$$S3_BUCKET" \
&& mc anonymous set none "local/$$S3_BUCKET" \
&& (mc admin user add local "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY" \
|| mc admin user info local "$$S3_ACCESS_KEY_ID") \
&& (mc admin policy create local messenger-attachments-rw /tmp/minio-api-policy.json \
|| mc admin policy info local messenger-attachments-rw) \
&& mc admin policy attach local messenger-attachments-rw --user "$$S3_ACCESS_KEY_ID"
do
attempt=$$((attempt + 1))
if [ "$$attempt" -ge 30 ]; then
echo "MinIO bootstrap did not complete after 30 attempts" >&2
exit 1
fi
sleep 2
done
restart: "no"
networks: [backend]
private-bucket-check:
profiles: ["verification"]
image: bitnamilegacy/minio-client:2025.4.16-debian-12-r0@sha256:28e38f52642c06a2344fa974cf55786b4c52601b6b9b0676c68d40ee0876a5bc
depends_on:
minio-init:
condition: service_completed_successfully
environment:
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
entrypoint: ["/bin/sh", "-c"]
command:
- |
set -eu
mc alias set anonymous http://minio:9000
if mc ls "anonymous/$$S3_BUCKET" >/dev/null 2>&1; then
echo "FAIL: anonymous bucket listing succeeded" >&2
exit 1
fi
mc alias set restricted http://minio:9000 "$$S3_ACCESS_KEY_ID" "$$S3_SECRET_ACCESS_KEY"
mc ls "restricted/$$S3_BUCKET" >/dev/null
if mc admin info restricted >/dev/null 2>&1; then
echo "FAIL: API credentials have administrative access" >&2
exit 1
fi
echo "PASS: bucket is private and API credentials are non-admin"
restart: "no"
networks: [backend]
# One-shot Flyway runner. It is invoked explicitly before API replicas.
migrate:
image: hla-lab5-app:local
build:
context: .
environment:
<<: *api-environment
FLYWAY_ENABLED: "true"
MIGRATE_ONLY: "true"
MESSENGER_ROLE: migration
SPRING_MAIN_WEB_APPLICATION_TYPE: none
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
rabbitmq:
condition: service_healthy
restart: "no"
cpus: 0.75
mem_limit: 768m
networks: [backend]
api:
image: hla-lab5-app:local
build:
context: .
environment:
<<: *api-environment
FLYWAY_ENABLED: "false"
MESSENGER_ROLE: api
FAULT_CRASH_AFTER_OUTBOX_COMMIT: ${FAULT_CRASH_AFTER_OUTBOX_COMMIT:-false}
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
minio-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "http://localhost:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 15
start_period: 45s
init: true
stop_grace_period: 12s
restart: unless-stopped
cpus: 0.75
mem_limit: 768m
networks: [backend]
worker:
image: hla-lab5-app:local
environment:
<<: *api-environment
FLYWAY_ENABLED: "false"
MESSENGER_ROLE: worker
SPRING_MAIN_WEB_APPLICATION_TYPE: none
JAVA_TOOL_OPTIONS: -XX:MaxRAMPercentage=70
FAULT_CRASH_BEFORE_SIDE_EFFECT: ${FAULT_CRASH_BEFORE_SIDE_EFFECT:-false}
FAULT_CRASH_AFTER_SIDE_EFFECT: ${FAULT_CRASH_AFTER_SIDE_EFFECT:-false}
FAULT_CRASH_AFTER_RESULT_COMMIT: ${FAULT_CRASH_AFTER_RESULT_COMMIT:-false}
depends_on:
migrate:
condition: service_completed_successfully
minio-init:
condition: service_completed_successfully
rabbitmq:
condition: service_healthy
healthcheck:
disable: true
init: true
stop_grace_period: 20s
restart: unless-stopped
cpus: 1.0
mem_limit: 1024m
networks: [backend]
nginx:
image: nginx:1.27.4-bookworm
ports:
- "127.0.0.1:${APP_PORT:-8080}:80"
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "nginx", "-t"]
interval: 2s
timeout: 2s
retries: 15
stop_grace_period: 10s
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [backend]
networks:
backend:
driver: bridge
volumes:
postgres-data:
redis-data:
minio-data:
rabbitmq-data:
+24
View File
@@ -0,0 +1,24 @@
{
"scenario": "sequential failover of each API replica",
"replicas": 2,
"results": [
{
"target_instance_id": "8e081627dfc3",
"window_seconds": 15.775,
"successful_reads": 3,
"successful_writes": 4,
"failed_attempts": 0,
"successful_rps": 0.444,
"first_recovery_seconds": 5.018
},
{
"target_instance_id": "95b5db854bee",
"window_seconds": 16.774,
"successful_reads": 3,
"successful_writes": 3,
"failed_attempts": 1,
"successful_rps": 0.358,
"first_recovery_seconds": 6.15
}
]
}
+79
View File
@@ -0,0 +1,79 @@
# Lab4 evidence — 2026-11-11
Стенд: Docker Compose, nginx на `127.0.0.1:18084`, API ×2, PostgreSQL, Redis, MinIO. Секреты и session tokens в evidence не сохранялись.
## Bootstrap и миграции
- MinIO server image: `bitnamilegacy/minio:2025.4.22-debian-12-r1@sha256:d7cd…e0cab`.
- `minio-init`: bucket `messenger-attachments` создан, anonymous access установлен `private`, пользователь `messenger-api` создан, policy `messenger-attachments-rw` назначена.
- Flyway: успешно проверены и применены 4 миграции, итоговая версия schema `v4`.
- `docker compose ps`: две здоровые API-реплики; MinIO/PostgreSQL/Redis/nginx healthy; наружу опубликован только nginx.
## Проверки
```text
AttachmentServiceTest + AttachmentReconcilerTest: BUILD SUCCESSFUL, 10 tests.
tests/smoke.py: PASS: lab4 public API scenario.
student_tests/lab4_negative.py:
PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption,
byte limit and filename checks.
```
Persistence hook останавливал nginx, обе API, Redis, PostgreSQL и MinIO, затем запускал те же named volumes и повторял idempotent bucket bootstrap:
```text
PASS: persistence preserved confirmed data and existing session.
```
Failover hook остановил replica из `X-Instance-Id`, обслужившую тестовый запрос:
```json
{"observed_instances_before":2,"successful_writes":102,"failed_attempts":0,"first_recovery_seconds":1.386}
```
```text
PASS: failover preserved confirmed data and existing session.
```
При остановленном MinIO:
```text
PASS: unavailable S3 returns contract JSON 503
```
После запуска того же MinIO volume публичный smoke повторно завершился PASS, checksum скачанного original совпал.
## Level 4: consistency, memory and private S3
Unit tests inject an interrupted request stream before S3 PUT and a repository
failure after S3 PUT. The first leaves storage empty; the second verifies the
compensating DELETE. Reconciler tests prove that an orphan younger than the
one-hour grace period survives, an older orphan is deleted, dry-run never
deletes, and a missing object is reported.
Bootstrap was executed again against the existing bucket and completed with
exit code 0. The direct S3 probe (without application API) produced:
```text
PASS: bucket is private and API credentials are non-admin
```
Full Gradle suite using the live Compose PostgreSQL/Redis/MinIO network:
```text
BUILD SUCCESSFUL in 11m 47s
```
Concurrent memory experiment (`evidence/upload-memory-2026-11-11.json`):
```json
{"concurrency":12,"successful_uploads":12,"payload_bytes_each":6835833,
"duration_seconds":10.156,"api_replicas":2,"memory_samples":5,
"baseline_api_memory_bytes":668467200,"peak_api_memory_bytes":839909375,
"peak_increase_bytes":171442175,"configured_max_concurrent_uploads_per_replica":3}
```
The memory values are the aggregate of both API containers from `docker stats`.
Sampling can miss a shorter peak, so this is an operational bound observation,
not a JVM heap proof. The code-level bounds are 10 MiB/request, 40M pixels and
three active uploads per replica.
+41
View File
@@ -0,0 +1,41 @@
# Lab5 verification — 2026-11-18
Среда: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23, Spring Boot 4.1.1, RabbitMQ 4.1.4.
## Статические и unit-проверки
```text
docker compose --env-file .env.example config --quiet
PASS (exit 0)
gradlew test --tests '*AttachmentServiceTest' --tests '*ThumbnailProcessorTest' --tests '*AttachmentReconcilerTest' --tests '*StagingCleanerTest'
BUILD SUCCESSFUL
```
## Публичный сценарий
```text
python tests/smoke.py --base-url http://localhost:8080
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
```
В момент проверки отдельные services `api`, `worker`, `rabbitmq`, `postgres`, `redis`, `minio` и `nginx` были запущены; RabbitMQ queue `attachments.process.v1` имела `durable=true`.
## Остановка worker, restart и redelivery
```text
python student_tests/lab5_worker_recovery.py
PASS: stopped worker, durable restart recovery and duplicate delivery
```
Сценарий теста:
1. останавливает worker;
2. получает upload `202 queued`, проверяет `409` content и привязку к сообщению;
3. перезапускает API и RabbitMQ без удаления volumes;
4. запускает worker и ждёт `ready`;
5. сверяет original bytes и единственный thumbnail;
6. публикует тот же `job_id` повторно;
7. проверяет terminal `ready` и ровно одну запись thumbnail.
Синтетические пароли и токены тест не печатает и в evidence не сохраняет.
+56
View File
@@ -0,0 +1,56 @@
# Lab5 level 4 verification — 2026-11-19
Среда: Windows, Docker Compose, Java 23, Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, MinIO.
## Статические и unit-проверки
```text
python -X utf8 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
docker compose --env-file .env.example config --quiet
exit 0
gradlew test --tests '*AttachmentServiceTest' --tests '*ImageVariantProcessorTest' \
--tests '*RetryPolicyTest' --tests '*AttachmentReconcilerTest' \
--tests '*StagingCleanerTest'
BUILD SUCCESSFUL
```
`ImageVariantProcessorTest` проверяет реальные JPEG/WebP signatures, ImageIO decode и размеры thumbnail/crop/optimized. `RetryPolicyTest` проверяет bounded exponential backoff и jitter.
## API и recovery
```text
python -X utf8 tests/smoke.py --base-url http://localhost:8080
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
python -X utf8 student_tests/lab5_worker_recovery.py
PASS: stopped worker, durable restart recovery and duplicate delivery
```
Recovery test останавливает worker, получает `202 queued`, перезапускает API и RabbitMQ без удаления volumes, запускает worker и сверяет original SHA-256, thumbnail 256×160, crop 128×128, WebP signature. Повторная публикация того же `job_id` оставляет `ready` и ровно три variant rows.
## Доказанные окна отказа
```text
python -X utf8 student_tests/lab5_fault_injection.py
PASS: outbox commit/publish crash and result commit/ack crash recovered
```
1. API завершается после COMMIT attachment/outbox до publish. Перезапущенный API republisher публикует сохранённое намерение; новый upload не создаётся вручную.
2. Worker завершается после записи объектов и atomic ready COMMIT до ack. Redelivery завершает тот же effect; variant rows остаются уникальными.
## DLQ, replay и lease
```text
python -X utf8 student_tests/lab5_dlq_replay_lease.py
PASS: DLQ, guarded replay and expired lease recovery
```
- malformed persistent JSON был permanent failure и увеличил `attachments.failed.v1` ровно на 1;
- failed job replayed один раз, audit count = 1; повтор в queued/ready дал no-op;
- после replay получены те же три variant names без второго набора;
- искусственно истёкший processing lease был requeued, attempt увеличился, job дошёл до ready.
Секреты, session tokens и contents сообщений в evidence не сохранялись. Docker volumes в проверках не удалялись.
+213
View File
@@ -0,0 +1,213 @@
# Lab5, трек 5.2 — доказанная идемпотентность pipeline
Дата: 2026-11-19. Исходная точка: `75d2f78` + рабочее дерево с изменениями lab5.
Стенд: Windows, Docker 28.0.1, Docker Compose 2.33.1, Java 23 (контейнеры и Gradle
toolchain), Spring Boot 4.1.1, PostgreSQL 16.4, RabbitMQ 4.1.4, Redis 7.4,
MinIO `bitnamilegacy/minio:2025.4.22-debian-12-r1`.
Все пользователи, сообщения и изображения синтетические. Секреты, session tokens,
содержимое сообщений и signed URL в evidence не сохранялись.
## Гипотеза и инварианты
Гипотеза: при versioned pipeline и fencing по `(attachment_id, pipeline_version,
lease)` повторная доставка, перестановка сообщений, crash в окнах до/после side
effect и устаревшая job не теряют подтверждённую работу и не заменяют результат
новой версии старым.
Проверяемые инварианты:
1. `attachments.status='ready'` для активной версии означает ровно три variant
rows этой версии (`thumbnail`, `crop`, `optimized`) и отсутствие
`queued/processing` jobs у вложения.
2. Для вложения существует не более одной активной (`queued/processing`) job.
3. Ключ variant rows — `(attachment_id, pipeline_version, name)`; повторная
запись не создаёт дубль.
4. Объектные ключи версионированы: `thumbnail-vN.jpg`, `crop-vN.jpg`,
`optimized-vN.webp`; `original` общий и неизменяемый. Поэтому v1 не может
перезаписать v2.
5. `attachments.pipeline_version` монотонно повышается только аудированной
транзакцией `promote-pipeline`; старая `queued/processing` job получает
`superseded` и `published=true`.
6. Все подтверждённые тестом задачи (`status='ready'`) остаются `ready` после
duplicate/reorder/stale доставок.
7. В active version нет variant rows с `pipeline_version` больше
`attachments.pipeline_version`.
8. Одна и та же `pipeline_version` детерминирована; изменение профиля
оформляется новой версией, потому что `ON CONFLICT DO NOTHING` не перезаписывает
уже зафиксированные variant rows.
## Команды и результаты
```text
python -X utf8 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
docker compose --env-file .env.example config --quiet
exit 0
.\gradlew.bat test
BUILD SUCCESSFUL (30 tests; full suite against live Compose PostgreSQL/Redis/MinIO)
```
Для полного Gradle suite хранилища Compose не публикуют порты наружу; на время
прогона использовались временные localhost-туннели к `postgres:5432`,
`redis:6379`, `rabbitmq:5672`, `minio:9000`. После прогона туннели удалены.
### Основной эксперимент трека
```text
python -X utf8 student_tests/lab5_pipeline_idempotency.py
PASS: version fencing, superseded, duplicate/reorder and pre/post-side-effect crashes
```
Тест автоматически:
1. создаёт v1 attachment, дожидается `ready` и снимает snapshot трёх variant rows;
2. останавливает worker, повышает версию до v2 через `scripts/promote-pipeline`,
публикует `v2, v2, v1` (порядок доставки не совпадает с порядком создания) и
запускает worker;
3. проверяет, что v2 дал отдельный результат, в БД ровно 6 rows (3 v1 + 3 v2),
active version = 2, а повторные stale v1 / duplicate v2 не меняют snapshot;
4. отдельно повышает ещё **queued** v1 job: она становится
`superseded|published=true|superseded_by_v2`, v1 variant rows не появляются,
v2 доходит до `ready` с тремя вариантами;
5. вставляет crash до S3 side effect, после S3 side effect и после result
COMMIT; каждый раз проверяет восстановление через lease/reaper/redelivery;
6. проверяет, что все 5 подтверждённых вложений остались `ready`.
### Наблюдаемое состояние после прогона
Пример versioned-attachment (сокращённые ID приведены для читаемости):
```text
attachment A (v1 -> v2, сначала завершён v1)
active=2 ready
job v1 ready published=t attempt=1
job v2 ready published=t attempt=1
variant keys: crop-v1, optimized-v1, thumbnail-v1,
crop-v2, optimized-v2, thumbnail-v2
attachment B (queued v1 повышен до v2)
active=2 ready
job v1 superseded published=t last_error=superseded_by_v2
job v2 ready published=t attempt=1
variant keys: crop-v2, optimized-v2, thumbnail-v2 (v1 rows отсутствуют)
```
SQL-инварианты по всем attachment, созданным 2026-11-19 в ходе финального
прогона и регрессии lab5:
```text
superseded_jobs|1
today_promotions|9
today_ready|55
today_ready_bad_variants|0
ready_with_nonterminal_job|0
attachments_with_two_active_jobs|0
duplicate_variant_keys|0
stale_results_for_active_version|0
```
Полный volume содержит также 3 attachment, созданных 2026-11-18 до миграции V7
(тогда уровень 3 сохранял только original + thumbnail). Это исторические данные
предыдущего стенда; они не являются результатом текущего versioned pipeline.
Запросы инвариантов выше ограничены `created_at >= '2026-11-19'`; на чистом
volume исторических строк нет.
### Точки отказа
| Fault | Где срабатывает | Наблюдение |
| --- | --- | --- |
| `FAULT_CRASH_BEFORE_SIDE_EFFECT` | worker, до чтения staging/S3 PUT, `halt(84)` | 0 variant rows, 0 объектов; reaper возвращает job в `queued`, attempt увеличивается, задача доходит до `ready` |
| `FAULT_CRASH_AFTER_SIDE_EFFECT` | worker, после 4 S3 PUT, до result COMMIT, `halt(85)` | objects=4, variant rows=0; повтор пишет те же deterministic keys и коммитит результат, objects остаётся 4 |
| `FAULT_CRASH_AFTER_RESULT_COMMIT` | worker, после atomic `ready` COMMIT, до ack, `halt(86)` | redelivery видит `ready` и завершает no-op; 3 variant rows не дублируются |
| `FAULT_CRASH_AFTER_OUTBOX_COMMIT` | API, после COMMIT attachment/outbox, до publish, `halt(87)` | republisher находит `published=false` и публикует сохранённое намерение; задача доходит до `ready` |
Crash-точки включаются только переменными окружения и по умолчанию выключены;
они предназначены для изолированного учебного стенда.
## Регрессия уровней 3–4
```text
python -X utf8 tests/smoke.py --base-url http://localhost:8080
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
python -X utf8 student_tests/lab5_worker_recovery.py
PASS: stopped worker, durable restart recovery and duplicate delivery
python -X utf8 student_tests/lab5_fault_injection.py
PASS: outbox commit/publish crash and result commit/ack crash recovered
python -X utf8 student_tests/lab5_dlq_replay_lease.py
PASS: DLQ, guarded replay and expired lease recovery
python -X utf8 student_tests/idempotency_concurrent.py
PASS: 32 concurrent requests used 2 replicas and created one message
python -X utf8 student_tests/readiness_gate.py
PASS: 2 unhealthy API containers refused business traffic with 503 and recovered after schema restoration
python -X utf8 student_tests/failover_each.py --action-script scripts/stop-one.cmd \
--restore-script scripts/restore-api.cmd --window 15 \
--output evidence/failover-each-2026-11-19.json
(JSON: 2 replicas sequentially stopped and restored; confirmed writes preserved)
python -X utf8 student_tests/rolling_update.py --action-script scripts/rolling-update.cmd \
--duration 180 --concurrency 4 \
--output evidence/rolling-update-2026-11-19.json
(JSON: 4 observed replacement instance IDs; 335 successful writes, 3 transitional failures)
python -X utf8 student_tests/lab4_negative.py
PASS: unknown/foreign attachment, foreign chat/user, MIME spoof, corruption, byte limit and filename checks
python -X utf8 student_tests/concurrent_upload_memory.py \
--output evidence/upload-memory-2026-11-19.json
PASS: concurrent upload memory experiment (12/12 uploads accepted, 2 API replicas)
```
`readiness_gate.py` проверяет Google-подобный gate по обязательной схеме:
`ReadinessService` теперь пробует обе таблицы — `message_idempotency` (lab3) и
`attachments` (lab4/lab5). `concurrent_upload_memory.py` обновлён с lab4-контракта
`201` на lab5-контракт `202`.
## Почему это at-least-once, а не exactly-once
RabbitMQ даёт at-least-once: сообщение может быть доставлено повторно после
потери соединения или падения worker до ack; порядок между разными сообщениями
не является бизнес-порядком. Поэтому duplicate и reorder — нормальные состояния,
а не аномалии.
Наш pipeline делает **эффект** идемпотентным, но не делает доставку exactly-once:
- publisher confirm подтверждает только приём сообщения брокером, а consumer ack
отправляется после PostgreSQL COMMIT; между S3 PUT и COMMIT нет распределённой
транзакции;
- crash после S3 PUT и до COMMIT оставляет объекты, а crash после COMMIT и до ack
вызывает redelivery; оба окна специально инжектируются тестом;
- повторная обработка может повторно выполнить S3 PUT, но deterministic keys,
`ON CONFLICT DO NOTHING`, уникальные PK и conditional `markReady` не создают
второго набора результатов и не меняют active version;
- stale job ограничена `pipeline_version` и lease в каждом переходе состояния,
поэтому не может записать результат поверх новой версии.
Корректная формулировка: **at-least-once delivery + idempotent effect =
effectively-once наблюдаемый результат для активной версии**. Exactly-once
доставка потребовала бы транзакционной связи RabbitMQ, PostgreSQL и S3 (или
эквивалентного распределённого протокола), которой в этой учебной архитектуре
нет и которая не нужна для данных инвариантов.
## Границы
- Один Docker-host, по одному экземпляру PostgreSQL, RabbitMQ, Redis и MinIO;
эксперимент не доказывает отказоустойчивость к потере host/диска.
- Versioned S3 objects старой версии после promotion «в полёте» могут остаться
orphan-объектами: active result они не перезаписывают, но отдельного GC для
versioned objects нет (чистится только staging).
- `superseded`/promotion — операторская SQL-транзакция через скрипт, не публичный
HTTP API.
- Числа failover/rolling в JSON — проверка сохранности и восстановления на одном
Windows-хосте, не benchmark пропускной способности; latency включает
особенность localhost dual-stack в Docker Desktop.
- Исторические 3 attachment от 2026-11-18 созданы до V7 и имеют один вариант;
на чистом volume и для новых данных действует инвариант трёх вариантов.
+23
View File
@@ -0,0 +1,23 @@
{
"scenario": "sequential API replacement under closed-loop writes",
"requested_duration_seconds": 180.0,
"elapsed_seconds": 182.022,
"rolling_action_seconds": 125.275,
"concurrency": 4,
"observed_instance_ids": [
"431acab3a50c",
"8e081627dfc3",
"d67820ebd09d",
"fdff9a67741c"
],
"successful_requests": 335,
"failed_requests": 3,
"error_ratio": 0.008876,
"successful_rps": 1.84,
"successful_latency_ms": {
"p50": 2092.164,
"p95": 2252.301,
"p99": 3064.667
},
"note": "Closed-loop result on one host; success latency excludes failures."
}
+14
View File
@@ -0,0 +1,14 @@
{
"concurrency": 12,
"successful_uploads": 12,
"payload_bytes_each": 6835833,
"duration_seconds": 10.156,
"api_replicas": 2,
"memory_samples": 5,
"baseline_api_memory_bytes": 668467200,
"peak_api_memory_bytes": 839909375,
"peak_increase_bytes": 171442175,
"configured_max_concurrent_uploads_per_replica": 3,
"configured_max_upload_bytes": 10485760,
"configured_max_image_pixels": 40000000
}
+14
View File
@@ -0,0 +1,14 @@
{
"concurrency": 12,
"successful_uploads": 12,
"payload_bytes_each": 6835833,
"duration_seconds": 31.241,
"api_replicas": 2,
"memory_samples": 10,
"baseline_api_memory_bytes": 665321471,
"peak_api_memory_bytes": 833198489,
"peak_increase_bytes": 167877018,
"configured_max_concurrent_uploads_per_replica": 3,
"configured_max_upload_bytes": 10485760,
"configured_max_image_pixels": 40000000
}
Binary file not shown.
+9
View File
@@ -0,0 +1,9 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-9.7.1-bin.zip
networkTimeout=10000
retries=0
retryBackOffMs=500
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
Vendored
+248
View File
@@ -0,0 +1,248 @@
#!/bin/sh
#
# Copyright © 2015 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#
##############################################################################
#
# gradlew start up script for POSIX generated by Gradle.
#
# Important for running:
#
# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
# noncompliant, but you have some other compliant shell such as ksh or
# bash, then to run this script, type that shell name before the whole
# command line, like:
#
# ksh gradlew
#
# Busybox and similar reduced shells will NOT work, because this script
# requires all of these POSIX shell features:
# * functions;
# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
# * compound commands having a testable exit status, especially «case»;
# * various built-in commands including «command», «set», and «ulimit».
#
# Important for patching:
#
# (2) This script targets any POSIX shell, so it avoids extensions provided
# by Bash, Ksh, etc; in particular arrays are avoided.
#
# The "traditional" practice of packing multiple parameters into a
# space-separated string is a well documented source of bugs and security
# problems, so this is (mostly) avoided, by progressively accumulating
# options in "$@", and eventually passing that to Java.
#
# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
# see the in-line comments for details.
#
# There are tweaks for specific operating systems such as AIX, CygWin,
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
#
##############################################################################
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
app_path=$0
# Need this for daisy-chained symlinks.
while
APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
[ -h "$app_path" ]
do
ls=$( ls -ld "$app_path" )
link=${ls#*' -> '}
case $link in #(
/*) app_path=$link ;; #(
*) app_path=$APP_HOME$link ;;
esac
done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
warn () {
echo "$*"
} >&2
die () {
echo
echo "$*"
echo
exit 1
} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
case "$( uname )" in #(
CYGWIN* ) cygwin=true ;; #(
Darwin* ) darwin=true ;; #(
MSYS* | MINGW* ) msys=true ;; #(
NONSTOP* ) nonstop=true ;;
esac
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD=$JAVA_HOME/jre/sh/java
else
JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD=java
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.
if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
case $MAX_FD in #(
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
fi
# Collect all arguments for the java command, stacking in reverse order:
# * args from the command line
# * the main class name
# * -classpath
# * -D...appname settings
# * --module-path (only if needed)
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
if "$cygwin" || "$msys" ; then
APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
JAVACMD=$( cygpath --unix "$JAVACMD" )
# Now convert the arguments - kludge to limit ourselves to /bin/sh
for arg do
if
case $arg in #(
-*) false ;; # don't mess with options #(
/?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
[ -e "$t" ] ;; #(
*) false ;;
esac
then
arg=$( cygpath --path --ignore --mixed "$arg" )
fi
# Roll the args list around exactly as many times as the number of
# args, so each arg winds up back in the position where it started, but
# possibly modified.
#
# NB: a `for` loop captures its iteration list before it begins, so
# changing the positional parameters here affects neither the number of
# iterations, nor the values presented in `arg`.
shift # remove old arg
set -- "$@" "$arg" # push replacement arg
done
fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \
-jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
"$@"
# Stop when "xargs" is not available.
if ! command -v xargs >/dev/null 2>&1
then
die "xargs is not available"
fi
# Use "xargs" to parse quoted args.
#
# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
#
# In Bash we could simply go:
#
# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
# set -- "${ARGS[@]}" "$@"
#
# but POSIX shell has neither arrays nor command substitution, so instead we
# post-process each arg (as a line of input to sed) to backslash-escape any
# character that might be a shell metacharacter, then use eval to reverse
# that process (while maintaining the separation between arguments), and wrap
# the whole thing up as a single "set" statement.
#
# This will of course break if any of these variables contains a newline or
# an unmatched quote.
#
eval "set -- $(
printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
xargs -n1 |
sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
tr '\n' ' '
)" '"$@"'
exec "$JAVACMD" "$@"
Vendored
+82
View File
@@ -0,0 +1,82 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem https://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@rem SPDX-License-Identifier: Apache-2.0
@rem
@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
@rem
@rem gradlew startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables, and ensure extensions are enabled
setlocal EnableExtensions
set DIRNAME=%~dp0
if "%DIRNAME%"=="" set DIRNAME=.
@rem This is normally unused
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Resolve any "." and ".." in APP_HOME to make it shorter.
for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if %ERRORLEVEL% equ 0 goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
"%COMSPEC%" /c exit 1
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto execute
echo. 1>&2
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
echo. 1>&2
echo Please set the JAVA_HOME variable in your environment to match the 1>&2
echo location of your Java installation. 1>&2
"%COMSPEC%" /c exit 1
:execute
@rem Setup the command line
@rem Execute gradlew
@rem endlocal doesn't take effect until after the line is parsed and variables are expanded
@rem which allows us to clear the local environment before executing the java command
endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel
:exitWithErrorLevel
@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts
"%COMSPEC%" /c exit %ERRORLEVEL%
+27
View File
@@ -0,0 +1,27 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::__S3_BUCKET__/attachments/*",
"arn:aws:s3:::__S3_BUCKET__/staging/*"
]
}
]
}
+103
View File
@@ -0,0 +1,103 @@
worker_processes auto;
worker_shutdown_timeout 10s;
events {
worker_connections 1024;
}
http {
# Let the application enforce the exact 10 MiB contract and return JSON.
client_max_body_size 11m;
client_body_timeout 10s;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log warn;
# Docker embedded DNS returns current Compose replicas. Nginx OSS 1.27.3+
# refreshes these addresses without a configuration reload.
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 1s;
upstream api_backend {
zone api_backend 64k;
least_conn;
server api:8080 resolve max_fails=1 fail_timeout=2s;
keepalive 32;
}
server {
listen 80;
server_name _;
proxy_intercept_errors on;
error_page 502 503 504 = @no_ready_api;
location = /nginx-health {
access_log off;
default_type application/json;
return 200 '{"status":"ok"}';
}
# After both peers have refused traffic or become unreachable, Nginx
# would normally emit its own HTML 502. Keep the public API contract:
# no ready backend is a JSON 503 with the required tracing headers.
location @no_ready_api {
internal;
default_type application/json;
add_header X-Request-Id $request_id always;
add_header X-Instance-Id nginx always;
add_header Cache-Control "no-store" always;
return 503 '{"error":{"code":"unavailable","message":"No ready API backend","request_id":"$request_id"}}';
}
# Message POST is the only non-idempotent HTTP method that is safe to
# replay here: the API requires Idempotency-Key and commits the message
# together with its shared PostgreSQL idempotency record.
location ~ ^/api/v1/chats/[0-9a-fA-F-]+/messages$ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_request_buffering on;
proxy_connect_timeout 500ms;
proxy_send_timeout 4s;
proxy_read_timeout 4s;
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
proxy_next_upstream_tries 2;
}
location ~ ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_request_buffering on;
proxy_connect_timeout 500ms;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
# Upload is not replayed: it has no idempotency key. The client may retry explicitly.
proxy_next_upstream off;
}
location / {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 500ms;
proxy_send_timeout 4s;
proxy_read_timeout 4s;
# No sticky sessions. Failed/unready peers are retried elsewhere.
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
}
}
+7
View File
@@ -0,0 +1,7 @@
@echo off
setlocal EnableExtensions
cd /d "%~dp0\.."
if not defined ENV_FILE set "ENV_FILE=.env"
docker compose --env-file "%ENV_FILE%" --profile verification run --rm private-bucket-check
exit /b %ERRORLEVEL%
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
cd "$PROJECT_DIR"
: "${ENV_FILE:=.env}"
docker compose --env-file "$ENV_FILE" --profile verification run --rm private-bucket-check
+73
View File
@@ -0,0 +1,73 @@
param(
[Parameter(Mandatory = $true)]
[string]$AttachmentId,
[Parameter(Mandatory = $true)]
[ValidateRange(1, 2)]
[int]$PipelineVersion,
[string]$RequestedBy = $env:USERNAME,
[string]$EnvFile = ".env"
)
$parsedAttachmentId = [guid]::Empty
if (-not [guid]::TryParse($AttachmentId, [ref]$parsedAttachmentId)) {
throw "AttachmentId must be a UUID"
}
if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') {
throw "RequestedBy contains unsupported characters"
}
$sql = @"
WITH candidate AS (
SELECT id, pipeline_version
FROM attachments
WHERE id = '$parsedAttachmentId'::uuid AND pipeline_version < $PipelineVersion
FOR UPDATE
), superseded AS (
UPDATE attachment_jobs j
SET status = 'superseded', published = true,
last_error_code = 'superseded_by_v$PipelineVersion',
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
FROM candidate c
WHERE j.attachment_id = c.id AND j.pipeline_version < $PipelineVersion
AND j.status IN ('queued', 'processing')
RETURNING j.id
), created AS (
INSERT INTO attachment_jobs
(id, attachment_id, pipeline_version, attempt, status, published,
next_attempt_at, created_at, updated_at)
SELECT gen_random_uuid(), id, $PipelineVersion, 1, 'queued', false,
clock_timestamp(), clock_timestamp(), clock_timestamp()
FROM candidate
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
RETURNING id, attachment_id
), promoted AS (
UPDATE attachments a
SET pipeline_version = $PipelineVersion, status = 'queued', error_code = NULL,
updated_at = clock_timestamp()
FROM candidate c, created n
WHERE a.id = c.id AND n.attachment_id = c.id
RETURNING a.id
), audited AS (
INSERT INTO attachment_pipeline_promotions
(attachment_id, from_version, to_version, job_id, requested_by)
SELECT c.id, c.pipeline_version, $PipelineVersion, n.id, '$RequestedBy'
FROM candidate c
JOIN created n ON n.attachment_id = c.id
JOIN promoted p ON p.id = c.id
RETURNING job_id
)
SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');
"@
$result = docker compose --env-file $EnvFile exec -T postgres `
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql
if ($LASTEXITCODE -ne 0) {
throw "Pipeline promotion transaction failed"
}
$last = ($result | Select-Object -Last 1).Trim()
if ($last -eq "no-op") {
Write-Output "No-op: attachment is missing, already at v$PipelineVersion, or that version already exists"
} else {
Write-Output "Promoted attachment $parsedAttachmentId to pipeline v$PipelineVersion with job $last"
}
+59
View File
@@ -0,0 +1,59 @@
#!/bin/sh
set -eu
attachment_id="${1:?usage: promote-pipeline.sh ATTACHMENT_UUID PIPELINE_VERSION [REQUESTED_BY] [ENV_FILE]}"
pipeline_version="${2:?pipeline version is required}"
requested_by="${3:-operator}"
env_file="${4:-.env}"
case "$attachment_id" in
????????-????-????-????-????????????) ;;
*) echo "AttachmentId must be a UUID" >&2; exit 2 ;;
esac
case "$pipeline_version" in
1|2) ;;
*) echo "PipelineVersion must be 1 or 2" >&2; exit 2 ;;
esac
case "$requested_by" in
*[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;;
esac
sql="WITH candidate AS (
SELECT id, pipeline_version FROM attachments
WHERE id = '$attachment_id'::uuid AND pipeline_version < $pipeline_version FOR UPDATE
), superseded AS (
UPDATE attachment_jobs j SET status = 'superseded', published = true,
last_error_code = 'superseded_by_v$pipeline_version',
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
FROM candidate c WHERE j.attachment_id = c.id
AND j.pipeline_version < $pipeline_version AND j.status IN ('queued', 'processing')
RETURNING j.id
), created AS (
INSERT INTO attachment_jobs
(id, attachment_id, pipeline_version, attempt, status, published,
next_attempt_at, created_at, updated_at)
SELECT gen_random_uuid(), id, $pipeline_version, 1, 'queued', false,
clock_timestamp(), clock_timestamp(), clock_timestamp() FROM candidate
ON CONFLICT (attachment_id, pipeline_version) DO NOTHING
RETURNING id, attachment_id
), promoted AS (
UPDATE attachments a SET pipeline_version = $pipeline_version,
status = 'queued', error_code = NULL, updated_at = clock_timestamp()
FROM candidate c, created n WHERE a.id = c.id AND n.attachment_id = c.id
RETURNING a.id
), audited AS (
INSERT INTO attachment_pipeline_promotions
(attachment_id, from_version, to_version, job_id, requested_by)
SELECT c.id, c.pipeline_version, $pipeline_version, n.id, '$requested_by'
FROM candidate c JOIN created n ON n.attachment_id = c.id
JOIN promoted p ON p.id = c.id RETURNING job_id
)
SELECT COALESCE((SELECT job_id::text FROM audited), 'no-op');"
result="$(docker compose --env-file "$env_file" exec -T postgres \
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")"
if [ "$result" = "no-op" ]; then
echo "No-op: attachment is missing, already at v$pipeline_version, or that version already exists"
else
echo "Promoted attachment $attachment_id to pipeline v$pipeline_version with job $result"
fi
+58
View File
@@ -0,0 +1,58 @@
param(
[Parameter(Mandatory = $true)]
[string]$JobId,
[string]$RequestedBy = $env:USERNAME,
[string]$EnvFile = ".env"
)
$parsedJobId = [guid]::Empty
if (-not [guid]::TryParse($JobId, [ref]$parsedJobId)) {
throw "JobId must be a UUID"
}
if ($RequestedBy -notmatch '^[A-Za-z0-9_.@-]{1,128}$') {
throw "RequestedBy contains unsupported characters"
}
$sql = @"
WITH candidate AS (
SELECT j.id, j.attachment_id, j.attempt
FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id
WHERE j.id = '$parsedJobId'::uuid
AND j.status = 'failed' AND a.status = 'failed'
AND j.pipeline_version = a.pipeline_version
FOR UPDATE OF j, a
), audited AS (
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
SELECT id, '$RequestedBy', attempt FROM candidate
RETURNING job_id
), reset_job AS (
UPDATE attachment_jobs j
SET status = 'queued', attempt = 1, published = false,
last_error_code = NULL, next_attempt_at = clock_timestamp(),
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
FROM candidate c, audited a
WHERE j.id = c.id AND a.job_id = c.id
RETURNING j.attachment_id
), reset_attachment AS (
UPDATE attachments a
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
FROM reset_job r
WHERE a.id = r.attachment_id AND a.status = 'failed'
RETURNING a.id
)
SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment)
THEN 'replayed' ELSE 'no-op' END;
"@
$result = docker compose --env-file $EnvFile exec -T postgres `
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c $sql
if ($LASTEXITCODE -ne 0) {
throw "Replay transaction failed"
}
if (($result | Select-Object -Last 1).Trim() -eq "replayed") {
Write-Output "Replay requested for failed job $parsedJobId by $RequestedBy"
} else {
Write-Output "No-op: job $parsedJobId is missing, not failed, or already recovered"
}
+46
View File
@@ -0,0 +1,46 @@
#!/bin/sh
set -eu
job_id="${1:?usage: replay-failed.sh JOB_UUID [REQUESTED_BY] [ENV_FILE]}"
requested_by="${2:-operator}"
env_file="${3:-.env}"
case "$job_id" in
????????-????-????-????-????????????) ;;
*) echo "JobId must be a UUID" >&2; exit 2 ;;
esac
case "$requested_by" in
*[!A-Za-z0-9_.@-]*|'') echo "RequestedBy contains unsupported characters" >&2; exit 2 ;;
esac
sql="WITH candidate AS (
SELECT j.id, j.attachment_id, j.attempt FROM attachment_jobs j
JOIN attachments a ON a.id = j.attachment_id
WHERE j.id = '$job_id'::uuid AND j.status = 'failed' AND a.status = 'failed'
AND j.pipeline_version = a.pipeline_version
FOR UPDATE OF j, a
), audited AS (
INSERT INTO attachment_job_replays (job_id, requested_by, previous_attempt)
SELECT id, '$requested_by', attempt FROM candidate RETURNING job_id
), reset_job AS (
UPDATE attachment_jobs j SET status = 'queued', attempt = 1, published = false,
last_error_code = NULL, next_attempt_at = clock_timestamp(),
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
FROM candidate c, audited a WHERE j.id = c.id AND a.job_id = c.id
RETURNING j.attachment_id
), reset_attachment AS (
UPDATE attachments a SET status = 'queued', error_code = NULL,
updated_at = clock_timestamp()
FROM reset_job r WHERE a.id = r.attachment_id AND a.status = 'failed'
RETURNING a.id
)
SELECT CASE WHEN EXISTS (SELECT 1 FROM reset_attachment)
THEN 'replayed' ELSE 'no-op' END;"
result="$(docker compose --env-file "$env_file" exec -T postgres \
psql -v ON_ERROR_STOP=1 -U messenger -d messenger -At -c "$sql")"
if [ "$result" = "replayed" ]; then
echo "Replay requested for failed job $job_id by $requested_by"
else
echo "No-op: job $job_id is missing, not failed, or already recovered"
fi
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal EnableExtensions
cd /d "%~dp0\.."
if not defined ENV_FILE set "ENV_FILE=.env"
docker compose --env-file "%ENV_FILE%" stop nginx api redis postgres minio || exit /b 1
docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 postgres redis minio || exit /b 1
docker compose --env-file "%ENV_FILE%" run --rm minio-init || exit /b 1
docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api || exit /b 1
docker compose --env-file "%ENV_FILE%" up -d --no-deps nginx || exit /b 1
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
set -eu
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
cd "$PROJECT_DIR"
: "${ENV_FILE:=.env}"
# Preserve named volumes and the network. The migration job is intentionally
# not re-run during a same-version restart.
docker compose --env-file "$ENV_FILE" stop nginx api redis postgres minio
docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 postgres redis minio
docker compose --env-file "$ENV_FILE" run --rm minio-init
docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api
docker compose --env-file "$ENV_FILE" up -d --no-deps nginx
+7
View File
@@ -0,0 +1,7 @@
@echo off
setlocal EnableExtensions
cd /d "%~dp0\.."
if not defined ENV_FILE set "ENV_FILE=.env"
docker compose --env-file "%ENV_FILE%" up -d --wait --wait-timeout 90 --no-deps --scale api=2 api
exit /b %ERRORLEVEL%
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
cd "$PROJECT_DIR"
: "${ENV_FILE:=.env}"
docker compose --env-file "$ENV_FILE" up -d --wait --wait-timeout 90 \
--no-deps --scale api=2 api
+16
View File
@@ -0,0 +1,16 @@
@echo off
setlocal EnableExtensions
cd /d "%~dp0\.."
set /a REPLICA_COUNT=0
for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do set /a REPLICA_COUNT+=1
if %REPLICA_COUNT% LSS 2 exit /b 2
for /f "usebackq delims=" %%C in (`docker compose --env-file .env ps -q api`) do call :replace_one %%C
exit /b 0
:replace_one
docker stop --time 10 %1 >nul || exit /b 1
docker rm %1 >nul || exit /b 1
docker compose --env-file .env up -d --wait --wait-timeout 90 --no-deps --scale api=2 api >nul || exit /b 1
exit /b 0
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
set -eu
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
cd "$PROJECT_DIR"
original_ids=$(docker compose --env-file .env ps -q api)
count=$(printf '%s\n' "$original_ids" | sed '/^$/d' | wc -l | tr -d ' ')
if [ "$count" -lt 2 ]; then
printf '%s\n' 'Rolling update requires at least two API replicas' >&2
exit 2
fi
for container_id in $original_ids; do
# Replace exactly one original replica and wait for the replacement before
# touching the next one. Named volumes and state stores are not changed.
docker stop --time 10 "$container_id" >/dev/null
docker rm "$container_id" >/dev/null
docker compose --env-file .env up -d --wait --wait-timeout 90 \
--no-deps --scale api=2 api >/dev/null
done
+26
View File
@@ -0,0 +1,26 @@
@echo off
setlocal EnableExtensions EnableDelayedExpansion
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
cd /d "%~dp0\.."
if not defined ENV_FILE set "ENV_FILE=.env"
set "TARGET_CONTAINER="
set /a MATCHES=0
for /f "usebackq delims=" %%C in (`docker compose --env-file "%ENV_FILE%" ps -q api`) do (
set "CONTAINER_ID=%%C"
set "SHORT_ID=!CONTAINER_ID:~0,12!"
set "HOSTNAME_VALUE="
for /f "usebackq delims=" %%H in (`docker inspect --format "{{.Config.Hostname}}" %%C`) do set "HOSTNAME_VALUE=%%H"
if "!HOSTNAME_VALUE!"=="%TARGET_INSTANCE_ID%" (
set "TARGET_CONTAINER=%%C"
set /a MATCHES+=1
) else if "!SHORT_ID!"=="%TARGET_INSTANCE_ID%" (
set "TARGET_CONTAINER=%%C"
set /a MATCHES+=1
)
)
if not "!MATCHES!"=="1" exit /b 2
docker stop --time 5 "!TARGET_CONTAINER!" >nul
exit /b %ERRORLEVEL%
+29
View File
@@ -0,0 +1,29 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PROJECT_DIR=$(dirname -- "$SCRIPT_DIR")
cd "$PROJECT_DIR"
: "${ENV_FILE:=.env}"
target=""
matches=0
for container_id in $(docker compose --env-file "$ENV_FILE" ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
short_id=$(printf '%s' "$container_id" | cut -c1-12)
if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$short_id" ]; then
target=$container_id
matches=$((matches + 1))
fi
done
if [ "$matches" -ne 1 ]; then
printf '%s\n' "Expected one API replica for X-Instance-Id, found $matches" >&2
exit 2
fi
# Five seconds keeps the operator hook below the test's ten-second budget while
# still delivering SIGTERM and allowing Spring's graceful shutdown phase.
docker stop --time 5 "$target" >/dev/null
+19
View File
@@ -0,0 +1,19 @@
$ErrorActionPreference = 'Continue'
if (Test-Path variable:PSNativeCommandUseErrorActionPreference) {
$PSNativeCommandUseErrorActionPreference = $false
}
$nodes = @('hla-swarm-manager', 'hla-swarm-worker1', 'hla-swarm-worker2', 'hla-swarm-registry')
foreach ($node in $nodes) {
$existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}')
if ($existing -eq $node) {
docker rm --force $node | Out-Null
if ($LASTEXITCODE -ne 0) { throw "Cannot remove $node" }
}
}
$existingNetwork = (& docker network ls --filter 'name=^hla-swarm-underlay$' --format '{{.Name}}')
if ($existingNetwork -eq 'hla-swarm-underlay') {
docker network rm hla-swarm-underlay | Out-Null
if ($LASTEXITCODE -ne 0) { throw 'Cannot remove hla-swarm-underlay' }
}
Write-Output 'Removed the disposable three-node Swarm lab and its nested volumes.'
+171
View File
@@ -0,0 +1,171 @@
$ErrorActionPreference = 'Continue'
if (Test-Path variable:PSNativeCommandUseErrorActionPreference) {
$PSNativeCommandUseErrorActionPreference = $false
}
$projectDir = Split-Path -Parent $PSScriptRoot
Set-Location $projectDir
$manager = 'hla-swarm-manager'
$nodes = @($manager, 'hla-swarm-worker1', 'hla-swarm-worker2')
$network = 'hla-swarm-underlay'
$registry = 'hla-swarm-registry'
function Invoke-Checked {
param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Command)
$executable = $Command[0]
$arguments = if ($Command.Length -gt 1) {
$Command[1..($Command.Length - 1)]
} else {
@()
}
& $executable @arguments
if ($LASTEXITCODE -ne 0) {
throw "Command failed: $($Command -join ' ')"
}
}
function Wait-InnerDocker([string]$Node) {
$deadline = (Get-Date).AddSeconds(90)
do {
& docker exec $Node docker info *> $null
if ($LASTEXITCODE -eq 0) { return }
Start-Sleep -Seconds 2
} while ((Get-Date) -lt $deadline)
throw "Docker daemon did not start in $Node"
}
function Wait-Service([string]$Service, [string]$Expected, [int]$Seconds = 180) {
$deadline = (Get-Date).AddSeconds($Seconds)
do {
$replicas = (& docker exec $manager docker service ls `
--filter "name=$Service" --format '{{.Replicas}}' 2>$null | Select-Object -First 1)
if ($replicas -like "$Expected*") { return }
Start-Sleep -Seconds 2
} while ((Get-Date) -lt $deadline)
throw "Service $Service did not reach $Expected replicas"
}
foreach ($node in $nodes) {
$existing = (& docker ps --all --filter "name=^/${node}$" --format '{{.Names}}')
if ($existing -eq $node) {
throw "Container $node already exists. Run scripts/swarm-lab-down.ps1 first."
}
}
$existingRegistry = (& docker ps --all --filter "name=^/${registry}$" --format '{{.Names}}')
if ($existingRegistry -eq $registry) {
throw "Container $registry already exists. Run scripts/swarm-lab-down.ps1 first."
}
$envValues = @{}
Get-Content -LiteralPath '.env' | ForEach-Object {
if ($_ -match '^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$') {
$envValues[$matches[1]] = $matches[2].Trim('"', "'")
}
}
$password = $envValues['DATABASE_PASSWORD']
if ([string]::IsNullOrWhiteSpace($password)) { throw 'DATABASE_PASSWORD is missing from .env' }
$database = if ($envValues['DATABASE_NAME']) { $envValues['DATABASE_NAME'] } else { 'messenger' }
$user = if ($envValues['DATABASE_USER']) { $envValues['DATABASE_USER'] } else { 'messenger' }
$minioRootUser = $envValues['MINIO_ROOT_USER']
$minioRootPassword = $envValues['MINIO_ROOT_PASSWORD']
$s3AccessKey = $envValues['S3_ACCESS_KEY_ID']
$s3SecretKey = $envValues['S3_SECRET_ACCESS_KEY']
$s3Bucket = if ($envValues['S3_BUCKET']) { $envValues['S3_BUCKET'] } else { 'messenger-attachments' }
$s3Region = if ($envValues['S3_REGION']) { $envValues['S3_REGION'] } else { 'us-east-1' }
foreach ($required in @{
MINIO_ROOT_USER = $minioRootUser
MINIO_ROOT_PASSWORD = $minioRootPassword
S3_ACCESS_KEY_ID = $s3AccessKey
S3_SECRET_ACCESS_KEY = $s3SecretKey
}.GetEnumerator()) {
if ([string]::IsNullOrWhiteSpace($required.Value)) { throw "$($required.Key) is missing from .env" }
}
$existingNetwork = (& docker network ls --filter "name=^${network}$" --format '{{.Name}}')
if ($existingNetwork -ne $network) { Invoke-Checked docker network create $network }
Invoke-Checked docker run --detach --name $registry --hostname $registry `
--network $network --publish 127.0.0.1:15000:5000 registry:2
Invoke-Checked docker run --privileged --detach --name $manager --hostname $manager `
--network $network --publish 127.0.0.1:18080:8080 `
--volume "${projectDir}:/project:ro" --env DOCKER_TLS_CERTDIR= docker:28-dind `
--storage-driver=overlay2 --insecure-registry=${registry}:5000
foreach ($worker in $nodes[1..2]) {
Invoke-Checked docker run --privileged --detach --name $worker --hostname $worker `
--network $network --env DOCKER_TLS_CERTDIR= docker:28-dind `
--storage-driver=overlay2 --insecure-registry=${registry}:5000
}
foreach ($node in $nodes) { Wait-InnerDocker $node }
$managerIp = (& docker inspect --format `
'{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $manager).Trim()
Invoke-Checked docker exec $manager docker swarm init --advertise-addr $managerIp
$token = (& docker exec $manager docker swarm join-token --quiet worker).Trim()
foreach ($worker in $nodes[1..2]) {
Invoke-Checked docker exec $worker docker swarm join --token $token "$managerIp`:2377"
}
Invoke-Checked docker exec $manager docker node update `
--label-add hla.state=true --label-add hla.lb=true $manager
Invoke-Checked docker compose --env-file .env build api
Invoke-Checked docker tag hla-lab4-api:local localhost:15000/hla-lab4-api:swarm
Invoke-Checked docker build --file swarm/nginx.Dockerfile `
--tag localhost:15000/hla-lab4-nginx:swarm .
Invoke-Checked docker push localhost:15000/hla-lab4-api:swarm
Invoke-Checked docker push localhost:15000/hla-lab4-nginx:swarm
Invoke-Checked docker exec --env "DATABASE_PASSWORD=$password" `
--env "DATABASE_NAME=$database" --env "DATABASE_USER=$user" `
--env "MINIO_ROOT_USER=$minioRootUser" --env "MINIO_ROOT_PASSWORD=$minioRootPassword" `
--env "S3_ACCESS_KEY_ID=$s3AccessKey" --env "S3_SECRET_ACCESS_KEY=$s3SecretKey" `
--env "S3_BUCKET=$s3Bucket" `
$manager docker stack deploy --compose-file /project/swarm/stack.state.yaml hla-state
Wait-Service 'hla-state_postgres' '1/1'
Wait-Service 'hla-state_redis' '1/1'
Wait-Service 'hla-state_minio' '1/1'
$deadline = (Get-Date).AddSeconds(180)
do {
$minioInitState = (& docker exec $manager docker service ps hla-state_minio-init `
--no-trunc --format '{{.CurrentState}}' | Select-Object -First 1)
if ($minioInitState -like 'Complete*') { break }
Start-Sleep -Seconds 2
} while ((Get-Date) -lt $deadline)
if ($minioInitState -notlike 'Complete*') { throw 'MinIO bucket bootstrap did not complete' }
Invoke-Checked docker exec $manager docker service create --name hla-migrate `
--mode replicated-job --replicas 1 --max-concurrent 1 `
--network hla-state_backend --constraint node.labels.hla.state==true `
--restart-condition on-failure --restart-delay 3s `
--limit-cpu 0.75 --limit-memory 768M `
--env "DATABASE_URL=jdbc:postgresql://hla-state_postgres:5432/$database" `
--env "DATABASE_USER=$user" --env "DATABASE_PASSWORD=$password" `
--env FLYWAY_ENABLED=true --env MIGRATE_ONLY=true `
--env SPRING_MAIN_WEB_APPLICATION_TYPE=none `
--env "S3_ENDPOINT=http://hla-state_minio:9000" `
--env "S3_BUCKET=$s3Bucket" --env "S3_REGION=$s3Region" `
--env "S3_ACCESS_KEY_ID=$s3AccessKey" --env "S3_SECRET_ACCESS_KEY=$s3SecretKey" `
hla-swarm-registry:5000/hla-lab4-api:swarm
$deadline = (Get-Date).AddSeconds(180)
do {
$state = (& docker exec $manager docker service ps hla-migrate `
--no-trunc --format '{{.CurrentState}}' | Select-Object -First 1)
if ($state -like 'Complete*') { break }
Start-Sleep -Seconds 2
} while ((Get-Date) -lt $deadline)
if ($state -notlike 'Complete*') { throw 'Migration job did not complete' }
Invoke-Checked docker exec $manager docker service rm hla-migrate
Invoke-Checked docker exec --env "DATABASE_PASSWORD=$password" `
--env "DATABASE_NAME=$database" --env "DATABASE_USER=$user" `
--env "S3_BUCKET=$s3Bucket" --env "S3_REGION=$s3Region" `
--env "S3_ACCESS_KEY_ID=$s3AccessKey" --env "S3_SECRET_ACCESS_KEY=$s3SecretKey" `
$manager docker stack deploy --compose-file /project/swarm/stack.app.yaml hla-app
Wait-Service 'hla-app_api' '2/2'
Wait-Service 'hla-app_nginx' '1/1'
Write-Output 'Swarm lab is ready at http://localhost:18080'
Invoke-Checked docker exec $manager docker node ls
Invoke-Checked docker exec $manager docker service ps hla-app_api
+1
View File
@@ -0,0 +1 @@
rootProject.name = 'mtusi-messenger'
@@ -0,0 +1,20 @@
package com.example.mtusi_messenger;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
public class MtusiMessengerApplication {
public static void main(String[] args) {
ConfigurableApplicationContext context = SpringApplication.run(
MtusiMessengerApplication.class, args);
if (Boolean.parseBoolean(System.getenv("MIGRATE_ONLY"))) {
System.exit(SpringApplication.exit(context));
}
}
}
@@ -0,0 +1,83 @@
package com.example.mtusi_messenger.cache;
import com.example.mtusi_messenger.model.User;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.util.Optional;
import java.util.UUID;
@Component
public class UserCache {
private static final Logger log = LoggerFactory.getLogger(UserCache.class);
private static final String KEY_PREFIX = "messenger:user:";
private final StringRedisTemplate redis;
private final ObjectMapper mapper;
private final Duration ttl;
public UserCache(
StringRedisTemplate redis,
ObjectMapper mapper,
@Value("${messenger.cache.user-ttl-seconds:300}") long ttlSeconds
) {
if (ttlSeconds <= 0) {
throw new IllegalArgumentException("User cache TTL must be positive");
}
this.redis = redis;
this.mapper = mapper;
this.ttl = Duration.ofSeconds(ttlSeconds);
}
public Optional<User> find(UUID userId) {
try {
String value = redis.opsForValue().get(key(userId));
if (value == null) {
return Optional.empty();
}
User cachedUser = mapper.readValue(value, User.class);
if (cachedUser == null) {
log.warn("Ignoring null Redis user cache entry for {}", userId);
evict(userId);
return Optional.empty();
}
return Optional.of(cachedUser);
} catch (DataAccessException e) {
log.warn("Redis user cache is unavailable; using PostgreSQL fallback");
return Optional.empty();
} catch (JsonProcessingException e) {
log.warn("Ignoring malformed Redis user cache entry for {}", userId);
evict(userId);
return Optional.empty();
}
}
public void put(User user) {
try {
redis.opsForValue().set(key(user.id()), mapper.writeValueAsString(user), ttl);
} catch (DataAccessException e) {
log.warn("Redis user cache is unavailable; user was not cached");
} catch (JsonProcessingException e) {
throw new IllegalStateException("Cannot serialize user cache entry", e);
}
}
public void evict(UUID userId) {
try {
redis.delete(key(userId));
} catch (DataAccessException e) {
log.warn("Redis user cache is unavailable; cache eviction was skipped");
}
}
private String key(UUID userId) {
return KEY_PREFIX + userId;
}
}
@@ -0,0 +1,28 @@
package com.example.mtusi_messenger.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.boot.jackson.autoconfigure.JsonMapperBuilderCustomizer;
import tools.jackson.databind.cfg.CoercionAction;
import tools.jackson.databind.cfg.CoercionInputShape;
import tools.jackson.databind.type.LogicalType;
@Configuration
public class JacksonConfig {
@Bean
JsonMapperBuilderCustomizer strictStringTypes() {
return builder -> builder.withCoercionConfig(
LogicalType.Textual,
config -> {
config.setCoercion(
CoercionInputShape.Integer, CoercionAction.Fail);
config.setCoercion(
CoercionInputShape.Float, CoercionAction.Fail);
config.setCoercion(
CoercionInputShape.Boolean, CoercionAction.Fail);
}
);
}
}
@@ -0,0 +1,21 @@
package com.example.mtusi_messenger.config;
import io.swagger.v3.oas.annotations.enums.SecuritySchemeType;
import io.swagger.v3.oas.annotations.security.SecurityScheme;
import org.springframework.context.annotation.Configuration;
@SecurityScheme(
name = "basicAuth",
type = SecuritySchemeType.HTTP,
scheme = "basic"
)
@SecurityScheme(
name = "bearerAuth",
type = SecuritySchemeType.HTTP,
scheme = "bearer",
bearerFormat = "opaque session token"
)
@Configuration
public class OpenApiConfig {
}
@@ -0,0 +1,14 @@
package com.example.mtusi_messenger.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.crypto.argon2.Argon2PasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
@Configuration
public class PasswordConfig {
@Bean
PasswordEncoder passwordEncoder() {
return Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8();
}
}
@@ -0,0 +1,57 @@
package com.example.mtusi_messenger.config;
import org.springframework.amqp.core.Binding;
import org.springframework.amqp.core.BindingBuilder;
import org.springframework.amqp.core.DirectExchange;
import org.springframework.amqp.core.Queue;
import org.springframework.amqp.core.QueueBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class RabbitJobConfig {
@Bean
DirectExchange attachmentExchange(@Value("${messenger.jobs.exchange}") String name) {
return new DirectExchange(name, true, false);
}
@Bean
Queue attachmentQueue(
@Value("${messenger.jobs.queue}") String name,
@Value("${messenger.jobs.dead-letter-exchange}") String deadLetterExchange,
@Value("${messenger.jobs.dead-letter-routing-key}") String deadLetterRoutingKey) {
return QueueBuilder.durable(name)
.withArgument("x-dead-letter-exchange", deadLetterExchange)
.withArgument("x-dead-letter-routing-key", deadLetterRoutingKey)
.build();
}
@Bean
DirectExchange attachmentDeadLetterExchange(
@Value("${messenger.jobs.dead-letter-exchange}") String name) {
return new DirectExchange(name, true, false);
}
@Bean
Queue attachmentDeadLetterQueue(@Value("${messenger.jobs.dead-letter-queue}") String name) {
return QueueBuilder.durable(name).build();
}
@Bean
Binding attachmentBinding(
@Qualifier("attachmentQueue") Queue attachmentQueue,
@Qualifier("attachmentExchange") DirectExchange attachmentExchange,
@Value("${messenger.jobs.routing-key}") String routingKey) {
return BindingBuilder.bind(attachmentQueue).to(attachmentExchange).with(routingKey);
}
@Bean
Binding attachmentDeadLetterBinding(
@Qualifier("attachmentDeadLetterQueue") Queue queue,
@Qualifier("attachmentDeadLetterExchange") DirectExchange exchange,
@Value("${messenger.jobs.dead-letter-routing-key}") String routingKey) {
return BindingBuilder.bind(queue).to(exchange).with(routingKey);
}
}
@@ -0,0 +1,107 @@
package com.example.mtusi_messenger.config;
import com.example.mtusi_messenger.security.ApiAuthenticationFilter;
import com.example.mtusi_messenger.security.DatabaseAuthenticationProvider;
import com.example.mtusi_messenger.security.RestAuthenticationEntryPoint;
import com.example.mtusi_messenger.service.AuthenticationService;
import com.example.mtusi_messenger.web.ReadinessGuardFilter;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.HttpMethod;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.ProviderManager;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
import org.springframework.security.config.http.SessionCreationPolicy;
import org.springframework.security.web.SecurityFilterChain;
import org.springframework.security.web.authentication.AnonymousAuthenticationFilter;
@Configuration
@ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET)
public class SecurityConfig {
@Bean
AuthenticationManager authenticationManager(DatabaseAuthenticationProvider provider) {
return new ProviderManager(provider);
}
@Bean
RestAuthenticationEntryPoint basicAuthenticationEntryPoint(ObjectMapper mapper) {
return new RestAuthenticationEntryPoint(
mapper,
"Basic realm=\"messenger\", charset=\"UTF-8\"");
}
@Bean
RestAuthenticationEntryPoint bearerAuthenticationEntryPoint(ObjectMapper mapper) {
return new RestAuthenticationEntryPoint(mapper, "Bearer realm=\"messenger\"");
}
@Bean
ApiAuthenticationFilter apiAuthenticationFilter(
AuthenticationManager authenticationManager,
AuthenticationService authenticationService,
@Qualifier("basicAuthenticationEntryPoint") RestAuthenticationEntryPoint basicAuthenticationEntryPoint,
@Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint
) {
return new ApiAuthenticationFilter(
authenticationManager,
authenticationService,
basicAuthenticationEntryPoint,
bearerAuthenticationEntryPoint);
}
@Bean
FilterRegistrationBean<ApiAuthenticationFilter> disableServletFilterRegistration(
ApiAuthenticationFilter filter
) {
var registration = new FilterRegistrationBean<>(filter);
registration.setEnabled(false);
return registration;
}
@Bean
FilterRegistrationBean<ReadinessGuardFilter> disableReadinessGuardServletRegistration(
ReadinessGuardFilter filter
) {
var registration = new FilterRegistrationBean<>(filter);
registration.setEnabled(false);
return registration;
}
@Bean
SecurityFilterChain securityFilterChain(
HttpSecurity http,
ApiAuthenticationFilter apiAuthenticationFilter,
ReadinessGuardFilter readinessGuardFilter,
@Qualifier("bearerAuthenticationEntryPoint") RestAuthenticationEntryPoint bearerAuthenticationEntryPoint
) throws Exception {
http
.csrf(csrf -> csrf.disable())
.formLogin(form -> form.disable())
.httpBasic(basic -> basic.disable())
.logout(logout -> logout.disable())
.requestCache(cache -> cache.disable())
.sessionManagement(session -> session
.sessionCreationPolicy(SessionCreationPolicy.STATELESS))
.exceptionHandling(errors -> errors
.authenticationEntryPoint(bearerAuthenticationEntryPoint))
.authorizeHttpRequests(authorize -> authorize
.requestMatchers(HttpMethod.GET, "/health/**").permitAll()
.requestMatchers(
"/swagger-ui.html",
"/swagger-ui/**",
"/v3/api-docs/**"
).permitAll()
.requestMatchers(HttpMethod.POST, "/api/v1/users").permitAll()
.anyRequest().authenticated())
// Refuse business traffic before authentication touches storage.
// Nginx retries this 503 on another peer and passively excludes
// the refusing replica according to max_fails/fail_timeout.
.addFilterBefore(readinessGuardFilter, AnonymousAuthenticationFilter.class)
.addFilterAfter(apiAuthenticationFilter, ReadinessGuardFilter.class);
return http.build();
}
}
@@ -0,0 +1,19 @@
package com.example.mtusi_messenger.config;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class WebConfig {
@Bean
ObjectMapper objectMapper() {
return JsonMapper.builder().findAndAddModules()
.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true)
.build();
}
}
@@ -0,0 +1,75 @@
package com.example.mtusi_messenger.controller;
import com.example.mtusi_messenger.dto.AttachmentResponse;
import com.example.mtusi_messenger.model.Attachment;
import com.example.mtusi_messenger.security.SessionPrincipal;
import com.example.mtusi_messenger.service.AttachmentService;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import org.springframework.http.ContentDisposition;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.UUID;
import static com.example.mtusi_messenger.service.UuidValidator.parse;
@RequiredArgsConstructor
@RestController
@RequestMapping("/api/v1")
@SecurityRequirement(name = "bearerAuth")
public class AttachmentController {
private final AttachmentService attachments;
@PostMapping(value = "/chats/{chatId}/attachments",
consumes = {"image/png", "image/jpeg"},
produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<AttachmentResponse> upload(
@AuthenticationPrincipal SessionPrincipal principal,
@PathVariable String chatId,
@RequestParam String filename,
@RequestHeader(HttpHeaders.CONTENT_TYPE) String contentType,
HttpServletRequest request
) throws IOException {
Attachment attachment = attachments.upload(parse(chatId), principal.user().id(), filename,
contentType, request.getContentLengthLong(), request.getInputStream());
return ResponseEntity.status(HttpStatus.ACCEPTED).body(AttachmentResponse.from(attachment));
}
@GetMapping("/attachments/{attachmentId}")
public AttachmentResponse metadata(
@AuthenticationPrincipal SessionPrincipal principal,
@PathVariable String attachmentId
) {
return AttachmentResponse.from(attachments.getAccessible(parse(attachmentId), principal.user().id()));
}
@GetMapping("/attachments/{attachmentId}/content")
public ResponseEntity<byte[]> download(
@AuthenticationPrincipal SessionPrincipal principal,
@PathVariable String attachmentId,
@RequestParam(defaultValue = "original") String variant
) {
AttachmentService.Download download = attachments.download(
parse(attachmentId), principal.user().id(), variant);
return ResponseEntity.ok()
.contentType(MediaType.parseMediaType(download.contentType()))
.contentLength(download.bytes().length)
.header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.inline()
.filename(download.filename(), StandardCharsets.UTF_8).build().toString())
.body(download.bytes());
}
}
@@ -0,0 +1,83 @@
package com.example.mtusi_messenger.controller;
import com.example.mtusi_messenger.exception.ApiExceptions;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.*;
import com.example.mtusi_messenger.dto.*;
import com.example.mtusi_messenger.model.Chat;
import com.example.mtusi_messenger.service.ChatService;
import com.example.mtusi_messenger.security.SessionPrincipal;
import java.util.List;
import java.util.UUID;
import static com.example.mtusi_messenger.service.UuidValidator.parse;
@RestController
@RequestMapping("/api/v1/chats")
@SecurityRequirement(name = "bearerAuth")
@RequiredArgsConstructor
public class ChatController {
private final ChatService chats;
@PostMapping
public ResponseEntity<ChatResponse> create(@AuthenticationPrincipal SessionPrincipal principal,
@Valid @RequestBody CreateChatRequest body
) {
UUID userId = principal.user().id();
Chat c = chats.create(body, userId);
return ResponseEntity.status(HttpStatus.CREATED).body(ChatResponse.from(c));
}
@GetMapping
public ChatPageResponse list(@AuthenticationPrincipal SessionPrincipal principal,
@RequestParam(required = false) String limit,
@RequestParam(required = false) String cursor
) {
UUID userId = principal.user().id();
int parsedLimit = validateLimit(limit);
validateCursorPresence(cursor);
List<Chat> page = chats.page(userId, parsedLimit, cursor);
boolean hasNext = page.size() > parsedLimit;
if (hasNext) page = page.subList(0, parsedLimit);
return new ChatPageResponse(page.stream().map(ChatResponse::from).toList(), page.isEmpty() ? null :
chats.nextCursor(userId, page.get(page.size() - 1), hasNext));
}
@GetMapping("/{chatId}")
public ChatResponse get(@AuthenticationPrincipal SessionPrincipal principal,
@PathVariable String chatId
) {
UUID userId = principal.user().id();
return ChatResponse.from(chats.getAccessible(parse(chatId), userId));
}
private static int validateLimit(String limit) {
if (limit == null) {
return 50;
}
if (!limit.matches("[0-9]+")) {
throw ApiExceptions.invalid("Invalid limit");
}
try {
int value = Integer.parseInt(limit);
if (value < 1 || value > 100) {
throw ApiExceptions.invalid("Invalid limit");
}
return value;
} catch (NumberFormatException ex) {
throw ApiExceptions.invalid("Invalid limit");
}
}
private static void validateCursorPresence(String c) {
if (c != null && c.isBlank()) throw ApiExceptions.invalid("Invalid cursor");
}
}
@@ -0,0 +1,32 @@
package com.example.mtusi_messenger.controller;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
import com.example.mtusi_messenger.dto.HealthResponse;
import com.example.mtusi_messenger.exception.ApiExceptions;
import com.example.mtusi_messenger.service.ReadinessService;
@RestController
public class HealthController {
private final ReadinessService readiness;
public HealthController(ReadinessService readiness) {
this.readiness = readiness;
}
@GetMapping("/health/live")
public ResponseEntity<HealthResponse> live() {
return ResponseEntity.ok(new HealthResponse("ok"));
}
@GetMapping("/health/ready")
public ResponseEntity<HealthResponse> ready() {
// Always probe dependencies, even after a failure, so a restored schema
// can move this process back to ACCEPTING_TRAFFIC without a restart.
if (!readiness.probeDependencies() || !readiness.acceptsBusinessTraffic()) {
throw ApiExceptions.unavailable("Application is not accepting traffic");
}
return ResponseEntity.ok(new HealthResponse("ok"));
}
}
@@ -0,0 +1,79 @@
package com.example.mtusi_messenger.controller;
import com.example.mtusi_messenger.exception.ApiExceptions;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.*;
import com.example.mtusi_messenger.dto.*;
import com.example.mtusi_messenger.model.Message;
import com.example.mtusi_messenger.service.MessageService;
import com.example.mtusi_messenger.security.SessionPrincipal;
import java.util.List;
import java.util.UUID;
import static com.example.mtusi_messenger.service.UuidValidator.parse;
@RequiredArgsConstructor
@RestController
@RequestMapping("/api/v1/chats/{chatId}/messages")
@SecurityRequirement(name = "bearerAuth")
public class MessageController {
private static final String IDEMPOTENCY_KEY_PATTERN = "[A-Za-z0-9._:-]{1,128}";
private final MessageService messages;
@PostMapping
public ResponseEntity<MessageResponse> create(@AuthenticationPrincipal SessionPrincipal principal,
@PathVariable String chatId,
@RequestHeader(name = "Idempotency-Key", required = false) String idempotencyKey,
@Valid @RequestBody CreateMessageRequest body
) {
if (idempotencyKey == null || !idempotencyKey.matches(IDEMPOTENCY_KEY_PATTERN)) {
throw ApiExceptions.invalid("Invalid or missing Idempotency-Key");
}
UUID userId = principal.user().id();
UUID parsedChatId = parse(chatId);
Message m = messages.create(parsedChatId, userId, body, idempotencyKey);
return ResponseEntity.status(HttpStatus.CREATED).body(MessageResponse.from(m));
}
@GetMapping
public MessagePageResponse list(@AuthenticationPrincipal SessionPrincipal principal,
@PathVariable String chatId,
@RequestParam(required = false) String limit,
@RequestParam(required=false) String cursor
) {
UUID userId = principal.user().id();
UUID parsedChatId = parse(chatId);
int parsedLimit = validateLimit(limit);
if (cursor != null && cursor.isBlank()) throw ApiExceptions.invalid("Invalid cursor");
List<Message> page = messages.page(parsedChatId, userId, parsedLimit, cursor);
boolean hasNext = page.size() > parsedLimit; if (hasNext) page = page.subList(0, parsedLimit);
return new MessagePageResponse(page.stream().map(MessageResponse::from).toList(),
page.isEmpty() ? null : messages.nextCursor(parsedChatId, page.get(page.size()-1), hasNext));
}
private static int validateLimit(String limit) {
if (limit == null) {
return 50;
}
if (!limit.matches("[0-9]+")) {
throw ApiExceptions.invalid("Invalid limit");
}
try {
int value = Integer.parseInt(limit);
if (value < 1 || value > 100) {
throw ApiExceptions.invalid("Invalid limit");
}
return value;
} catch (NumberFormatException ex) {
throw ApiExceptions.invalid("Invalid limit");
}
}
}
@@ -0,0 +1,41 @@
package com.example.mtusi_messenger.controller;
import com.example.mtusi_messenger.dto.SessionResponse;
import com.example.mtusi_messenger.dto.UserResponse;
import com.example.mtusi_messenger.service.AuthenticationService;
import com.example.mtusi_messenger.model.User;
import com.example.mtusi_messenger.security.SessionPrincipal;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequestMapping("/api/v1/auth/sessions")
@RequiredArgsConstructor
public class SessionController {
private final AuthenticationService authentication;
@PostMapping
@SecurityRequirement(name = "basicAuth")
public ResponseEntity<SessionResponse> create(@AuthenticationPrincipal User user) {
var issued = authentication.issueSession(user);
return ResponseEntity.status(HttpStatus.CREATED).body(new SessionResponse(
issued.token(),
"Bearer",
issued.expiresAt(),
UserResponse.from(issued.user())));
}
@DeleteMapping("/current")
@SecurityRequirement(name = "bearerAuth")
public ResponseEntity<Void> revoke(@AuthenticationPrincipal SessionPrincipal principal) {
authentication.revoke(principal);
return ResponseEntity.noContent().build();
}
}
@@ -0,0 +1,32 @@
package com.example.mtusi_messenger.controller;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.security.core.annotation.AuthenticationPrincipal;
import org.springframework.web.bind.annotation.*;
import com.example.mtusi_messenger.dto.*;
import com.example.mtusi_messenger.service.UserService;
import com.example.mtusi_messenger.security.SessionPrincipal;
@RestController
@RequestMapping("/api/v1/users")
@RequiredArgsConstructor
public class UserController {
private final UserService users;
@PostMapping
public ResponseEntity<UserResponse> create(@Valid @RequestBody CreateUserRequest request) {
var user = users.create(request);
return ResponseEntity.status(HttpStatus.CREATED).body(UserResponse.from(user));
}
@GetMapping("/me")
@SecurityRequirement(name = "bearerAuth")
public UserResponse me(@AuthenticationPrincipal SessionPrincipal principal) {
return UserResponse.from(principal.user());
}
}
@@ -0,0 +1,36 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.model.Attachment;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
public record AttachmentResponse(UUID id,
UUID chat_id,
UUID owner_id,
String filename,
String content_type,
long size_bytes,
String sha256,
String status,
List<AttachmentVariantResponse> variants,
String error_code,
Instant created_at,
Instant updated_at) {
public static AttachmentResponse from(Attachment attachment) {
return new AttachmentResponse(
attachment.id(),
attachment.chatId(),
attachment.ownerId(),
attachment.filename(),
attachment.contentType(),
attachment.sizeBytes(),
attachment.sha256(),
attachment.status(),
attachment.variants().stream().map(AttachmentVariantResponse::from).toList(),
attachment.errorCode(),
attachment.createdAt(),
attachment.updatedAt());
}
}
@@ -0,0 +1,16 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.model.AttachmentVariant;
public record AttachmentVariantResponse(String name,
String content_type,
long size_bytes,
int width,
int height,
String sha256) {
public static AttachmentVariantResponse from(AttachmentVariant variant) {
return new AttachmentVariantResponse(
variant.name(), variant.contentType(), variant.sizeBytes(),
variant.width(), variant.height(), variant.sha256());
}
}
@@ -0,0 +1,8 @@
package com.example.mtusi_messenger.dto;
import java.util.List;
public record ChatPageResponse(
List<ChatResponse> items,
String next_cursor
) {}
@@ -0,0 +1,18 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.model.Chat;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
public record ChatResponse(UUID id,
String title,
List<UUID> member_ids,
Instant created_at) {
public static ChatResponse from(Chat c) {
return new ChatResponse(
c.id(),
c.title(),
c.memberIds(),
c.createdAt()); }
}
@@ -0,0 +1,19 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.web.NotUnicodeBlank;
import com.example.mtusi_messenger.web.UnicodeSize;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import java.util.List;
import java.util.UUID;
public record CreateChatRequest(
@NotUnicodeBlank
@UnicodeSize(min = 1, max = 100)
String title,
@NotEmpty
@Size(min = 1, max = 100)
List<UUID> member_ids
) {}
@@ -0,0 +1,31 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.web.UnicodeSize;
import jakarta.validation.constraints.AssertTrue;
import jakarta.validation.constraints.NotNull;
import jakarta.validation.constraints.Size;
import java.util.HashSet;
import java.util.List;
import java.util.UUID;
public record CreateMessageRequest(
@NotNull @UnicodeSize(max = 4000) String text,
@Size(max = 10) List<UUID> attachment_ids
) {
public CreateMessageRequest {
attachment_ids = attachment_ids == null ? List.of() : List.copyOf(attachment_ids);
}
@AssertTrue(message = "text or attachment_ids must be present")
public boolean isContentPresent() {
return text != null && (!text.isBlank() || !attachment_ids.isEmpty());
}
@AssertTrue(message = "attachment_ids must be unique and non-null")
public boolean isAttachmentIdsValid() {
return attachment_ids != null
&& attachment_ids.stream().noneMatch(java.util.Objects::isNull)
&& new HashSet<>(attachment_ids).size() == attachment_ids.size();
}
}
@@ -0,0 +1,20 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.web.NotUnicodeBlank;
import com.example.mtusi_messenger.web.UnicodeSize;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Pattern;
public record CreateUserRequest(
@NotBlank
@Pattern(regexp = "[a-z0-9_]{3,32}")
String username,
@NotUnicodeBlank
@UnicodeSize(min = 1, max = 100)
String display_name,
@NotUnicodeBlank
@UnicodeSize(min = 12, max = 128)
String password
) {}
@@ -0,0 +1,8 @@
package com.example.mtusi_messenger.dto;
public record ErrorResponse(ErrorBody error) {
public record ErrorBody(String code,
String message,
String request_id)
{}
}
@@ -0,0 +1,3 @@
package com.example.mtusi_messenger.dto;
public record HealthResponse(String status) {}
@@ -0,0 +1,8 @@
package com.example.mtusi_messenger.dto;
import java.util.List;
public record MessagePageResponse(
List<MessageResponse> items,
String next_cursor
) {}
@@ -0,0 +1,22 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.model.Message;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
public record MessageResponse(UUID id,
UUID chat_id,
UUID sender_id,
String text,
Instant created_at,
List<UUID> attachment_ids) {
public static MessageResponse from(Message m) {
return new MessageResponse(m.id(),
m.chatId(),
m.senderId(),
m.text(),
m.createdAt(),
m.attachmentIds());
}
}
@@ -0,0 +1,10 @@
package com.example.mtusi_messenger.dto;
import java.time.Instant;
public record SessionResponse(
String token,
String token_type,
Instant expires_at,
UserResponse user
) {}
@@ -0,0 +1,18 @@
package com.example.mtusi_messenger.dto;
import com.example.mtusi_messenger.model.User;
import java.time.Instant;
import java.util.UUID;
public record UserResponse(UUID id,
String username,
String display_name,
Instant created_at) {
public static UserResponse from(User u) {
return new UserResponse(
u.id(),
u.username(),
u.displayName(),
u.createdAt());
}
}
@@ -0,0 +1,14 @@
package com.example.mtusi_messenger.exception;
import org.springframework.http.HttpStatus;
public class ApiException extends RuntimeException {
private final String code; private final HttpStatus status; private final String authenticate;
public ApiException(String code, HttpStatus status, String message) { this(code, status, message, null); }
public ApiException(String code, HttpStatus status, String message, String authenticate) {
super(message); this.code = code; this.status = status; this.authenticate = authenticate;
}
public String code() { return code; }
public HttpStatus status() { return status; }
public String authenticate() { return authenticate; }
}
@@ -0,0 +1,17 @@
package com.example.mtusi_messenger.exception;
import org.springframework.http.HttpStatus;
public final class ApiExceptions {
private ApiExceptions() {}
public static ApiException invalid(String m) { return new ApiException("invalid_request", HttpStatus.BAD_REQUEST, m); }
public static ApiException unauthorized(String m) { return unauthorizedBearer(m); }
public static ApiException unauthorizedBasic(String m) { return new ApiException("unauthorized", HttpStatus.UNAUTHORIZED, m, "Basic realm=\"messenger\", charset=\"UTF-8\""); }
public static ApiException unauthorizedBearer(String m) { return new ApiException("unauthorized", HttpStatus.UNAUTHORIZED, m, "Bearer realm=\"messenger\""); }
public static ApiException notFound(String m) { return new ApiException("not_found", HttpStatus.NOT_FOUND, m); }
public static ApiException conflict(String m) { return new ApiException("conflict", HttpStatus.CONFLICT, m); }
public static ApiException payloadTooLarge(String m) { return new ApiException("payload_too_large", HttpStatus.PAYLOAD_TOO_LARGE, m); }
public static ApiException unsupportedMediaType(String m) { return new ApiException("unsupported_media_type", HttpStatus.UNSUPPORTED_MEDIA_TYPE, m); }
public static ApiException tooManyRequests(String m) { return new ApiException("too_many_requests", HttpStatus.TOO_MANY_REQUESTS, m); }
public static ApiException unavailable(String m) { return new ApiException("unavailable", HttpStatus.SERVICE_UNAVAILABLE, m); }
}
@@ -0,0 +1,38 @@
package com.example.mtusi_messenger.model;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
public record Attachment(UUID id,
UUID chatId,
UUID ownerId,
String filename,
String contentType,
long sizeBytes,
String sha256,
String objectKey,
String stagingKey,
String status,
List<AttachmentVariant> variants,
String errorCode,
Instant createdAt,
Instant updatedAt) {
public Attachment {
variants = variants == null ? List.of() : List.copyOf(variants);
}
public Attachment(UUID id,
UUID chatId,
UUID ownerId,
String filename,
String contentType,
long sizeBytes,
String sha256,
String objectKey,
Instant createdAt,
Instant updatedAt) {
this(id, chatId, ownerId, filename, contentType, sizeBytes, sha256, objectKey,
null, "ready", List.of(), null, createdAt, updatedAt);
}
}
@@ -0,0 +1,40 @@
package com.example.mtusi_messenger.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.Instant;
import java.util.UUID;
public record AttachmentJob(
@JsonProperty("schema_version") int schemaVersion,
@JsonProperty("job_id") UUID jobId,
String type,
@JsonProperty("attachment_id") UUID attachmentId,
@JsonProperty("pipeline_version") int pipelineVersion,
int attempt,
@JsonProperty("created_at") Instant createdAt) {
public static final int SCHEMA_VERSION = 1;
public static final int MIN_PIPELINE_VERSION = 1;
public static final int CURRENT_PIPELINE_VERSION = 2;
public static final String TYPE = "attachment.process.v1";
public AttachmentJob {
if (schemaVersion != SCHEMA_VERSION || !TYPE.equals(type)
|| pipelineVersion < MIN_PIPELINE_VERSION
|| pipelineVersion > CURRENT_PIPELINE_VERSION || attempt < 1) {
throw new IllegalArgumentException("Unsupported attachment job message");
}
}
public static AttachmentJob first(
UUID jobId, UUID attachmentId, int pipelineVersion, Instant createdAt) {
return new AttachmentJob(SCHEMA_VERSION, jobId, TYPE, attachmentId,
pipelineVersion, 1, createdAt);
}
public AttachmentJob nextAttempt(int nextAttempt) {
return new AttachmentJob(schemaVersion, jobId, type, attachmentId,
pipelineVersion, nextAttempt, createdAt);
}
}
@@ -0,0 +1,10 @@
package com.example.mtusi_messenger.model;
public record AttachmentVariant(String name,
String contentType,
long sizeBytes,
int width,
int height,
String sha256,
String objectKey) {
}
@@ -0,0 +1,11 @@
package com.example.mtusi_messenger.model;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
public record Chat(UUID id, String title, List<UUID> memberIds, Instant createdAt) {
public Chat {
memberIds = List.copyOf(memberIds);
}
}
@@ -0,0 +1,3 @@
package com.example.mtusi_messenger.model;
public record IdempotentMessage(byte[] requestHash, Message message) {}
@@ -0,0 +1,16 @@
package com.example.mtusi_messenger.model;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
public record Message(UUID id,
UUID chatId,
UUID senderId,
String text,
Instant createdAt,
List<UUID> attachmentIds) {
public Message {
attachmentIds = attachmentIds == null ? List.of() : List.copyOf(attachmentIds);
}
}
@@ -0,0 +1,35 @@
package com.example.mtusi_messenger.model;
import java.time.Instant;
import java.util.UUID;
public record MessageKey(Instant createdAt, UUID id)
implements Comparable<MessageKey> {
private static int compareUuidV7(UUID a, UUID b) {
int result = Long.compareUnsigned(
a.getMostSignificantBits(),
b.getMostSignificantBits()
);
if (result != 0) {
return result;
}
return Long.compareUnsigned(
a.getLeastSignificantBits(),
b.getLeastSignificantBits()
);
}
@Override
public int compareTo(MessageKey other) {
int result = createdAt.compareTo(other.createdAt);
if (result != 0) {
return result;
}
return compareUuidV7(id, other.id);
}
}
@@ -0,0 +1,15 @@
package com.example.mtusi_messenger.model;
import java.time.Instant;
import java.util.UUID;
public record Session(UUID id, UUID userId, byte[] tokenHash, Instant createdAt, Instant expiresAt) {
public Session {
tokenHash = tokenHash.clone();
}
@Override
public byte[] tokenHash() {
return tokenHash.clone();
}
}
@@ -0,0 +1,6 @@
package com.example.mtusi_messenger.model;
import java.time.Instant;
import java.util.UUID;
public record User(UUID id, String username, String displayName, Instant createdAt) {}
@@ -0,0 +1,65 @@
package com.example.mtusi_messenger.queue;
import com.example.mtusi_messenger.model.AttachmentJob;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.amqp.core.Message;
import org.springframework.amqp.core.MessageDeliveryMode;
import org.springframework.amqp.core.MessageProperties;
import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.time.Duration;
@Component
public class AttachmentJobPublisher {
private final RabbitTemplate rabbit;
private final ObjectMapper mapper;
private final String exchange;
private final String routingKey;
private final long confirmTimeoutMillis;
public AttachmentJobPublisher(
RabbitTemplate rabbit,
ObjectMapper mapper,
@Value("${messenger.jobs.exchange}") String exchange,
@Value("${messenger.jobs.routing-key}") String routingKey,
@Value("${messenger.jobs.publish-confirm-timeout:5s}") Duration confirmTimeout) {
this.rabbit = rabbit;
this.mapper = mapper;
this.exchange = exchange;
this.routingKey = routingKey;
this.confirmTimeoutMillis = confirmTimeout.toMillis();
this.rabbit.setMandatory(true);
}
public void publish(AttachmentJob job) {
try {
byte[] body = mapper.writeValueAsBytes(job);
MessageProperties properties = new MessageProperties();
properties.setContentType(MessageProperties.CONTENT_TYPE_JSON);
properties.setContentEncoding("UTF-8");
properties.setDeliveryMode(MessageDeliveryMode.PERSISTENT);
properties.setMessageId(job.jobId().toString());
properties.setHeader("schema_version", job.schemaVersion());
Message message = new Message(body, properties);
rabbit.invoke(operations -> {
operations.send(exchange, routingKey, message);
operations.waitForConfirmsOrDie(confirmTimeoutMillis);
return null;
});
} catch (JsonProcessingException error) {
throw new JobPublishException("Cannot encode attachment job", error);
} catch (RuntimeException error) {
throw new JobPublishException("RabbitMQ did not confirm attachment job", error);
}
}
public static final class JobPublishException extends RuntimeException {
public JobPublishException(String message, Throwable cause) {
super(message, cause);
}
}
}
@@ -0,0 +1,57 @@
package com.example.mtusi_messenger.queue;
import com.example.mtusi_messenger.repository.AttachmentRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.support.TransactionTemplate;
import java.time.Clock;
@Component
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
public class ExpiredLeaseReaper {
private static final Logger log = LoggerFactory.getLogger(ExpiredLeaseReaper.class);
private final AttachmentRepository attachments;
private final RetryPolicy retryPolicy;
private final TransactionTemplate transaction;
private final int maxAttempts;
private final Clock clock = Clock.systemUTC();
public ExpiredLeaseReaper(
AttachmentRepository attachments,
RetryPolicy retryPolicy,
TransactionTemplate transaction,
@Value("${messenger.jobs.max-attempts:5}") int maxAttempts) {
this.attachments = attachments;
this.retryPolicy = retryPolicy;
this.transaction = transaction;
this.maxAttempts = maxAttempts;
}
@Scheduled(fixedDelayString = "${messenger.jobs.lease-reaper-interval:5s}")
public void recoverExpired() {
for (var job : attachments.findExpiredJobs(50)) {
try {
transaction.executeWithoutResult(status -> {
if (job.attempt() >= maxAttempts) {
attachments.failExpired(job.jobId(), job.attachmentId(),
job.pipelineVersion(), "retry_exhausted");
} else {
int nextAttempt = job.attempt() + 1;
attachments.requeueExpired(
job.jobId(), job.attachmentId(), job.pipelineVersion(),
nextAttempt, "processing_timeout",
clock.instant().plus(retryPolicy.delayForAttempt(nextAttempt)));
}
});
} catch (RuntimeException error) {
log.warn("Expired attachment lease recovery postponed for job {}", job.jobId());
return;
}
}
}
}
@@ -0,0 +1,39 @@
package com.example.mtusi_messenger.queue;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.util.concurrent.ThreadLocalRandom;
@Component
public class RetryPolicy {
private final long baseDelayMillis;
private final long maxDelayMillis;
public RetryPolicy(
@Value("${messenger.jobs.retry-base-delay:1s}") Duration baseDelay,
@Value("${messenger.jobs.retry-max-delay:30s}") Duration maxDelay) {
if (baseDelay.isNegative() || baseDelay.isZero() || maxDelay.compareTo(baseDelay) < 0) {
throw new IllegalArgumentException("Invalid retry delay configuration");
}
this.baseDelayMillis = baseDelay.toMillis();
this.maxDelayMillis = maxDelay.toMillis();
}
public Duration delayForAttempt(int nextAttempt) {
int exponent = Math.max(0, Math.min(30, nextAttempt - 2));
long exponential;
try {
exponential = Math.multiplyExact(baseDelayMillis, 1L << exponent);
} catch (ArithmeticException overflow) {
exponential = maxDelayMillis;
}
long ceiling = Math.min(maxDelayMillis, exponential);
long floor = Math.max(1, ceiling / 2);
long jittered = floor == ceiling
? ceiling
: ThreadLocalRandom.current().nextLong(floor, ceiling + 1);
return Duration.ofMillis(jittered);
}
}
@@ -0,0 +1,34 @@
package com.example.mtusi_messenger.queue;
import com.example.mtusi_messenger.repository.AttachmentRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Component
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
public class UnpublishedJobRepublisher {
private static final Logger log = LoggerFactory.getLogger(UnpublishedJobRepublisher.class);
private final AttachmentRepository attachments;
private final AttachmentJobPublisher publisher;
public UnpublishedJobRepublisher(AttachmentRepository attachments, AttachmentJobPublisher publisher) {
this.attachments = attachments;
this.publisher = publisher;
}
@Scheduled(fixedDelayString = "${messenger.jobs.republish-interval:5s}")
public void publishPending() {
for (var job : attachments.findUnpublishedJobs(50)) {
try {
publisher.publish(job);
attachments.markPublished(job.jobId());
} catch (RuntimeException error) {
log.warn("Attachment job publication postponed for job {}", job.jobId());
return;
}
}
}
}
@@ -0,0 +1,41 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Attachment;
import com.example.mtusi_messenger.model.AttachmentJob;
import com.example.mtusi_messenger.model.AttachmentVariant;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.Set;
import java.time.Duration;
import java.time.Instant;
public interface AttachmentRepository {
Attachment saveQueued(Attachment attachment, AttachmentJob job);
Optional<Attachment> findById(UUID id);
List<Attachment> findAllByIdsInChat(UUID chatId, List<UUID> ids);
Set<String> findAllObjectKeys();
Set<String> findActiveStagingKeys();
void markPublished(UUID jobId);
boolean markPublishFaultInjected(UUID jobId);
List<AttachmentJob> findUnpublishedJobs(int limit);
int currentAttempt(UUID jobId);
boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion,
int attempt, UUID leaseToken, Duration leaseDuration);
boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken);
boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken);
boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion, UUID leaseToken,
int nextAttempt, String errorCode, Instant nextAttemptAt);
CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion,
List<AttachmentVariant> variants, boolean injectCrashAfterCommit);
boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion,
UUID leaseToken, String errorCode);
List<AttachmentJob> findExpiredJobs(int limit);
boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion, int nextAttempt,
String errorCode, Instant nextAttemptAt);
boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion, String errorCode);
record CompletionResult(boolean completed, boolean crashAfterCommit) {
}
}
@@ -0,0 +1,14 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Chat;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
public interface ChatRepository {
Chat save(Chat chat);
Optional<Chat> findById(UUID id);
boolean isMember(UUID chatId, UUID userId);
List<Chat> findAllForUser(UUID userId);
List<Chat> findPageForUser(UUID userId, UUID afterId, java.time.Instant afterCreatedAt, int limitPlusOne);
}
@@ -0,0 +1,18 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Message;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
public interface MessageRepository {
Message save(Message message);
void lockIdempotencyScope(UUID userId, String method, String path, String key);
void deleteExpiredIdempotency(UUID userId, String method, String path, String key, Instant expiresBefore);
Optional<com.example.mtusi_messenger.model.IdempotentMessage> findIdempotent(
UUID userId, String method, String path, String key);
void saveIdempotency(UUID userId, String method, String path, String key,
byte[] requestHash, UUID messageId, Instant createdAt);
List<Message> findPage(UUID chatId, Instant afterCreatedAt, UUID afterId, int limitPlusOne);
}
@@ -0,0 +1,382 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Attachment;
import com.example.mtusi_messenger.model.AttachmentJob;
import com.example.mtusi_messenger.model.AttachmentVariant;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@Repository
public class PostgresAttachmentRepository implements AttachmentRepository {
private static final String COLUMNS = """
id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, staging_key, status, error_code, created_at, updated_at
""";
private final JdbcTemplate jdbc;
public PostgresAttachmentRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc;
}
@Override
public Attachment saveQueued(Attachment attachment, AttachmentJob job) {
jdbc.update("""
INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, size_bytes,
sha256, object_key, staging_key, pipeline_version,
status, error_code, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', NULL, ?, ?)
""", attachment.id(), attachment.chatId(), attachment.ownerId(),
attachment.filename(), attachment.contentType(), attachment.sizeBytes(),
attachment.sha256(), attachment.objectKey(), attachment.stagingKey(),
job.pipelineVersion(),
Timestamp.from(attachment.createdAt()), Timestamp.from(attachment.updatedAt()));
jdbc.update("""
INSERT INTO attachment_jobs
(id, attachment_id, pipeline_version, attempt, status, published, created_at, updated_at)
VALUES (?, ?, ?, ?, 'queued', false, ?, ?)
""", job.jobId(), job.attachmentId(), job.pipelineVersion(), job.attempt(),
Timestamp.from(job.createdAt()), Timestamp.from(job.createdAt()));
return attachment;
}
@Override
public Optional<Attachment> findById(UUID id) {
return jdbc.query("SELECT " + COLUMNS + " FROM attachments WHERE id = ?",
(rs, rowNum) -> map(rs), id).stream().findFirst();
}
@Override
public List<Attachment> findAllByIdsInChat(UUID chatId, List<UUID> ids) {
if (ids.isEmpty()) {
return List.of();
}
String placeholders = String.join(",", Collections.nCopies(ids.size(), "?"));
Object[] args = new Object[ids.size() + 1];
args[0] = chatId;
for (int i = 0; i < ids.size(); i++) {
args[i + 1] = ids.get(i);
}
return jdbc.query("SELECT " + COLUMNS + " FROM attachments WHERE chat_id = ? AND id IN ("
+ placeholders + ") AND status <> 'failed'",
(rs, rowNum) -> map(rs), args);
}
@Override
public Set<String> findAllObjectKeys() {
Set<String> keys = new HashSet<>(jdbc.query(
"SELECT object_key FROM attachments WHERE status = 'ready'",
(rs, rowNum) -> rs.getString("object_key")));
keys.addAll(jdbc.query("SELECT object_key FROM attachment_variants",
(rs, rowNum) -> rs.getString("object_key")));
return keys;
}
@Override
public Set<String> findActiveStagingKeys() {
return new HashSet<>(jdbc.query("""
SELECT staging_key
FROM attachments
WHERE status IN ('queued', 'processing') AND staging_key IS NOT NULL
""", (rs, rowNum) -> rs.getString("staging_key")));
}
@Override
public void markPublished(UUID jobId) {
jdbc.update("""
UPDATE attachment_jobs
SET published = true, updated_at = clock_timestamp()
WHERE id = ? AND status = 'queued'
""", jobId);
}
@Override
public boolean markPublishFaultInjected(UUID jobId) {
return jdbc.update("""
UPDATE attachment_jobs
SET publish_fault_injected = true, updated_at = clock_timestamp()
WHERE id = ? AND publish_fault_injected = false AND status = 'queued'
""", jobId) == 1;
}
@Override
public List<AttachmentJob> findUnpublishedJobs(int limit) {
return jdbc.query("""
SELECT id, attachment_id, pipeline_version, attempt, created_at
FROM attachment_jobs
WHERE status = 'queued' AND published = false
AND next_attempt_at <= clock_timestamp()
ORDER BY next_attempt_at, created_at, id
LIMIT ?
""", (rs, rowNum) -> mapJob(rs), limit);
}
@Override
public int currentAttempt(UUID jobId) {
Integer attempt = jdbc.query("SELECT attempt FROM attachment_jobs WHERE id = ?",
(rs, rowNum) -> rs.getInt("attempt"), jobId).stream().findFirst().orElse(null);
return attempt == null ? 1 : attempt;
}
@Override
public boolean claim(UUID jobId, UUID attachmentId, int pipelineVersion, int attempt,
UUID leaseToken, Duration leaseDuration) {
int claimed = jdbc.update("""
UPDATE attachment_jobs
SET status = 'processing', attempt = GREATEST(attempt, ?),
lease_token = ?,
lease_until = clock_timestamp() + (? * interval '1 millisecond'),
updated_at = clock_timestamp()
WHERE id = ? AND attachment_id = ?
AND pipeline_version = ?
AND (status = 'queued'
OR (status = 'processing' AND lease_until < clock_timestamp()))
AND EXISTS (
SELECT 1 FROM attachments a
WHERE a.id = ? AND a.pipeline_version = ?
AND a.status IN ('queued', 'processing'))
""", attempt, leaseToken, leaseDuration.toMillis(), jobId, attachmentId,
pipelineVersion, attachmentId, pipelineVersion);
if (claimed == 0) {
return false;
}
jdbc.update("""
UPDATE attachments
SET status = 'processing', error_code = NULL, updated_at = clock_timestamp()
WHERE id = ? AND pipeline_version = ?
AND status IN ('queued', 'processing')
""", attachmentId, pipelineVersion);
return true;
}
@Override
public boolean markBeforeEffectFaultInjected(UUID jobId, UUID leaseToken) {
return jdbc.update("""
UPDATE attachment_jobs
SET before_effect_fault_injected = true, updated_at = clock_timestamp()
WHERE id = ? AND status = 'processing' AND lease_token = ?
AND before_effect_fault_injected = false
""", jobId, leaseToken) == 1;
}
@Override
public boolean markAfterEffectFaultInjected(UUID jobId, UUID leaseToken) {
return jdbc.update("""
UPDATE attachment_jobs
SET after_effect_fault_injected = true, updated_at = clock_timestamp()
WHERE id = ? AND status = 'processing' AND lease_token = ?
AND after_effect_fault_injected = false
""", jobId, leaseToken) == 1;
}
@Override
public boolean markQueuedForRetry(UUID jobId, UUID attachmentId, int pipelineVersion,
UUID leaseToken,
int nextAttempt, String errorCode, Instant nextAttemptAt) {
int updated = jdbc.update("""
UPDATE attachment_jobs
SET status = 'queued', attempt = ?, published = false,
last_error_code = ?, next_attempt_at = ?,
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
WHERE id = ? AND attachment_id = ? AND status = 'processing'
AND pipeline_version = ? AND lease_token = ?
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt),
jobId, attachmentId, pipelineVersion, leaseToken);
if (updated == 0) {
return false;
}
jdbc.update("""
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = ? AND pipeline_version = ?
AND status IN ('queued', 'processing')
""", attachmentId, pipelineVersion);
return true;
}
@Override
public CompletionResult markReady(UUID jobId, UUID attachmentId, UUID leaseToken, int pipelineVersion,
List<AttachmentVariant> variants, boolean injectCrashAfterCommit) {
boolean triggerCrash = injectCrashAfterCommit && Boolean.TRUE.equals(jdbc.query("""
SELECT NOT result_fault_injected
FROM attachment_jobs
WHERE id = ? AND attachment_id = ? AND status = 'processing' AND lease_token = ?
FOR UPDATE
""", (rs, rowNum) -> rs.getBoolean(1), jobId, attachmentId, leaseToken)
.stream().findFirst().orElse(false));
int completed = jdbc.update("""
UPDATE attachment_jobs
SET status = 'ready', published = true, last_error_code = NULL,
lease_token = NULL, lease_until = NULL,
result_fault_injected = result_fault_injected OR ?,
updated_at = clock_timestamp()
WHERE id = ? AND attachment_id = ? AND status = 'processing'
AND pipeline_version = ? AND lease_token = ?
AND EXISTS (
SELECT 1 FROM attachments a
WHERE a.id = ? AND a.pipeline_version = ?
AND a.status = 'processing')
""", triggerCrash, jobId, attachmentId, pipelineVersion, leaseToken,
attachmentId, pipelineVersion);
if (completed == 0) {
return new CompletionResult(false, false);
}
for (AttachmentVariant variant : variants) {
jdbc.update("""
INSERT INTO attachment_variants
(attachment_id, pipeline_version, name, content_type, size_bytes,
width, height, sha256, object_key, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, clock_timestamp())
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
""", attachmentId, pipelineVersion, variant.name(), variant.contentType(),
variant.sizeBytes(), variant.width(), variant.height(), variant.sha256(),
variant.objectKey());
}
jdbc.update("""
UPDATE attachments
SET status = 'ready', error_code = NULL, updated_at = clock_timestamp()
WHERE id = ? AND pipeline_version = ? AND status = 'processing'
""", attachmentId, pipelineVersion);
return new CompletionResult(true, triggerCrash);
}
@Override
public boolean markFailed(UUID jobId, UUID attachmentId, int pipelineVersion,
UUID leaseToken, String errorCode) {
int updated = jdbc.update("""
UPDATE attachment_jobs
SET status = 'failed', published = true, last_error_code = ?,
lease_token = NULL, lease_until = NULL,
updated_at = clock_timestamp()
WHERE id = ? AND attachment_id = ? AND status = 'processing'
AND pipeline_version = ? AND lease_token = ?
AND EXISTS (SELECT 1 FROM attachments a
WHERE a.id = ? AND a.pipeline_version = ?)
""", errorCode, jobId, attachmentId, pipelineVersion, leaseToken,
attachmentId, pipelineVersion);
if (updated == 0) {
return false;
}
jdbc.update("""
UPDATE attachments
SET status = 'failed', error_code = ?, updated_at = clock_timestamp()
WHERE id = ? AND pipeline_version = ? AND status <> 'ready'
""", errorCode, attachmentId, pipelineVersion);
return true;
}
@Override
public List<AttachmentJob> findExpiredJobs(int limit) {
return jdbc.query("""
SELECT id, attachment_id, pipeline_version, attempt, created_at
FROM attachment_jobs
WHERE status = 'processing' AND lease_until < clock_timestamp()
ORDER BY lease_until, id
LIMIT ?
""", (rs, rowNum) -> mapJob(rs), limit);
}
@Override
public boolean requeueExpired(UUID jobId, UUID attachmentId, int pipelineVersion,
int nextAttempt,
String errorCode, Instant nextAttemptAt) {
int updated = jdbc.update("""
UPDATE attachment_jobs
SET status = 'queued', attempt = ?, published = false,
last_error_code = ?, next_attempt_at = ?,
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
WHERE id = ? AND attachment_id = ? AND status = 'processing'
AND pipeline_version = ? AND lease_until < clock_timestamp()
AND EXISTS (SELECT 1 FROM attachments a
WHERE a.id = ? AND a.pipeline_version = ?)
""", nextAttempt, errorCode, Timestamp.from(nextAttemptAt), jobId, attachmentId,
pipelineVersion, attachmentId, pipelineVersion);
if (updated == 1) {
jdbc.update("""
UPDATE attachments SET status = 'queued', error_code = NULL,
updated_at = clock_timestamp()
WHERE id = ? AND pipeline_version = ? AND status = 'processing'
""", attachmentId, pipelineVersion);
}
return updated == 1;
}
@Override
public boolean failExpired(UUID jobId, UUID attachmentId, int pipelineVersion,
String errorCode) {
int updated = jdbc.update("""
UPDATE attachment_jobs
SET status = 'failed', published = true, last_error_code = ?,
lease_token = NULL, lease_until = NULL, updated_at = clock_timestamp()
WHERE id = ? AND attachment_id = ? AND status = 'processing'
AND pipeline_version = ? AND lease_until < clock_timestamp()
AND EXISTS (SELECT 1 FROM attachments a
WHERE a.id = ? AND a.pipeline_version = ?)
""", errorCode, jobId, attachmentId, pipelineVersion,
attachmentId, pipelineVersion);
if (updated == 1) {
jdbc.update("""
UPDATE attachments SET status = 'failed', error_code = ?,
updated_at = clock_timestamp()
WHERE id = ? AND pipeline_version = ? AND status <> 'ready'
""", errorCode, attachmentId, pipelineVersion);
}
return updated == 1;
}
private Attachment map(ResultSet rs) throws SQLException {
UUID id = rs.getObject("id", UUID.class);
return new Attachment(
id,
rs.getObject("chat_id", UUID.class),
rs.getObject("owner_id", UUID.class),
rs.getString("filename"),
rs.getString("content_type"),
rs.getLong("size_bytes"),
rs.getString("sha256"),
rs.getString("object_key"),
rs.getString("staging_key"),
rs.getString("status"),
findVariants(id),
rs.getString("error_code"),
rs.getTimestamp("created_at").toInstant(),
rs.getTimestamp("updated_at").toInstant());
}
private List<AttachmentVariant> findVariants(UUID attachmentId) {
return jdbc.query("""
SELECT name, content_type, size_bytes, width, height, sha256, object_key
FROM attachment_variants
WHERE attachment_id = ?
AND pipeline_version = (
SELECT pipeline_version FROM attachments WHERE id = ?)
ORDER BY name
""", (rs, rowNum) -> new AttachmentVariant(
rs.getString("name"), rs.getString("content_type"), rs.getLong("size_bytes"),
rs.getInt("width"), rs.getInt("height"), rs.getString("sha256"),
rs.getString("object_key")), attachmentId, attachmentId);
}
private static AttachmentJob mapJob(ResultSet rs) throws SQLException {
return new AttachmentJob(
AttachmentJob.SCHEMA_VERSION,
rs.getObject("id", UUID.class),
AttachmentJob.TYPE,
rs.getObject("attachment_id", UUID.class),
rs.getInt("pipeline_version"),
rs.getInt("attempt"),
rs.getTimestamp("created_at").toInstant());
}
}
@@ -0,0 +1,106 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Chat;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.transaction.annotation.Transactional;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
@Repository
public class PostgresChatRepository implements ChatRepository {
private final JdbcTemplate jdbc;
public PostgresChatRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc;
}
@Override
@Transactional(timeout = 5)
public Chat save(Chat chat) {
jdbc.update("""
INSERT INTO chats (id, title, created_at)
VALUES (?, ?, ?)
""", chat.id(), chat.title(), Timestamp.from(chat.createdAt()));
for (UUID memberId : chat.memberIds()) {
jdbc.update("""
INSERT INTO chat_members (chat_id, user_id)
VALUES (?, ?)
""", chat.id(), memberId);
}
return chat;
}
@Override
public Optional<Chat> findById(UUID id) {
return jdbc.query("""
SELECT id, title, created_at
FROM chats
WHERE id = ?
""", (rs, rowNum) -> new ChatRow(
rs.getObject("id", UUID.class),
rs.getString("title"),
rs.getTimestamp("created_at").toInstant()), id)
.stream().findFirst()
.map(this::toChat);
}
@Override
public boolean isMember(UUID chatId, UUID userId) {
return Boolean.TRUE.equals(jdbc.queryForObject("""
SELECT EXISTS (
SELECT 1 FROM chat_members
WHERE chat_id = ? AND user_id = ?
)
""", Boolean.class, chatId, userId));
}
@Override
public List<Chat> findAllForUser(UUID userId) {
return findPageForUser(userId, null, null, Integer.MAX_VALUE);
}
@Override
public List<Chat> findPageForUser(UUID userId, UUID afterId, Instant afterCreatedAt, int limitPlusOne) {
StringBuilder sql = new StringBuilder("""
SELECT c.id, c.title, c.created_at
FROM chats c
JOIN chat_members cm ON cm.chat_id = c.id
WHERE cm.user_id = ?
""");
List<Object> parameters = new ArrayList<>();
parameters.add(userId);
if (afterCreatedAt != null) {
sql.append(" AND (c.created_at, c.id) > (?, ?) ");
parameters.add(Timestamp.from(afterCreatedAt));
parameters.add(afterId);
}
sql.append(" ORDER BY c.created_at, c.id LIMIT ? ");
parameters.add(limitPlusOne);
return jdbc.query(sql.toString(), (rs, rowNum) -> new ChatRow(
rs.getObject("id", UUID.class),
rs.getString("title"),
rs.getTimestamp("created_at").toInstant()), parameters.toArray())
.stream().map(this::toChat).toList();
}
private Chat toChat(ChatRow row) {
List<UUID> members = jdbc.queryForList("""
SELECT user_id
FROM chat_members
WHERE chat_id = ?
ORDER BY joined_at, user_id
""", UUID.class, row.id());
return new Chat(row.id(), row.title(), members, row.createdAt());
}
private record ChatRow(UUID id, String title, Instant createdAt) {}
}
@@ -0,0 +1,172 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Message;
import com.example.mtusi_messenger.model.IdempotentMessage;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@Repository
public class PostgresMessageRepository implements MessageRepository {
private final JdbcTemplate jdbc;
public PostgresMessageRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc;
}
@Override
public Message save(Message message) {
jdbc.update("""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
VALUES (?, ?, ?, ?, ?)
""", message.id(), message.chatId(), message.senderId(), message.text(),
Timestamp.from(message.createdAt()));
if (!message.attachmentIds().isEmpty()) {
jdbc.batchUpdate("""
INSERT INTO message_attachments
(message_id, chat_id, attachment_id, position)
VALUES (?, ?, ?, ?)
""", new BatchPreparedStatementSetter() {
@Override
public void setValues(PreparedStatement statement, int index) throws SQLException {
statement.setObject(1, message.id());
statement.setObject(2, message.chatId());
statement.setObject(3, message.attachmentIds().get(index));
statement.setInt(4, index);
}
@Override
public int getBatchSize() {
return message.attachmentIds().size();
}
});
}
return message;
}
@Override
public void lockIdempotencyScope(UUID userId, String method, String path, String key) {
String scope = userId + "\n" + method + "\n" + path + "\n" + key;
jdbc.query(
"SELECT pg_advisory_xact_lock(hashtextextended(?, 0))",
rs -> {
rs.next();
return null;
},
scope);
}
@Override
public void deleteExpiredIdempotency(
UUID userId,
String method,
String path,
String key,
Instant expiresBefore
) {
jdbc.update("""
DELETE FROM message_idempotency
WHERE user_id = ?
AND method = ?
AND request_path = ?
AND idempotency_key = ?
AND created_at < ?
""", userId, method, path, key, Timestamp.from(expiresBefore));
}
@Override
public Optional<IdempotentMessage> findIdempotent(
UUID userId,
String method,
String path,
String key
) {
Optional<IdempotentMessage> result = jdbc.query("""
SELECT i.request_hash,
m.id, m.chat_id, m.sender_id, m.text, m.created_at
FROM message_idempotency i
JOIN messages m ON m.id = i.message_id
WHERE i.user_id = ?
AND i.method = ?
AND i.request_path = ?
AND i.idempotency_key = ?
""", (rs, rowNum) -> new IdempotentMessage(
rs.getBytes("request_hash"),
new Message(
rs.getObject("id", UUID.class),
rs.getObject("chat_id", UUID.class),
rs.getObject("sender_id", UUID.class),
rs.getString("text"),
rs.getTimestamp("created_at").toInstant(),
List.of())),
userId, method, path, key).stream().findFirst();
return result.map(value -> new IdempotentMessage(
value.requestHash(), withAttachments(value.message())));
}
@Override
public void saveIdempotency(
UUID userId,
String method,
String path,
String key,
byte[] requestHash,
UUID messageId,
Instant createdAt
) {
jdbc.update("""
INSERT INTO message_idempotency
(user_id, method, request_path, idempotency_key,
request_hash, message_id, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
""", userId, method, path, key, requestHash, messageId,
Timestamp.from(createdAt));
}
@Override
public List<Message> findPage(UUID chatId, Instant afterCreatedAt, UUID afterId, int limitPlusOne) {
StringBuilder sql = new StringBuilder("""
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = ?
""");
List<Object> parameters = new ArrayList<>();
parameters.add(chatId);
if (afterCreatedAt != null) {
sql.append(" AND (created_at, id) > (?, ?) ");
parameters.add(Timestamp.from(afterCreatedAt));
parameters.add(afterId);
}
sql.append(" ORDER BY created_at, id LIMIT ? ");
parameters.add(limitPlusOne);
return jdbc.query(sql.toString(), (rs, rowNum) -> new Message(
rs.getObject("id", UUID.class),
rs.getObject("chat_id", UUID.class),
rs.getObject("sender_id", UUID.class),
rs.getString("text"),
rs.getTimestamp("created_at").toInstant(),
List.of()), parameters.toArray()).stream().map(this::withAttachments).toList();
}
private Message withAttachments(Message message) {
List<UUID> attachmentIds = jdbc.query("""
SELECT attachment_id
FROM message_attachments
WHERE message_id = ?
ORDER BY position
""", (rs, rowNum) -> rs.getObject("attachment_id", UUID.class), message.id());
return new Message(message.id(), message.chatId(), message.senderId(), message.text(),
message.createdAt(), attachmentIds);
}
}
@@ -0,0 +1,51 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Session;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import java.sql.Timestamp;
import java.util.Optional;
import java.util.UUID;
@Repository
public class PostgresSessionRepository implements SessionRepository {
private final JdbcTemplate jdbc;
public PostgresSessionRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc;
}
@Override
public Session save(Session session) {
jdbc.update("""
INSERT INTO sessions (id, user_id, token_hash, created_at, expires_at)
VALUES (?, ?, ?, ?, ?)
""", session.id(), session.userId(), session.tokenHash(),
Timestamp.from(session.createdAt()), Timestamp.from(session.expiresAt()));
return session;
}
@Override
public Optional<UUID> findActiveUserId(byte[] tokenHash) {
return jdbc.query("""
SELECT user_id
FROM sessions
WHERE token_hash = ?
AND revoked_at IS NULL
AND expires_at > CURRENT_TIMESTAMP
""", (rs, rowNum) -> rs.getObject("user_id", UUID.class), tokenHash)
.stream().findFirst();
}
@Override
public boolean revoke(byte[] tokenHash) {
return jdbc.update("""
UPDATE sessions
SET revoked_at = CURRENT_TIMESTAMP
WHERE token_hash = ?
AND revoked_at IS NULL
AND expires_at > CURRENT_TIMESTAMP
""", tokenHash) == 1;
}
}
@@ -0,0 +1,72 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.User;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import java.sql.Timestamp;
import java.util.Optional;
import java.util.UUID;
@Repository
public class PostgresUserRepository implements UserRepository {
private final JdbcTemplate jdbc;
public PostgresUserRepository(JdbcTemplate jdbc) {
this.jdbc = jdbc;
}
@Override
public User save(User user, String passwordHash) {
jdbc.update("""
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (?, ?, ?, ?, ?)
""", user.id(), user.username(), user.displayName(), passwordHash,
Timestamp.from(user.createdAt()));
return user;
}
@Override
public Optional<User> findById(UUID id) {
return jdbc.query("""
SELECT id, username, display_name, created_at
FROM users
WHERE id = ?
""", (rs, rowNum) -> new User(
rs.getObject("id", UUID.class),
rs.getString("username"),
rs.getString("display_name"),
rs.getTimestamp("created_at").toInstant()), id)
.stream().findFirst();
}
@Override
public Optional<User> findByUsername(String username) {
return jdbc.query("""
SELECT id, username, display_name, created_at
FROM users
WHERE username = ?
""", (rs, rowNum) -> new User(
rs.getObject("id", UUID.class),
rs.getString("username"),
rs.getString("display_name"),
rs.getTimestamp("created_at").toInstant()), username)
.stream().findFirst();
}
@Override
public Optional<UserCredentials> findCredentialsByUsername(String username) {
return jdbc.query("""
SELECT id, username, display_name, password_hash, created_at
FROM users
WHERE username = ?
""", (rs, rowNum) -> new UserCredentials(
new User(
rs.getObject("id", UUID.class),
rs.getString("username"),
rs.getString("display_name"),
rs.getTimestamp("created_at").toInstant()),
rs.getString("password_hash")), username)
.stream().findFirst();
}
}
@@ -0,0 +1,12 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.Session;
import java.util.Optional;
import java.util.UUID;
public interface SessionRepository {
Session save(Session session);
Optional<UUID> findActiveUserId(byte[] tokenHash);
boolean revoke(byte[] tokenHash);
}
@@ -0,0 +1,14 @@
package com.example.mtusi_messenger.repository;
import com.example.mtusi_messenger.model.User;
import java.util.Optional;
import java.util.UUID;
public interface UserRepository {
User save(User user, String passwordHash);
Optional<User> findById(UUID id);
Optional<User> findByUsername(String username);
Optional<UserCredentials> findCredentialsByUsername(String username);
record UserCredentials(User user, String passwordHash) {}
}
@@ -0,0 +1,130 @@
package com.example.mtusi_messenger.security;
import com.example.mtusi_messenger.service.AuthenticationService;
import jakarta.servlet.FilterChain;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.HttpMethod;
import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.web.filter.OncePerRequestFilter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Enumeration;
import java.util.List;
public class ApiAuthenticationFilter extends OncePerRequestFilter {
private static final String AUTHORIZATION = "Authorization";
private static final String SESSION_PATH = "/api/v1/auth/sessions";
private final AuthenticationManager authenticationManager;
private final AuthenticationService sessions;
private final RestAuthenticationEntryPoint basicEntryPoint;
private final RestAuthenticationEntryPoint bearerEntryPoint;
public ApiAuthenticationFilter(
AuthenticationManager authenticationManager,
AuthenticationService sessions,
RestAuthenticationEntryPoint basicEntryPoint,
RestAuthenticationEntryPoint bearerEntryPoint
) {
this.authenticationManager = authenticationManager;
this.sessions = sessions;
this.basicEntryPoint = basicEntryPoint;
this.bearerEntryPoint = bearerEntryPoint;
}
@Override
protected void doFilterInternal(
HttpServletRequest request,
HttpServletResponse response,
FilterChain filterChain
) throws ServletException, IOException {
boolean basicLogin = HttpMethod.POST.matches(request.getMethod())
&& SESSION_PATH.equals(request.getRequestURI());
if (isPublicRequest(request) && !basicLogin) {
filterChain.doFilter(request, response);
return;
}
try {
Authentication authentication = basicLogin
? authenticateBasic(request)
: authenticateBearer(request);
var context = SecurityContextHolder.createEmptyContext();
context.setAuthentication(authentication);
SecurityContextHolder.setContext(context);
filterChain.doFilter(request, response);
} catch (AuthenticationException e) {
SecurityContextHolder.clearContext();
(basicLogin ? basicEntryPoint : bearerEntryPoint)
.commence(request, response, e);
} finally {
SecurityContextHolder.clearContext();
}
}
private Authentication authenticateBasic(HttpServletRequest request) {
String header = singleAuthorizationHeader(request);
if (!header.startsWith("Basic ") || header.length() == 6) {
throw new BadCredentialsException("Invalid Basic credentials");
}
try {
byte[] decoded = Base64.getDecoder().decode(header.substring(6));
String credentials = new String(decoded, StandardCharsets.UTF_8);
int separator = credentials.indexOf(':');
if (separator <= 0) {
throw new BadCredentialsException("Invalid Basic credentials");
}
return authenticationManager.authenticate(
UsernamePasswordAuthenticationToken.unauthenticated(
credentials.substring(0, separator),
credentials.substring(separator + 1)));
} catch (IllegalArgumentException e) {
throw new BadCredentialsException("Invalid Basic credentials", e);
}
}
private Authentication authenticateBearer(HttpServletRequest request) {
String header = singleAuthorizationHeader(request);
if (!header.startsWith("Bearer ") || header.length() == 7) {
throw new BadCredentialsException("Invalid Bearer token");
}
String token = header.substring(7);
if (!token.matches("[A-Za-z0-9_-]{43}")) {
throw new BadCredentialsException("Invalid Bearer token");
}
SessionPrincipal principal = sessions.authenticateBearer(token);
return UsernamePasswordAuthenticationToken.authenticated(principal, null, List.of());
}
private String singleAuthorizationHeader(HttpServletRequest request) {
Enumeration<String> values = request.getHeaders(AUTHORIZATION);
if (values == null || !values.hasMoreElements()) {
throw new BadCredentialsException("Authentication required");
}
String value = values.nextElement();
if (values.hasMoreElements()) {
throw new BadCredentialsException("Multiple Authorization headers");
}
return value;
}
private boolean isPublicRequest(HttpServletRequest request) {
String path = request.getRequestURI();
return path.startsWith("/health/")
|| path.equals("/swagger-ui.html")
|| path.startsWith("/swagger-ui/")
|| path.equals("/v3/api-docs")
|| path.startsWith("/v3/api-docs/")
|| (HttpMethod.POST.matches(request.getMethod())
&& "/api/v1/users".equals(path));
}
}
@@ -0,0 +1,63 @@
package com.example.mtusi_messenger.security;
import com.example.mtusi_messenger.repository.UserRepository;
import org.springframework.security.authentication.AuthenticationProvider;
import org.springframework.security.authentication.AuthenticationServiceException;
import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Component;
import org.springframework.dao.DataAccessException;
import java.util.List;
@Component
public class DatabaseAuthenticationProvider implements AuthenticationProvider {
private final UserRepository users;
private final PasswordEncoder passwordEncoder;
private final String dummyPasswordHash;
public DatabaseAuthenticationProvider(UserRepository users, PasswordEncoder passwordEncoder) {
this.users = users;
this.passwordEncoder = passwordEncoder;
this.dummyPasswordHash = passwordEncoder.encode("dummy-password-used-only-for-timing");
}
@Override
public Authentication authenticate(Authentication authentication) throws AuthenticationException {
String username = authentication.getName();
String password = String.valueOf(authentication.getCredentials());
final java.util.Optional<UserRepository.UserCredentials> stored;
try {
stored = users.findCredentialsByUsername(username);
} catch (DataAccessException e) {
throw new AuthenticationServiceException("Authentication database is unavailable", e);
}
String passwordHash = stored.map(UserRepository.UserCredentials::passwordHash)
.filter(hash -> hash.startsWith("$argon2"))
.orElse(dummyPasswordHash);
boolean matches;
try {
matches = passwordEncoder.matches(password, passwordHash);
} catch (IllegalArgumentException e) {
matches = false;
}
if (stored.isEmpty() || !matches) {
throw new BadCredentialsException("Invalid credentials");
}
return UsernamePasswordAuthenticationToken.authenticated(
stored.get().user(),
null,
List.of());
}
@Override
public boolean supports(Class<?> authentication) {
return UsernamePasswordAuthenticationToken.class.isAssignableFrom(authentication);
}
}
@@ -0,0 +1,46 @@
package com.example.mtusi_messenger.security;
import com.example.mtusi_messenger.dto.ErrorResponse;
import com.example.mtusi_messenger.web.RequestIdFilter;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.http.MediaType;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.authentication.AuthenticationServiceException;
import org.springframework.security.web.AuthenticationEntryPoint;
import java.io.IOException;
public class RestAuthenticationEntryPoint implements AuthenticationEntryPoint {
private final ObjectMapper mapper;
private final String challenge;
public RestAuthenticationEntryPoint(ObjectMapper mapper, String challenge) {
this.mapper = mapper;
this.challenge = challenge;
}
@Override
public void commence(
HttpServletRequest request,
HttpServletResponse response,
AuthenticationException exception
) throws IOException {
String requestId = (String) request.getAttribute(RequestIdFilter.ATTRIBUTE);
boolean unavailable = exception instanceof AuthenticationServiceException;
response.setStatus(unavailable
? HttpServletResponse.SC_SERVICE_UNAVAILABLE
: HttpServletResponse.SC_UNAUTHORIZED);
response.setContentType(MediaType.APPLICATION_JSON_VALUE);
response.setCharacterEncoding("UTF-8");
if (!unavailable) {
response.setHeader("WWW-Authenticate", challenge);
}
mapper.writeValue(response.getOutputStream(), new ErrorResponse(
new ErrorResponse.ErrorBody(
unavailable ? "unavailable" : "unauthorized",
unavailable ? "Authentication service temporarily unavailable" : "Authentication required",
requestId)));
}
}
@@ -0,0 +1,14 @@
package com.example.mtusi_messenger.security;
import com.example.mtusi_messenger.model.User;
public record SessionPrincipal(User user, byte[] tokenHash) {
public SessionPrincipal {
tokenHash = tokenHash.clone();
}
@Override
public byte[] tokenHash() {
return tokenHash.clone();
}
}
@@ -0,0 +1,106 @@
package com.example.mtusi_messenger.service;
import com.example.mtusi_messenger.repository.AttachmentRepository;
import com.example.mtusi_messenger.storage.ObjectStorage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.HashSet;
import java.util.Set;
@Component
@ConditionalOnProperty(name = "messenger.role", havingValue = "api", matchIfMissing = true)
public class AttachmentReconciler {
private static final Logger log = LoggerFactory.getLogger(AttachmentReconciler.class);
private static final String PREFIX = "attachments/";
private final AttachmentRepository attachments;
private final ObjectStorage storage;
private final Duration gracePeriod;
private final boolean dryRun;
private final Clock clock;
@Autowired
public AttachmentReconciler(
AttachmentRepository attachments,
ObjectStorage storage,
@Value("${messenger.attachments.cleanup-grace-seconds:3600}") long graceSeconds,
@Value("${messenger.attachments.cleanup-dry-run:false}") boolean dryRun
) {
this(attachments, storage, Duration.ofSeconds(graceSeconds), dryRun, Clock.systemUTC());
}
AttachmentReconciler(AttachmentRepository attachments,
ObjectStorage storage,
Duration gracePeriod,
boolean dryRun,
Clock clock) {
if (gracePeriod.isNegative() || gracePeriod.isZero()) {
throw new IllegalArgumentException("Attachment cleanup grace period must be positive");
}
this.attachments = attachments;
this.storage = storage;
this.gracePeriod = gracePeriod;
this.dryRun = dryRun;
this.clock = clock;
}
@Scheduled(
fixedDelayString = "${messenger.attachments.reconcile-interval-millis:600000}",
initialDelayString = "${messenger.attachments.reconcile-initial-delay-millis:60000}")
public void scheduledReconcile() {
try {
Report report = reconcile();
if (report.orphanObjects() > 0 || report.missingObjects() > 0) {
log.warn("Attachment reconciliation: {} orphan, {} deleted, {} missing, dryRun={}",
report.orphanObjects(), report.deletedObjects(), report.missingObjects(), dryRun);
}
} catch (RuntimeException error) {
log.warn("Attachment reconciliation postponed: {}", error.getMessage());
}
}
public Report reconcile() {
Set<String> databaseKeys = attachments.findAllObjectKeys();
var storedObjects = storage.list(PREFIX);
Set<String> storageKeys = new HashSet<>();
Instant deleteBefore = clock.instant().minus(gracePeriod);
int orphans = 0;
int deleted = 0;
for (ObjectStorage.StoredObject object : storedObjects) {
storageKeys.add(object.objectKey());
if (!databaseKeys.contains(object.objectKey()) && object.lastModified().isBefore(deleteBefore)) {
orphans++;
if (!dryRun) {
storage.delete(object.objectKey());
deleted++;
}
}
}
int missing = 0;
for (String databaseKey : databaseKeys) {
if (!storageKeys.contains(databaseKey)) {
missing++;
log.error("Attachment metadata references missing object {}", databaseKey);
}
}
return new Report(databaseKeys.size(), storedObjects.size(), orphans, deleted, missing, dryRun);
}
public record Report(int metadataObjects,
int storageObjects,
int orphanObjects,
int deletedObjects,
int missingObjects,
boolean dryRun) {
}
}
@@ -0,0 +1,295 @@
package com.example.mtusi_messenger.service;
import com.example.mtusi_messenger.exception.ApiExceptions;
import com.example.mtusi_messenger.model.Attachment;
import com.example.mtusi_messenger.model.AttachmentJob;
import com.example.mtusi_messenger.model.AttachmentVariant;
import com.example.mtusi_messenger.queue.AttachmentJobPublisher;
import com.example.mtusi_messenger.repository.AttachmentRepository;
import com.example.mtusi_messenger.storage.ObjectStorage;
import com.example.mtusi_messenger.storage.ObjectStorageException;
import com.github.f4b6a3.uuid.UuidCreator;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HexFormat;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@Service
public class AttachmentService {
private static final List<String> ALLOWED_TYPES = List.of("image/png", "image/jpeg");
private final AttachmentRepository attachments;
private final ChatService chats;
private final ObjectStorage storage;
private final AttachmentJobPublisher publisher;
private final TransactionTemplate transaction;
private final long maxUploadBytes;
private final long maxImagePixels;
private final Semaphore uploadPermits;
private final long uploadPermitWaitMillis;
private final boolean injectCrashAfterOutboxCommit;
private final int pipelineVersion;
public AttachmentService(
AttachmentRepository attachments,
ChatService chats,
ObjectStorage storage,
AttachmentJobPublisher publisher,
TransactionTemplate transaction,
@Value("${messenger.attachments.max-upload-bytes:10485760}") long maxUploadBytes,
@Value("${messenger.attachments.max-image-pixels:40000000}") long maxImagePixels,
@Value("${messenger.attachments.max-concurrent-uploads:3}") int maxConcurrentUploads,
@Value("${messenger.attachments.upload-permit-wait-millis:10000}") long uploadPermitWaitMillis,
@Value("${messenger.jobs.fault-crash-after-outbox-commit:false}") boolean injectCrashAfterOutboxCommit,
@Value("${messenger.jobs.pipeline-version:2}") int pipelineVersion
) {
this.attachments = attachments;
this.chats = chats;
this.storage = storage;
this.publisher = publisher;
this.transaction = transaction;
this.maxUploadBytes = maxUploadBytes;
this.maxImagePixels = maxImagePixels;
if (maxConcurrentUploads < 1 || uploadPermitWaitMillis < 1) {
throw new IllegalArgumentException("Upload concurrency limits must be positive");
}
this.uploadPermits = new Semaphore(maxConcurrentUploads, true);
this.uploadPermitWaitMillis = uploadPermitWaitMillis;
this.injectCrashAfterOutboxCommit = injectCrashAfterOutboxCommit;
if (pipelineVersion < AttachmentJob.MIN_PIPELINE_VERSION
|| pipelineVersion > AttachmentJob.CURRENT_PIPELINE_VERSION) {
throw new IllegalArgumentException("Unsupported attachment pipeline version");
}
this.pipelineVersion = pipelineVersion;
}
public Attachment upload(UUID chatId,
UUID ownerId,
String filename,
String declaredContentType,
long declaredLength,
InputStream input) {
chats.getAccessible(chatId, ownerId);
String safeFilename = validateFilename(filename);
String contentType = normalizeContentType(declaredContentType);
if (declaredLength > maxUploadBytes) {
throw ApiExceptions.payloadTooLarge("Attachment exceeds 10 MiB");
}
try {
try {
if (!uploadPermits.tryAcquire(uploadPermitWaitMillis, TimeUnit.MILLISECONDS)) {
throw ApiExceptions.tooManyRequests("Upload capacity is temporarily exhausted");
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
throw ApiExceptions.unavailable("Upload was interrupted");
}
try {
return storeValidated(chatId, ownerId, safeFilename, contentType, input);
} finally {
uploadPermits.release();
}
} catch (com.example.mtusi_messenger.exception.ApiException error) {
throw error;
}
}
private Attachment storeValidated(UUID chatId,
UUID ownerId,
String safeFilename,
String contentType,
InputStream input) {
byte[] bytes = readBounded(input);
if (bytes.length == 0) {
throw ApiExceptions.unsupportedMediaType("Image is empty");
}
validateImage(bytes, contentType);
UUID id = UuidCreator.getTimeOrderedEpochPlus1();
String objectKey = "attachments/" + id + "/original";
String stagingKey = "staging/" + id + "/original";
Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS);
AttachmentJob job = AttachmentJob.first(UUID.randomUUID(), id, pipelineVersion, now);
Attachment attachment = new Attachment(id, chatId, ownerId, safeFilename, contentType,
bytes.length, sha256(bytes), objectKey, stagingKey, "queued", List.of(),
null, now, now);
storage.put(stagingKey, bytes, contentType);
try {
Attachment saved = transaction.execute(status -> attachments.saveQueued(attachment, job));
if (injectCrashAfterOutboxCommit && attachments.markPublishFaultInjected(job.jobId())) {
// Opt-in fault injection for the exact DB COMMIT -> broker publish window.
// The persisted outbox row remains unpublished and is recovered after restart.
Runtime.getRuntime().halt(87);
}
try {
publisher.publish(job);
attachments.markPublished(job.jobId());
return saved;
} catch (RuntimeException publishFailure) {
// The database job and staging object are intentionally retained.
// The scheduled republisher closes the DB-commit -> broker window.
throw ApiExceptions.unavailable("Attachment was staged; queue publication is pending");
}
} catch (RuntimeException databaseFailure) {
if (databaseFailure instanceof com.example.mtusi_messenger.exception.ApiException) {
throw databaseFailure;
}
try {
storage.delete(stagingKey);
} catch (RuntimeException cleanupFailure) {
databaseFailure.addSuppressed(cleanupFailure);
}
throw databaseFailure;
}
}
public Attachment getAccessible(UUID attachmentId, UUID userId) {
Attachment attachment = attachments.findById(attachmentId)
.orElseThrow(() -> ApiExceptions.notFound("Attachment not found"));
chats.getAccessible(attachment.chatId(), userId);
return attachment;
}
public Download download(UUID attachmentId, UUID userId) {
return download(attachmentId, userId, "original");
}
public Download download(UUID attachmentId, UUID userId, String variantName) {
Attachment attachment = getAccessible(attachmentId, userId);
if (!"ready".equals(attachment.status())) {
throw ApiExceptions.conflict("Attachment content is not ready");
}
if ("original".equals(variantName)) {
return new Download(attachment, attachment.contentType(), attachment.filename(),
storage.get(attachment.objectKey()));
}
if (!List.of("thumbnail", "crop", "optimized").contains(variantName)) {
throw ApiExceptions.invalid("Unsupported attachment variant");
}
AttachmentVariant variant = attachment.variants().stream()
.filter(candidate -> candidate.name().equals(variantName))
.findFirst()
.orElseThrow(() -> ApiExceptions.notFound("Attachment variant not found"));
String extension = "image/webp".equals(variant.contentType()) ? ".webp" : ".jpg";
return new Download(attachment, variant.contentType(), variantName + extension,
storage.get(variant.objectKey()));
}
public void validateForMessage(UUID chatId, List<UUID> attachmentIds) {
if (attachmentIds.isEmpty()) {
return;
}
if (attachments.findAllByIdsInChat(chatId, attachmentIds).size() != attachmentIds.size()) {
throw ApiExceptions.notFound("Attachment not found");
}
}
private byte[] readBounded(InputStream input) {
try {
ByteArrayOutputStream output = new ByteArrayOutputStream(
(int) Math.min(maxUploadBytes, 1024 * 1024));
byte[] chunk = new byte[8192];
long total = 0;
int read;
while ((read = input.read(chunk)) != -1) {
total += read;
if (total > maxUploadBytes) {
throw ApiExceptions.payloadTooLarge("Attachment exceeds 10 MiB");
}
output.write(chunk, 0, read);
}
return output.toByteArray();
} catch (IOException error) {
throw ApiExceptions.invalid("Cannot read request body");
}
}
private void validateImage(byte[] bytes, String declaredContentType) {
ImageIO.setUseCache(false);
try (var imageInput = ImageIO.createImageInputStream(new ByteArrayInputStream(bytes))) {
if (imageInput == null) {
throw ApiExceptions.unsupportedMediaType("Invalid image");
}
Iterator<ImageReader> readers = ImageIO.getImageReaders(imageInput);
if (!readers.hasNext()) {
throw ApiExceptions.unsupportedMediaType("Invalid PNG/JPEG image");
}
ImageReader reader = readers.next();
try {
reader.setInput(imageInput, true, true);
String actualType = switch (reader.getFormatName().toLowerCase(Locale.ROOT)) {
case "png" -> "image/png";
case "jpeg", "jpg" -> "image/jpeg";
default -> throw ApiExceptions.unsupportedMediaType("Only PNG and JPEG are supported");
};
if (!actualType.equals(declaredContentType)) {
throw ApiExceptions.unsupportedMediaType("Content-Type does not match image bytes");
}
int width = reader.getWidth(0);
int height = reader.getHeight(0);
if (width <= 0 || height <= 0) {
throw ApiExceptions.unsupportedMediaType("Invalid image dimensions");
}
if ((long) width * height > maxImagePixels) {
throw ApiExceptions.payloadTooLarge("Image exceeds 40 million pixels");
}
if (reader.read(0) == null) {
throw ApiExceptions.unsupportedMediaType("Image cannot be decoded");
}
} finally {
reader.dispose();
}
} catch (com.example.mtusi_messenger.exception.ApiException error) {
throw error;
} catch (IOException | RuntimeException error) {
throw ApiExceptions.unsupportedMediaType("Corrupted PNG/JPEG image");
}
}
private static String validateFilename(String filename) {
if (filename == null || filename.isBlank() || filename.length() > 255
|| filename.equals(".") || filename.equals("..")
|| filename.indexOf('/') >= 0 || filename.indexOf('\\') >= 0
|| filename.codePoints().anyMatch(Character::isISOControl)) {
throw ApiExceptions.invalid("Invalid filename");
}
return filename;
}
private static String normalizeContentType(String contentType) {
String normalized = contentType == null ? "" : contentType.split(";", 2)[0].trim().toLowerCase(Locale.ROOT);
if (!ALLOWED_TYPES.contains(normalized)) {
throw ApiExceptions.unsupportedMediaType("Only image/png and image/jpeg are supported");
}
return normalized;
}
private static String sha256(byte[] bytes) {
try {
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
} catch (NoSuchAlgorithmException impossible) {
throw new IllegalStateException("SHA-256 is unavailable", impossible);
}
}
public record Download(Attachment attachment, String contentType, String filename, byte[] bytes) {
}
}
@@ -0,0 +1,104 @@
package com.example.mtusi_messenger.service;
import com.example.mtusi_messenger.cache.UserCache;
import com.example.mtusi_messenger.model.Session;
import com.example.mtusi_messenger.model.User;
import com.example.mtusi_messenger.repository.SessionRepository;
import com.example.mtusi_messenger.repository.UserRepository;
import com.example.mtusi_messenger.security.SessionPrincipal;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.dao.DataAccessException;
import org.springframework.security.authentication.AuthenticationServiceException;
import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Base64;
import java.util.UUID;
@Service
public class AuthenticationService {
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
private final UserRepository users;
private final SessionRepository sessions;
private final UserCache userCache;
private final long ttlSeconds;
public AuthenticationService(
UserRepository users,
SessionRepository sessions,
UserCache userCache,
@Value("${messenger.session.ttl-seconds:3600}") long ttlSeconds
) {
if (ttlSeconds <= 0) {
throw new IllegalArgumentException("Session TTL must be positive");
}
this.users = users;
this.sessions = sessions;
this.userCache = userCache;
this.ttlSeconds = ttlSeconds;
}
public IssuedSession issueSession(User user) {
String token = newToken();
Instant createdAt = Instant.now().truncatedTo(ChronoUnit.MICROS);
Instant expiresAt = createdAt.plusSeconds(ttlSeconds);
sessions.save(new Session(
UUID.randomUUID(),
user.id(),
hashToken(token),
createdAt,
expiresAt));
return new IssuedSession(token, expiresAt, user);
}
public SessionPrincipal authenticateBearer(String token) {
byte[] tokenHash = hashToken(token);
try {
UUID userId = sessions.findActiveUserId(tokenHash)
.orElseThrow(() -> new BadCredentialsException("Invalid or expired session"));
User user = userCache.find(userId).orElseGet(() -> {
User loaded = users.findById(userId)
.orElseThrow(() -> new BadCredentialsException("Invalid or expired session"));
userCache.put(loaded);
return loaded;
});
return new SessionPrincipal(user, tokenHash);
} catch (DataAccessException e) {
throw new AuthenticationServiceException("Authentication database is unavailable", e);
}
}
public void revoke(SessionPrincipal principal) {
try {
if (!sessions.revoke(principal.tokenHash())) {
throw new BadCredentialsException("Invalid or expired session");
}
} catch (DataAccessException e) {
throw new AuthenticationServiceException("Authentication database is unavailable", e);
}
}
private String newToken() {
byte[] bytes = new byte[32];
SECURE_RANDOM.nextBytes(bytes);
return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes);
}
private byte[] hashToken(String token) {
try {
return MessageDigest.getInstance("SHA-256")
.digest(token.getBytes(StandardCharsets.US_ASCII));
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 is unavailable", e);
}
}
public record IssuedSession(String token, Instant expiresAt, User user) {}
}
@@ -0,0 +1,38 @@
package com.example.mtusi_messenger.service;
import org.springframework.stereotype.Service;
import com.example.mtusi_messenger.dto.CreateChatRequest;
import com.example.mtusi_messenger.exception.ApiExceptions;
import com.example.mtusi_messenger.model.Chat;
import com.example.mtusi_messenger.repository.ChatRepository;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
@Service
public class ChatService {
private final ChatRepository chats; private final UserService users; private final CursorService cursors;
public ChatService(ChatRepository chats, UserService users, CursorService cursors) { this.chats=chats; this.users=users; this.cursors=cursors; }
public Chat create(CreateChatRequest r, UUID caller) {
Set<UUID> ids = new LinkedHashSet<>(r.member_ids());
if (ids.size() != r.member_ids().size() || !ids.contains(caller)) throw ApiExceptions.invalid("Invalid member_ids");
for (UUID id : ids) if (id == null || !users.exists(id)) throw ApiExceptions.invalid("Invalid member_ids");
return chats.save(new Chat(UUID.randomUUID(), r.title(), List.copyOf(ids),
Instant.now().truncatedTo(ChronoUnit.MICROS)));
}
public Chat getAccessible(UUID chatId, UUID userId) {
users.get(userId);
if (!chats.isMember(chatId, userId)) throw ApiExceptions.notFound("Chat not found");
return chats.findById(chatId).orElseThrow(() -> ApiExceptions.notFound("Chat not found"));
}
public List<Chat> page(UUID userId, int limit, String cursor) {
CursorService.Cursor c = cursor == null ? null : cursors.decode(cursor, "chats", userId);
return chats.findPageForUser(userId, c == null ? null : c.id(), c == null ? null : c.createdAt(), limit + 1);
}
public String nextCursor(UUID userId,
Chat last,
boolean hasNext
) {
return hasNext ? cursors.encode("chats", userId, last.createdAt(), last.id()) : null;
}
}
@@ -0,0 +1,40 @@
package com.example.mtusi_messenger.service;
import com.example.mtusi_messenger.exception.ApiException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.stereotype.Service;
import com.example.mtusi_messenger.exception.ApiExceptions;
import java.time.Instant;
import java.util.Base64;
import java.util.UUID;
@Service
public class CursorService {
private final ObjectMapper mapper;
public CursorService(ObjectMapper mapper) { this.mapper = mapper; }
public String encode(String resource, UUID resourceId, Instant createdAt, UUID id) {
try {
byte[] json = mapper.writeValueAsBytes(new CursorPayload(1, resource, resourceId, createdAt.toString(), id));
return Base64.getUrlEncoder().withoutPadding().encodeToString(json);
} catch (Exception e) { throw new IllegalStateException(e); }
}
public Cursor decode(String raw, String resource, UUID resourceId) {
if (raw == null || raw.isBlank()) throw ApiExceptions.invalid("Invalid cursor");
try {
CursorPayload p = mapper.readValue(Base64.getUrlDecoder().decode(raw), CursorPayload.class);
if (p.v != 1
|| !resource.equals(p.resource)
|| !Objects.equals(resourceId, p.resourceId)
|| p.createdAt() == null
|| p.createdAt().isBlank()
|| p.id() == null
) throw ApiExceptions.invalid("Invalid cursor");
return new Cursor(Instant.parse(p.createdAt), p.id);
} catch (ApiException e) { throw e; }
catch (Exception e) { throw ApiExceptions.invalid("Invalid cursor"); }
}
public record Cursor(Instant createdAt, UUID id) {}
private record CursorPayload(int v, String resource, UUID resourceId, String createdAt, UUID id) {}
private static final class Objects { static boolean equals(Object a, Object b) { return java.util.Objects.equals(a,b); } }
}
@@ -0,0 +1,102 @@
package com.example.mtusi_messenger.service;
import com.github.f4b6a3.uuid.UuidCreator;
import com.example.mtusi_messenger.exception.ApiExceptions;
import com.example.mtusi_messenger.model.IdempotentMessage;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import com.example.mtusi_messenger.dto.CreateMessageRequest;
import com.example.mtusi_messenger.model.Message;
import com.example.mtusi_messenger.repository.MessageRepository;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@Service
public class MessageService {
private static final String METHOD = "POST";
private final MessageRepository messages;
private final ChatService chats;
private final CursorService cursors;
private final AttachmentService attachments;
private final Duration idempotencyTtl;
public MessageService(
MessageRepository messages,
ChatService chats,
CursorService cursors,
AttachmentService attachments,
@Value("${messenger.idempotency.ttl-seconds:86400}") long idempotencyTtlSeconds
) {
if (idempotencyTtlSeconds < 86400) {
throw new IllegalArgumentException("Idempotency TTL must be at least 24 hours");
}
this.messages = messages;
this.chats = chats;
this.cursors = cursors;
this.attachments = attachments;
this.idempotencyTtl = Duration.ofSeconds(idempotencyTtlSeconds);
}
@Transactional
public Message create(UUID chatId, UUID senderId, CreateMessageRequest r, String idempotencyKey) {
chats.getAccessible(chatId, senderId);
attachments.validateForMessage(chatId, r.attachment_ids());
String path = "/api/v1/chats/" + chatId + "/messages";
byte[] requestHash = hash(r.text(), r.attachment_ids());
Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS);
// The transaction-scoped PostgreSQL advisory lock serializes the same
// idempotency scope across every API replica. A hash collision only
// causes harmless extra serialization; the full scope is still checked.
messages.lockIdempotencyScope(senderId, METHOD, path, idempotencyKey);
messages.deleteExpiredIdempotency(
senderId, METHOD, path, idempotencyKey, now.minus(idempotencyTtl));
var existing = messages.findIdempotent(senderId, METHOD, path, idempotencyKey);
if (existing.isPresent()) {
IdempotentMessage value = existing.get();
if (!Arrays.equals(value.requestHash(), requestHash)) {
throw ApiExceptions.conflict("Idempotency-Key was already used with another request body");
}
return value.message();
}
Message message = messages.save(new Message(
UuidCreator.getTimeOrderedEpochPlus1(), chatId, senderId, r.text(), now,
r.attachment_ids()));
messages.saveIdempotency(
senderId, METHOD, path, idempotencyKey, requestHash, message.id(), now);
return message;
}
private static byte[] hash(String text, List<UUID> attachmentIds) {
try {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
byte[] textBytes = text.getBytes(StandardCharsets.UTF_8);
digest.update(java.nio.ByteBuffer.allocate(Integer.BYTES).putInt(textBytes.length).array());
digest.update(textBytes);
for (UUID id : attachmentIds) {
digest.update(java.nio.ByteBuffer.allocate(16)
.putLong(id.getMostSignificantBits())
.putLong(id.getLeastSignificantBits()).array());
}
return digest.digest();
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-256 is unavailable", e);
}
}
public List<Message> page(UUID chatId, UUID userId, int limit, String cursor) {
chats.getAccessible(chatId, userId);
CursorService.Cursor c = cursor == null ? null : cursors.decode(cursor, "messages", chatId);
return messages.findPage(chatId, c == null ? null : c.createdAt(), c == null ? null : c.id(), limit + 1);
}
public String nextCursor(UUID chatId, Message last, boolean hasNext) { return hasNext ? cursors.encode("messages", chatId, last.createdAt(), last.id()) : null; }
}
@@ -0,0 +1,85 @@
package com.example.mtusi_messenger.service;
import org.springframework.boot.availability.ApplicationAvailability;
import org.springframework.boot.availability.AvailabilityChangeEvent;
import org.springframework.boot.availability.ReadinessState;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.context.event.ContextClosedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Service;
import java.util.concurrent.atomic.AtomicBoolean;
@Service
public class ReadinessService {
private final JdbcTemplate jdbc;
private final StringRedisTemplate redis;
private final ApplicationAvailability availability;
private final ApplicationEventPublisher events;
private final AtomicBoolean dependencyUnavailable = new AtomicBoolean(false);
private final AtomicBoolean shuttingDown = new AtomicBoolean(false);
public ReadinessService(
JdbcTemplate jdbc,
StringRedisTemplate redis,
ApplicationAvailability availability,
ApplicationEventPublisher events
) {
this.jdbc = jdbc;
this.redis = redis;
this.availability = availability;
this.events = events;
}
public boolean probeDependencies() {
try {
// Redis is an optional cache. Ping only warms the lazy connection;
// its failure must not make the authoritative API unavailable.
redis.execute((RedisCallback<String>) connection -> connection.ping());
} catch (DataAccessException ignored) {
// PostgreSQL remains the source of truth.
}
try {
// The schema-level probe proves that both the shared idempotency
// schema (lab3) and the attachment schema (lab4/lab5) completed
// before this replica is allowed to serve business traffic.
jdbc.queryForObject("""
SELECT (SELECT count(*) FROM message_idempotency)
+ (SELECT count(*) FROM attachments)
""", Long.class);
markDependenciesAvailable();
return true;
} catch (DataAccessException e) {
markDependenciesUnavailable();
return false;
}
}
public boolean acceptsBusinessTraffic() {
return !shuttingDown.get()
&& !dependencyUnavailable.get()
&& availability.getReadinessState() == ReadinessState.ACCEPTING_TRAFFIC;
}
@EventListener(ContextClosedEvent.class)
void onContextClosed() {
shuttingDown.set(true);
}
private void markDependenciesUnavailable() {
dependencyUnavailable.set(true);
AvailabilityChangeEvent.publish(events, this, ReadinessState.REFUSING_TRAFFIC);
}
private void markDependenciesAvailable() {
boolean recovered = dependencyUnavailable.getAndSet(false);
if (recovered && !shuttingDown.get()) {
AvailabilityChangeEvent.publish(events, this, ReadinessState.ACCEPTING_TRAFFIC);
}
}
}

Some files were not shown because too many files have changed in this diff Show More