160 changed files with 16901 additions and 27 deletions
+4
View File
@@ -16,5 +16,9 @@ backups
.venv
__pycache__
node_modules
**/bin
**/obj
evidence
.DS_Store
.local
+3 -1
View File
@@ -11,7 +11,7 @@ certs/private/
*.pem
*.tfstate*
volumes/
data/
/data/
backups/
# Runtime/build artifacts
.venv/
@@ -30,3 +30,5 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
/.local/
+39
View File
@@ -0,0 +1,39 @@
FROM mcr.microsoft.com/dotnet/sdk:9.0.306 AS build
WORKDIR /src
COPY src/Messenger.Api/Messenger.Api.csproj src/Messenger.Api/
RUN dotnet restore src/Messenger.Api/Messenger.Api.csproj
COPY src/Messenger.Api/ src/Messenger.Api/
RUN dotnet publish src/Messenger.Api/Messenger.Api.csproj \
-c Release \
--no-restore \
-o /app/publish
FROM mcr.microsoft.com/dotnet/aspnet:9.0.10 AS runtime
RUN apt-get update \
&& apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
ARG BUILD_REVISION=local
LABEL org.opencontainers.image.revision=$BUILD_REVISION
COPY --from=build /app/publish .
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
ENV APP_ENV=lab
EXPOSE 8080
USER app
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD curl --fail --silent http://127.0.0.1:8080/health/ready > /dev/null || exit 1
ENTRYPOINT ["dotnet", "Messenger.Api.dll"]
+48 -2
View File
@@ -1,5 +1,5 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
PYTHON ?= .venv/bin/python
BASE_URL ?= http://localhost:18085
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -28,3 +28,49 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: setup up down build restore-api stop-worker start-worker replay reconcile
setup:
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt -r requirements-student.txt -r requirements-research.txt
./scripts/init-secrets.sh
up:
./scripts/init-secrets.sh
./scripts/compose.sh build api
./scripts/compose.sh up -d --wait --wait-timeout 150
build:
dotnet build src/Messenger.Api
restore-api:
./scripts/compose.sh up -d --wait api lb
stop-worker:
./scripts/compose.sh stop worker
start-worker:
./scripts/compose.sh up -d worker
replay:
@test -n "$(JOB_ID)" || (echo "Set JOB_ID"; exit 2)
./scripts/compose.sh exec -T dispatcher dotnet Messenger.Api.dll --replay "$(JOB_ID)"
reconcile:
./scripts/compose.sh exec -T dispatcher dotnet Messenger.Api.dll --reconcile
down:
./scripts/compose.sh down
.PHONY: student-test
student-test:
$(PYTHON) student-tests/pipeline_tests.py --base-url "$(BASE_URL)"
.PHONY: research-grade5
research-grade5:
$(PYTHON) student-tests/overload_research.py --base-url "$(BASE_URL)"
$(PYTHON) scripts/plot_overload.py
.PHONY: share stop-share operations-test
share:
python3 scripts/share.py
stop-share:
python3 scripts/share.py --stop
operations-test:
$(PYTHON) student-tests/operations_panel.py
.PHONY: operator
operator:
python3 scripts/configure-operator.py "$(USERNAME)"
+292 -24
View File
@@ -1,37 +1,305 @@
# Отчёт — лабораторная 5
# Лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Автор: Каменский Никита, БВТ-2302.
Целевая оценка: 5. Трек 1: очередь под перегрузкой.
PR: [ЛР 5 - БВТ-2302 - Каменский - на 5](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/38).
## Запуск из чистого clone
## Запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Desktop, Python 3.9+ и доступ в интернет для сборки.
Для сборки вне Docker нужен .NET SDK 9.
## Архитектура и решения
```sh
make setup
make up
make check
make validate
make test
make student-test
make research-grade5
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
make restore-api
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Сайт: http://localhost:18085/. Lab5 использует отдельную базу и volumes.
Для входа сначала зарегистрировать аккаунт с паролем 12–128 символов.
Остановка с сохранением данных: make down.
Намеренное удаление учебных данных: ./scripts/compose.sh down -v.
Секреты создаёт scripts/init-secrets.py в исключённой папке secrets/.
Повторный setup сохраняет существующие ключи. Общий volume RabbitMQ закреплён
за hostname broker; миграции запускаются отдельно с блокировкой PostgreSQL.
## Доказательства критериев
Версии: .NET SDK 9.0.306/runtime 9.0.10, PostgreSQL 17.6, Redis 7.4.2,
HAProxy 3.2.7-alpine, SeaweedFS 3.97, RabbitMQ 4.2.0-alpine,
RabbitMQ.Client 7.1.2, AWSSDK.S3 4.0.0, SkiaSharp 3.119.1.
Pillow 12.0.0 используется только для тестовых изображений и проверки результатов.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Изолированная сборка из копии исходников без .env, secrets, bin/obj и старых
volumes проверена отдельно: evidence/clean-build.txt. Созданы новые secrets,
проверен Compose config и собран Docker-образ.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Архитектура
## Эксперименты и отказы
Перенесены собственные исходники и интерфейс Lab4, Dockerfile, конфигурация
HAProxy и операционные скрипты. Skeleton, contracts и публичные tests взяты
из Lab5. Сопоставление архитектуры: docs/ARCHITECTURE.md.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Две API за HAProxy обслуживают общий PostgreSQL, Redis и приватный S3.
Добавлены отдельные RabbitMQ, worker и outbox dispatcher. Один C# образ
используется для нескольких ролей, но worker и API являются разными процессами.
Интерфейс опрашивает метаданные; queued/processing показываются пользователю,
после ready работает прежняя галерея с оригиналами. В браузере проверен переход
при остановленном и затем запущенном worker: evidence/ui-queued.jpg и ui-ready.jpg.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
## Границы сохранности
## Ограничения и следующие шаги
API полностью принимает PNG/JPEG до 10 MiB, проверяет MIME, имя и максимум
40 млн пикселей. Проверка декодируемости остаётся на API для совместимости
с ошибкой 415; resize выполняет только worker. Копирование ограничено буфером
64 KiB и временным файлом. Один upload/decoder на API, deadline 90 секунд.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Под блокировкой приёма проверяется количество queued/processing jobs. Если
лимит PIPELINE_CAPACITY достигнут, возвращается 429 и Retry-After: 3, без
записи нового объекта. По умолчанию лимит 128. Входящие HTTP-передачи и
валидация занимают ограниченные ресурсы до этой проверки; лимит защищает
принятую работу, а не означает отсутствие стоимости отклонённых запросов.
Полный оригинал сохраняется в приватный staging/<attachment UUID>. В одной
транзакции PostgreSQL записываются Attachment queued, job и outbox. Только
после S3 PUT и COMMIT API возвращает снимок 202 queued. Даже если worker уже
успел завершиться, ответ upload остаётся queued по контракту. Broker и worker
не входят в readiness API: при их отказе работа принимается до лимита outbox.
PostgreSQL и S3 остаются критичными для синхронного приёма.
Outbox dispatcher выбирает доступную запись FOR UPDATE SKIP LOCKED, отправляет
persistent сообщение в durable quorum queue attachments.v1 с mandatory и
publisher confirms. Отметка published_at записывается только после confirm.
Сбой между publish и COMMIT приводит к дублю, но не теряет намерение.
Брокер получает только поля jobs.schema.json, без байтов и credentials.
Job ID сохраняется при управляемых повторах и redelivery. Реальный carrier
проверен по jobs.schema.json с FormatChecker: evidence/job-envelope.json.
Worker получает job с ручным ack и проверяет состояние в PostgreSQL.
Для job используется session advisory lock; параллельный дубль не начинает
второй эффект. queued атомарно становится processing с attempt и lease_token.
Работа скачивается из доверенного staging_key в БД, проверяются размер и SHA-256.
Клиентский filename не используется как путь или ключ.
Имена результатов определены attachment_id/pipeline_version/variant.
Оригинал: originals/<id>/v1; варианты: variants/<id>/v1/<name>. Оригинальные
байты не перекодируются. После записи всех объектов одна транзакция публикует
метаданные вариантов и ready. Фиксация проверяет lease_token и срок аренды.
Ack выполняется после устойчивого результата или устойчивого решения о retry/failed.
Дубль ready не меняет completed_at, attempts или варианты.
## Изображения и ограничения worker
Thumbnail: JPEG 85, вписан в 256×256 без увеличения. Crop: центральный квадрат
128×128, для маленьких изображений возможно увеличение. Optimized: WebP 80,
вписан в 1280×1280 без увеличения. Размеры округляются вниз с минимумом 1 px.
EXIF orientation применяется перед расчётом размеров и crop. Alpha компонуется
на белый. Перекодирование из bitmap удаляет EXIF. Для fixture 320×200 thumbnail
имеет 256×160. Не гарантируется, что WebP каждого маленького файла меньше PNG.
Worker повторно проверяет bytes/checksum, заголовок, пиксели и декодирование.
Нативное преобразование идёт в дочернем процессе, который принудительно
останавливается по JOB_TIMEOUT_SECONDS, по умолчанию 30. Аренда дольше на 5 s.
Один decoder хранит RGBA bitmap до 160 млн байт и небольшие выходные bitmap.
Варианты формируются по очереди. Общие CPU/RAM worker ограничены Docker:
1 CPU, 1024 MiB. Время/память ограничены и при ошибках нативной библиотеки.
Переменные WORKER_CONCURRENCY и WORKER_PREFETCH по умолчанию 2 и 1;
каждый канал обрабатывает одно сообщение одновременно, prefetch ограничивает
число полученных, но ещё не подтверждённых сообщений на канале.
## Ошибки, повторы и replay
Невалидный или потерянный staging становится failed с безопасным invalid_image.
Временные ошибки S3/БД/таймаута приводят к queued и следующей попытке.
Backoff: min(30, 2^attempt) секунд плюс случайный jitter [0,1).
По умолчанию JOB_MAX_ATTEMPTS=5, затем failed с retry_exhausted.
Реестр failed находится в attachment_jobs, журнал в attachment_job_events.
Секреты, внутренние адреса и stack trace не выдаются клиенту.
```sh
make replay JOB_ID=<UUID задания>
```
Replay разрешён только для failed с ещё сохранённым staging. Он открывает
новый цикл попыток, увеличивает replay_count и сохраняет audit-событие;
идентификатор логического задания и ключ результата остаются прежними.
Replay уже ready является no-op. Контроль доступа к replay осуществляется
операторским доступом к Docker, публичного HTTP endpoint replay нет.
После гибели worker просроченное processing восстанавливается dispatcher.
Прежде чем менять его, проверяется advisory lock: работа живого владельца
не захватывается вторым worker. Lease ограничивает восстановление зависшего
состояния; нативный subprocess имеет отдельный принудительный timeout.
Потерянная доставка queued восстанавливается из сохранённого намерения по
истечении 30 s. Это at-least-once доставка с идемпотентным эффектом, не exactly-once.
## Очистка staging
После ready dispatcher удаляет staging только под блокировкой job и после
проверки устойчивого завершения. Отказ между DELETE и отметкой повторяется
безопасно. Failed staging сохраняется 86400 секунд для replay; минимум
retention 3600. Текущие queued/processing не очищаются.
```sh
make reconcile
./scripts/compose.sh exec -T dispatcher dotnet Messenger.Api.dll --reconcile --apply
```
Сверка untracked staging имеет grace 1 час и общую блокировку с приёмом upload.
Она не удаляет staging, известный job, в том числе текущую работу. Это закрывает
сбой S3 PUT до COMMIT приёма. Автоматически копировать отсутствующие байты
без резервной копии невозможно; такое задание получает failed.
## Проверки
| Критерий | Сценарий | Доказательство |
| --- | --- | --- |
| Весь обязательный HTTP, 202, оригинал, thumbnail | make test | evidence/checks.txt |
| Worker остановлен, API и broker перезапущены | make student-test | evidence/pipeline.json |
| Сохранность опубликованного задания в broker volume | make student-test | evidence/pipeline.json |
| Crop/WebP, EXIF orientation, alpha, отсутствие EXIF | make student-test | evidence/pipeline.json |
| Bytes/pixels/MIME/filename и worker pixel guard | make student-test | evidence/pipeline.json |
| COMMIT без publish, confirm без отметки outbox | make student-test | evidence/pipeline.json |
| Повтор готовой job, crash после S3 и после COMMIT до ack | make student-test | evidence/pipeline.json |
| Ограниченные retries, failed, явный replay и no-op ready | make student-test | evidence/pipeline.json |
| Очистка готового staging и retention failed | make student-test | evidence/pipeline.json |
| Три настройки, два размера, lag/p95/RSS | make research-grade5 | evidence/overload.json, docs/OVERLOAD.md |
| Backpressure и восстановление принятой работы | make research-grade5 | evidence/overload.json |
| Сессия и оригинал после restart всех хранилищ | make test-persistence | evidence/persistence.txt |
| Переключение API и общая идемпотентность | make test-failover | evidence/failover.txt |
Fault injection включается только оператором через PIPELINE_FAULT_FILE для
worker/dispatcher. Локальный файл содержит выбранный job ID и точку остановки;
он одноразовый. Публичный клиент не может включить паузу. Тест ждёт marker,
убивает настоящий контейнер и проверяет восстановление. По умолчанию путь
пустой и инструментация не исполняется. Тесты создают только синтетические данные.
## Исследование
Гипотеза: при одинаковых CPU/RAM рост concurrency может повысить расход памяти
без пропорционального роста скорости. Методика: 1/1, 2/2, 4/4 для concurrency /
prefetch; два размера PNG; три повтора каждой точки. Порядок меняется между
повторами. Каждая точка прогревается одним изображением. Затем batch 8 файлов,
четыре HTTP-клиента, ожидание ready всех восьми. Метрики и ограничения измерения
описаны в docs/OVERLOAD.md, индивидуальные job/HTTP и выборки в overload.json.
Отдельная серия останавливает worker, устанавливает лимит 4 и отправляет
8 файлов одновременно. Проверяются ровно 4 принятых и 4 отклонённых с 429,
непревышение лимита в БД и последующая готовность всех принятых после запуска.
Для крупных PNG медиана throughput составила 0.808, 0.951 и 0.813 готовых/с
для профилей 1/1, 2/2 и 4/4. Их медианы p95 готовности: 7.693, 6.520 и
8.308 s; RSS процессов worker: 189.7, 268.6 и 378.8 MiB. У маленьких PNG
лучше оказался 1/1. Рост concurrency не оказался универсальным ускорением.
Профили меняют concurrency и prefetch вместе; отдельный эффект каждого параметра
из этой серии не выводится. По умолчанию оставлено 2/1 как ограниченный
рабочий вариант; этот профиль отдельно не измерен и не объявляется оптимальным.
Пилотная серия выявила нехватку свободных томов S3 при настройке volume.max=8,
перенесённой из маленького стенда Lab4. До окончательных измерений лимит
поднят до 64; volumes сохранены, неудавшиеся пилотные задания восстановлены
явным replay. Пилотные результаты отделены от основной серии и не включаются
в её статистику. Это изменение ёмкости хранилища, не результат ускорения worker.
Отдельный запуск публичного smoke во время намеренного рестарта API получил
503; этот вывод сохранён в evidence/checks-during-restart.txt. Финальные проверки
на готовом стенде записаны в evidence/checks.txt.
## Ограничения и настройки
Один Docker-host и единственные PostgreSQL, S3, RabbitMQ и HAProxy являются
точками отказа. Несколько API и durable queue не защищают от потери host/диска.
Для исходников и broker volume требуются резервные копии. Внешний доступ
и настройка туннеля не входят в Lab5; наружу опубликован только localhost:18085.
Из .env.example используются APP_HOST/PORT, SESSION_TTL_SECONDS, API_REPLICAS,
WORKER_CONCURRENCY, JOB_MAX_ATTEMPTS, JOB_TIMEOUT_SECONDS и STAGING_RETENTION_SECONDS.
BROKER_URL_FILE сопоставлен BROKER_HOST и BROKER_PASSWORD_FILE.
S3_ACCESS_KEY_ID_FILE/S3_SECRET_ACCESS_KEY_FILE сопоставлены S3_ACCESS_KEY_FILE/
S3_SECRET_KEY_FILE. Endpoint S3 http://s3:8333, регион us-east-1, bucket messenger.
Redis остаётся кэшем, PostgreSQL источником сессий. Byte/pixel/Idempotency TTL
фиксированы контрактом в коде. Отдельно реализованы WORKER_PREFETCH и PIPELINE_CAPACITY.
Источники: контракт Lab5; официальная документация RabbitMQ publisher confirms,
consumer acknowledgements и .NET Client API Guide; AWS SDK for .NET; SkiaSharp.
ChatGPT использован при реализации и проверках.
## Панель обработки изображений
Кнопка «Обработка» под изображением открывает состояния, число попыток, время всего цикла и события задания из PostgreSQL. После готовности доступны оригинал, миниатюра, crop и WebP с размерами и скачиванием. Endpoint GET /api/v1/attachments/{id}/processing доступен только участникам чата; ключи S3 и служебные lease не возвращаются. Панель обновляется только пока открыта и задание выполняется. Проверка: .venv/bin/python student-tests/processing_panel.py. Результаты нагрузочного эксперимента выше относятся к версии до добавления этого интерфейса.
## Панель системы и внешний доступ
Кнопка «Система» доступна только аккаунту, UUID которого задан в OPS_USER_ID.
После регистрации владельца: `make operator USERNAME=ваш_логин`. Команда записывает
UUID в локальный .env и обновляет API. Если OPS_USER_ID не настроен, панель закрыта
для всех. Клиентское скрытие кнопки дополнено проверкой каждой серверной операции.
Панель показывает две API-реплики по heartbeat Redis, PostgreSQL, Redis, S3,
RabbitMQ, worker и dispatcher. Heartbeat процессов хранится в pipeline_presence;
срок свежести 10 секунд. Это признак работающего процесса, а не гарантия успешной
обработки любого задания. Проверка HAProxy основана на ответе текущего адреса.
Графики содержат последние 60 замеров открытой панели: активные задания в БД
и длительность запроса проверки зависимостей. CPU и память здесь не измеряются.
При ошибке запроса старые состояния отмечаются как неизвестные.
Последние 30 заданий и событий доступны владельцу. Кнопка «Повторить» использует
существующий механизм replay: только failed с сохранённым staging. Проверка состояния
и изменение задания выполняются в одной транзакции под advisory lock; второй
запрос не создаёт ещё один replay. Обычный пользователь получает 404, анонимный 401.
Проверка: `OPS_TEST_PASSWORD=пароль_владельца make operations-test` (логин по умолчанию
n1k0der, другой задаётся OPS_TEST_USERNAME). Тест использует контролируемый failed
fixture и проверяет завершение настоящим worker, восстанавливая worker в finally.
Создать чат можно по @логину или UUID. Поиск выполняется только по точному логину,
после входа; возвращает публичный User DTO без password_hash.
Временный внешний доступ: установить cloudflared, затем `make share`.
Отключение: `make stop-share`. Ссылка и состояние процесса остаются в .local вне Git.
Ссылка работает, пока включён компьютер, запущены Docker и туннель; при следующем
запуске адрес может измениться. Снаружи доступен только HTTP-вход мессенджера,
порты БД и RabbitMQ не публикуются. Собеседник регистрирует свой аккаунт и создаёт
чат с @логином владельца. Учётная запись собеседника не получает права оператора.
Через HTTPS-туннель пустой POST входа может приходить с Transfer-Encoding: chunked.
API проверяет фактическое отсутствие тела (с таймаутом 3 секунды), а не отклоняет
такой способ передачи целиком. Непустое тело по-прежнему возвращает 400.
Проверка: `.venv/bin/python student-tests/chunked_login.py`.
Внешняя регистрация, вход, точный поиск, создание общего чата и сообщение проверены
через публичный HTTPS-адрес; результат без токенов записан в evidence/operations.txt.
## Оформление интерфейса
Общие стили мессенджера находятся в wwwroot/design.css, панель системы в
operations.css и operations.js. Добавлены 16 SVG на общей сетке и спрайт,
две новые иллюстрации персонажа в WebP с прозрачностью. Использование графики
описано в docs/ASSETS.md. Изображения загружаются в размерах 320/640, без
исходных PNG в webroot. На ширине 1366 и 390 проверены отсутствие горизонтальной
прокрутки, загрузка иллюстраций, навигация панели и доступность поля сообщения.
В браузере проверены реальные состояния восьми карточек API/зависимостей.
SVG разобраны XML-парсером, все новые asset URL отвечают 200. make check,
make validate и публичный smoke-сценарий прошли после изменений.
## Удобство и устойчивость интерфейса
В чате добавлены копирование текста, вставка PNG/JPEG из буфера и перенос файла
мышью. Поиск чатов открывается сочетанием ⌘K/Ctrl+K, результаты можно очистить
кнопкой или Escape. В панели задания фильтруются среди последних 30 записей;
состояние кнопки повтора не перерисовывается без изменения данных. При смене
пользователя содержимое панели удаляется из страницы. Анимации уважают настройку
уменьшения движения.
Формы входа и регистрации ожидают загрузки скриптов; их резервный метод POST не
помещает поля в адрес страницы. HAProxy проверяет API с тайм-аутом 2 секунды и
интервалом 1 секунда. Docker healthcheck RabbitMQ проверяет TCP-порт 5672 без
запуска Erlang-процесса на каждый опрос; проверка панели дополнительно выполняет
AMQP-запрос. Лимит RabbitMQ по умолчанию 512 МиБ, настраивается BROKER_MEMORY.
При прежней проверке регулярные Erlang-процессы исчерпывали лимит при запуске.
Результаты повторных проверок и переключения API: evidence/ui-reliability.txt.
Предыдущие измерения нагрузки относятся к настройкам и версии, указанным в их
исходных результатах; повторный эксперимент для новых параметров не проводился.
+263
View File
@@ -0,0 +1,263 @@
name: mtusi2026lab5
x-api: &api
image: messenger-lab5:local
build: .
environment:
OPS_USER_ID: ${OPS_USER_ID:-}
BROKER_HOST: broker
BROKER_PASSWORD_FILE: /run/secrets/broker_password
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL_FILE: /run/secrets/database_url
REDIS_URL: redis:6379
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_SIZE: ${DB_POOL_SIZE:-20}
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger
S3_ACCESS_KEY_FILE: /run/secrets/s3_access
S3_SECRET_KEY_FILE: /run/secrets/s3_secret
PIPELINE_CAPACITY: ${PIPELINE_CAPACITY:-128}
MESSAGE_BATCHING: ${MESSAGE_BATCHING:-true}
Logging__LogLevel__Default: Warning
Logging__LogLevel__Microsoft.Hosting.Lifetime: Information
secrets:
- broker_password
- database_url
- s3_access
- s3_secret
cpus: ${API_CPUS:-1}
mem_limit: ${API_MEMORY:-512m}
stop_grace_period: 10s
healthcheck:
test: ["CMD", "curl", "--fail", "--silent", "--max-time", "1", "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 5
start_period: 5s
restart: unless-stopped
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
services:
postgres:
image: postgres:17.6
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets:
- postgres_password
volumes:
- postgres_data:/var/lib/postgresql/data
cpus: ${POSTGRES_CPUS:-0.5}
mem_limit: 512m
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 2s
timeout: 2s
retries: 30
restart: unless-stopped
redis:
image: redis:7.4.2-alpine
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "64mb", "--maxmemory-policy", "allkeys-lru"]
cpus: 0.25
mem_limit: 96m
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 2s
timeout: 2s
retries: 15
restart: unless-stopped
migrate:
image: messenger-lab5:local
command: ["--migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
cpus: 0.5
mem_limit: 256m
restart: "no"
healthcheck:
disable: true
depends_on:
postgres:
condition: service_healthy
s3:
image: chrislusf/seaweedfs:3.97
command: ["server", "-dir=/data", "-ip=s3", "-master.volumeSizeLimitMB=100", "-volume.max=64", "-filer", "-s3", "-s3.config=/run/secrets/s3_config"]
secrets:
- s3_config
volumes:
- s3_data:/data
cpus: 0.5
mem_limit: 512m
restart: unless-stopped
healthcheck:
test: ["CMD-SHELL", "wget -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q '403'"]
interval: 2s
timeout: 2s
retries: 40
start_period: 35s
s3-bootstrap:
image: messenger-lab5:local
command: ["--bootstrap"]
environment:
S3_ENDPOINT: http://s3:8333
S3_ACCESS_KEY_FILE: /run/secrets/s3_admin_access
S3_SECRET_KEY_FILE: /run/secrets/s3_admin_secret
secrets:
- s3_admin_access
- s3_admin_secret
depends_on:
s3:
condition: service_healthy
restart: on-failure
healthcheck:
disable: true
ulimits:
core: 0
cpus: 0.25
mem_limit: 128m
broker:
image: rabbitmq:4.2.0-alpine
hostname: broker
environment:
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2 +sbwt none"
secrets:
- source: broker_config
target: /etc/rabbitmq/conf.d/20-messenger.conf
volumes:
- broker_data:/var/lib/rabbitmq
cpus: 0.5
mem_limit: ${BROKER_MEMORY:-512m}
restart: unless-stopped
healthcheck:
test: ["CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/127.0.0.1/5672'"]
interval: 3s
timeout: 5s
retries: 30
start_period: 15s
dispatcher:
image: messenger-lab5:local
command: ["--dispatcher"]
environment: &pipeline-env
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:8333
S3_BUCKET: messenger
S3_ACCESS_KEY_FILE: /run/secrets/s3_access
S3_SECRET_KEY_FILE: /run/secrets/s3_secret
BROKER_HOST: broker
BROKER_PASSWORD_FILE: /run/secrets/broker_password
PIPELINE_FAULT_FILE: ${PIPELINE_FAULT_FILE:-}
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-30}
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
volumes:
- ./.local/faults:/run/faults
secrets: &pipeline-secrets
- database_url
- s3_access
- s3_secret
- broker_password
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
disable: true
cpus: 0.25
mem_limit: 192m
restart: unless-stopped
stop_grace_period: 10s
worker:
image: messenger-lab5:local
command: ["--worker"]
environment:
<<: *pipeline-env
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
volumes:
- ./.local/faults:/run/faults
secrets: *pipeline-secrets
depends_on:
migrate:
condition: service_completed_successfully
s3-bootstrap:
condition: service_completed_successfully
broker:
condition: service_healthy
healthcheck:
disable: true
cpus: 1
mem_limit: 1024m
restart: unless-stopped
stop_grace_period: 10s
ulimits:
core: 0
api:
<<: *api
deploy:
replicas: ${API_REPLICAS:-2}
lb:
image: haproxy:3.2.7-alpine
volumes:
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${API_PORT:-18085}:8080"
cpus: 0.5
mem_limit: 96m
stop_grace_period: 10s
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
retries: 30
start_period: 5s
restart: unless-stopped
depends_on:
api:
condition: service_healthy
secrets:
broker_config:
file: ./secrets/broker.conf
broker_password:
file: ./secrets/broker_password
s3_config:
file: ./secrets/s3_config
s3_admin_access:
file: ./secrets/s3_admin_access
s3_admin_secret:
file: ./secrets/s3_admin_secret
s3_access:
file: ./secrets/s3_access
s3_secret:
file: ./secrets/s3_secret
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
volumes:
broker_data:
s3_data:
postgres_data:
+38
View File
@@ -0,0 +1,38 @@
# Переход Lab4 → Lab5
| Часть | Lab4 | Lab5 |
| --- | --- | --- |
| Приём файла | S3 PUT и 201 ready | Приватный staging, метаданные, job и outbox, затем 202 queued |
| Преобразования | Нет | Worker: JPEG thumbnail, JPEG crop, WebP optimized |
| Брокер | Нет | RabbitMQ, durable quorum queue, persistent messages, publisher confirms |
| Надёжность публикации | Не требуется | OutboxDispatcher читает PostgreSQL и публикует с подтверждением |
| Повторная доставка | Только отправка сообщений | Дополнительно job_id, состояние, advisory lock, lease, детерминированные S3 keys |
| Ошибки | Синхронный HTTP | Ограниченные попытки, backoff/jitter, failed registry и явный replay |
| Интерфейс | Превью готового оригинала | Опрос queued/processing, затем прежняя галерея и скачивание |
## Процессы
Один образ C# запускается в нескольких ролях. API обрабатывает HTTP. Worker
запускается с --worker, dispatcher с --dispatcher. Это отдельные контейнеры
и процессы. Миграции и создание bucket выполняются одноразовыми сервисами.
Worker запускает короткоживущий дочерний процесс --render-image для SkiaSharp:
его можно принудительно остановить при превышении времени декодирования.
PostgreSQL хранит метаданные, состояния, аренду работы, outbox и журнал событий.
SeaweedFS хранит staging, оригинал и варианты. Redis сохраняет прежнюю роль
кэша сессий. RabbitMQ переносит только идентификаторы и служебные поля.
## Файлы
- Services/AttachmentService.cs: приём, проверка, лимит очереди, staging и транзакция приёма.
- Data/Migrations/005_pipeline.sql: jobs, outbox, журнал, новые статусы и варианты.
- Pipeline/OutboxDispatcher.cs: надёжная публикация в RabbitMQ.
- Pipeline/PipelineWorker.cs: получение, ограниченная обработка, запись S3 и результата.
- Pipeline/ImageRenderer.cs: EXIF orientation, resize, crop, JPEG/WebP.
- Pipeline/JobStore.cs: захват работы, lease, повтор, восстановление и replay.
- Pipeline/PipelineMaintenance.cs: очистка staging с проверкой состояния.
- Pipeline/FaultInjection.cs: выключенные по умолчанию локальные точки остановки для тестов.
- wwwroot/attachments.js: ожидание обработки и просмотр изображения.
Program.cs выбирает режим процесса и подключает зависимости. HTTP-контракт,
авторизация, доступ к чатам и идемпотентность сообщений продолжают работать.
+41
View File
@@ -0,0 +1,41 @@
# Графика интерфейса
Набор: src/Messenger.Api/wwwroot/assets/.
## SVG
16 отдельных файлов в icons/: api, database, queue, storage, worker, dispatcher,
cache, proxy, browser, chat, image, activity, check, clock, error, overview.
Размерная сетка 48×48, обводка 2, скруглённые окончания, прозрачный фон.
Для обычной картинки: `<img src="/assets/icons/database.svg" alt="База данных">`.
Для управления цветом через CSS используйте спрайт:
```html
<svg viewBox="0 0 48 48" width="24" height="24" aria-hidden="true"
style="color:#FFE135">
<use href="/assets/system-icons.svg#database"></use>
</svg>
```
Файл system-icons.svg должен находиться на том же origin. В другом проекте
измените путь к спрайту. Для кнопки задайте текст или aria-label; декоративному
SVG оставьте aria-hidden. Исходные SVG не зависят от JavaScript или библиотеки.
conversation-grid.svg используется как малоконтрастный фон переписки.
## Иллюстрации
kolovertysh-message-320.webp / kolovertysh-message-640.webp: пустой экран чатов
и боковая панель системы. kolovertysh-reconnect-320.webp: ошибка подключения.
Все изображения содержат alpha; прозрачность сохранена при экспорте.
Созданы с помощью ChatGPT по существующему образу персонажа. Для UI используются
уменьшенные WebP, исходные PNG оставлены локально вне webroot.
## Цвета и движение
Основной фон #262626, акцент #FFE135, основной текст #EEEEE2.
Рабочие поверхности #282A25 и #30312C, вторичный текст #A8AC9B.
Состояния: зелёный #A6CF9A, ошибка #EEB09F.
Анимация точек между узлами запускается после реального ответа панели.
Выбранный API определяется по X-Instance-Id. При prefers-reduced-motion
переходы, появление окон и движение точек отключены. Графики строят прямые
отрезки между измерениями, без сглаживания или искусственных данных.
+35
View File
@@ -0,0 +1,35 @@
# Очередь под нагрузкой
Три повтора; одинаковые CPU/RAM worker; batch 8, четыре HTTP-клиента.
| Файл | Concurrency / prefetch | Готовых/с, медиана | p95 до ready, s | p95 ожидания, s | RSS процессов, MiB |
| --- | --- | ---: | ---: | ---: | ---: |
| small | 1/1 | 4.411 | 1.524 | 1.291 | 157.6 |
| small | 2/2 | 3.625 | 1.675 | 1.319 | 165.2 |
| small | 4/4 | 3.904 | 1.788 | 1.052 | 281.9 |
| large | 1/1 | 0.808 | 7.693 | 6.868 | 189.7 |
| large | 2/2 | 0.951 | 6.520 | 4.749 | 268.6 |
| large | 4/4 | 0.813 | 8.308 | 4.477 | 378.8 |
p95: nearest rank; при batch 8 это максимальная задержка. RSS: сумма VmRSS
процессов dotnet внутри worker, включая дочерние декодеры; общие страницы могут
считаться несколько раз. memory.current измерен отдельно. Период выборок 200 ms
плюс docker exec; короткие пики могут быть пропущены. Время включает загрузку
batch и ожидание последнего ready, без регистрации и прогрева. Queue lag: от
создания записи задания до первой processing. Время до ready: по часам PostgreSQL.
Backpressure: при остановленном worker лимит 4 принял ровно 4 из 8 файлов;
остальные получили 429. После запуска обработались все четыре принятых задания.
Исходные результаты: evidence/overload.json. Один Docker-host, небольшие batch,
параллельно могут работать другие локальные приложения. Эти значения не являются
пределом производительности сервера. Вывод сравнения относится к этому профилю.
![Результаты](../evidence/overload.png)
В этом профиле для small один обработчик оказался быстрее двух и четырёх.
Для large вариант 2/2 дал лучшую медиану throughput и меньшую p95 готовности,
чем 1/1, при большем RSS. Четыре обработчика уменьшили время ожидания начала,
но увеличили полное время обработки и память. По умолчанию оставлены 2/1:
два параллельных обработчика с небольшим prefetch. Пара 2/1 отдельно не входит
в сравнение, поэтому не заявляется как измеренный оптимум.
+23
View File
@@ -0,0 +1,23 @@
ЛР 5 - БВТ-2302 - Каменский - на 5
Автор / группа: Каменский Никита, БВТ-2302.
Целевая оценка: 5. Трек 1: очередь под перегрузкой.
Из Lab4 перенесены собственные API, PostgreSQL, Redis, S3, HAProxy и интерфейс.
Загрузка теперь возвращает 202 queued после сохранения staging и транзакции
Attachment/job/outbox. Отдельные RabbitMQ, dispatcher и worker выполняют
доставку и обработку: thumbnail JPEG, crop JPEG, optimized WebP.
Добавлены ограниченные retries с backoff/jitter, failed registry, replay,
lease recovery и очистка staging после устойчивого результата. Дубли не
создают второй набор вариантов. UI показывает ожидание и обработку изображения.
Чистый запуск: make setup, make up. Проверки: make check, make validate,
make test, make student-test, make research-grade5; persistence/failover
по REPORT.md. Результаты: evidence/, docs/OVERLOAD.md и REPORT.md.
Публичный контракт, tests и README задания сохранены. Один Docker-host,
единственные БД, S3, RabbitMQ и HAProxy остаются точками отказа. Измерения
получены на локальном стенде с ограниченными ресурсами и небольшими batch.
ChatGPT использован при реализации и проверках.
+9
View File
@@ -0,0 +1,9 @@
.venv/bin/python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/python tests/smoke.py --base-url "http://localhost:18085"
FAIL: GET /api/v1/attachments/bb21267f-b7da-491b-96b3-7f55f8c3d884: expected [200], received 503
make: *** [test] Error 1
.venv/bin/python tests/smoke.py --base-url "http://localhost:18085"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+6
View File
@@ -0,0 +1,6 @@
.venv/bin/python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/python tests/smoke.py --base-url "http://localhost:18085"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+67
View File
@@ -0,0 +1,67 @@
Local secrets ready
#0 building with "desktop-linux" instance using docker driver
#1 [internal] load build definition from Dockerfile
#1 transferring dockerfile: 969B done
#1 DONE 0.0s
#2 [internal] load metadata for mcr.microsoft.com/dotnet/sdk:9.0.306
#2 DONE 0.6s
#3 [internal] load metadata for mcr.microsoft.com/dotnet/aspnet:9.0.10
#3 DONE 0.6s
#4 [internal] load .dockerignore
#4 transferring context: 218B done
#4 DONE 0.0s
#5 [build 1/6] FROM mcr.microsoft.com/dotnet/sdk:9.0.306@sha256:81f6d622fe21ed9d31375167f62a3538ff4d6835f9d5e6da9c2defa8a84b7687
#5 resolve mcr.microsoft.com/dotnet/sdk:9.0.306@sha256:81f6d622fe21ed9d31375167f62a3538ff4d6835f9d5e6da9c2defa8a84b7687 0.0s done
#5 DONE 0.0s
#6 [runtime 1/4] FROM mcr.microsoft.com/dotnet/aspnet:9.0.10@sha256:3dcb33395722d14c80d19107158293ed677b2c07841100d51df07275ae2b2682
#6 resolve mcr.microsoft.com/dotnet/aspnet:9.0.10@sha256:3dcb33395722d14c80d19107158293ed677b2c07841100d51df07275ae2b2682 0.0s done
#6 DONE 0.0s
#7 [internal] load build context
#7 transferring context: 420.67kB 0.0s done
#7 DONE 0.0s
#8 [build 3/6] COPY src/Messenger.Api/Messenger.Api.csproj src/Messenger.Api/
#8 CACHED
#9 [runtime 2/4] RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
#9 CACHED
#10 [runtime 3/4] WORKDIR /app
#10 CACHED
#11 [build 4/6] RUN dotnet restore src/Messenger.Api/Messenger.Api.csproj
#11 CACHED
#12 [build 6/6] RUN dotnet publish src/Messenger.Api/Messenger.Api.csproj -c Release --no-restore -o /app/publish
#12 CACHED
#13 [build 2/6] WORKDIR /src
#13 CACHED
#14 [build 5/6] COPY src/Messenger.Api/ src/Messenger.Api/
#14 CACHED
#15 [runtime 4/4] COPY --from=build /app/publish .
#15 CACHED
#16 exporting to image
#16 exporting layers done
#16 exporting manifest sha256:8032c23ca82abca0a98189e25912a472fa9f9766aaadfb1dc72e3a8ffd87797d done
#16 exporting config sha256:6d5f902dbe672c56753a47a14e780cedd15d0a6134b91f7e1bfcb0da75d784e6 0.0s done
#16 exporting attestation manifest sha256:1c91c52f053bf7e2022c725dfdc1906fc181127ae049e472f4f61cbc5255c452 0.0s done
#16 exporting manifest list sha256:bc7b3964de3dcbf7f81a4313518695817ae781561b99f22cf02e8114b41271ba 0.0s done
#16 naming to docker.io/library/messenger-lab5:clean-check done
#16 unpacking to docker.io/library/messenger-lab5:clean-check
#16 unpacking to docker.io/library/messenger-lab5:clean-check done
#16 DONE 0.1s
View build details: docker-desktop://dashboard/build/desktop-linux/desktop-linux/mwlwenbjkzp9ukzg8a92tezj0
PASS: isolated source context, fresh generated secrets, compose config and Docker build. No existing .env, volumes or build outputs copied.
+55
View File
@@ -0,0 +1,55 @@
.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18085"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 1, "first_recovery_seconds": 1.87}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab5-api-2 messenger-lab5:local "dotnet Messenger.Ap…" api 3 minutes ago Up 30 seconds (healthy) 8080/tcp
mtusi2026lab5-broker-1 rabbitmq:4.2.0-alpine "docker-entrypoint.s…" broker 15 minutes ago Up 54 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab5-dispatcher-1 messenger-lab5:local "dotnet Messenger.Ap…" dispatcher About a minute ago Up 30 seconds 8080/tcp
mtusi2026lab5-lb-1 haproxy:3.2.7-alpine "docker-entrypoint.s…" lb 15 minutes ago Up 30 seconds (healthy) 127.0.0.1:18085->8080/tcp
mtusi2026lab5-postgres-1 postgres:17.6 "docker-entrypoint.s…" postgres 15 minutes ago Up 55 seconds (healthy) 5432/tcp
mtusi2026lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 55 seconds (healthy) 6379/tcp
mtusi2026lab5-s3-1 chrislusf/seaweedfs:3.97 "/entrypoint.sh serv…" s3 7 minutes ago Up 52 seconds (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
mtusi2026lab5-worker-1 messenger-lab5:local "dotnet Messenger.Ap…" worker About a minute ago Up 30 seconds 8080/tcp
./scripts/compose.sh up -d --wait api lb
Container mtusi2026lab5-redis-1 Running
Container mtusi2026lab5-api-2 Running
Container mtusi2026lab5-s3-1 Running
Container mtusi2026lab5-lb-1 Running
Container mtusi2026lab5-postgres-1 Running
Container mtusi2026lab5-s3-1 Waiting
Container mtusi2026lab5-postgres-1 Waiting
Container mtusi2026lab5-s3-1 Healthy
Container mtusi2026lab5-postgres-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Starting
Container mtusi2026lab5-migrate-1 Starting
Container mtusi2026lab5-s3-bootstrap-1 Started
Container mtusi2026lab5-migrate-1 Started
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-redis-1 Waiting
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-redis-1 Healthy
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-api-1 Starting
Container mtusi2026lab5-api-1 Started
Container mtusi2026lab5-api-2 Waiting
Container mtusi2026lab5-api-1 Waiting
Container mtusi2026lab5-api-2 Healthy
Container mtusi2026lab5-api-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Waiting
Container mtusi2026lab5-postgres-1 Waiting
Container mtusi2026lab5-redis-1 Waiting
Container mtusi2026lab5-api-2 Waiting
Container mtusi2026lab5-api-1 Waiting
Container mtusi2026lab5-migrate-1 Waiting
Container mtusi2026lab5-s3-1 Waiting
Container mtusi2026lab5-lb-1 Waiting
Container mtusi2026lab5-migrate-1 Exited
Container mtusi2026lab5-lb-1 Healthy
Container mtusi2026lab5-s3-bootstrap-1 Exited
Container mtusi2026lab5-redis-1 Healthy
Container mtusi2026lab5-s3-1 Healthy
Container mtusi2026lab5-postgres-1 Healthy
Container mtusi2026lab5-api-2 Healthy
Container mtusi2026lab5-api-1 Healthy
+9
View File
@@ -0,0 +1,9 @@
{
"schema_version": 1,
"job_id": "3590b5cc-4392-4c8e-819f-a44e67ca7c64",
"type": "attachment.process.v1",
"attachment_id": "80fc41c9-e25b-4bcb-8475-e08d62822e78",
"pipeline_version": 1,
"attempt": 1,
"created_at": "2026-11-09T22:22:38.7816100Z"
}
+9
View File
@@ -0,0 +1,9 @@
Проверки дополнения панели системы:
- Сборка .NET: 0 ошибок, 0 предупреждений.
- make check, make validate, make test: PASS.
- operations_panel.py: PASS: operator-only metrics and replay; anonymous 401; regular user 404; exact login lookup; one replay completes with real worker.
- Повторный POST replay для queued: 409; replay_count остался 1.
- Интерфейс проверен в браузере: экземпляры API, службы, графики, события и повторный запуск доступны владельцу.
Первый прогон теста после рестарта API встретил временный 503 при ожидании ready; само задание завершилось. Ожидание учитывает 503 до общего таймаута, финальное состояние ready остаётся обязательным.
PASS: public HTTPS registration, login, lookup @n1k0der, shared chat and message; operations denied to guest
PASS: empty chunked login accepted; nonempty chunked login rejected (student-tests/chunked_login.py).
+20
View File
@@ -0,0 +1,20 @@
.venv/bin/python student-tests/overload_research.py --base-url "http://localhost:18085"
run 1/18: small, 1/1, 4.41 ready/s
run 2/18: small, 2/2, 3.62 ready/s
run 3/18: small, 4/4, 3.91 ready/s
run 4/18: large, 1/1, 0.78 ready/s
run 5/18: large, 2/2, 0.95 ready/s
run 6/18: large, 4/4, 0.81 ready/s
run 7/18: large, 2/2, 0.99 ready/s
run 8/18: large, 4/4, 0.97 ready/s
run 9/18: large, 1/1, 0.88 ready/s
run 10/18: small, 2/2, 3.57 ready/s
run 11/18: small, 4/4, 3.62 ready/s
run 12/18: small, 1/1, 4.65 ready/s
run 13/18: small, 4/4, 3.90 ready/s
run 14/18: small, 1/1, 3.86 ready/s
run 15/18: small, 2/2, 4.66 ready/s
run 16/18: large, 4/4, 0.69 ready/s
run 17/18: large, 1/1, 0.81 ready/s
run 18/18: large, 2/2, 0.73 ready/s
PASS: overload research and bounded admission
+14
View File
@@ -0,0 +1,14 @@
.venv/bin/python student-tests/overload_research.py --base-url "http://localhost:18085"
run 1/18: small, 1/1, 4.60 ready/s
run 2/18: small, 2/2, 4.06 ready/s
run 3/18: small, 4/4, 0.84 ready/s
run 4/18: large, 1/1, 0.86 ready/s
FAIL: Infrastructure command failed: 2cad4d5d_mtusi2026lab5-api-2 Stopping
Container 74e89bf6ba55_mtusi2026lab5-worker-1 Recreate
Container 5c6d2cad4d5d_mtusi2026lab5-api-2 Stopped
Container 5c6d2cad4d5d_mtusi2026lab5-api-2 Removing
Container mtusi2026lab5-api-2 Error response from daemon: No such container: 5c6d2cad4d5db9f150fce5fc6f432103bddadddad8b1bf0d5f723466384f4843
Container mtusi2026lab5-api-2 Removing
Error response from daemon: No such container: 5c6d2cad4d5db9f150fce5fc6f432103bddadddad8b1bf0d5f723466384f4843
make: *** [research-grade5] Error 1
+693
View File
@@ -0,0 +1,693 @@
{
"hypothesis": "With fixed worker CPU, more simultaneous image processes need not improve throughput and can increase RSS; bounded admission protects the queue when the worker stops.",
"profile": {
"settings": [
[
1,
1
],
[
2,
2
],
[
4,
4
]
],
"repeats": 3,
"batch": 8,
"upload_clients": 4,
"worker_cpu": 1,
"worker_memory_limit_mib": 1024,
"warmup_images": 1,
"image_sizes_bytes": {
"small": 111817,
"large": 3319965
}
},
"runs": [
{
"repeat": 1,
"image": "small",
"concurrency": 1,
"prefetch": 1,
"successful_jobs": 8,
"errors": 0,
"seconds": 1.7385662909946404,
"throughput": 4.601492644507199,
"p95_ready_seconds": 1.355825,
"p95_queue_lag_seconds": 1.221817,
"peak_rss_bytes": 152162304,
"peak_cgroup_memory_bytes": 53309440,
"worker_cpu_seconds": 1.329851,
"requests": [
{
"upload_seconds": 0.04113916699861875,
"status": 202
},
{
"upload_seconds": 0.10876779200043529,
"status": 202
},
{
"upload_seconds": 0.12739412499649916,
"status": 202
},
{
"upload_seconds": 0.1493414579963428,
"status": 202
},
{
"upload_seconds": 0.11392083399550756,
"status": 202
},
{
"upload_seconds": 0.05546583300019847,
"status": 202
},
{
"upload_seconds": 0.04235250000056112,
"status": 202
},
{
"upload_seconds": 0.02778862499690149,
"status": 202
}
],
"jobs": [
{
"id": "d2b88786-66d5-44f9-8398-afd799333b0d",
"ready_seconds": 0.27748,
"queue_lag_seconds": 0.115927,
"attempts": 1
},
{
"id": "746abbed-2f2a-44a7-8054-8e2fb4e40d60",
"ready_seconds": 0.405159,
"queue_lag_seconds": 0.269961,
"attempts": 1
},
{
"id": "a4fbcad3-77a5-402f-8ce9-a286c3c50998",
"ready_seconds": 0.52426,
"queue_lag_seconds": 0.395915,
"attempts": 1
},
{
"id": "bbc83e93-f179-4f84-a5a1-4bfb334ecf5d",
"ready_seconds": 0.629692,
"queue_lag_seconds": 0.459954,
"attempts": 1
},
{
"id": "8d5f7573-9a93-4f75-9871-fe57beaafa15",
"ready_seconds": 0.909593,
"queue_lag_seconds": 0.620331,
"attempts": 1
},
{
"id": "8280ed8d-da55-40a0-bde3-36ee21893e72",
"ready_seconds": 1.068921,
"queue_lag_seconds": 0.902328,
"attempts": 1
},
{
"id": "d8dbae97-6243-4f95-b9cc-4a3e8fdfefbb",
"ready_seconds": 1.225459,
"queue_lag_seconds": 1.068507,
"attempts": 1
},
{
"id": "dcd72436-5215-4aca-900d-74767f0385b2",
"ready_seconds": 1.355825,
"queue_lag_seconds": 1.221817,
"attempts": 1
}
],
"samples": [
{
"time": 1794262118.743776,
"memory_bytes": 42041344,
"cpu_usec": 795694,
"rss_bytes": 103636992
},
{
"time": 1794262119.0674748,
"memory_bytes": 48332800,
"cpu_usec": 1134009,
"rss_bytes": 127852544
},
{
"time": 1794262119.545117,
"memory_bytes": 50176000,
"cpu_usec": 1544976,
"rss_bytes": 152162304
},
{
"time": 1794262119.89533,
"memory_bytes": 53309440,
"cpu_usec": 1889921,
"rss_bytes": 147939328
},
{
"time": 1794262120.2800698,
"memory_bytes": 46026752,
"cpu_usec": 2125545,
"rss_bytes": 107089920
}
]
},
{
"repeat": 1,
"image": "small",
"concurrency": 2,
"prefetch": 2,
"successful_jobs": 8,
"errors": 0,
"seconds": 1.969336124995607,
"throughput": 4.062282663919469,
"p95_ready_seconds": 1.525365,
"p95_queue_lag_seconds": 1.016688,
"peak_rss_bytes": 219119616,
"peak_cgroup_memory_bytes": 62783488,
"worker_cpu_seconds": 1.534243,
"requests": [
{
"upload_seconds": 0.03250100000150269,
"status": 202
},
{
"upload_seconds": 0.044958750004298054,
"status": 202
},
{
"upload_seconds": 0.05604845799825853,
"status": 202
},
{
"upload_seconds": 0.06452524999622256,
"status": 202
},
{
"upload_seconds": 0.0502204169970355,
"status": 202
},
{
"upload_seconds": 0.04115625000122236,
"status": 202
},
{
"upload_seconds": 0.03952566699445015,
"status": 202
},
{
"upload_seconds": 0.0422521250002319,
"status": 202
}
],
"jobs": [
{
"id": "9353ef12-3388-4f17-a52e-7788d9631d5e",
"ready_seconds": 0.394761,
"queue_lag_seconds": 0.116528,
"attempts": 1
},
{
"id": "5ff9fdf3-e9e6-446e-95c7-978c9b3fd22b",
"ready_seconds": 0.404182,
"queue_lag_seconds": 0.129876,
"attempts": 1
},
{
"id": "314ef166-fd5e-4c85-abf7-15185aea7576",
"ready_seconds": 0.692893,
"queue_lag_seconds": 0.388414,
"attempts": 1
},
{
"id": "52a91f37-b649-4896-8051-5f39802fb3ad",
"ready_seconds": 0.751849,
"queue_lag_seconds": 0.382157,
"attempts": 1
},
{
"id": "052cd532-0568-47b6-a95c-e35bc68c6d81",
"ready_seconds": 0.976998,
"queue_lag_seconds": 0.669662,
"attempts": 1
},
{
"id": "5df94c7d-97a6-4ca1-a3bb-c663ea5a7e41",
"ready_seconds": 1.032436,
"queue_lag_seconds": 0.738607,
"attempts": 1
},
{
"id": "926d6d55-a9b6-408f-a7e2-2bf2b98871b0",
"ready_seconds": 1.523118,
"queue_lag_seconds": 1.005413,
"attempts": 1
},
{
"id": "dad6cf86-7200-4468-b927-aca54ae14db9",
"ready_seconds": 1.525365,
"queue_lag_seconds": 1.016688,
"attempts": 1
}
],
"samples": [
{
"time": 1794262122.505246,
"memory_bytes": 42008576,
"cpu_usec": 782744,
"rss_bytes": 103116800
},
{
"time": 1794262122.863539,
"memory_bytes": 54919168,
"cpu_usec": 1204770,
"rss_bytes": 161333248
},
{
"time": 1794262123.2467399,
"memory_bytes": 57008128,
"cpu_usec": 1566113,
"rss_bytes": 175915008
},
{
"time": 1794262123.8038552,
"memory_bytes": 62783488,
"cpu_usec": 2036485,
"rss_bytes": 219119616
},
{
"time": 1794262124.115168,
"memory_bytes": 45998080,
"cpu_usec": 2316987,
"rss_bytes": 106729472
}
]
},
{
"repeat": 1,
"image": "small",
"concurrency": 4,
"prefetch": 4,
"successful_jobs": 8,
"errors": 0,
"seconds": 9.549317416000122,
"throughput": 0.8377562135065066,
"p95_ready_seconds": 9.209337,
"p95_queue_lag_seconds": 7.286448,
"peak_rss_bytes": 289599488,
"peak_cgroup_memory_bytes": 87416832,
"worker_cpu_seconds": 5.107425,
"requests": [
{
"upload_seconds": 0.03872108399809804,
"status": 202
},
{
"upload_seconds": 0.06579587499436457,
"status": 202
},
{
"upload_seconds": 0.08007283299957635,
"status": 202
},
{
"upload_seconds": 0.09368379199440824,
"status": 202
},
{
"upload_seconds": 0.040345542001887225,
"status": 202
},
{
"upload_seconds": 0.07635662500251783,
"status": 202
},
{
"upload_seconds": 0.032701916999940295,
"status": 202
},
{
"upload_seconds": 0.05977749999874504,
"status": 202
}
],
"jobs": [
{
"id": "c70a41eb-b3da-40f7-9669-fbc0adf914d1",
"ready_seconds": 9.074966,
"queue_lag_seconds": 7.278132,
"attempts": 1
},
{
"id": "e799ce82-e00a-4e15-aacc-daa6fa757335",
"ready_seconds": 9.150763,
"queue_lag_seconds": 7.224741,
"attempts": 1
},
{
"id": "d248b714-1eeb-4668-b497-4a30b3b0aad8",
"ready_seconds": 9.16808,
"queue_lag_seconds": 7.267137,
"attempts": 1
},
{
"id": "a540b150-be0e-4d38-9077-92802293c12d",
"ready_seconds": 9.209337,
"queue_lag_seconds": 7.286448,
"attempts": 1
},
{
"id": "d62a309a-a54e-4a7d-aebc-864fd251bed4",
"ready_seconds": 7.164243,
"queue_lag_seconds": 0.116455,
"attempts": 1
},
{
"id": "063093ae-38a4-4fc4-a841-d3e5db5624b3",
"ready_seconds": 7.164926,
"queue_lag_seconds": 0.135483,
"attempts": 1
},
{
"id": "06613954-2472-4efe-b2a7-42644a1d8a30",
"ready_seconds": 7.149471,
"queue_lag_seconds": 0.127605,
"attempts": 1
},
{
"id": "f593fbcd-f166-4e3e-9d89-09faeaa592bb",
"ready_seconds": 7.127115,
"queue_lag_seconds": 0.132045,
"attempts": 1
}
],
"samples": [
{
"time": 1794262126.910836,
"memory_bytes": 44044288,
"cpu_usec": 1063223,
"rss_bytes": 143413248
},
{
"time": 1794262127.9235861,
"memory_bytes": 87416832,
"cpu_usec": 1873587,
"rss_bytes": 284831744
},
{
"time": 1794262130.018199,
"memory_bytes": 42848256,
"cpu_usec": 3399041,
"rss_bytes": 106557440
},
{
"time": 1794262130.52949,
"memory_bytes": 43167744,
"cpu_usec": 3541243,
"rss_bytes": 106573824
},
{
"time": 1794262131.362358,
"memory_bytes": 42827776,
"cpu_usec": 3715963,
"rss_bytes": 106573824
},
{
"time": 1794262133.449707,
"memory_bytes": 44789760,
"cpu_usec": 4203236,
"rss_bytes": 108687360
},
{
"time": 1794262133.8703759,
"memory_bytes": 47022080,
"cpu_usec": 4272996,
"rss_bytes": 108298240
},
{
"time": 1794262134.622085,
"memory_bytes": 77737984,
"cpu_usec": 5000530,
"rss_bytes": 289599488
},
{
"time": 1794262135.440066,
"memory_bytes": 55914496,
"cpu_usec": 5808140,
"rss_bytes": 108367872
},
{
"time": 1794262135.956406,
"memory_bytes": 44863488,
"cpu_usec": 6170648,
"rss_bytes": 109416448
}
]
},
{
"repeat": 1,
"image": "large",
"concurrency": 1,
"prefetch": 1,
"successful_jobs": 8,
"errors": 0,
"seconds": 9.295377750000625,
"throughput": 0.8606428071198572,
"p95_ready_seconds": 7.433391,
"p95_queue_lag_seconds": 6.388493,
"peak_rss_bytes": 193171456,
"peak_cgroup_memory_bytes": 102318080,
"worker_cpu_seconds": 7.074789,
"requests": [
{
"upload_seconds": 0.7639207920001354,
"status": 202
},
{
"upload_seconds": 0.9345893749996321,
"status": 202
},
{
"upload_seconds": 1.1469821249993402,
"status": 202
},
{
"upload_seconds": 1.436994916999538,
"status": 202
},
{
"upload_seconds": 0.7003262910002377,
"status": 202
},
{
"upload_seconds": 0.9740857910001068,
"status": 202
},
{
"upload_seconds": 0.39281854200089583,
"status": 202
},
{
"upload_seconds": 0.8782962080003927,
"status": 202
}
],
"jobs": [
{
"id": "2be2644d-861c-43c4-8085-5fbba90a4d64",
"ready_seconds": 1.712623,
"queue_lag_seconds": 0.363585,
"attempts": 1
},
{
"id": "ad0083b2-3bf8-4f42-8043-4a57ad7b84a9",
"ready_seconds": 2.629908,
"queue_lag_seconds": 1.699148,
"attempts": 1
},
{
"id": "407c2a2e-e5aa-4437-ab5b-49a2ec05829b",
"ready_seconds": 3.021709,
"queue_lag_seconds": 2.407417,
"attempts": 1
},
{
"id": "854309f9-5d4f-4305-ad65-bcb005b3844f",
"ready_seconds": 3.852419,
"queue_lag_seconds": 2.844126,
"attempts": 1
},
{
"id": "fae78450-5058-4101-8cd9-4dcbcecd9e2c",
"ready_seconds": 4.70857,
"queue_lag_seconds": 3.753701,
"attempts": 1
},
{
"id": "fe283bf1-cdff-4ee2-867d-369338758dd4",
"ready_seconds": 5.265006,
"queue_lag_seconds": 4.502767,
"attempts": 1
},
{
"id": "86228db3-70a0-4118-bd9c-1f0aeb13ccf4",
"ready_seconds": 6.477127,
"queue_lag_seconds": 5.104128,
"attempts": 1
},
{
"id": "3fea2d31-ad7b-4282-b815-23adaa068c7a",
"ready_seconds": 7.433391,
"queue_lag_seconds": 6.388493,
"attempts": 1
}
],
"samples": [
{
"time": 1794262140.0249028,
"memory_bytes": 41619456,
"cpu_usec": 1623573,
"rss_bytes": 106139648
},
{
"time": 1794262140.4498188,
"memory_bytes": 41545728,
"cpu_usec": 1659252,
"rss_bytes": 106139648
},
{
"time": 1794262140.9149399,
"memory_bytes": 60805120,
"cpu_usec": 2022910,
"rss_bytes": 150097920
},
{
"time": 1794262141.32444,
"memory_bytes": 88137728,
"cpu_usec": 2454146,
"rss_bytes": 191004672
},
{
"time": 1794262141.7649221,
"memory_bytes": 54976512,
"cpu_usec": 2788341,
"rss_bytes": 106950656
},
{
"time": 1794262142.153526,
"memory_bytes": 82583552,
"cpu_usec": 3145199,
"rss_bytes": 180039680
},
{
"time": 1794262142.4781618,
"memory_bytes": 98701312,
"cpu_usec": 3485405,
"rss_bytes": 192589824
},
{
"time": 1794262142.818176,
"memory_bytes": 54738944,
"cpu_usec": 3670691,
"rss_bytes": 106897408
},
{
"time": 1794262143.150275,
"memory_bytes": 92516352,
"cpu_usec": 4041841,
"rss_bytes": 187322368
},
{
"time": 1794262143.4721,
"memory_bytes": 58019840,
"cpu_usec": 4330075,
"rss_bytes": 106909696
},
{
"time": 1794262143.816778,
"memory_bytes": 91193344,
"cpu_usec": 4691490,
"rss_bytes": 185057280
},
{
"time": 1794262144.319894,
"memory_bytes": 54292480,
"cpu_usec": 4990675,
"rss_bytes": 106909696
},
{
"time": 1794262144.7009401,
"memory_bytes": 66936832,
"cpu_usec": 5267047,
"rss_bytes": 147943424
},
{
"time": 1794262145.049203,
"memory_bytes": 98369536,
"cpu_usec": 5612259,
"rss_bytes": 192471040
},
{
"time": 1794262145.4414349,
"memory_bytes": 63508480,
"cpu_usec": 5901715,
"rss_bytes": 109088768
},
{
"time": 1794262145.81682,
"memory_bytes": 92336128,
"cpu_usec": 6171315,
"rss_bytes": 182304768
},
{
"time": 1794262146.1910982,
"memory_bytes": 75481088,
"cpu_usec": 6539191,
"rss_bytes": 109092864
},
{
"time": 1794262146.612237,
"memory_bytes": 73048064,
"cpu_usec": 6912398,
"rss_bytes": 154517504
},
{
"time": 1794262147.0016918,
"memory_bytes": 96288768,
"cpu_usec": 7288823,
"rss_bytes": 190681088
},
{
"time": 1794262147.491589,
"memory_bytes": 92078080,
"cpu_usec": 7759923,
"rss_bytes": 179814400
},
{
"time": 1794262147.9121048,
"memory_bytes": 64536576,
"cpu_usec": 8079635,
"rss_bytes": 128561152
},
{
"time": 1794262148.251782,
"memory_bytes": 102318080,
"cpu_usec": 8385623,
"rss_bytes": 193171456
},
{
"time": 1794262148.6732469,
"memory_bytes": 67047424,
"cpu_usec": 8698362,
"rss_bytes": 115306496
}
]
}
]
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 107 KiB

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 85 KiB

+23
View File
@@ -0,0 +1,23 @@
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab5-api-1 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp
mtusi2026lab5-api-2 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp
mtusi2026lab5-broker-1 rabbitmq:4.2.0-alpine "docker-entrypoint.s…" broker 14 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab5-dispatcher-1 messenger-lab5:local "dotnet Messenger.Ap…" dispatcher 2 seconds ago Up Less than a second 8080/tcp
mtusi2026lab5-lb-1 haproxy:3.2.7-alpine "docker-entrypoint.s…" lb 14 minutes ago Up 13 minutes (healthy) 127.0.0.1:18085->8080/tcp
mtusi2026lab5-postgres-1 postgres:17.6 "docker-entrypoint.s…" postgres 14 minutes ago Up 14 minutes (healthy) 5432/tcp
mtusi2026lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 14 minutes ago Up 14 minutes (healthy) 6379/tcp
mtusi2026lab5-s3-1 chrislusf/seaweedfs:3.97 "/entrypoint.sh serv…" s3 6 minutes ago Up 6 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
mtusi2026lab5-worker-1 messenger-lab5:local "dotnet Messenger.Ap…" worker 2 seconds ago Up Less than a second 8080/tcp
.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18085"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab5-api-1 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up 7 seconds (healthy) 8080/tcp
mtusi2026lab5-api-2 messenger-lab5:local "dotnet Messenger.Ap…" api 2 minutes ago Up 7 seconds (healthy) 8080/tcp
mtusi2026lab5-broker-1 rabbitmq:4.2.0-alpine "docker-entrypoint.s…" broker 15 minutes ago Up 30 seconds (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
mtusi2026lab5-dispatcher-1 messenger-lab5:local "dotnet Messenger.Ap…" dispatcher 44 seconds ago Up 7 seconds 8080/tcp
mtusi2026lab5-lb-1 haproxy:3.2.7-alpine "docker-entrypoint.s…" lb 15 minutes ago Up 7 seconds (healthy) 127.0.0.1:18085->8080/tcp
mtusi2026lab5-postgres-1 postgres:17.6 "docker-entrypoint.s…" postgres 15 minutes ago Up 32 seconds (healthy) 5432/tcp
mtusi2026lab5-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 15 minutes ago Up 32 seconds (healthy) 6379/tcp
mtusi2026lab5-s3-1 chrislusf/seaweedfs:3.97 "/entrypoint.sh serv…" s3 7 minutes ago Up 29 seconds (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
mtusi2026lab5-worker-1 messenger-lab5:local "dotnet Messenger.Ap…" worker 44 seconds ago Up 7 seconds 8080/tcp
+2
View File
@@ -0,0 +1,2 @@
.venv/bin/python student-tests/pipeline_tests.py --base-url "http://localhost:18085"
PASS: 11 pipeline scenarios
+18
View File
@@ -0,0 +1,18 @@
{
"checks": [
"HTTP size/MIME/corruption/filename validation remains compatible with Lab4",
"202 while worker stopped; durable outbox survives API restart; published queue survives broker restart; same ID becomes ready",
"thumbnail/crop/WebP bytes and dimensions; no EXIF; orientation applied; transparent small image remains small with white background; private variants",
"three duplicate ready deliveries are no-op",
"after_objects: worker killed, same logical job recovered, one published result",
"after_commit: worker killed, same logical job recovered, one published result",
"publisher confirm before outbox COMMIT crash recovers without lost task or duplicate effect",
"invalid staging checksum becomes failed after one attempt; failed attachment cannot be linked",
"worker independently rejects >40 million pixels before bitmap allocation",
"transient failure retries with bounded attempts then failed; explicit replay succeeds; replay of ready is no-op",
"ready staging cleaned after durable publication; failed staging retained for replay"
],
"broker_queue_before_restart": "[\n{\"name\":\"attachments.v1\",\"messages_ready\":1}\n]\n",
"retry_test_max_attempts": 3,
"default_max_attempts": 5
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

+19
View File
@@ -0,0 +1,19 @@
Проверка после изменений интерфейса и настроек сервисов:
make check: PASS
make validate: PASS
make test: PASS
student-tests/chunked_login.py: PASS
student-tests/processing_panel.py: PASS
make operations-test: PASS (повторное задание обработано worker)
make test-failover ACTION_SCRIPT=scripts/stop-one.sh:
observed_instances_before: 2
successful_writes: 109
failed_attempts: 1
first_recovery_seconds: 2.756
PASS: подтверждённые данные и сессия сохранены.
Первый запуск failover остановился до сценария из-за тайм-аута регистрации,
пока RabbitMQ запускался. После готовности сервисов повторный запуск прошёл.
В браузере проверены поиск, копирование сообщения, вставка PNG, превью и удаление
черновика, фильтр ошибок и отображение состояний сервисов.
+40
View File
@@ -0,0 +1,40 @@
global
maxconn 2000
stats socket ipv4@127.0.0.1:9999 level admin
hard-stop-after 8s
defaults
mode http
timeout connect 2s
timeout check 2s
timeout client 15s
timeout server 12s
timeout http-request 5s
timeout http-keep-alive 5s
timeout queue 2s
retries 1
resolvers docker
nameserver dns 127.0.0.11:53
accepted_payload_size 8192
resolve_retries 2
timeout resolve 500ms
timeout retry 500ms
hold valid 1s
hold nx 1s
hold other 1s
hold refused 1s
hold timeout 1s
hold obsolete 0s
frontend public
bind :8080
default_backend messenger
backend messenger
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2
server-template api 1-8 api:8080 resolvers docker resolve-prefer ipv4 init-addr none
+1
View File
@@ -0,0 +1 @@
matplotlib==3.11.2
+1
View File
@@ -0,0 +1 @@
Pillow==12.0.0
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
root="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
exec docker compose --project-name mtusi2026lab5 --project-directory "$root" "$@"
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env python3
"""Grant the local operations UI to an existing messenger account."""
import re,subprocess,sys
from pathlib import Path
root=Path(__file__).resolve().parents[1]
if len(sys.argv)!=2 or not re.fullmatch('[a-z0-9_]{3,32}',sys.argv[1]):raise SystemExit('Укажите логин: make operator USERNAME=n1k0der')
result=subprocess.check_output([str(root/'scripts/compose.sh'),'exec','-T','postgres','psql','-U','messenger','-d','messenger','-At','-c',"SELECT id FROM users WHERE username='"+sys.argv[1]+"'"],cwd=root,text=True).strip()
if not result:raise SystemExit('Сначала зарегистрируйте этот аккаунт в мессенджере')
p=root/'.env';lines=p.read_text().splitlines() if p.exists() else []
lines=[line for line in lines if not line.startswith('OPS_USER_ID=')];lines.append('OPS_USER_ID='+result);p.write_text('\n'.join(lines)+'\n');p.chmod(0o600)
subprocess.run([str(root/'scripts/compose.sh'),'up','-d','--no-deps','--wait','api'],cwd=root,check=True)
print('Панель доступна аккаунту '+sys.argv[1]+'. Перезайдите в мессенджер.')
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env python3
from pathlib import Path
import secrets,json,os
os.umask(0o077)
p=Path(__file__).resolve().parents[1]/'secrets';p.mkdir(exist_ok=True)
def secret(name,size=24):
f=p/name
if not f.exists(): f.write_text(secrets.token_hex(size))
return f.read_text().strip()
pw=secret('postgres_password')
f=p/'database_url'
if not f.exists(): f.write_text('Host=postgres;Port=5432;Database=messenger;Username=messenger;Password='+pw)
admin=secret('s3_admin_access',12); admin_secret=secret('s3_admin_secret')
access=secret('s3_access',12); key=secret('s3_secret')
config={'identities':[
{'name':'bootstrap','credentials':[{'accessKey':admin,'secretKey':admin_secret}],'actions':['Admin','Read','Write','List','Tagging']},
{'name':'messenger','credentials':[{'accessKey':access,'secretKey':key}],'actions':['Read:messenger','Write:messenger','List:messenger']}]}
(p/'s3_config').write_text(json.dumps(config))
print('Local secrets ready')
broker_password=secret('broker_password')
(p/'broker.conf').write_text('default_user = messenger\ndefault_pass = '+broker_password+'\ndefault_vhost = messenger\n')
(p.parent/'.local/faults').mkdir(parents=True,exist_ok=True)
(p.parent/'.local/faults').chmod(0o777)
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
set -eu
exec python3 "$(dirname "$0")/init-secrets.py"
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env python3
from pathlib import Path
import json,statistics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT=Path(__file__).resolve().parents[1]
runs=json.loads((ROOT/'evidence/overload.json').read_text())['runs']
fig,axes=plt.subplots(1,3,figsize=(13,4.3),layout='constrained')
settings=[1,2,4]
for label,color in [('small','#4b6fba'),('large','#719036')]:
for ax,key,title,unit in zip(axes,['throughput','p95_ready_seconds','peak_rss_bytes'],['Скорость обработки','Время до ready: p95 batch','Память процессов worker'],['готовых файлов/с','секунды','MiB']):
groups=[[r[key]/(1048576 if key=='peak_rss_bytes' else 1) for r in runs if r['image']==label and r['concurrency']==n] for n in settings]
mid=[statistics.median(g) for g in groups]
ax.errorbar(range(3),mid,yerr=[[m-min(g) for m,g in zip(mid,groups)],[max(g)-m for m,g in zip(mid,groups)]],color=color,marker='o',capsize=4,label=label)
ax.set_xticks(range(3),['1 / 1','2 / 2','4 / 4']);ax.set_xlabel('Concurrency / prefetch');ax.set_ylabel(unit);ax.set_title(title,fontsize=11);ax.set_ylim(bottom=0);ax.grid(alpha=.2)
for ax,key in zip(axes,['throughput','p95_ready_seconds','peak_rss_bytes']):
ax.set_ylim(0,max(r[key] for r in runs)/(1048576 if key=='peak_rss_bytes' else 1)*1.12)
axes[0].legend(title='Размер PNG')
fig.suptitle('Lab5: одинаковые 1 CPU / 1024 MiB worker; медианы и диапазон 3 повторов',fontsize=12)
fig.savefig(ROOT/'evidence/overload.png',dpi=170)
fig.savefig(ROOT/'evidence/overload.svg')
print('Saved evidence/overload.png and overload.svg')
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
./scripts/compose.sh restart postgres redis s3 broker
./scripts/compose.sh up -d --wait --wait-timeout 60 postgres redis s3 broker
./scripts/compose.sh restart api lb dispatcher worker
./scripts/compose.sh up -d --wait --wait-timeout 60 api lb dispatcher worker
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""Manage a temporary HTTPS tunnel for this project's HTTP entry point."""
import fcntl,os,re,shutil,signal,socket,subprocess,sys,time,urllib.request
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
STATE=ROOT/'.local';STATE.mkdir(exist_ok=True)
PID=STATE/'share.pid';URL=STATE/'share.url';origin='http://127.0.0.1:18085'
def running():
if not PID.exists():return None
try:pid=int(PID.read_text())
except ValueError:return None
command=subprocess.run(['ps','-p',str(pid),'-o','command='],capture_output=True,text=True).stdout
return pid if 'cloudflared tunnel' in command and '--url '+origin in command else None
with (STATE/'share.lock').open('w') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
pid=running()
if '--stop' in sys.argv:
if pid:
os.kill(pid,signal.SIGTERM)
for _ in range(50):
if not running():break
time.sleep(.1)
else:
try:os.kill(pid,signal.SIGKILL)
except ProcessLookupError:pass
PID.unlink(missing_ok=True);URL.unlink(missing_ok=True)
print('Внешний доступ отключён. Localhost продолжает работать.');raise SystemExit
if pid:
print(URL.read_text().strip() if URL.exists() else 'Туннель запущен, ссылка ещё создаётся.');raise SystemExit
binary=shutil.which('cloudflared')
if not binary:raise SystemExit('Установите cloudflared: brew install cloudflared')
try:urllib.request.urlopen(origin+'/health/ready',timeout=5).close()
except Exception:raise SystemExit('Сначала запустите мессенджер: make up')
URL.unlink(missing_ok=True)
(STATE/'share.log').unlink(missing_ok=True)
with (STATE/'share.log').open('w') as log:
edge=socket.gethostbyname('region1.v2.argotunnel.com')
process=subprocess.Popen([binary,'tunnel','--no-autoupdate','--protocol','http2','--edge',edge+':7844','--url',origin],stdout=log,stderr=subprocess.STDOUT,start_new_session=True,env={**os.environ,'GODEBUG':'netdns=cgo'})
PID.write_text(str(process.pid));PID.chmod(0o600)
deadline=time.monotonic()+120
while time.monotonic()<deadline and process.poll() is None:
text=(STATE/'share.log').read_text()
match=re.search(r'https://[a-z0-9-]+\.trycloudflare\.com',text)
if match:URL.write_text(match.group(0)+'\n')
if match and 'Registered tunnel connection' in text:
check=subprocess.run(['curl','--fail','--silent','--max-time','8',match.group(0)+'/health/ready'],capture_output=True)
if check.returncode==0:
print(match.group(0));print('Отключение: make stop-share');raise SystemExit
time.sleep(.25)
if process.poll() is None:process.terminate()
PID.unlink(missing_ok=True)
raise SystemExit('Не удалось получить ссылку. Подробности в .local/share.log')
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
test -n "${TARGET_INSTANCE_ID:-}" || { echo "TARGET_INSTANCE_ID is required" >&2; exit 1; }
for id in $(./scripts/compose.sh ps -q api); do
instance="$(docker exec "$id" curl -fsS -D - -o /dev/null http://127.0.0.1:8080/health/live | tr -d '\r' | awk 'tolower($1) == "x-instance-id:" { print $2 }')"
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 8 "$id" >/dev/null
exit 0
fi
done
echo "Requested API instance was not found" >&2
exit 1
@@ -0,0 +1,4 @@
namespace Messenger.Api.Contracts;
public sealed record AttachmentDto(Guid Id, Guid ChatId, Guid OwnerId, string Filename,
string ContentType, long SizeBytes, string Sha256, string Status, object[] Variants,
string? ErrorCode, string CreatedAt, string UpdatedAt);
@@ -0,0 +1,7 @@
namespace Messenger.Api.Contracts;
public sealed record ChatCursor(
string Scope,
string CreatedAt,
Guid Id
);
+3
View File
@@ -0,0 +1,3 @@
namespace Messenger.Api.Contracts;
public sealed record ChatDto(Guid Id, string Title, List<Guid> MemberIds, string CreatedAt);
@@ -0,0 +1,4 @@
namespace Messenger.Api.Contracts;
public sealed record ChatPreviewDto(
Guid Id, string Title, List<Guid> MemberIds, string CreatedAt, MessageDto? LastMessage);
@@ -0,0 +1,6 @@
namespace Messenger.Api.Contracts;
public sealed record CreateChatRequest(
string? Title,
List<Guid>? MemberIds
);
@@ -0,0 +1,8 @@
using System.Text.Json;
namespace Messenger.Api.Contracts;
public sealed record CreateMessageRequest(
string? Text,
JsonElement AttachmentIds = default
);
@@ -0,0 +1,7 @@
namespace Messenger.Api.Contracts;
public sealed record CreateUserRequest(
string? Username,
string? DisplayName,
string? Password
);
@@ -0,0 +1,3 @@
namespace Messenger.Api.Contracts;
public sealed record MessageDto(Guid Id, Guid ChatId, Guid SenderId, string Text, string CreatedAt, Guid[] AttachmentIds);
@@ -0,0 +1,3 @@
namespace Messenger.Api.Contracts;
public sealed record SessionDto(string Token, string TokenType, string ExpiresAt, UserDto User);
+3
View File
@@ -0,0 +1,3 @@
namespace Messenger.Api.Contracts;
public sealed record UserDto(Guid Id, string Username, string DisplayName, string CreatedAt);
@@ -0,0 +1,2 @@
namespace Messenger.Api.Contracts;
public sealed record VariantDto(string Name, string ContentType, long SizeBytes, int Width, int Height, string Sha256);
+206
View File
@@ -0,0 +1,206 @@
using Messenger.Api.Data.Entities;
using System.Net.Sockets;
using Npgsql;
namespace Messenger.Api.Data;
public sealed partial class AppDatabase(NpgsqlDataSource dataSource, IConfiguration configuration)
{
public async Task<bool> IsReadyAsync()
{
try
{
await using var command = dataSource.CreateCommand("SELECT 1");
return (int)(await command.ExecuteScalarAsync())! == 1;
}
catch (Exception exception) when (exception is NpgsqlException or SocketException or TimeoutException)
{
return false;
}
}
public async Task<UserEntity?> FindUserByUsernameAsync(string username)
{
await using var command = dataSource.CreateCommand(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = @username");
command.Parameters.AddWithValue("username", username);
await using var reader = await command.ExecuteReaderAsync();
return await reader.ReadAsync() ? ReadUser(reader) : null;
}
public async Task<UserEntity?> FindUserByIdAsync(Guid id)
{
await using var command = dataSource.CreateCommand(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE id = @id");
command.Parameters.AddWithValue("id", id);
await using var reader = await command.ExecuteReaderAsync();
return await reader.ReadAsync() ? ReadUser(reader) : null;
}
public async Task<List<UserEntity>> ListChatUsersAsync(Guid chatId)
{
await using var command = dataSource.CreateCommand("""
SELECT u.id, u.username, u.display_name, u.password_hash, u.created_at
FROM users u JOIN chat_members m ON m.user_id = u.id
WHERE m.chat_id = @chat_id ORDER BY u.display_name, u.id
""");
command.Parameters.AddWithValue("chat_id", chatId);
await using var reader = await command.ExecuteReaderAsync();
var result = new List<UserEntity>();
while (await reader.ReadAsync()) result.Add(ReadUser(reader));
return result;
}
public async Task<UserEntity> CreateUserAsync(string username, string displayName, string passwordHash)
{
var user = new UserEntity(Guid.NewGuid(), username, displayName, passwordHash, UtcClock.Now());
await using var command = dataSource.CreateCommand(
"INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (@id, @username, @display_name, @password_hash, @created_at)");
command.Parameters.AddWithValue("id", user.Id);
command.Parameters.AddWithValue("username", user.Username);
command.Parameters.AddWithValue("display_name", user.DisplayName);
command.Parameters.AddWithValue("password_hash", user.PasswordHash);
command.Parameters.AddWithValue("created_at", user.CreatedAt);
await command.ExecuteNonQueryAsync();
return user;
}
public async Task CreateSessionAsync(byte[] tokenHash, Guid userId, DateTime expiresAt)
{
await using var command = dataSource.CreateCommand(
"INSERT INTO sessions(token_hash, user_id, expires_at, created_at) VALUES (@token_hash, @user_id, @expires_at, @created_at)");
command.Parameters.AddWithValue("token_hash", tokenHash);
command.Parameters.AddWithValue("user_id", userId);
command.Parameters.AddWithValue("expires_at", expiresAt);
command.Parameters.AddWithValue("created_at", UtcClock.Now());
await command.ExecuteNonQueryAsync();
}
public async Task<SessionEntity?> FindActiveSessionAsync(byte[] tokenHash)
{
await using var command = dataSource.CreateCommand("""
SELECT user_id, expires_at FROM sessions
WHERE token_hash = @token_hash AND revoked_at IS NULL AND expires_at > now()
""");
command.Parameters.AddWithValue("token_hash", tokenHash);
await using var reader = await command.ExecuteReaderAsync();
return await reader.ReadAsync() ? new SessionEntity(reader.GetGuid(0), reader.GetDateTime(1)) : null;
}
public async Task<bool> RevokeSessionAsync(byte[] tokenHash)
{
await using var command = dataSource.CreateCommand("""
UPDATE sessions SET revoked_at = now()
WHERE token_hash = @token_hash AND revoked_at IS NULL AND expires_at > now()
""");
command.Parameters.AddWithValue("token_hash", tokenHash);
return await command.ExecuteNonQueryAsync() == 1;
}
public async Task<ChatEntity?> CreateChatAsync(string title, List<Guid> memberIds)
{
await using var connection = await dataSource.OpenConnectionAsync();
await using var transaction = await connection.BeginTransactionAsync();
await using (var membersCommand = new NpgsqlCommand(
"SELECT count(*) FROM users WHERE id = ANY(@member_ids)", connection, transaction))
{
membersCommand.Parameters.AddWithValue("member_ids", memberIds.ToArray());
if ((long)(await membersCommand.ExecuteScalarAsync())! != memberIds.Count)
{
await transaction.RollbackAsync();
return null;
}
}
var chat = new ChatEntity(Guid.NewGuid(), title, memberIds, UtcClock.Now());
await using (var chatCommand = new NpgsqlCommand(
"INSERT INTO chats(id, title, created_at) VALUES (@id, @title, @created_at)", connection, transaction))
{
chatCommand.Parameters.AddWithValue("id", chat.Id);
chatCommand.Parameters.AddWithValue("title", chat.Title);
chatCommand.Parameters.AddWithValue("created_at", chat.CreatedAt);
await chatCommand.ExecuteNonQueryAsync();
}
await using (var membershipCommand = new NpgsqlCommand(
"INSERT INTO chat_members(chat_id, user_id) SELECT @chat_id, unnest(CAST(@member_ids AS uuid[]))", connection, transaction))
{
membershipCommand.Parameters.AddWithValue("chat_id", chat.Id);
membershipCommand.Parameters.AddWithValue("member_ids", memberIds.ToArray());
await membershipCommand.ExecuteNonQueryAsync();
}
await transaction.CommitAsync();
return chat;
}
public async Task<ChatEntity?> FindChatForMemberAsync(Guid chatId, Guid userId)
{
await using var command = dataSource.CreateCommand("""
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id)
FROM chats c
WHERE c.id = @chat_id AND EXISTS (
SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = @user_id)
""");
command.Parameters.AddWithValue("chat_id", chatId);
command.Parameters.AddWithValue("user_id", userId);
await using var reader = await command.ExecuteReaderAsync();
return await reader.ReadAsync() ? ReadChat(reader) : null;
}
public async Task<List<ChatEntity>> ListChatsAsync(Guid userId, DateTime? cursorTime, Guid? cursorId, int count)
{
var sql = """
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id)
FROM chats c JOIN chat_members m ON m.chat_id = c.id
WHERE m.user_id = @user_id
""";
if (cursorTime is not null)
sql += " AND (c.created_at, c.id) > (@cursor_time, @cursor_id)";
sql += " ORDER BY c.created_at, c.id LIMIT @count";
await using var command = dataSource.CreateCommand(sql);
command.Parameters.AddWithValue("user_id", userId);
if (cursorTime is not null)
{
command.Parameters.AddWithValue("cursor_time", cursorTime.Value);
command.Parameters.AddWithValue("cursor_id", cursorId!.Value);
}
command.Parameters.AddWithValue("count", count);
await using var reader = await command.ExecuteReaderAsync();
var result = new List<ChatEntity>();
while (await reader.ReadAsync()) result.Add(ReadChat(reader));
return result;
}
public async Task<List<MessageEntity>> ListMessagesAsync(Guid chatId, DateTime? cursorTime, Guid? cursorId, int count, bool newestFirst = false)
{
var sql = "SELECT id, chat_id, sender_id, text, created_at, attachment_ids FROM messages WHERE chat_id = @chat_id";
if (cursorTime is not null)
sql += newestFirst ? " AND (created_at, id) < (@cursor_time, @cursor_id)" : " AND (created_at, id) > (@cursor_time, @cursor_id)";
sql += newestFirst ? " ORDER BY created_at DESC, id DESC LIMIT @count" : " ORDER BY created_at, id LIMIT @count";
await using var command = dataSource.CreateCommand(sql);
command.Parameters.AddWithValue("chat_id", chatId);
if (cursorTime is not null)
{
command.Parameters.AddWithValue("cursor_time", cursorTime.Value);
command.Parameters.AddWithValue("cursor_id", cursorId!.Value);
}
command.Parameters.AddWithValue("count", count);
await using var reader = await command.ExecuteReaderAsync();
var result = new List<MessageEntity>();
while (await reader.ReadAsync()) result.Add(new MessageEntity(
reader.GetGuid(0), reader.GetGuid(1), reader.GetGuid(2), reader.GetString(3), reader.GetDateTime(4), reader.GetFieldValue<Guid[]>(5)));
return result;
}
private static UserEntity ReadUser(NpgsqlDataReader reader) => new(
reader.GetGuid(0), reader.GetString(1), reader.GetString(2), reader.GetString(3), reader.GetDateTime(4));
private static ChatEntity ReadChat(NpgsqlDataReader reader) => new(
reader.GetGuid(0), reader.GetString(1), reader.GetFieldValue<Guid[]>(3).ToList(), reader.GetDateTime(2));
}
+23
View File
@@ -0,0 +1,23 @@
using Messenger.Api.Contracts;
using Npgsql;
namespace Messenger.Api.Data;
public sealed partial class AppDatabase
{
public const long AttachmentGate = 48260104;
public async Task<(AttachmentDto Dto, string Key)?> FindAttachmentAsync(Guid id, Guid actor)
{
await using var cmd = dataSource.CreateCommand("""
SELECT a.id,a.chat_id,a.owner_id,a.filename,a.content_type,a.size_bytes,a.sha256,
a.status,a.created_at,a.updated_at,a.object_key,a.variants::text,a.error_code
FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id
WHERE a.id=@id AND m.user_id=@actor
""");
cmd.Parameters.AddWithValue("id",id); cmd.Parameters.AddWithValue("actor",actor);
await using var r=await cmd.ExecuteReaderAsync();
return await r.ReadAsync() ? (ReadAttachment(r),r.GetString(10)) : null;
}
public static AttachmentDto ReadAttachment(NpgsqlDataReader r) => new(r.GetGuid(0),r.GetGuid(1),
r.GetGuid(2),r.GetString(3),r.GetString(4),r.GetInt64(5),r.GetString(6),r.GetString(7),
System.Text.Json.JsonSerializer.Deserialize<VariantDto[]>(r.GetString(11), Pipeline.JobJson.Options) ?? [],
r.IsDBNull(12)?null:r.GetString(12),r.GetDateTime(8).ToString("O"),r.GetDateTime(9).ToString("O"));
}
+51
View File
@@ -0,0 +1,51 @@
using Messenger.Api.Contracts;
using Npgsql;
namespace Messenger.Api.Data;
public sealed partial class AppDatabase
{
public async Task<List<ChatPreviewDto>> ListChatPreviewsAsync(Guid userId, DateTime? cursorTime, Guid? cursorId, int count)
{
var query = """
WITH page AS (
SELECT c.id, c.title, c.created_at,
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids
FROM chats c JOIN chat_members membership ON membership.chat_id = c.id
WHERE membership.user_id = @user_id
""";
if (cursorTime is not null) query += " AND (c.created_at, c.id) > (@cursor_time, @cursor_id)";
query += """
ORDER BY c.created_at, c.id LIMIT @count
)
SELECT p.id, p.title, p.created_at, p.member_ids,
latest.id, latest.sender_id, latest.text, latest.created_at, latest.attachment_ids
FROM page p
LEFT JOIN LATERAL (
SELECT id, sender_id, text, created_at, attachment_ids
FROM messages WHERE chat_id = p.id
ORDER BY created_at DESC, id DESC LIMIT 1
) latest ON true
ORDER BY p.created_at, p.id
""";
await using var command = dataSource.CreateCommand(query);
command.Parameters.AddWithValue("user_id", userId);
command.Parameters.AddWithValue("count", count);
if (cursorTime is not null)
{
command.Parameters.AddWithValue("cursor_time", cursorTime.Value);
command.Parameters.AddWithValue("cursor_id", cursorId!.Value);
}
await using var reader = await command.ExecuteReaderAsync();
var result = new List<ChatPreviewDto>();
while (await reader.ReadAsync())
{
var id = reader.GetGuid(0);
var message = reader.IsDBNull(4) ? null : new MessageDto(reader.GetGuid(4), id,
reader.GetGuid(5), reader.GetString(6), reader.GetDateTime(7).ToString("O"), reader.GetFieldValue<Guid[]>(8));
result.Add(new ChatPreviewDto(id, reader.GetString(1), reader.GetFieldValue<Guid[]>(3).ToList(),
reader.GetDateTime(2).ToString("O"), message));
}
return result;
}
}
@@ -0,0 +1,40 @@
using Npgsql;
namespace Messenger.Api.Data;
public static class DatabaseMigrator
{
public static async Task ApplyAsync(NpgsqlDataSource dataSource)
{
await using var connection = await dataSource.OpenConnectionAsync();
await using var transaction = await connection.BeginTransactionAsync();
await using (var command = new NpgsqlCommand("SELECT pg_advisory_xact_lock(202602)", connection, transaction))
await command.ExecuteNonQueryAsync();
await using (var command = new NpgsqlCommand(
"CREATE TABLE IF NOT EXISTS schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL)",
connection, transaction))
await command.ExecuteNonQueryAsync();
var assembly = typeof(DatabaseMigrator).Assembly;
var migrations = assembly.GetManifestResourceNames()
.Where(name => name.Contains(".Data.Migrations.", StringComparison.Ordinal) && name.EndsWith(".sql", StringComparison.Ordinal))
.Order(StringComparer.Ordinal);
foreach (var name in migrations)
{
var version = int.Parse(name.Split('.')[^2].Split('_')[0]);
await using var check = new NpgsqlCommand("SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = @version)", connection, transaction);
check.Parameters.AddWithValue("version", version);
if ((bool)(await check.ExecuteScalarAsync())!) continue;
await using var stream = assembly.GetManifestResourceStream(name)!;
using var reader = new StreamReader(stream);
await using (var migration = new NpgsqlCommand(await reader.ReadToEndAsync(), connection, transaction))
await migration.ExecuteNonQueryAsync();
await using var record = new NpgsqlCommand(
"INSERT INTO schema_migrations(version, applied_at) VALUES (@version, now())", connection, transaction);
record.Parameters.AddWithValue("version", version);
await record.ExecuteNonQueryAsync();
}
await transaction.CommitAsync();
}
}
@@ -0,0 +1,6 @@
namespace Messenger.Api.Data.Entities;
public sealed record ChatEntity(Guid Id, string Title, List<Guid> MemberIds, DateTime CreatedAt)
{
public Contracts.ChatDto ToDto() => new(Id, Title, MemberIds, CreatedAt.ToString("O"));
}
@@ -0,0 +1,6 @@
namespace Messenger.Api.Data.Entities;
public sealed record MessageEntity(Guid Id, Guid ChatId, Guid SenderId, string Text, DateTime CreatedAt, Guid[]? AttachmentIds = null)
{
public Contracts.MessageDto ToDto() => new(Id, ChatId, SenderId, Text, CreatedAt.ToString("O"), AttachmentIds ?? []);
}
@@ -0,0 +1,3 @@
namespace Messenger.Api.Data.Entities;
public sealed record SessionEntity(Guid UserId, DateTime ExpiresAt);
@@ -0,0 +1,6 @@
namespace Messenger.Api.Data.Entities;
public sealed record UserEntity(Guid Id, string Username, string DisplayName, string PasswordHash, DateTime CreatedAt)
{
public Contracts.UserDto ToDto() => new(Id, Username, DisplayName, CreatedAt.ToString("O"));
}
+91
View File
@@ -0,0 +1,91 @@
using System.Security.Cryptography;
using System.Text;
using Messenger.Api.Data.Entities;
using Npgsql;
namespace Messenger.Api.Data;
public sealed partial class AppDatabase
{
private bool BatchCommands => !string.Equals(configuration["MESSAGE_BATCHING"], "false", StringComparison.OrdinalIgnoreCase);
public async Task<MessageEntity?> CreateMessageOnceAsync(Guid chatId, Guid senderId, string text, string key, Guid[] attachmentIds)
{
var hash = SHA256.HashData(System.Text.Json.JsonSerializer.SerializeToUtf8Bytes(new { text, attachment_ids = attachmentIds }));
await using var connection = await dataSource.OpenConnectionAsync();
await using var transaction = await connection.BeginTransactionAsync();
await using (var gate = new NpgsqlCommand($"SELECT pg_advisory_xact_lock_shared({AttachmentGate})",connection,transaction))
await gate.ExecuteNonQueryAsync();
if (attachmentIds.Length>0)
{
await using var attachments = new NpgsqlCommand("SELECT id,status FROM attachments WHERE chat_id=@chat AND id=ANY(@ids)",connection,transaction);
attachments.Parameters.AddWithValue("chat",chatId); attachments.Parameters.AddWithValue("ids",attachmentIds);
await using var found = await attachments.ExecuteReaderAsync();
var count=0; var ready=true;
while(await found.ReadAsync()) { count++; ready &= found.GetString(1)!="failed"; }
if(count!=attachmentIds.Length) throw new Services.AttachmentException(404,"not_found","Вложение не найдено");
if(!ready) throw new Services.AttachmentException(409,"conflict","Вложение недоступно");
}
// A transaction-scoped lock coordinates the same request across all API processes.
if (!BatchCommands)
await using (var gate = new NpgsqlCommand("SELECT pg_advisory_xact_lock(hashtextextended(@scope, 0))", connection, transaction))
{
gate.Parameters.AddWithValue("scope", $"{senderId}:POST:/api/v1/chats/{chatId}/messages:{key}");
await gate.ExecuteNonQueryAsync();
}
await using (var previous = new NpgsqlCommand((BatchCommands ? "SELECT pg_advisory_xact_lock(hashtextextended(@scope, 0));" : "") + """
SELECT r.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, m.attachment_ids
FROM message_requests r JOIN messages m ON m.id = r.message_id
WHERE r.user_id = @user_id AND r.chat_id = @chat_id AND r.request_key = @key
AND r.expires_at > clock_timestamp()
""", connection, transaction))
{
previous.Parameters.AddWithValue("scope", $"{senderId}:POST:/api/v1/chats/{chatId}/messages:{key}");
previous.Parameters.AddWithValue("user_id", senderId);
previous.Parameters.AddWithValue("chat_id", chatId);
previous.Parameters.AddWithValue("key", key);
await using var reader = await previous.ExecuteReaderAsync();
if (BatchCommands) await reader.NextResultAsync();
if (await reader.ReadAsync())
{
var same = reader.GetFieldValue<byte[]>(0).AsSpan().SequenceEqual(hash);
var result = same ? new MessageEntity(reader.GetGuid(1), reader.GetGuid(2), reader.GetGuid(3), reader.GetString(4), reader.GetDateTime(5), reader.GetFieldValue<Guid[]>(6)) : null;
await reader.DisposeAsync();
await transaction.CommitAsync();
return result;
}
}
if (!BatchCommands)
await using (var cleanup = new NpgsqlCommand("""
DELETE FROM message_requests
WHERE user_id = @user_id AND chat_id = @chat_id AND request_key = @key
""", connection, transaction))
{
cleanup.Parameters.AddWithValue("user_id", senderId);
cleanup.Parameters.AddWithValue("chat_id", chatId);
cleanup.Parameters.AddWithValue("key", key);
await cleanup.ExecuteNonQueryAsync();
}
var message = new MessageEntity(Guid.NewGuid(), chatId, senderId, text, UtcClock.Now(), attachmentIds);
await using (var insert = new NpgsqlCommand((BatchCommands ? "DELETE FROM message_requests WHERE user_id = @sender_id AND chat_id = @chat_id AND request_key = @key;" : "") + """
INSERT INTO messages(id, chat_id, sender_id, text, created_at, attachment_ids)
VALUES (@id, @chat_id, @sender_id, @text, @created_at, @attachments);
INSERT INTO message_requests(user_id, chat_id, request_key, request_hash, message_id, expires_at)
VALUES (@sender_id, @chat_id, @key, @hash, @id, clock_timestamp() + interval '24 hours');
""", connection, transaction))
{
insert.Parameters.AddWithValue("attachments", attachmentIds);
insert.Parameters.AddWithValue("id", message.Id);
insert.Parameters.AddWithValue("chat_id", chatId);
insert.Parameters.AddWithValue("sender_id", senderId);
insert.Parameters.AddWithValue("text", text);
insert.Parameters.AddWithValue("created_at", message.CreatedAt);
insert.Parameters.AddWithValue("key", key);
insert.Parameters.AddWithValue("hash", hash);
await insert.ExecuteNonQueryAsync();
}
await transaction.CommitAsync();
return message;
}
}
@@ -0,0 +1,43 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE INDEX chats_created_id_idx ON chats (created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL,
revoked_at timestamptz,
created_at timestamptz NOT NULL
);
CREATE INDEX sessions_user_id_idx ON sessions (user_id);
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at);
@@ -0,0 +1,11 @@
CREATE TABLE message_requests (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
request_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, request_key)
);
CREATE INDEX message_requests_expires_idx ON message_requests (expires_at);
@@ -0,0 +1,6 @@
CREATE TABLE message_reactions (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
reaction text NOT NULL CHECK (reaction IN ('like', 'love', 'laugh', 'mushroom')),
PRIMARY KEY (message_id, user_id, reaction)
);
@@ -0,0 +1,11 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY, chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id), filename text NOT NULL,
content_type text NOT NULL, size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL, object_key text NOT NULL UNIQUE,
status text NOT NULL CHECK (status IN ('ready','missing')),
created_at timestamptz NOT NULL, updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_created_idx ON attachments(created_at, id);
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
CREATE INDEX messages_attachments_idx ON messages USING gin(attachment_ids);
@@ -0,0 +1,25 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
UPDATE attachments SET status='failed' WHERE status='missing';
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check CHECK(status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
ALTER TABLE attachments ADD COLUMN error_code text;
UPDATE attachments SET error_code='original_missing' WHERE status='failed';
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version int NOT NULL DEFAULT 1, staging_key text NOT NULL UNIQUE,
state text NOT NULL CHECK(state IN ('queued','processing','ready','failed')),
attempts int NOT NULL DEFAULT 0, next_at timestamptz NOT NULL DEFAULT now(),
lease_token uuid, lease_until timestamptz, error_code text,
created_at timestamptz NOT NULL DEFAULT now(), completed_at timestamptz,
staging_deleted_at timestamptz, replay_count int NOT NULL DEFAULT 0
);
CREATE INDEX attachment_jobs_active_idx ON attachment_jobs(state,next_at);
CREATE TABLE attachment_outbox (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id),
available_at timestamptz NOT NULL DEFAULT now(), published_at timestamptz
);
CREATE INDEX attachment_outbox_pending_idx ON attachment_outbox(available_at) WHERE published_at IS NULL;
CREATE TABLE attachment_job_events (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
event text NOT NULL, attempt int NOT NULL, created_at timestamptz NOT NULL DEFAULT now()
);
@@ -0,0 +1 @@
CREATE TABLE pipeline_presence (role text PRIMARY KEY, seen_at timestamptz NOT NULL);
+48
View File
@@ -0,0 +1,48 @@
using Npgsql;
namespace Messenger.Api.Data;
public sealed record ReactionDto(Guid MessageId, string Reaction, int Count, bool Mine);
public sealed partial class AppDatabase
{
public async Task<bool> SetReactionAsync(Guid chatId, Guid messageId, Guid userId, string reaction, bool present)
{
await using var command = dataSource.CreateCommand(present ? """
WITH target AS (SELECT id FROM messages WHERE id = @message_id AND chat_id = @chat_id),
written AS (
INSERT INTO message_reactions(message_id, user_id, reaction)
SELECT id, @user_id, @reaction FROM target ON CONFLICT DO NOTHING
) SELECT EXISTS(SELECT 1 FROM target)
""" : """
WITH target AS (SELECT id FROM messages WHERE id = @message_id AND chat_id = @chat_id),
removed AS (
DELETE FROM message_reactions WHERE message_id IN (SELECT id FROM target)
AND user_id = @user_id AND reaction = @reaction
) SELECT EXISTS(SELECT 1 FROM target)
""");
command.Parameters.AddWithValue("chat_id", chatId);
command.Parameters.AddWithValue("message_id", messageId);
command.Parameters.AddWithValue("user_id", userId);
command.Parameters.AddWithValue("reaction", reaction);
return (bool)(await command.ExecuteScalarAsync())!;
}
public async Task<List<ReactionDto>> ListReactionsAsync(Guid chatId, Guid[] messageIds, Guid userId)
{
await using var command = dataSource.CreateCommand("""
SELECT r.message_id, r.reaction, count(*)::integer, bool_or(r.user_id = @user_id)
FROM message_reactions r JOIN messages m ON m.id = r.message_id
WHERE m.chat_id = @chat_id AND m.id = ANY(@message_ids)
GROUP BY r.message_id, r.reaction ORDER BY r.message_id, r.reaction
""");
command.Parameters.AddWithValue("chat_id", chatId);
command.Parameters.AddWithValue("message_ids", messageIds);
command.Parameters.AddWithValue("user_id", userId);
await using var reader = await command.ExecuteReaderAsync();
var items = new List<ReactionDto>();
while (await reader.ReadAsync())
items.Add(new(reader.GetGuid(0), reader.GetString(1), reader.GetInt32(2), reader.GetBoolean(3)));
return items;
}
}
+11
View File
@@ -0,0 +1,11 @@
namespace Messenger.Api.Data;
public static class UtcClock
{
// PostgreSQL timestamptz stores microseconds; keep the response identical to the saved value.
public static DateTime Now()
{
var ticks = DateTime.UtcNow.Ticks;
return new DateTime(ticks - ticks % 10, DateTimeKind.Utc);
}
}
@@ -0,0 +1,79 @@
using Messenger.Api.Data;
using Messenger.Api.Infrastructure;
using Messenger.Api.Services;
using Amazon.S3;
using Amazon.S3.Model;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class AttachmentEndpoints
{
public static IEndpointRouteBuilder MapAttachmentEndpoints(this IEndpointRouteBuilder app)
{
app.MapPost("/api/v1/chats/{chat_id}/attachments",async(string chat_id,HttpContext ctx,
SessionService sessions,ChatService chats,AttachmentService files)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
if(!Guid.TryParse(chat_id,out var chat)) return ApiError(ctx,400,"invalid_request","Некорректный chat_id");
if(await chats.FindForMemberAsync(chat,actor.Id) is null) return ApiError(ctx,404,"not_found","Чат не найден");
var names=ctx.Request.Query["filename"]; var name=names.ToString();
if(names.Count!=1 || string.IsNullOrWhiteSpace(name) || name.EnumerateRunes().Count()>255 || name.Any(c=>char.IsControl(c)||c is '/' or '\\'))
return ApiError(ctx,400,"invalid_request","Некорректное имя файла");
if(ctx.Request.ContentType?.Split(';')[0].Trim() is not "image/png" and not "image/jpeg")
return ApiError(ctx,415,"unsupported_media_type","Требуется PNG или JPEG");
return Results.Json(await files.UploadAsync(ctx.Request,chat,actor.Id,name),statusCode:202);
});
app.MapGet("/api/v1/attachments/{attachment_id}",async(string attachment_id,HttpContext ctx,
SessionService sessions,AppDatabase db)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
if(!Guid.TryParse(attachment_id,out var id)) return ApiError(ctx,400,"invalid_request","Некорректный ID");
var file=await db.FindAttachmentAsync(id,actor.Id);
if(file is null) return ApiError(ctx,404,"not_found","Вложение не найдено");
return Results.Ok(file.Value.Dto);
});
app.MapGet("/api/v1/attachments/{attachment_id}/processing",async(string attachment_id,HttpContext ctx,
SessionService sessions,AppDatabase db,Npgsql.NpgsqlDataSource source)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
if(!Guid.TryParse(attachment_id,out var id)) return ApiError(ctx,400,"invalid_request","Некорректный ID");
if(await db.FindAttachmentAsync(id,actor.Id) is null) return ApiError(ctx,404,"not_found","Вложение не найдено");
await using var cmd=source.CreateCommand("""
SELECT j.state,j.attempts,j.error_code,j.created_at,j.completed_at,
e.event,e.attempt,e.created_at
FROM attachment_jobs j LEFT JOIN attachment_job_events e ON e.job_id=j.id
WHERE j.attachment_id=@id ORDER BY e.id
""");
cmd.Parameters.AddWithValue("id",id);
await using var reader=await cmd.ExecuteReaderAsync(ctx.RequestAborted);
string? state=null,error=null;int attempts=0;DateTime? created=null,completed=null;
var events=new List<object>();
while(await reader.ReadAsync(ctx.RequestAborted)) {
state=reader.GetString(0);attempts=reader.GetInt32(1);
error=reader.IsDBNull(2)?null:reader.GetString(2);
created=reader.GetDateTime(3);completed=reader.IsDBNull(4)?null:reader.GetDateTime(4);
if(!reader.IsDBNull(5)) events.Add(new {name=reader.GetString(5),attempt=reader.GetInt32(6),at=reader.GetDateTime(7)});
}
ctx.Response.Headers.CacheControl="no-store";
return Results.Ok(new {state,attempts,error_code=error,created_at=created,completed_at=completed,events});
});
app.MapGet("/api/v1/attachments/{attachment_id}/content",async(string attachment_id,HttpContext ctx,
SessionService sessions,AppDatabase db,S3Storage storage)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx); if(actor is null) return Unauthorized(ctx);
if(!Guid.TryParse(attachment_id,out var id)) return ApiError(ctx,400,"invalid_request","Некорректный ID");
var file=await db.FindAttachmentAsync(id,actor.Id);
if(file is null) return ApiError(ctx,404,"not_found","Вложение не найдено");
if(file.Value.Dto.Status!="ready") return ApiError(ctx,409,"conflict","Оригинал недоступен");
var variant=ctx.Request.Query.TryGetValue("variant",out var value)?value.ToString():"original";
var selected=file.Value.Dto.Variants.OfType<Messenger.Api.Contracts.VariantDto>().FirstOrDefault(v=>v.Name==variant);
if(variant!="original" && selected is null) return ApiError(ctx,404,"not_found","Вариант не найден");
var key=variant=="original"?file.Value.Key:$"variants/{id:N}/v1/{variant}";
var mime=selected?.ContentType ?? file.Value.Dto.ContentType;
var filename=variant=="original"?file.Value.Dto.Filename:variant+(mime=="image/webp"?".webp":".jpg");
try {
var original=await storage.Client.GetObjectAsync(storage.Bucket,key,ctx.RequestAborted);
return Results.Stream(async stream=> {using(original) {await original.ResponseStream.CopyToAsync(stream,ctx.RequestAborted);}},
mime,filename);
} catch(AmazonS3Exception ex) when(ex.StatusCode==System.Net.HttpStatusCode.NotFound) {
return ApiError(ctx,503,"unavailable","Оригинал временно недоступен");
}
});
return app;
}
}
@@ -0,0 +1,72 @@
using System.Text;
using System.Text.RegularExpressions;
using Messenger.Api.Services;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class AuthEndpoints
{
private static readonly UTF8Encoding StrictUtf8 = new(false, true);
public static IEndpointRouteBuilder MapAuthEndpoints(this IEndpointRouteBuilder app)
{
app.MapPost("/api/v1/auth/sessions", async (HttpContext context, SessionService sessions) =>
{
if (!context.Request.Headers.TryGetValue("Authorization", out var header))
return Unauthorized(context, basic: true);
var authorization = header.ToString();
if (!authorization.StartsWith("Basic ", StringComparison.OrdinalIgnoreCase))
return Unauthorized(context, basic: true);
string credentials;
try
{
credentials = StrictUtf8.GetString(Convert.FromBase64String(authorization[6..]));
}
catch (Exception exception) when (exception is FormatException or DecoderFallbackException)
{
return Unauthorized(context, basic: true);
}
var delimiter = credentials.IndexOf(':');
if (delimiter < 0) return Unauthorized(context, basic: true);
var username = credentials[..delimiter];
var password = credentials[(delimiter + 1)..];
if (!Regex.IsMatch(username, "\\A[a-z0-9_]{3,32}\\z"))
return Unauthorized(context, basic: true);
if (context.Request.ContentLength > 0)
return ApiError(context, 400, "invalid_request", "Вход принимает Basic без тела запроса");
if (context.Request.ContentLength is null && context.Request.Headers.ContainsKey("Transfer-Encoding"))
{
using var deadline=CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted);
deadline.CancelAfter(TimeSpan.FromSeconds(3));
try {
if(await context.Request.Body.ReadAsync(new byte[1],deadline.Token)>0)
return ApiError(context,400,"invalid_request","Вход принимает Basic без тела запроса");
} catch(OperationCanceledException) when(!context.RequestAborted.IsCancellationRequested) {
return ApiError(context,400,"invalid_request","Тело запроса не завершено");
}
}
var session = await sessions.LoginAsync(username, password);
return session is null
? Unauthorized(context, basic: true)
: Results.Json(session, statusCode: StatusCodes.Status201Created);
});
app.MapDelete("/api/v1/auth/sessions/current", async (HttpContext context, SessionService sessions) =>
{
var (user, tokenHash) = await sessions.AuthenticateAsync(context);
if (user is null || tokenHash is null) return Unauthorized(context);
return await sessions.RevokeAsync(tokenHash)
? Results.NoContent()
: Unauthorized(context);
});
return app;
}
}
@@ -0,0 +1,67 @@
using Messenger.Api.Contracts;
using Messenger.Api.Services;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class ChatEndpoints
{
public static IEndpointRouteBuilder MapChatEndpoints(this IEndpointRouteBuilder app)
{
app.MapPost("/api/v1/chats", async (HttpContext context, SessionService sessions, ChatService chats) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
var (request, error) = await ReadJsonAsync<CreateChatRequest>(context);
if (error is not null) return error;
if (string.IsNullOrWhiteSpace(request!.Title) || request.Title.EnumerateRunes().Count() > 100)
return ApiError(context, 400, "invalid_request", "Некорректное название чата");
if (request.MemberIds is null || request.MemberIds.Count is < 1 or > 100 ||
request.MemberIds.Distinct().Count() != request.MemberIds.Count ||
!request.MemberIds.Contains(actor.Id))
return ApiError(context, 400, "invalid_request", "Некорректный список участников");
var chat = await chats.CreateAsync(request.Title, request.MemberIds);
return chat is null
? ApiError(context, 400, "invalid_request", "Один из участников не существует")
: Results.Json(chat, statusCode: StatusCodes.Status201Created);
});
app.MapGet("/api/v1/chats", async (HttpContext context, SessionService sessions, ChatService chats) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
var (limit, limitError) = ParseLimit(context);
if (limitError is not null) return limitError;
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"user:{actor.Id}");
if (cursorError is not null) return cursorError;
var page = await chats.ListAsync(actor.Id, cursorTime, cursor?.Id, limit + 1);
string? nextCursor = null;
if (page.Count > limit)
{
var last = page[limit - 1];
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"user:{actor.Id}", last.CreatedAt, last.Id));
page = page.Take(limit).ToList();
}
return Results.Ok(new { items = page, next_cursor = nextCursor });
});
app.MapGet("/api/v1/chats/{chat_id}", async (
string chat_id, HttpContext context, SessionService sessions, ChatService chats) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId))
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
var chat = await chats.FindForMemberAsync(chatId, actor.Id);
return chat is null
? ApiError(context, 404, "not_found", "Чат не найден")
: Results.Ok(chat);
});
return app;
}
}
@@ -0,0 +1,18 @@
using Messenger.Api.Data;
using Messenger.Api.Services;
using Messenger.Api.Infrastructure;
namespace Messenger.Api.Endpoints;
public static class HealthEndpoints
{
public static IEndpointRouteBuilder MapHealthEndpoints(this IEndpointRouteBuilder app)
{
app.MapGet("/health/live", () => Results.Ok(new { status = "ok" }));
app.MapGet("/health/ready", async (HttpContext context, AppDatabase database, ReadinessState readiness, S3Storage storage) =>
!readiness.IsStopping && await database.IsReadyAsync() && await storage.ReadyAsync()
? Results.Ok(new { status = "ok" })
: ApiSupport.ApiError(context, 503, "unavailable", "Хранилище недоступно"));
return app;
}
}
@@ -0,0 +1,80 @@
using Messenger.Api.Contracts;
using Messenger.Api.Services;
using System.Text.Json;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class MessageEndpoints
{
public static IEndpointRouteBuilder MapMessageEndpoints(this IEndpointRouteBuilder app)
{
app.MapPost("/api/v1/chats/{chat_id}/messages", async (
string chat_id, HttpContext context, SessionService sessions, ChatService chats, MessageService messages) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId))
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
return ApiError(context, 404, "not_found", "Чат не найден");
var keys = context.Request.Headers["Idempotency-Key"];
var key = keys.ToString();
if (keys.Count != 1 || key.Length is < 1 or > 128 ||
key.Any(c => !char.IsAsciiLetterOrDigit(c) && c is not '.' and not '_' and not ':' and not '-'))
return ApiError(context, 400, "invalid_request", "Требуется корректный Idempotency-Key");
var (request, error) = await ReadJsonAsync<CreateMessageRequest>(context);
if (error is not null) return error;
if (request!.Text is null || request.Text.EnumerateRunes().Count() > 4000)
return ApiError(context, 400, "invalid_request", "Некорректный текст сообщения");
var ids = new List<Guid>();
if (request.AttachmentIds.ValueKind != JsonValueKind.Undefined)
{
if (request.AttachmentIds.ValueKind != JsonValueKind.Array || request.AttachmentIds.GetArrayLength()>10)
return ApiError(context,400,"invalid_request","Некорректный список вложений");
foreach(var item in request.AttachmentIds.EnumerateArray())
{
if(item.ValueKind!=JsonValueKind.String || !item.TryGetGuid(out var id) || ids.Contains(id))
return ApiError(context,400,"invalid_request","Некорректный ID вложения");
ids.Add(id);
}
}
if(string.IsNullOrWhiteSpace(request.Text) && ids.Count==0)
return ApiError(context,400,"invalid_request","Нужен текст или вложение");
var message = await messages.CreateAsync(chatId, actor.Id, request.Text, key, ids.ToArray());
if (message is null)
return ApiError(context, 409, "conflict", "Этот ключ уже использован для другого сообщения");
return Results.Json(message, statusCode: StatusCodes.Status201Created);
});
app.MapGet("/api/v1/chats/{chat_id}/messages", async (
string chat_id, HttpContext context, SessionService sessions, ChatService chats, MessageService messages) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId))
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
return ApiError(context, 404, "not_found", "Чат не найден");
var (limit, limitError) = ParseLimit(context);
if (limitError is not null) return limitError;
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"chat:{chatId}");
if (cursorError is not null) return cursorError;
var page = await messages.ListAsync(chatId, cursorTime, cursor?.Id, limit + 1);
string? nextCursor = null;
if (page.Count > limit)
{
var last = page[limit - 1];
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"chat:{chatId}", last.CreatedAt, last.Id));
page = page.Take(limit).ToList();
}
return Results.Ok(new { items = page, next_cursor = nextCursor });
});
return app;
}
}
@@ -0,0 +1,79 @@
using Messenger.Api.Data;
using Messenger.Api.Infrastructure;
using Messenger.Api.Pipeline;
using Messenger.Api.Services;
using Npgsql;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class OperationsEndpoints
{
public static bool IsOperator(Guid id,IConfiguration config)=>Guid.TryParse(config["OPS_USER_ID"],out var owner)&&owner==id;
public static IEndpointRouteBuilder MapOperationsEndpoints(this IEndpointRouteBuilder app)
{
app.MapGet("/api/ui/operator",async(HttpContext ctx,SessionService sessions,IConfiguration config)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
ctx.Response.Headers.CacheControl="no-store";
return Results.Ok(new {enabled=IsOperator(actor.Id,config)});
});
app.MapGet("/api/ui/operations",async(HttpContext ctx,SessionService sessions,IConfiguration config,
NpgsqlDataSource source,ClusterPresence cluster,S3Storage storage)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
if(!IsOperator(actor.Id,config))return ApiError(ctx,404,"not_found","Доступ только владельцу системы");
ctx.Response.Headers.CacheControl="no-store";
using var deadline=CancellationTokenSource.CreateLinkedTokenSource(ctx.RequestAborted);deadline.CancelAfter(TimeSpan.FromSeconds(7));
var ct=deadline.Token;
var counts=new Dictionary<string,long>{{"queued",0},{"processing",0},{"ready",0},{"failed",0}};
var recent=new List<object>();var history=new List<object>();var heartbeats=new Dictionary<string,DateTime>();
long pending=0;double? latency=null;
await using(var cmd=source.CreateCommand("SELECT state,count(*) FROM attachment_jobs GROUP BY state"))
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))counts[rd.GetString(0)]=rd.GetInt64(1);
await using(var cmd=source.CreateCommand("SELECT count(*) FROM attachment_outbox WHERE published_at IS NULL"))pending=(long)(await cmd.ExecuteScalarAsync(ct))!;
await using(var cmd=source.CreateCommand("SELECT avg(extract(epoch FROM (completed_at-created_at)))::double precision FROM attachment_jobs WHERE state='ready' AND completed_at>now()-interval '15 minutes'")) {
var value=await cmd.ExecuteScalarAsync(ct);if(value is double seconds)latency=seconds;
}
await using(var cmd=source.CreateCommand("SELECT role,seen_at FROM pipeline_presence"))
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))heartbeats[rd.GetString(0)]=rd.GetDateTime(1);
await using(var cmd=source.CreateCommand("""
SELECT a.id,a.filename,j.state,j.attempts,j.error_code,j.created_at,j.completed_at,
j.state='failed' AND j.staging_deleted_at IS NULL
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id ORDER BY j.created_at DESC LIMIT 30
"""))
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))recent.Add(new {
attachment_id=rd.GetGuid(0),filename=rd.GetString(1),state=rd.GetString(2),attempts=rd.GetInt32(3),
error_code=rd.IsDBNull(4)?null:rd.GetString(4),created_at=rd.GetDateTime(5),
completed_at=rd.IsDBNull(6)?(DateTime?)null:rd.GetDateTime(6),can_replay=rd.GetBoolean(7)});
await using(var cmd=source.CreateCommand("""
SELECT e.event,e.attempt,e.created_at,a.filename FROM attachment_job_events e
JOIN attachment_jobs j ON j.id=e.job_id JOIN attachments a ON a.id=j.attachment_id ORDER BY e.id DESC LIMIT 30
"""))
await using(var rd=await cmd.ExecuteReaderAsync(ct))while(await rd.ReadAsync(ct))history.Add(new {
name=rd.GetString(0),attempt=rd.GetInt32(1),at=rd.GetDateTime(2),filename=rd.GetString(3)});
bool? s3=null,broker=null;uint? messages=null,consumers=null;
async Task ProbeStorage() {try {await storage.Client.GetBucketLocationAsync(storage.Bucket,ct);s3=true;}catch {s3=false;}}
async Task ProbeBroker() {try {
await using var connection=await BrokerConnection.OpenAsync(config,ct);
await using var channel=await connection.CreateChannelAsync(cancellationToken:ct);
var result=await channel.QueueDeclarePassiveAsync(BrokerConnection.Queue,ct);
broker=true;messages=result.MessageCount;consumers=result.ConsumerCount;
}catch {broker=false;}}
var snapshotTask=cluster.ReadAsync();await Task.WhenAll(ProbeStorage(),ProbeBroker());var snapshot=await snapshotTask;
var now=DateTime.UtcNow;
return Results.Ok(new {checked_at=now,counts,pending_outbox=pending,average_cycle_seconds=latency,
services=new {postgres=true,redis=snapshot.RedisAvailable,s3,broker,
worker=heartbeats.TryGetValue("worker",out var worker)&&(now-worker).TotalSeconds<10,
dispatcher=heartbeats.TryGetValue("dispatcher",out var dispatcher)&&(now-dispatcher).TotalSeconds<10},
instances=snapshot.Instances,broker_messages=messages,broker_consumers=consumers,jobs=recent,events=history});
});
app.MapPost("/api/ui/operations/attachments/{id:guid}/replay",async(Guid id,HttpContext ctx,
SessionService sessions,IConfiguration config,NpgsqlDataSource source)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
if(!IsOperator(actor.Id,config))return ApiError(ctx,404,"not_found","Доступ только владельцу системы");
await using var cmd=source.CreateCommand("SELECT id FROM attachment_jobs WHERE attachment_id=@id");cmd.Parameters.AddWithValue("id",id);
var job=await cmd.ExecuteScalarAsync(ctx.RequestAborted);
if(job is not Guid jobId)return ApiError(ctx,404,"not_found","Задание не найдено");
var changed=await new JobStore(source,config).ReplayAsync(jobId,ctx.RequestAborted);
return changed?Results.Accepted(value:new {status="queued"}):ApiError(ctx,409,"conflict","Повтор недоступен: задание не завершилось ошибкой или исходник уже удалён");
});
return app;
}
}
@@ -0,0 +1,39 @@
using Messenger.Api.Data;
using Messenger.Api.Services;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class ReactionEndpoints
{
public static IEndpointRouteBuilder MapReactionEndpoints(this IEndpointRouteBuilder app)
{
app.MapGet("/api/ui/chats/{chat_id}/reactions", async (
string chat_id, HttpContext context, SessionService sessions, ChatService chats, AppDatabase database) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId)) return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
if (await chats.FindForMemberAsync(chatId, actor.Id) is null) return ApiError(context, 404, "not_found", "Чат не найден");
var values = context.Request.Query["message_ids"].ToString().Split(',');
if (values.Length is < 1 or > 100 || values.Any(value => !Guid.TryParse(value, out _)))
return ApiError(context, 400, "invalid_request", "Укажите от 1 до 100 ID сообщений");
var ids = values.Select(Guid.Parse).Distinct().ToArray();
return Results.Ok(new { items = await database.ListReactionsAsync(chatId, ids, actor.Id) });
});
app.MapMethods("/api/ui/chats/{chat_id}/messages/{message_id}/reactions/{reaction}", ["PUT", "DELETE"], async (
string chat_id, string message_id, string reaction, HttpContext context,
SessionService sessions, ChatService chats, AppDatabase database) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId) || !Guid.TryParse(message_id, out var messageId)
|| reaction is not ("like" or "love" or "laugh" or "mushroom"))
return ApiError(context, 400, "invalid_request", "Некорректная реакция или ID");
if (await chats.FindForMemberAsync(chatId, actor.Id) is null) return ApiError(context, 404, "not_found", "Чат не найден");
return await database.SetReactionAsync(chatId, messageId, actor.Id, reaction, context.Request.Method == "PUT")
? Results.NoContent() : ApiError(context, 404, "not_found", "Сообщение не найдено");
});
return app;
}
}
@@ -0,0 +1,87 @@
using Messenger.Api.Contracts;
using Messenger.Api.Services;
using Messenger.Api.Data;
using Messenger.Api.Infrastructure;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class UiEndpoints
{
// The required /api/v1 history stays ascending. The web client has a separate,
// optional route to open a large chat at its newest page and page backwards.
public static IEndpointRouteBuilder MapUiEndpoints(this IEndpointRouteBuilder app)
{
app.MapGet("/api/ui/cluster", async (HttpContext context, SessionService sessions, ClusterPresence presence) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
context.Response.Headers.CacheControl = "no-store";
return Results.Ok(await presence.ReadAsync());
});
app.MapGet("/api/ui/users/{username}",async(string username,HttpContext ctx,SessionService sessions,AppDatabase db)=> {
var(actor,_)=await sessions.AuthenticateAsync(ctx);if(actor is null)return Unauthorized(ctx);
if(!System.Text.RegularExpressions.Regex.IsMatch(username,"\\A[a-z0-9_]{3,32}\\z"))return ApiError(ctx,400,"invalid_request","Некорректный логин");
var user=await db.FindUserByUsernameAsync(username);
return user is null?ApiError(ctx,404,"not_found","Пользователь не найден"):Results.Ok(user.ToDto());
});
app.MapGet("/api/ui/chats", async (HttpContext context, SessionService sessions, AppDatabase database) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
var (limit, limitError) = ParseLimit(context);
if (limitError is not null) return limitError;
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"ui-chats:{actor.Id}");
if (cursorError is not null) return cursorError;
var page = await database.ListChatPreviewsAsync(actor.Id, cursorTime, cursor?.Id, limit + 1);
string? nextCursor = null;
if (page.Count > limit)
{
var last = page[limit - 1];
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"ui-chats:{actor.Id}", last.CreatedAt, last.Id));
page = page.Take(limit).ToList();
}
return Results.Ok(new { items = page, next_cursor = nextCursor });
});
app.MapGet("/api/ui/chats/{chat_id}/members", async (
string chat_id, HttpContext context, SessionService sessions, ChatService chats, UserService users) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId))
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
return ApiError(context, 404, "not_found", "Чат не найден");
return Results.Ok(new { items = await users.ListForChatAsync(chatId) });
});
app.MapGet("/api/ui/chats/{chat_id}/messages", async (
string chat_id, HttpContext context, SessionService sessions, ChatService chats, MessageService messages) =>
{
var (actor, _) = await sessions.AuthenticateAsync(context);
if (actor is null) return Unauthorized(context);
if (!Guid.TryParse(chat_id, out var chatId))
return ApiError(context, 400, "invalid_request", "Некорректный chat_id");
if (await chats.FindForMemberAsync(chatId, actor.Id) is null)
return ApiError(context, 404, "not_found", "Чат не найден");
var (limit, limitError) = ParseLimit(context);
if (limitError is not null) return limitError;
var (cursor, cursorTime, cursorError) = ParseCursor(context, $"recent-chat:{chatId}");
if (cursorError is not null) return cursorError;
var page = await messages.ListAsync(chatId, cursorTime, cursor?.Id, limit + 1, newestFirst: true);
string? nextCursor = null;
if (page.Count > limit)
{
var last = page[limit - 1];
nextCursor = CursorService.EncodeChatCursor(new ChatCursor($"recent-chat:{chatId}", last.CreatedAt, last.Id));
page = page.Take(limit).ToList();
}
return Results.Ok(new { items = page, next_cursor = nextCursor });
});
return app;
}
}
@@ -0,0 +1,40 @@
using System.Text.RegularExpressions;
using Messenger.Api.Contracts;
using Messenger.Api.Services;
using static Messenger.Api.Services.ApiSupport;
namespace Messenger.Api.Endpoints;
public static class UserEndpoints
{
public static IEndpointRouteBuilder MapUserEndpoints(this IEndpointRouteBuilder app)
{
app.MapPost("/api/v1/users", async (HttpContext context, UserService users) =>
{
var (request, error) = await ReadJsonAsync<CreateUserRequest>(context);
if (error is not null) return error;
if (request!.Username is null || !Regex.IsMatch(request.Username, "\\A[a-z0-9_]{3,32}\\z"))
return ApiError(context, 400, "invalid_request", "Некорректный username");
if (string.IsNullOrWhiteSpace(request.DisplayName) ||
request.DisplayName.EnumerateRunes().Count() is < 1 or > 100)
return ApiError(context, 400, "invalid_request", "Некорректный display_name");
if (string.IsNullOrWhiteSpace(request.Password) ||
request.Password.EnumerateRunes().Count() is < 12 or > 128)
return ApiError(context, 400, "invalid_request", "Пароль должен содержать 12-128 символов");
var user = await users.RegisterAsync(request.Username, request.DisplayName, request.Password);
return user is null
? ApiError(context, 409, "conflict", "Username уже занят")
: Results.Json(user, statusCode: StatusCodes.Status201Created);
});
app.MapGet("/api/v1/users/me", async (HttpContext context, SessionService sessions) =>
{
var (user, _) = await sessions.AuthenticateAsync(context);
return user is null ? Unauthorized(context) : Results.Ok(user);
});
return app;
}
}
@@ -0,0 +1,55 @@
using Messenger.Api.Data;
using StackExchange.Redis;
namespace Messenger.Api.Infrastructure;
// Registry entries live ten minutes; an eight-second heartbeat gap means unavailable.
// A Redis outage is reported as unknown, never as proof that an API has stopped.
public sealed class ClusterPresence(IConnectionMultiplexer redis, InstanceIdentity identity,
ReadinessState readiness, AppDatabase database, ILogger<ClusterPresence> logger) : BackgroundService
{
private const string Registry = "messenger:ui:instances";
public sealed record Node(string Id, bool Ready, long LastSeen);
public sealed record Snapshot(bool RedisAvailable, bool PostgresAvailable, long CheckedAt, List<Node> Instances);
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var timer = new PeriodicTimer(TimeSpan.FromSeconds(2));
try
{
do
{
try
{
if (redis.IsConnected && !readiness.IsStopping && await database.IsReadyAsync())
{
var db = redis.GetDatabase();
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
await db.SortedSetAddAsync(Registry, identity.Id, now).WaitAsync(TimeSpan.FromSeconds(1), stoppingToken);
await db.SortedSetRemoveRangeByScoreAsync(Registry, double.NegativeInfinity, now - 600_000);
await db.KeyExpireAsync(Registry, TimeSpan.FromMinutes(15));
}
}
catch (Exception error) when (error is RedisException or TimeoutException)
{ logger.LogDebug("Cluster heartbeat unavailable"); }
} while (await timer.WaitForNextTickAsync(stoppingToken));
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { }
}
public async Task<Snapshot> ReadAsync()
{
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var postgres = await database.IsReadyAsync();
try
{
if (!redis.IsConnected) return new(false, postgres, now, []);
var entries = await redis.GetDatabase().SortedSetRangeByScoreWithScoresAsync(Registry,
now - 600_000, double.PositiveInfinity).WaitAsync(TimeSpan.FromSeconds(1));
return new(true, postgres, now, entries.Select(entry => new Node(entry.Element.ToString(),
now - entry.Score < 8000, (long)entry.Score)).ToList());
}
catch (Exception error) when (error is RedisException or TimeoutException)
{ return new(false, postgres, now, []); }
}
}
@@ -0,0 +1,6 @@
namespace Messenger.Api.Infrastructure;
public sealed class InstanceIdentity
{
public string Id { get; } = Guid.NewGuid().ToString("N");
}
@@ -0,0 +1,8 @@
namespace Messenger.Api.Infrastructure;
public sealed class ReadinessState
{
private int _stopping;
public bool IsStopping => Volatile.Read(ref _stopping) != 0;
public void Stop() => Interlocked.Exchange(ref _stopping, 1);
}
@@ -0,0 +1,58 @@
using System.Text.Json;
using Messenger.Api.Contracts;
using StackExchange.Redis;
namespace Messenger.Api.Infrastructure;
// PostgreSQL validates the session on every request. This cache saves the user lookup
// and never grants access independently of the durable session row.
public sealed class RedisSessionCache(IConnectionMultiplexer connection, ILogger<RedisSessionCache> logger)
{
private static readonly TimeSpan OperationBudget = TimeSpan.FromMilliseconds(750);
private static string Key(byte[] hash) => "messenger:session:" + Convert.ToHexString(hash).ToLowerInvariant();
public sealed record Entry(UserDto User, DateTime ExpiresAt);
public async Task<Entry?> GetAsync(byte[] hash)
{
if (!connection.IsConnected) return null;
try
{
var value = await connection.GetDatabase().StringGetAsync(Key(hash)).WaitAsync(OperationBudget);
return value.IsNullOrEmpty ? null : JsonSerializer.Deserialize<Entry>(value.ToString());
}
catch (Exception exception) when (exception is RedisException or TimeoutException or OperationCanceledException or JsonException)
{
logger.LogWarning("Session cache read unavailable; using PostgreSQL");
return null;
}
}
public async Task SetAsync(byte[] hash, UserDto user, DateTime expiresAt)
{
var remaining = expiresAt - DateTime.UtcNow;
if (!connection.IsConnected || remaining <= TimeSpan.Zero) return;
try
{
await connection.GetDatabase().StringSetAsync(Key(hash),
JsonSerializer.Serialize(new Entry(user, expiresAt)), remaining).WaitAsync(OperationBudget);
}
catch (Exception exception) when (exception is RedisException or TimeoutException or OperationCanceledException)
{
logger.LogWarning("Session cache write unavailable; using PostgreSQL");
}
}
public async Task RemoveAsync(byte[] hash)
{
if (!connection.IsConnected) return;
try
{
await connection.GetDatabase().KeyDeleteAsync(Key(hash)).WaitAsync(OperationBudget);
}
catch (Exception exception) when (exception is RedisException or TimeoutException or OperationCanceledException)
{
logger.LogWarning("Session cache invalidation unavailable; PostgreSQL still enforces revocation");
}
}
}
@@ -0,0 +1,26 @@
using Amazon.S3;
using Amazon.S3.Model;
using Amazon.Runtime;
namespace Messenger.Api.Infrastructure;
public sealed class S3Storage : IDisposable
{
public AmazonS3Client Client { get; }
public string Bucket { get; }
public S3Storage(IConfiguration config)
{
Bucket=config["S3_BUCKET"] ?? "messenger";
Client=new AmazonS3Client(new BasicAWSCredentials(
File.ReadAllText(config["S3_ACCESS_KEY_FILE"]!).Trim(),
File.ReadAllText(config["S3_SECRET_KEY_FILE"]!).Trim()),new AmazonS3Config {
ServiceURL=config["S3_ENDPOINT"] ?? "http://s3:9000", ForcePathStyle=true,
AuthenticationRegion="us-east-1", MaxErrorRetry=0, Timeout=TimeSpan.FromSeconds(20)
});
}
public async Task<bool> ReadyAsync()
{
using var timeout=new CancellationTokenSource(TimeSpan.FromSeconds(1));
try { await Client.ListObjectsV2Async(new ListObjectsV2Request {BucketName=Bucket,MaxKeys=1},timeout.Token); return true; }
catch { return false; }
}
public void Dispose() => Client.Dispose();
}
+23
View File
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk.Web">
<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<EmbeddedResource Include="Data/Migrations/*.sql" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="AWSSDK.S3" Version="4.0.0" />
<PackageReference Include="Isopoh.Cryptography.Argon2" Version="2.0.0" />
<PackageReference Include="Npgsql" Version="9.0.5" />
<PackageReference Include="RabbitMQ.Client" Version="7.1.2" />
<PackageReference Include="SkiaSharp" Version="3.119.1" />
<PackageReference Include="SkiaSharp.NativeAssets.Linux.NoDependencies" Version="3.119.1" />
<PackageReference Include="StackExchange.Redis" Version="2.8.58" />
</ItemGroup>
</Project>
@@ -0,0 +1,42 @@
using Messenger.Api.Services;
using System.Net.Sockets;
using Npgsql;
namespace Messenger.Api.Middleware;
public sealed class ExceptionHandlingMiddleware(RequestDelegate next, ILogger<ExceptionHandlingMiddleware> logger)
{
public async Task InvokeAsync(HttpContext context)
{
try
{
await next(context);
}
catch (AttachmentException exception)
{
if(!context.Response.HasStarted)
await ApiSupport.ApiError(context,exception.Status,exception.Code,exception.Message).ExecuteAsync(context);
}
catch (OperationCanceledException) when (!context.RequestAborted.IsCancellationRequested)
{
if(!context.Response.HasStarted)
await ApiSupport.ApiError(context,503,"unavailable","Время ожидания истекло").ExecuteAsync(context);
}
catch (BadHttpRequestException exception) when(exception.StatusCode==413)
{
await ApiSupport.ApiError(context,413,"payload_too_large","Слишком большой запрос").ExecuteAsync(context);
}
catch (Exception exception) when (exception is NpgsqlException or SocketException or TimeoutException or Amazon.S3.AmazonS3Exception or HttpRequestException)
{
logger.LogError(exception, "Database request failed");
if (!context.Response.HasStarted)
await ApiSupport.ApiError(context, 503, "unavailable", "Сервис временно недоступен").ExecuteAsync(context);
}
catch (Exception exception)
{
logger.LogError(exception, "Request failed");
if (!context.Response.HasStarted)
await ApiSupport.ApiError(context, 500, "internal_error", "Внутренняя ошибка").ExecuteAsync(context);
}
}
}
@@ -0,0 +1,18 @@
using Messenger.Api.Infrastructure;
using Messenger.Api.Services;
namespace Messenger.Api.Middleware;
public sealed class InstanceMiddleware(RequestDelegate next)
{
public async Task InvokeAsync(HttpContext context, InstanceIdentity identity, ReadinessState readiness)
{
context.Response.Headers["X-Instance-Id"] = identity.Id;
if (readiness.IsStopping && context.Request.Path != "/health/live")
{
await ApiSupport.ApiError(context, 503, "unavailable", "Экземпляр API останавливается").ExecuteAsync(context);
return;
}
await next(context);
}
}
@@ -0,0 +1,16 @@
namespace Messenger.Api.Middleware;
public sealed class RequestIdMiddleware
{
private readonly RequestDelegate _next;
public RequestIdMiddleware(RequestDelegate next) => _next = next;
public async Task InvokeAsync(HttpContext context)
{
var requestId = Guid.NewGuid().ToString();
context.Items["RequestId"] = requestId;
context.Response.Headers["X-Request-Id"] = requestId;
await _next(context);
}
}
@@ -0,0 +1,25 @@
namespace Messenger.Api.Middleware;
public sealed class RequestLoggingMiddleware
{
private readonly RequestDelegate _next;
public RequestLoggingMiddleware(RequestDelegate next) => _next = next;
public async Task InvokeAsync(HttpContext context)
{
var logger = context.RequestServices
.GetRequiredService<ILoggerFactory>()
.CreateLogger("Http");
await _next(context);
logger.LogInformation(
"{Method} {Path} -> {StatusCode} request_id={RequestId}",
context.Request.Method,
context.Request.Path,
context.Response.StatusCode,
context.Items["RequestId"]
);
}
}
@@ -0,0 +1,21 @@
using RabbitMQ.Client;
namespace Messenger.Api.Pipeline;
public static class BrokerConnection
{
public const string Queue="attachments.v1";
public static Task<IConnection> OpenAsync(IConfiguration config,CancellationToken ct)
{
var factory=new ConnectionFactory {
HostName=config["BROKER_HOST"]??"broker", UserName="messenger",
Password=File.ReadAllText(config["BROKER_PASSWORD_FILE"]!).Trim(),
VirtualHost="messenger",AutomaticRecoveryEnabled=true,
NetworkRecoveryInterval=TimeSpan.FromSeconds(2),RequestedHeartbeat=TimeSpan.FromSeconds(10),
RequestedConnectionTimeout=TimeSpan.FromSeconds(5)
};
return factory.CreateConnectionAsync(ct);
}
public static Task DeclareAsync(IChannel channel,CancellationToken ct)=>channel.QueueDeclareAsync(
Queue,durable:true,exclusive:false,autoDelete:false,arguments:new Dictionary<string,object?> {
["x-queue-type"]="quorum",["x-max-length"]=256,["x-overflow"]="reject-publish"
},cancellationToken:ct);
}
@@ -0,0 +1,19 @@
using System.Text.Json;
namespace Messenger.Api.Pipeline;
public static class FaultInjection
{
// Explicit local operator instrumentation; disabled in the normal Compose configuration.
public static async Task HitAsync(IConfiguration config,string point,Guid job,CancellationToken ct)
{
var path=config["PIPELINE_FAULT_FILE"];
if(string.IsNullOrEmpty(path)||!File.Exists(path))return;
try {
using var json=JsonDocument.Parse(await File.ReadAllTextAsync(path,ct));
if(json.RootElement.GetProperty("point").GetString()!=point || json.RootElement.GetProperty("job_id").GetGuid()!=job)return;
File.Move(path,path+".used",true);
await File.WriteAllTextAsync(path+".hit",point,ct);
}catch(IOException){return;}
Console.WriteLine($"Operator fault pause: {point}, job {job}");
await Task.Delay(Timeout.InfiniteTimeSpan,ct);
}
}
@@ -0,0 +1,57 @@
using System.Text.Json;
using System.Security.Cryptography;
using Messenger.Api.Contracts;
using SkiaSharp;
namespace Messenger.Api.Pipeline;
public static class ImageRenderer
{
public static void Render(string input,string output)
{
using var stream=File.OpenRead(input);using var managed=new SKManagedStream(stream,false);
using var codec=SKCodec.Create(managed);
if(codec is null || codec.EncodedFormat is not SKEncodedImageFormat.Png and not SKEncodedImageFormat.Jpeg)
throw new InvalidDataException("invalid_image");
var info=codec.Info;
if((long)info.Width*info.Height>40_000_000 || info.Width<1 || info.Height<1)throw new InvalidDataException("invalid_image");
using var bitmap=new SKBitmap(new SKImageInfo(info.Width,info.Height,SKColorType.Rgba8888,SKAlphaType.Premul));
if(codec.GetPixels(bitmap.Info,bitmap.GetPixels())!=SKCodecResult.Success)throw new InvalidDataException("invalid_image");
var rotated=(int)codec.EncodedOrigin>=5;
int width=rotated?info.Height:info.Width,height=rotated?info.Width:info.Height;
var matrix=codec.EncodedOrigin switch {
SKEncodedOrigin.TopRight=>new SKMatrix(-1,0,info.Width,0,1,0,0,0,1),
SKEncodedOrigin.BottomRight=>new SKMatrix(-1,0,info.Width,0,-1,info.Height,0,0,1),
SKEncodedOrigin.BottomLeft=>new SKMatrix(1,0,0,0,-1,info.Height,0,0,1),
SKEncodedOrigin.LeftTop=>new SKMatrix(0,1,0,1,0,0,0,0,1),
SKEncodedOrigin.RightTop=>new SKMatrix(0,-1,info.Height,1,0,0,0,0,1),
SKEncodedOrigin.RightBottom=>new SKMatrix(0,-1,info.Height,-1,0,info.Width,0,0,1),
SKEncodedOrigin.LeftBottom=>new SKMatrix(0,1,0,-1,0,info.Width,0,0,1),
_=>SKMatrix.Identity
};
Directory.CreateDirectory(output);var variants=new List<VariantDto>();
foreach(var name in new[]{"thumbnail","crop","optimized"})
{
var crop=name=="crop";var limit=name=="thumbnail"?256:1280;
var scale=Math.Min(1,(double)limit/Math.Max(width,height));
int targetWidth=crop?128:Math.Max(1,(int)Math.Floor(width*scale));
int targetHeight=crop?128:Math.Max(1,(int)Math.Floor(height*scale));
float side=Math.Min(width,height);
var rect=crop?new SKRect((width-side)/2,(height-side)/2,(width+side)/2,(height+side)/2):new SKRect(0,0,width,height);
using var target=new SKBitmap(new SKImageInfo(targetWidth,targetHeight,SKColorType.Rgba8888,SKAlphaType.Premul));
using(var canvas=new SKCanvas(target))
{
canvas.Clear(SKColors.White);
canvas.Scale(targetWidth/rect.Width,targetHeight/rect.Height);canvas.Translate(-rect.Left,-rect.Top);
canvas.Concat(matrix);
using var paint=new SKPaint { IsAntialias=true };
canvas.DrawBitmap(bitmap,0,0,paint);
}
using var image=SKImage.FromBitmap(target);
var webp=name=="optimized";
using var data=image.Encode(webp?SKEncodedImageFormat.Webp:SKEncodedImageFormat.Jpeg,webp?80:85);
var bytes=data.ToArray();File.WriteAllBytes(Path.Combine(output,name),bytes);
variants.Add(new(name,webp?"image/webp":"image/jpeg",bytes.Length,targetWidth,targetHeight,
Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant()));
}
File.WriteAllText(Path.Combine(output,"variants.json"),JsonSerializer.Serialize(variants,JobJson.Options));
}
}
@@ -0,0 +1,8 @@
using System.Text.Json;
namespace Messenger.Api.Pipeline;
public sealed record JobEnvelope(int SchemaVersion, Guid JobId, string Type, Guid AttachmentId,
int PipelineVersion, int Attempt, string CreatedAt);
public static class JobJson
{
public static readonly JsonSerializerOptions Options=new() {PropertyNamingPolicy=JsonNamingPolicy.SnakeCaseLower};
}
@@ -0,0 +1,22 @@
using System.Text.Json;
using Npgsql;
using RabbitMQ.Client;
namespace Messenger.Api.Pipeline;
public static class JobOperations
{
public static async Task RedeliverAsync(NpgsqlDataSource db,IConfiguration config,Guid id,CancellationToken ct)
{
await using var connection=await db.OpenConnectionAsync(ct);
await using var cmd=JobStore.Command(connection,"SELECT attachment_id,pipeline_version,attempts,state,created_at FROM attachment_jobs WHERE id=@id",("id",id));
JobEnvelope job;
await using(var r=await cmd.ExecuteReaderAsync(ct)) {
if(!await r.ReadAsync(ct))throw new InvalidOperationException("Job not found");
job=new(1,id,"attachment.process.v1",r.GetGuid(0),r.GetInt32(1),Math.Max(1,r.GetInt32(2)+(r.GetString(3)=="queued"?1:0)),r.GetDateTime(4).ToString("O"));
}
await using var broker=await BrokerConnection.OpenAsync(config,ct);
await using var channel=await broker.CreateChannelAsync(new CreateChannelOptions(true,true),ct);
await BrokerConnection.DeclareAsync(channel,ct);
await channel.BasicPublishAsync("",BrokerConnection.Queue,true,new BasicProperties {Persistent=true,ContentType="application/json",MessageId=id.ToString()},JsonSerializer.SerializeToUtf8Bytes(job,JobJson.Options),ct);
Console.WriteLine(JsonSerializer.Serialize(job,JobJson.Options));
}
}
+130
View File
@@ -0,0 +1,130 @@
using Npgsql;
namespace Messenger.Api.Pipeline;
public sealed record ClaimedJob(Guid Id,Guid AttachmentId,string StagingKey,string Sha256,long Size,string ContentType,int Attempt,Guid Lease);
public sealed class JobStore(NpgsqlDataSource db,IConfiguration config)
{
public int MaxAttempts=>Math.Clamp(config.GetValue("JOB_MAX_ATTEMPTS",5),1,20);
public int TimeoutSeconds=>Math.Clamp(config.GetValue("JOB_TIMEOUT_SECONDS",30),5,180);
public static NpgsqlCommand Command(NpgsqlConnection connection,string sql,params (string,object)[] values)
{
var cmd=new NpgsqlCommand(sql,connection);
foreach(var(name,value) in values) cmd.Parameters.AddWithValue(name,value);
return cmd;
}
public async Task<ClaimedJob?> ClaimAsync(NpgsqlConnection connection,JobEnvelope envelope,CancellationToken ct)
{
await using var tx=await connection.BeginTransactionAsync(ct);
var lease=Guid.NewGuid();
await using var cmd=Command(connection,"""
UPDATE attachment_jobs SET state='processing',attempts=attempts+1,lease_token=@lease,
lease_until=clock_timestamp()+make_interval(secs=>@seconds),error_code=NULL
WHERE id=@id AND attachment_id=@attachment AND pipeline_version=@version AND state='queued'
AND attempts+1=@attempt AND next_at<=clock_timestamp()
RETURNING staging_key,attempts
""",("id",envelope.JobId),("attachment",envelope.AttachmentId),("version",envelope.PipelineVersion),
("attempt",envelope.Attempt),("lease",lease),("seconds",TimeoutSeconds+5));
string staging;int attempt;
await using(var r=await cmd.ExecuteReaderAsync(ct))
{ if(!await r.ReadAsync(ct))return null;staging=r.GetString(0);attempt=r.GetInt32(1); }
await using var attachment=Command(connection,"""
UPDATE attachments SET status='processing',error_code=NULL,updated_at=clock_timestamp() WHERE id=@id
RETURNING sha256,size_bytes,content_type
""",("id",envelope.AttachmentId));
string hash,mime;long size;
await using(var r=await attachment.ExecuteReaderAsync(ct))
{await r.ReadAsync(ct);hash=r.GetString(0);size=r.GetInt64(1);mime=r.GetString(2);}
await EventAsync(connection,envelope.JobId,"processing",attempt,ct);
await tx.CommitAsync(ct);
return new(envelope.JobId,envelope.AttachmentId,staging,hash,size,mime,attempt,lease);
}
public static async Task EventAsync(NpgsqlConnection connection,Guid id,string name,int attempt,CancellationToken ct)
{
await using var cmd=Command(connection,"INSERT INTO attachment_job_events(job_id,event,attempt) VALUES (@id,@event,@attempt)",
("id",id),("event",name),("attempt",attempt));await cmd.ExecuteNonQueryAsync(ct);
}
public async Task FailAsync(NpgsqlConnection connection,ClaimedJob job,bool permanent,string code,CancellationToken ct)
{
var final=permanent||job.Attempt>=MaxAttempts;
var status=final?"failed":"queued";
var error=final && !permanent?"retry_exhausted":code;
var delay=Math.Min(30,Math.Pow(2,job.Attempt))+Random.Shared.NextDouble();
await using var tx=await connection.BeginTransactionAsync(ct);
await using var cmd=Command(connection,"""
UPDATE attachment_jobs SET state=@status,error_code=@code,lease_token=NULL,lease_until=NULL,
next_at=clock_timestamp()+make_interval(secs=>@delay)
WHERE id=@id AND state='processing' AND lease_token=@lease
""",("id",job.Id),("status",status),("code",error),("delay",delay),("lease",job.Lease));
if(await cmd.ExecuteNonQueryAsync(ct)==1)
{
await using var attachment=Command(connection,"UPDATE attachments SET status=@status,error_code=@code,updated_at=clock_timestamp() WHERE id=@id",
("id",job.AttachmentId),("status",status),("code",final?(object)error:DBNull.Value));
await attachment.ExecuteNonQueryAsync(ct);
if(!final)
{
await using var outbox=Command(connection,"""
INSERT INTO attachment_outbox(job_id,available_at) SELECT id,next_at FROM attachment_jobs WHERE id=@id
ON CONFLICT(job_id) DO UPDATE SET published_at=NULL,available_at=EXCLUDED.available_at
""",("id",job.Id));await outbox.ExecuteNonQueryAsync(ct);
}
await EventAsync(connection,job.Id,final?"failed":"retry_scheduled",job.Attempt,ct);
}
await tx.CommitAsync(ct);
}
public async Task RecoverAsync(CancellationToken ct)
{
await using var connection=await db.OpenConnectionAsync(ct);
var ids=new List<Guid>();
await using(var query=Command(connection,"SELECT id FROM attachment_jobs WHERE state='processing' AND lease_until<clock_timestamp() LIMIT 50"))
await using(var r=await query.ExecuteReaderAsync(ct))while(await r.ReadAsync(ct))ids.Add(r.GetGuid(0));
foreach(var id in ids)
{
await using var tx=await connection.BeginTransactionAsync(ct);
await using var gate=Command(connection,"SELECT pg_try_advisory_xact_lock(hashtextextended(@scope,0))",("scope","job:"+id));
if(!(bool)(await gate.ExecuteScalarAsync(ct))!)continue;
await using var cmd=Command(connection,"""
UPDATE attachment_jobs SET state=CASE WHEN attempts>=@max THEN 'failed' ELSE 'queued' END,
error_code=CASE WHEN attempts>=@max THEN 'retry_exhausted' ELSE 'worker_lost' END,
lease_token=NULL,lease_until=NULL,next_at=clock_timestamp()
WHERE id=@id AND state='processing' AND lease_until<clock_timestamp()
RETURNING attachment_id,state,attempts
""",("id",id),("max",MaxAttempts));
Guid attachment;string status;int attempt;
await using(var r=await cmd.ExecuteReaderAsync(ct))
{ if(!await r.ReadAsync(ct))continue;attachment=r.GetGuid(0);status=r.GetString(1);attempt=r.GetInt32(2); }
await using(var update=Command(connection,"UPDATE attachments SET status=@state,error_code=@code,updated_at=clock_timestamp() WHERE id=@id",
("id",attachment),("state",status),("code",status=="failed"?(object)"retry_exhausted":DBNull.Value)))await update.ExecuteNonQueryAsync(ct);
if(status=="queued")
{
await using var outbox=Command(connection,"UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=@id",("id",id));
await outbox.ExecuteNonQueryAsync(ct);
}
await EventAsync(connection,id,"lease_recovered",attempt,ct);await tx.CommitAsync(ct);
}
// A durable intention also recovers a dropped delivery, without changing job identity.
await using var rescue=Command(connection,"""
UPDATE attachment_outbox o SET published_at=NULL
FROM attachment_jobs j WHERE o.job_id=j.id AND j.state='queued'
AND o.published_at<clock_timestamp()-interval '30 seconds' AND j.next_at<=clock_timestamp()
""");await rescue.ExecuteNonQueryAsync(ct);
}
public async Task<bool> ReplayAsync(Guid id,CancellationToken ct)
{
await using var connection=await db.OpenConnectionAsync(ct);
await using var tx=await connection.BeginTransactionAsync(ct);
await using var gate=Command(connection,"SELECT pg_advisory_xact_lock(hashtextextended(@scope,0))",("scope","job:"+id));await gate.ExecuteNonQueryAsync(ct);
await using var cmd=Command(connection,"""
UPDATE attachment_jobs SET state='queued',attempts=0,error_code=NULL,next_at=clock_timestamp(),replay_count=replay_count+1,
lease_token=NULL,lease_until=NULL WHERE id=@id AND state='failed' AND staging_deleted_at IS NULL
RETURNING attachment_id
""",("id",id));
var attachment=await cmd.ExecuteScalarAsync(ct);
if(attachment is Guid attachmentId)
{
await using(var update=Command(connection,"UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=@id",("id",attachmentId)))await update.ExecuteNonQueryAsync(ct);
await using(var outbox=Command(connection,"UPDATE attachment_outbox SET published_at=NULL,available_at=clock_timestamp() WHERE job_id=@id",("id",id)))await outbox.ExecuteNonQueryAsync(ct);
await EventAsync(connection,id,"operator_replay",0,ct);
}
await tx.CommitAsync(ct);Console.WriteLine(attachment is Guid?"Replay scheduled":"No-op: job is not failed or staging has expired");
return attachment is Guid;
}
}
@@ -0,0 +1,49 @@
using System.Text.Json;
using Messenger.Api.Infrastructure;
using Npgsql;
using RabbitMQ.Client;
namespace Messenger.Api.Pipeline;
public sealed class OutboxDispatcher(NpgsqlDataSource db,S3Storage storage,IConfiguration config)
{
public async Task RunAsync(CancellationToken ct)
{
var jobs=new JobStore(db,config);var maintenance=new PipelineMaintenance(db,storage,config);
var nextMaintenance=DateTime.MinValue;
while(!ct.IsCancellationRequested)
{
try {
await using var broker=await BrokerConnection.OpenAsync(config,ct);
await using var channel=await broker.CreateChannelAsync(new CreateChannelOptions(publisherConfirmationsEnabled:true,publisherConfirmationTrackingEnabled:true),ct);
await BrokerConnection.DeclareAsync(channel,ct);
while(!ct.IsCancellationRequested && channel.IsOpen)
{
if(DateTime.UtcNow>=nextMaintenance)
{
await jobs.RecoverAsync(ct);await maintenance.CleanupAsync(ct);
nextMaintenance=DateTime.UtcNow.AddSeconds(5);
}
await using var connection=await db.OpenConnectionAsync(ct);
await using var tx=await connection.BeginTransactionAsync(ct);
JobEnvelope? job=null;
await using(var select=JobStore.Command(connection,"""
SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at
FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id
WHERE o.published_at IS NULL AND o.available_at<=clock_timestamp() AND j.state='queued'
ORDER BY o.available_at,o.job_id LIMIT 1 FOR UPDATE OF o SKIP LOCKED
"""))
await using(var r=await select.ExecuteReaderAsync(ct))
if(await r.ReadAsync(ct))job=new(1,r.GetGuid(0),"attachment.process.v1",r.GetGuid(1),r.GetInt32(2),r.GetInt32(3),r.GetDateTime(4).ToString("O"));
if(job is null) {await tx.CommitAsync(ct);await Task.Delay(100,ct);continue;}
using var confirmation=CancellationTokenSource.CreateLinkedTokenSource(ct);confirmation.CancelAfter(TimeSpan.FromSeconds(5));
await channel.BasicPublishAsync("",BrokerConnection.Queue,true,
new BasicProperties {Persistent=true,ContentType="application/json",MessageId=job.JobId.ToString()},
JsonSerializer.SerializeToUtf8Bytes(job,JobJson.Options),confirmation.Token);
await FaultInjection.HitAsync(config,"after_publish",job.JobId,ct);
await using var mark=JobStore.Command(connection,"UPDATE attachment_outbox SET published_at=clock_timestamp() WHERE job_id=@id",("id",job.JobId));
await mark.ExecuteNonQueryAsync(ct);await JobStore.EventAsync(connection,job.JobId,"published",job.Attempt,ct);await tx.CommitAsync(ct);
}
} catch(OperationCanceledException) when(ct.IsCancellationRequested) {break;}
catch(Exception ex) {Console.WriteLine("Outbox deferred: "+ex.GetType().Name);await Task.Delay(2000,ct);}
}
}
}
@@ -0,0 +1,54 @@
using Amazon.S3.Model;
using Messenger.Api.Data;
using Messenger.Api.Infrastructure;
using Npgsql;
namespace Messenger.Api.Pipeline;
public sealed class PipelineMaintenance(NpgsqlDataSource db,S3Storage storage,IConfiguration config)
{
public async Task CleanupAsync(CancellationToken ct)
{
await using var connection=await db.OpenConnectionAsync(ct);
var rows=new List<(Guid Id,string Key)>();
var retention=Math.Max(3600,config.GetValue("STAGING_RETENTION_SECONDS",86400));
await using(var cmd=JobStore.Command(connection,"""
SELECT id,staging_key FROM attachment_jobs WHERE staging_deleted_at IS NULL
AND (state='ready' OR state='failed' AND created_at<clock_timestamp()-make_interval(secs=>@retention)) LIMIT 20
""",("retention",retention)))
await using(var r=await cmd.ExecuteReaderAsync(ct))while(await r.ReadAsync(ct))rows.Add((r.GetGuid(0),r.GetString(1)));
foreach(var row in rows)
{
await using var tx=await connection.BeginTransactionAsync(ct);
await using var gate=JobStore.Command(connection,"SELECT pg_try_advisory_xact_lock(hashtextextended(@scope,0))",("scope","job:"+row.Id));
if(!(bool)(await gate.ExecuteScalarAsync(ct))!)continue;
await using var check=JobStore.Command(connection,"""
SELECT state FROM attachment_jobs WHERE id=@id AND staging_deleted_at IS NULL
AND (state='ready' OR state='failed' AND created_at<clock_timestamp()-make_interval(secs=>@retention)) FOR UPDATE
""",("id",row.Id),("retention",retention));
if(await check.ExecuteScalarAsync(ct) is not string)continue;
await storage.Client.DeleteObjectAsync(storage.Bucket,row.Key,ct);
await using var mark=JobStore.Command(connection,"UPDATE attachment_jobs SET staging_deleted_at=clock_timestamp() WHERE id=@id",("id",row.Id));
await mark.ExecuteNonQueryAsync(ct);await tx.CommitAsync(ct);
}
}
public async Task ReconcileStagingAsync(bool apply,CancellationToken ct)
{
var actions=new List<string>();
await using var connection=await db.OpenConnectionAsync(ct);
await using var tx=await connection.BeginTransactionAsync(ct);
await using(var gate=JobStore.Command(connection,$"SELECT pg_advisory_xact_lock({AppDatabase.AttachmentGate})"))await gate.ExecuteNonQueryAsync(ct);
string? continuation=null;
do {
var page=await storage.Client.ListObjectsV2Async(new ListObjectsV2Request {BucketName=storage.Bucket,Prefix="staging/",MaxKeys=1000,ContinuationToken=continuation},ct);
foreach(var obj in page.S3Objects ?? [])
{
if(obj.LastModified is null || obj.LastModified.Value.ToUniversalTime()>DateTime.UtcNow.AddHours(-1))continue;
await using var check=JobStore.Command(connection,"SELECT EXISTS(SELECT 1 FROM attachment_jobs WHERE staging_key=@key)",("key",obj.Key));
if(!(bool)(await check.ExecuteScalarAsync(ct))!) {
actions.Add(obj.Key);if(apply)await storage.Client.DeleteObjectAsync(storage.Bucket,obj.Key,ct);
}
}
continuation=page.IsTruncated==true?page.NextContinuationToken:null;
}while(continuation is not null);
await tx.CommitAsync(ct);Console.WriteLine(System.Text.Json.JsonSerializer.Serialize(new {mode=apply?"apply":"dry-run",old_untracked_staging=actions}));
}
}
@@ -0,0 +1,125 @@
using System.Diagnostics;
using System.Text.Json;
using System.Security.Cryptography;
using Amazon.S3.Model;
using Messenger.Api.Contracts;
using Messenger.Api.Infrastructure;
using Npgsql;
using RabbitMQ.Client;
using RabbitMQ.Client.Events;
namespace Messenger.Api.Pipeline;
public sealed class PipelineWorker(NpgsqlDataSource db,S3Storage storage,IConfiguration config)
{
private readonly JobStore jobs=new(db,config);
public async Task RunAsync(CancellationToken ct)
{
var concurrency=Math.Clamp(config.GetValue("WORKER_CONCURRENCY",2),1,8);
var prefetch=(ushort)Math.Clamp(config.GetValue("WORKER_PREFETCH",1),1,16);
while(!ct.IsCancellationRequested)
{
try {
await using var connection=await BrokerConnection.OpenAsync(config,ct);
var channels=new List<IChannel>();
try {
for(var i=0;i<concurrency;i++)
{
var channel=await connection.CreateChannelAsync(cancellationToken:ct);channels.Add(channel);
await BrokerConnection.DeclareAsync(channel,ct);
await channel.BasicQosAsync(0,prefetch,false,ct);
var consumer=new AsyncEventingBasicConsumer(channel);
consumer.ReceivedAsync+=async(_,delivery)=> {
try {
var envelope=JsonSerializer.Deserialize<JobEnvelope>(delivery.Body.Span,JobJson.Options);
if(envelope is not null && envelope.SchemaVersion==1 && envelope.Type=="attachment.process.v1" && envelope.PipelineVersion==1)
await ProcessAsync(envelope,ct);
await channel.BasicAckAsync(delivery.DeliveryTag,false,ct);
} catch(JsonException) { await channel.BasicRejectAsync(delivery.DeliveryTag,false,ct); }
catch(OperationCanceledException) when(ct.IsCancellationRequested) { }
catch(Exception ex) {
Console.WriteLine("Delivery deferred: "+ex.GetType().Name);
try {await channel.BasicNackAsync(delivery.DeliveryTag,false,true,ct);}catch { }
await Task.Delay(500,ct);
}
};
await channel.BasicConsumeAsync(BrokerConnection.Queue,false,consumer,ct);
}
Console.WriteLine($"Worker ready: concurrency={concurrency}, prefetch={prefetch}");
while(!ct.IsCancellationRequested && connection.IsOpen && channels.All(c=>c.IsOpen))await Task.Delay(500,ct);
} finally { foreach(var channel in channels)await channel.DisposeAsync(); }
} catch(OperationCanceledException) when(ct.IsCancellationRequested) {break;}
catch(Exception ex) {Console.WriteLine("Worker connection unavailable: "+ex.GetType().Name);await Task.Delay(2000,ct);}
}
}
private async Task ProcessAsync(JobEnvelope envelope,CancellationToken stop)
{
await using var connection=await db.OpenConnectionAsync(stop);
await using var gate=JobStore.Command(connection,"SELECT pg_try_advisory_lock(hashtextextended(@scope,0))",("scope","job:"+envelope.JobId));
if(!(bool)(await gate.ExecuteScalarAsync(stop))!)return;
try {
var job=await jobs.ClaimAsync(connection,envelope,stop);if(job is null)return;
Console.WriteLine($"Processing job {job.Id}, attempt {job.Attempt}");
var directory=Path.Combine(Path.GetTempPath(),"job-"+Guid.NewGuid().ToString("N"));Directory.CreateDirectory(directory);
using var timeout=CancellationTokenSource.CreateLinkedTokenSource(stop);timeout.CancelAfter(TimeSpan.FromSeconds(jobs.TimeoutSeconds));
var ct=timeout.Token;
try {
var input=Path.Combine(directory,"original");
using(var source=await storage.Client.GetObjectAsync(storage.Bucket,job.StagingKey,ct))
{
if(source.ContentLength>10485760)throw new InvalidDataException("invalid_image");
await using var file=File.Create(input);var buffer=new byte[65536];long count=0;
while(true) {
var n=await source.ResponseStream.ReadAsync(buffer,ct);if(n==0)break;
count+=n;if(count>10485760)throw new InvalidDataException("invalid_image");
await file.WriteAsync(buffer.AsMemory(0,n),ct);
}
}
await using(var source=File.OpenRead(input))
if(source.Length!=job.Size || Convert.ToHexString(await SHA256.HashDataAsync(source,ct)).ToLowerInvariant()!=job.Sha256)
throw new InvalidDataException("invalid_image");
var render=new ProcessStartInfo("dotnet") {UseShellExecute=false,RedirectStandardOutput=true,RedirectStandardError=true};
render.ArgumentList.Add(typeof(PipelineWorker).Assembly.Location);render.ArgumentList.Add("--render-image");
render.ArgumentList.Add(input);render.ArgumentList.Add(directory);
using(var process=Process.Start(render)!)
{
var stdout=process.StandardOutput.ReadToEndAsync(ct);var stderr=process.StandardError.ReadToEndAsync(ct);
try { await process.WaitForExitAsync(ct);await Task.WhenAll(stdout,stderr); }
catch { if(!process.HasExited)process.Kill(true);await process.WaitForExitAsync(CancellationToken.None);throw; }
if(process.ExitCode==2)throw new InvalidDataException("invalid_image");
if(process.ExitCode!=0)throw new IOException("renderer_unavailable");
}
var variants=JsonSerializer.Deserialize<VariantDto[]>(await File.ReadAllTextAsync(Path.Combine(directory,"variants.json"),ct),JobJson.Options)!;
await PutAsync($"originals/{job.AttachmentId:N}/v1",input,job.ContentType,ct);
foreach(var variant in variants)await PutAsync($"variants/{job.AttachmentId:N}/v1/{variant.Name}",Path.Combine(directory,variant.Name),variant.ContentType,ct);
await FaultInjection.HitAsync(config,"after_objects",job.Id,ct);
await using(var tx=await connection.BeginTransactionAsync(ct))
{
await using var done=JobStore.Command(connection,"""
UPDATE attachment_jobs SET state='ready',completed_at=clock_timestamp(),error_code=NULL,lease_token=NULL,lease_until=NULL
WHERE id=@id AND state='processing' AND lease_token=@lease AND lease_until>clock_timestamp()
""",("id",job.Id),("lease",job.Lease));
if(await done.ExecuteNonQueryAsync(ct)!=1)throw new OperationCanceledException("Lease expired");
await using var result=JobStore.Command(connection,"UPDATE attachments SET status='ready',variants=CAST(@variants AS jsonb),error_code=NULL,updated_at=clock_timestamp() WHERE id=@id",
("id",job.AttachmentId),("variants",JsonSerializer.Serialize(variants,JobJson.Options)));await result.ExecuteNonQueryAsync(ct);
await JobStore.EventAsync(connection,job.Id,"ready",job.Attempt,ct);await tx.CommitAsync(ct);
}
await FaultInjection.HitAsync(config,"after_commit",job.Id,stop);
Console.WriteLine($"Ready job {job.Id}");
} catch(Exception ex) when(!stop.IsCancellationRequested) {
var permanent=ex is InvalidDataException || ex is Amazon.S3.AmazonS3Exception s3 && s3.StatusCode==System.Net.HttpStatusCode.NotFound;
var code=permanent?"invalid_image":ex is OperationCanceledException?"processing_timeout":"dependency_unavailable";
await jobs.FailAsync(connection,job,permanent,code,stop);
Console.WriteLine($"Job {job.Id}: {code} ({ex.GetType().Name})");
}
finally {Directory.Delete(directory,true);}
}
finally {
await using var unlock=JobStore.Command(connection,"SELECT pg_advisory_unlock(hashtextextended(@scope,0))",("scope","job:"+envelope.JobId));
await unlock.ExecuteNonQueryAsync(CancellationToken.None);
}
}
private async Task PutAsync(string key,string path,string mime,CancellationToken ct)
{
await using var file=File.OpenRead(path);
await storage.Client.PutObjectAsync(new PutObjectRequest {BucketName=storage.Bucket,Key=key,InputStream=file,ContentType=mime,UseChunkEncoding=false,AutoCloseStream=false},ct);
}
}
@@ -0,0 +1,17 @@
using Npgsql;
namespace Messenger.Api.Pipeline;
public static class ProcessPresence
{
public static async Task RunAsync(NpgsqlDataSource source,string role,CancellationToken ct)
{
try {
while(!ct.IsCancellationRequested) {
try {
await using var cmd=source.CreateCommand("INSERT INTO pipeline_presence(role,seen_at) VALUES(@role,clock_timestamp()) ON CONFLICT(role) DO UPDATE SET seen_at=EXCLUDED.seen_at");
cmd.Parameters.AddWithValue("role",role);await cmd.ExecuteNonQueryAsync(ct);
} catch(Exception) when(!ct.IsCancellationRequested) { }
await Task.Delay(2000,ct);
}
} catch(OperationCanceledException) when(ct.IsCancellationRequested) { }
}
}
+131
View File
@@ -0,0 +1,131 @@
using Messenger.Api.Pipeline;
using System.Text.Json;
using System.Text.Json.Serialization;
using Messenger.Api.Data;
using Messenger.Api.Endpoints;
using Messenger.Api.Infrastructure;
using Messenger.Api.Middleware;
using Messenger.Api.Services;
using Npgsql;
using StackExchange.Redis;
ThreadPool.GetMinThreads(out var workerThreads, out var completionThreads);
ThreadPool.SetMinThreads(Math.Max(8, workerThreads), completionThreads);
if(args.Length==3 && args[0]=="--render-image")
{
try {ImageRenderer.Render(args[1],args[2]);}
catch(InvalidDataException) {Environment.ExitCode=2;}
catch {Environment.ExitCode=3;}
return;
}
var builder = WebApplication.CreateBuilder(args);
if(args.Contains("--bootstrap"))
{
using var storage=new S3Storage(builder.Configuration);
try { await storage.Client.GetBucketLocationAsync(storage.Bucket); }
catch(Amazon.S3.AmazonS3Exception ex) when(ex.StatusCode==System.Net.HttpStatusCode.NotFound)
{ await storage.Client.PutBucketAsync(storage.Bucket); }
Console.WriteLine("Private bucket ready");
return;
}
var host = Environment.GetEnvironmentVariable("APP_HOST") ?? "0.0.0.0";
var port = Environment.GetEnvironmentVariable("APP_PORT") ?? "8080";
builder.WebHost.UseUrls($"http://{host}:{port}");
builder.Services.ConfigureHttpJsonOptions(options =>
{
options.SerializerOptions.PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower;
options.SerializerOptions.UnmappedMemberHandling = JsonUnmappedMemberHandling.Disallow;
});
var databaseUrlFile = Environment.GetEnvironmentVariable("DATABASE_URL_FILE");
var databaseUrl = databaseUrlFile is not null
? File.ReadAllText(databaseUrlFile).Trim()
: Environment.GetEnvironmentVariable("DATABASE_URL");
if (string.IsNullOrWhiteSpace(databaseUrl))
throw new InvalidOperationException("Set DATABASE_URL_FILE or DATABASE_URL before starting the API");
var connectionString = new NpgsqlConnectionStringBuilder(databaseUrl)
{
MaxPoolSize = int.TryParse(builder.Configuration["DB_POOL_SIZE"], out var poolSize) && poolSize > 0 ? poolSize : 20,
Timeout = 5,
ApplicationName = "messenger-api",
CommandTimeout = 10
};
var dataSource = NpgsqlDataSource.Create(connectionString.ConnectionString);
if (args.Contains("--migrate", StringComparer.Ordinal))
{
await DatabaseMigrator.ApplyAsync(dataSource);
await dataSource.DisposeAsync();
return;
}
if(args.Contains("--worker") || args.Contains("--dispatcher") || args.Contains("--replay") || args.Contains("--redeliver") || args.Contains("--reconcile"))
{
using var storage=new S3Storage(builder.Configuration);
using var cancellation=new CancellationTokenSource();
Console.CancelKeyPress+=(_,e)=>{e.Cancel=true;cancellation.Cancel();};
using var signal=System.Runtime.InteropServices.PosixSignalRegistration.Create(System.Runtime.InteropServices.PosixSignal.SIGTERM,ctx=>{ctx.Cancel=true;cancellation.Cancel();});
var heartbeat=(args.Contains("--worker") || args.Contains("--dispatcher"))
? ProcessPresence.RunAsync(dataSource,args.Contains("--worker")?"worker":"dispatcher",cancellation.Token) : Task.CompletedTask;
try {
if(args.Contains("--worker"))await new PipelineWorker(dataSource,storage,builder.Configuration).RunAsync(cancellation.Token);
else if(args.Contains("--dispatcher"))await new OutboxDispatcher(dataSource,storage,builder.Configuration).RunAsync(cancellation.Token);
else if(args.Contains("--redeliver"))await JobOperations.RedeliverAsync(dataSource,builder.Configuration,Guid.Parse(args[Array.IndexOf(args,"--redeliver")+1]),cancellation.Token);
else if(args.Contains("--replay"))await new JobStore(dataSource,builder.Configuration).ReplayAsync(Guid.Parse(args[Array.IndexOf(args,"--replay")+1]),cancellation.Token);
else await new PipelineMaintenance(dataSource,storage,builder.Configuration).ReconcileStagingAsync(args.Contains("--apply"),cancellation.Token);
}catch(OperationCanceledException) when(cancellation.IsCancellationRequested) { }
cancellation.Cancel();await heartbeat;
await dataSource.DisposeAsync();return;
}
builder.Services.AddSingleton(dataSource);
builder.Services.AddSingleton<InstanceIdentity>();
builder.Services.AddSingleton<ReadinessState>();
builder.Services.Configure<HostOptions>(options => options.ShutdownTimeout = TimeSpan.FromSeconds(8));
builder.Services.AddSingleton<AppDatabase>();
builder.Services.AddSingleton<S3Storage>();
builder.Services.AddSingleton<AttachmentService>();
var redisOptions = ConfigurationOptions.Parse(builder.Configuration["REDIS_URL"] ?? "localhost:6379");
redisOptions.AbortOnConnectFail = false;
redisOptions.ConnectTimeout = 500;
redisOptions.ConnectRetry = 0;
redisOptions.AsyncTimeout = 500;
redisOptions.SyncTimeout = 500;
redisOptions.BacklogPolicy = BacklogPolicy.FailFast;
builder.Services.AddSingleton<IConnectionMultiplexer>(await ConnectionMultiplexer.ConnectAsync(redisOptions));
builder.Services.AddSingleton<RedisSessionCache>();
builder.Services.AddSingleton<PasswordService>();
builder.Services.AddSingleton<UserService>();
builder.Services.AddSingleton<SessionService>();
builder.Services.AddSingleton<ChatService>();
builder.Services.AddSingleton<MessageService>();
builder.Services.AddSingleton<ClusterPresence>();
builder.Services.AddHostedService(services => services.GetRequiredService<ClusterPresence>());
var app = builder.Build();
app.Lifetime.ApplicationStopping.Register(app.Services.GetRequiredService<ReadinessState>().Stop);
app.UseMiddleware<RequestIdMiddleware>();
app.UseMiddleware<InstanceMiddleware>();
app.UseMiddleware<RequestLoggingMiddleware>();
app.UseMiddleware<ExceptionHandlingMiddleware>();
app.UseDefaultFiles();
app.UseStaticFiles(new StaticFileOptions
{
OnPrepareResponse = context => context.Context.Response.Headers.CacheControl = "no-cache"
});
app.MapHealthEndpoints();
app.MapUserEndpoints();
app.MapAuthEndpoints();
app.MapChatEndpoints();
app.MapMessageEndpoints();
app.MapAttachmentEndpoints();
app.MapUiEndpoints();
app.MapOperationsEndpoints();
app.MapReactionEndpoints();
app.Run();
@@ -0,0 +1,23 @@
{
"$schema": "https://json.schemastore.org/launchsettings.json",
"profiles": {
"http": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"applicationUrl": "http://localhost:5052",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
},
"https": {
"commandName": "Project",
"dotnetRunMessages": true,
"launchBrowser": true,
"applicationUrl": "https://localhost:7253;http://localhost:5052",
"environmentVariables": {
"ASPNETCORE_ENVIRONMENT": "Development"
}
}
}
}
+93
View File
@@ -0,0 +1,93 @@
using System.Globalization;
using System.Text.Json;
using Messenger.Api.Contracts;
namespace Messenger.Api.Services;
public static class ApiSupport
{
public static async Task<(T? Value, IResult? Error)> ReadJsonAsync<T>(HttpContext context)
{
if (!context.Request.HasJsonContentType())
return (default, ApiError(context, 400, "invalid_request", "Требуется application/json"));
try
{
var value = await context.Request.ReadFromJsonAsync<T>();
return value is null
? (default, ApiError(context, 400, "invalid_request", "Тело запроса обязательно"))
: (value, null);
}
catch (Exception exception) when (exception is JsonException or NotSupportedException or BadHttpRequestException)
{
return (default, ApiError(context, 400, "invalid_request", "Некорректный JSON"));
}
}
public static IResult Unauthorized(HttpContext context, bool basic = false)
{
context.Response.Headers["WWW-Authenticate"] = basic
? "Basic realm=\"messenger\", charset=\"UTF-8\""
: "Bearer realm=\"messenger\"";
return ApiError(context, 401, "unauthorized", "Требуется действительная сессия");
}
public static (ChatCursor? Cursor, DateTime? Time, IResult? Error) ParseCursor(
HttpContext context, string scope)
{
if (!context.Request.Query.TryGetValue("cursor", out var value))
return (null, null, null);
var cursor = CursorService.DecodeChatCursor(value.ToString());
if (cursor is null || cursor.Scope != scope || cursor.Id == Guid.Empty ||
!DateTime.TryParse(cursor.CreatedAt, CultureInfo.InvariantCulture,
DateTimeStyles.RoundtripKind, out var time) || time.Kind != DateTimeKind.Utc)
return (null, null, ApiError(context, 400, "invalid_request", "Некорректный cursor"));
return (cursor, time, null);
}
public static (int Limit, IResult? Error) ParseLimit(HttpContext context)
{
const int defaultLimit = 50;
if (!context.Request.Query.TryGetValue("limit", out var value))
return (defaultLimit, null);
if (!int.TryParse(value.ToString(), out var limit) ||
limit < 1 ||
limit > 100)
{
return (
0,
ApiError(context, 400, "invalid_request", "limit должен быть от 1 до 100")
);
}
return (limit, null);
}
public static IResult ApiError(
HttpContext context,
int status,
string code,
string message)
{
var requestId =
context.Items["RequestId"]?.ToString()
?? Guid.NewGuid().ToString();
return Results.Json(
new
{
error = new
{
code,
message,
request_id = requestId
}
},
statusCode: status
);
}
}

Some files were not shown because too many files have changed in this diff Show More